Add LayoutComplexity struct to PdfProcessResult so callers can detect when a PDF has complex layout (tables or multi-column text) and decide whether to use the extracted markdown or fall back to OCR. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
252 lines
8.3 KiB
Rust
252 lines
8.3 KiB
Rust
//! Shared types used across the extraction and markdown pipelines.
|
|
//!
|
|
//! Centralises `TextItem`, `TextLine`, `PdfRect`, font-width / encoding
|
|
//! type aliases, and the `ItemType` enum so that every module can import
|
|
//! them from one place.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use crate::text_utils::should_join_items;
|
|
|
|
// ── Font types (crate-internal) ──────────────────────────────────────
|
|
|
|
/// Font encoding map: maps byte codes to Unicode characters
|
|
pub(crate) type FontEncodingMap = HashMap<u8, char>;
|
|
|
|
/// All font encodings for a page
|
|
pub(crate) type PageFontEncodings = HashMap<String, FontEncodingMap>;
|
|
|
|
/// Font width information extracted from PDF font dictionaries
|
|
#[derive(Debug, Clone)]
|
|
#[allow(dead_code)]
|
|
pub(crate) struct FontWidthInfo {
|
|
/// Glyph widths: maps character code to width in font units
|
|
pub(crate) widths: HashMap<u16, u16>,
|
|
/// Default width for glyphs not in the widths table
|
|
pub(crate) default_width: u16,
|
|
/// Width of the space character (code 32) if known
|
|
pub(crate) space_width: u16,
|
|
/// Whether this is a CID font (2-byte character codes)
|
|
pub(crate) is_cid: bool,
|
|
/// Scale factor to convert font units to text space units.
|
|
/// For Type1/TrueType: 0.001 (widths in 1000ths of em)
|
|
/// For Type3: FontMatrix[0] (e.g., 0.00048828125 for 2048-unit grid)
|
|
pub(crate) units_scale: f32,
|
|
/// Writing mode: 0 = horizontal (default), 1 = vertical
|
|
pub(crate) wmode: u8,
|
|
}
|
|
|
|
/// All font width info for a page, keyed by font resource name
|
|
pub(crate) type PageFontWidths = HashMap<String, FontWidthInfo>;
|
|
|
|
// ── Public types ─────────────────────────────────────────────────────
|
|
|
|
/// Type of extracted item
|
|
#[derive(Debug, Clone, Default)]
|
|
pub enum ItemType {
|
|
/// Regular text content
|
|
#[default]
|
|
Text,
|
|
/// Image placeholder
|
|
Image,
|
|
/// Hyperlink (with URL)
|
|
Link(String),
|
|
/// Form field (name: value)
|
|
FormField,
|
|
}
|
|
|
|
/// Layout complexity analysis result.
|
|
///
|
|
/// Callers can use this to decide whether the extracted markdown is reliable
|
|
/// or whether the PDF should be routed to an OCR pipeline instead.
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct LayoutComplexity {
|
|
/// True if any page has tables or multi-column text.
|
|
pub is_complex: bool,
|
|
/// 1-indexed pages where table borders were detected (rect count > 6).
|
|
pub pages_with_tables: Vec<u32>,
|
|
/// 1-indexed pages where 2+ text columns were detected.
|
|
pub pages_with_columns: Vec<u32>,
|
|
}
|
|
|
|
/// A rectangle from a PDF `re` operator (cell boundary, border, etc.)
|
|
#[derive(Debug, Clone)]
|
|
pub struct PdfRect {
|
|
pub x: f32,
|
|
pub y: f32,
|
|
pub width: f32,
|
|
pub height: f32,
|
|
pub page: u32,
|
|
}
|
|
|
|
/// A text item with position information
|
|
#[derive(Debug, Clone)]
|
|
pub struct TextItem {
|
|
/// The text content
|
|
pub text: String,
|
|
/// X position on page
|
|
pub x: f32,
|
|
/// Y position on page (PDF coordinates, origin at bottom-left)
|
|
pub y: f32,
|
|
/// Width of text
|
|
pub width: f32,
|
|
/// Height (approximated from font size)
|
|
pub height: f32,
|
|
/// Font name
|
|
pub font: String,
|
|
/// Font size
|
|
pub font_size: f32,
|
|
/// Page number (1-indexed)
|
|
pub page: u32,
|
|
/// Whether the font is bold
|
|
pub is_bold: bool,
|
|
/// Whether the font is italic
|
|
pub is_italic: bool,
|
|
/// Type of item (text, image, link)
|
|
pub item_type: ItemType,
|
|
}
|
|
|
|
/// A line of text (grouped text items)
|
|
#[derive(Debug, Clone)]
|
|
pub struct TextLine {
|
|
pub items: Vec<TextItem>,
|
|
pub y: f32,
|
|
pub page: u32,
|
|
}
|
|
|
|
impl TextLine {
|
|
pub fn text(&self) -> String {
|
|
self.text_with_formatting(false, false)
|
|
}
|
|
|
|
/// Get text with optional bold/italic markdown formatting
|
|
pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String {
|
|
if !format_bold && !format_italic {
|
|
return self.text_plain();
|
|
}
|
|
|
|
let mut result = String::new();
|
|
let mut current_bold = false;
|
|
let mut current_italic = false;
|
|
|
|
for (i, item) in self.items.iter().enumerate() {
|
|
let text = item.text.as_str();
|
|
let text_trimmed = text.trim();
|
|
|
|
// Skip empty items
|
|
if text_trimmed.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Determine spacing
|
|
let needs_space = if i == 0 || result.is_empty() {
|
|
false
|
|
} else {
|
|
let prev_item = &self.items[i - 1];
|
|
self.needs_space_between(prev_item, item, &result)
|
|
};
|
|
|
|
// Preserve leading whitespace from the item text.
|
|
// Items like " means any person" have a leading space that indicates
|
|
// a word boundary. needs_space_between returns false for these (because
|
|
// space_already_exists), but we still need to emit the space since
|
|
// we push text_trimmed below (which strips it).
|
|
let has_leading_space = text.starts_with(' ');
|
|
|
|
// Check for style changes
|
|
let item_bold = format_bold && item.is_bold;
|
|
let item_italic = format_italic && item.is_italic;
|
|
|
|
// Close previous styles if they change
|
|
if current_italic && !item_italic {
|
|
result.push('*');
|
|
current_italic = false;
|
|
}
|
|
if current_bold && !item_bold {
|
|
result.push_str("**");
|
|
current_bold = false;
|
|
}
|
|
|
|
// Add space: either from spacing logic or preserved from item text
|
|
if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) {
|
|
result.push(' ');
|
|
}
|
|
|
|
// Open new styles
|
|
if item_bold && !current_bold {
|
|
result.push_str("**");
|
|
current_bold = true;
|
|
}
|
|
if item_italic && !current_italic {
|
|
result.push('*');
|
|
current_italic = true;
|
|
}
|
|
|
|
result.push_str(text_trimmed);
|
|
}
|
|
|
|
// Close any remaining open styles
|
|
if current_italic {
|
|
result.push('*');
|
|
}
|
|
if current_bold {
|
|
result.push_str("**");
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
/// Get plain text without formatting
|
|
fn text_plain(&self) -> String {
|
|
let mut result = String::new();
|
|
for (i, item) in self.items.iter().enumerate() {
|
|
let text = item.text.as_str();
|
|
if i == 0 {
|
|
result.push_str(text);
|
|
} else {
|
|
let prev_item = &self.items[i - 1];
|
|
if self.needs_space_between(prev_item, item, &result) {
|
|
result.push(' ');
|
|
}
|
|
result.push_str(text);
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Determine if a space is needed between two items
|
|
fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
|
|
let text = item.text.as_str();
|
|
|
|
// Don't add space before/after hyphens for hyphenated words
|
|
let prev_ends_with_hyphen = result.ends_with('-');
|
|
let curr_is_hyphen = text.trim() == "-";
|
|
let curr_starts_with_hyphen = text.starts_with('-');
|
|
|
|
// Detect subscript/superscript: smaller font size and/or Y offset
|
|
let font_ratio = item.font_size / prev_item.font_size;
|
|
let reverse_font_ratio = prev_item.font_size / item.font_size;
|
|
let y_diff = (item.y - prev_item.y).abs();
|
|
|
|
let is_sub_super = font_ratio < 0.85 && y_diff > 1.0;
|
|
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
|
|
|
|
// Use position-based spacing detection
|
|
let should_join = should_join_items(prev_item, item);
|
|
|
|
// Check if space already exists
|
|
let prev_ends_with_space = result.ends_with(' ');
|
|
let curr_starts_with_space = text.starts_with(' ');
|
|
let space_already_exists = prev_ends_with_space || curr_starts_with_space;
|
|
|
|
// Add space unless one of these conditions applies
|
|
!(prev_ends_with_hyphen
|
|
|| curr_is_hyphen
|
|
|| curr_starts_with_hyphen
|
|
|| is_sub_super
|
|
|| was_sub_super
|
|
|| should_join
|
|
|| space_already_exists)
|
|
}
|
|
}
|