From 7328af91bdfa6b1013b666e1070d47fd5d07b552 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Feb 2026 10:16:11 -0800 Subject: [PATCH] chore(refactor): Better organization of the codebase, split modules, update docs, add AGENTS.md --- AGENTS.md | 138 ++ README.md | 72 +- src/extractor.rs | 3998 ------------------------------- src/extractor/content_stream.rs | 608 +++++ src/extractor/fonts.rs | 582 +++++ src/extractor/layout.rs | 635 +++++ src/extractor/links.rs | 320 +++ src/extractor/mod.rs | 893 +++++++ src/extractor/xobjects.rs | 474 ++++ src/lib.rs | 7 +- src/markdown.rs | 1805 -------------- src/markdown/analysis.rs | 201 ++ src/markdown/classify.rs | 180 ++ src/markdown/convert.rs | 670 ++++++ src/markdown/mod.rs | 382 +++ src/markdown/postprocess.rs | 275 +++ src/markdown/preprocess.rs | 142 ++ src/tables.rs | 2418 ------------------- src/tables/detect_heuristic.rs | 1061 ++++++++ src/tables/detect_rects.rs | 357 +++ src/tables/financial.rs | 115 + src/tables/format.rs | 148 ++ src/tables/grid.rs | 308 +++ src/tables/mod.rs | 452 ++++ src/text_utils.rs | 368 +++ src/types.rs | 237 ++ tests/integration_tests.rs | 5 +- 27 files changed, 8619 insertions(+), 8232 deletions(-) create mode 100644 AGENTS.md delete mode 100644 src/extractor.rs create mode 100644 src/extractor/content_stream.rs create mode 100644 src/extractor/fonts.rs create mode 100644 src/extractor/layout.rs create mode 100644 src/extractor/links.rs create mode 100644 src/extractor/mod.rs create mode 100644 src/extractor/xobjects.rs delete mode 100644 src/markdown.rs create mode 100644 src/markdown/analysis.rs create mode 100644 src/markdown/classify.rs create mode 100644 src/markdown/convert.rs create mode 100644 src/markdown/mod.rs create mode 100644 src/markdown/postprocess.rs create mode 100644 src/markdown/preprocess.rs delete mode 100644 src/tables.rs create mode 100644 src/tables/detect_heuristic.rs create mode 100644 src/tables/detect_rects.rs create mode 100644 src/tables/financial.rs create mode 100644 src/tables/format.rs create mode 100644 src/tables/grid.rs create mode 100644 src/tables/mod.rs create mode 100644 src/text_utils.rs create mode 100644 src/types.rs diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..059b73d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,138 @@ +# AGENTS.md — pdf-inspector Codebase Guide + +## Project Overview + +Rust crate (`pdf-inspector`) that extracts text from PDFs and converts it to structured Markdown. Ships a CLI binary `pdf2md`. + +- **Crate name:** `pdf-inspector` +- **Binary:** `pdf2md` (`src/bin/pdf2md.rs`) +- **PDF parsing:** `lopdf` crate (v0.39.0, git dependency) +- **Test PDFs:** `/Users/abimaelmartell/Code/pdf-evals/pdfs/` + +## Module Map + +``` +src/ + lib.rs — Public API, re-exports + types.rs — Shared types: TextItem, TextLine, PdfRect, ItemType + text_utils.rs — Character/text helpers: CJK, RTL, ligatures, bold/italic + detector.rs — Fast PDF type detection (text vs scanned) without full load + glyph_names.rs — Adobe Glyph List → Unicode mapping + tounicode.rs — ToUnicode CMap parsing for CID-encoded text + + extractor/ + mod.rs — Public API: extract_text, extract_text_with_positions + fonts.rs — Font width parsing, encoding, text decoding + content_stream.rs — PDF operator state machine (Tm, Td, Tj, TJ, etc.) + xobjects.rs — Form XObject and image XObject extraction + links.rs — Hyperlink and AcroForm field extraction + layout.rs — Column detection, line grouping, reading order + + tables/ + mod.rs — Table struct, TableDetectionMode, re-exports + detect_rects.rs — Rectangle-based table detection (union-find clustering) + detect_heuristic.rs — Heuristic table detection + validation + financial.rs — Financial token splitting for consolidated values + grid.rs — Column/row boundaries, cell assignment + format.rs — Table → Markdown formatting, footnotes + + markdown/ + mod.rs — MarkdownOptions, public API (to_markdown, to_markdown_from_items) + convert.rs — Core line-to-markdown loop, table/image interleaving + analysis.rs — Font statistics, heading tiers, paragraph thresholds + classify.rs — Caption, list, code detection + preprocess.rs — Heading merging, drop cap handling + postprocess.rs — Cleanup: dot leaders, hyphenation, page numbers, URLs + + bin/ + pdf2md.rs — CLI: PDF → Markdown + detect_pdf.rs — CLI: detect PDF type + debug_spaces.rs — Debug: dump text items with x/y/width per page + dump_ops.rs — Debug: dump raw PDF content stream operators + debug_ygaps.rs — Debug: Y-gap analysis between lines + debug_fonts.rs — Debug: font information + debug_ligatures.rs — Debug: ligature expansion + debug_order.rs — Debug: reading order + debug_pages.rs — Debug: page-level info + detection_report.rs — Batch detection report on PDF directory + profile_stages.rs — Performance profiling of pipeline stages +``` + +## Data Flow + +``` +PDF bytes + │ + ├─► detector.rs → PdfType (TextBased / Scanned / ImageBased) + │ + └─► extractor/ + ├─ fonts.rs → font widths, encodings + ├─ content_stream.rs → walk operators → Vec + Vec + ├─ xobjects.rs → Form XObject text, image placeholders + ├─ links.rs → hyperlinks, AcroForm fields + └─ layout.rs → column detection → group_into_lines → Vec + │ + ├─► tables/ + │ ├─ detect_rects.rs → rect-based tables (PdfRect clusters) + │ ├─ detect_heuristic.rs → heuristic tables (font-size + alignment) + │ ├─ grid.rs → column/row assignment → cells + │ └─ format.rs → Table → Markdown string + │ + └─► markdown/ + ├─ analysis.rs → font stats, heading tiers + ├─ preprocess.rs → merge headings, drop caps + ├─ convert.rs → line loop + table/image insertion + ├─ classify.rs → captions, lists, code + └─ postprocess.rs → cleanup → final Markdown string +``` + +## Critical Implementation Details + +### Text Matrix Math +PDF text positioning uses two matrices: `text_matrix` (Tm) and `line_matrix`. The `Td`/`TD` operators provide offsets in **text space**, which must be scaled by `line_matrix`: + +``` +e += tx * a + ty * c +f += tx * b + ty * d +``` + +When `Tm` has scaling (e.g., `[12,0,0,12,x,y]`), failing to apply this scaling produces incorrect positions. The `T*` and `'` operators are equivalent to `0 -TL Td` and need the same treatment. + +### Font Size +Font size can come from the `Tf` operand **or** the `Tm` matrix scaling. Use `effective_font_size()` from `text_utils.rs` to get the correct value. + +### White-Fill Text +Text drawn with white fill (`1 g` before text ops) should be skipped during extraction but the text matrix must still advance to keep positions correct. + +### CID Fonts +Fonts named `C2_*` or `C0_*` are CID fonts that emit one word per `Tj` operator. Spaces must be inserted between consecutive `Tj` items. + +### lopdf Quirks +- `lopdf::error::ParseError` is private — match by string for `InvalidFileHeader` +- Clippy enforces `-D warnings` — use `is_some_and(...)` instead of `map_or(false, ...)` + +## Testing + +```bash +cargo test # Run all 66 unit tests +cargo clippy -- -D warnings # Lint (enforced in CI) +cargo fmt --check # Format check +cargo run --release --bin pdf2md -- # Smoke test +``` + +## Common Tasks + +| Task | Where to Edit | +|------|--------------| +| Fix text positioning bugs | `extractor/content_stream.rs` | +| Add font encoding support | `extractor/fonts.rs`, `tounicode.rs` | +| Fix column/reading order | `extractor/layout.rs` | +| Improve table detection | `tables/detect_heuristic.rs` | +| Fix table formatting | `tables/format.rs`, `tables/grid.rs` | +| Add rectangle-based tables | `tables/detect_rects.rs` | +| Change heading detection | `markdown/analysis.rs` | +| Fix list/code detection | `markdown/classify.rs` | +| Fix paragraph breaks | `markdown/convert.rs`, `markdown/analysis.rs` | +| Fix URL/hyphenation cleanup | `markdown/postprocess.rs` | +| Add new PDF type detection | `detector.rs` | +| Add new text item type | `types.rs`, then update consumers | diff --git a/README.md b/README.md index 9d48419..c52c03e 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in - **Smart classification** — Detect TextBased, Scanned, ImageBased, or Mixed PDFs in ~10-50ms by sampling content streams. Returns a confidence score (0.0-1.0) and per-page OCR routing. - **Text extraction** — Position-aware extraction with font info, X/Y coordinates, and automatic multi-column reading order. -- **Markdown conversion** — Headings (H1-H4 via font size ratios), bullet/numbered/letter lists, code blocks (monospace font detection), tables, subscript/superscript, URL linking, and page breaks. -- **CID font support** — Proper ToUnicode CMap decoding for Type0/Identity-H fonts, UTF-16BE, UTF-8, and Latin-1 encodings. +- **Markdown conversion** — Headings (H1-H4 via font size ratios), bullet/numbered/letter lists, code blocks (monospace font detection), tables (rectangle-based and heuristic), bold/italic formatting, URL linking, and page breaks. +- **Table detection** — Dual-mode: rectangle-based detection from PDF drawing ops, plus heuristic detection from text alignment. Handles financial tables, footnotes, and continuation tables across pages. +- **CID font support** — ToUnicode CMap decoding for Type0/Identity-H fonts, UTF-16BE, UTF-8, and Latin-1 encodings. +- **Multi-column layout** — Automatic detection of newspaper-style columns, sequential reading order, and RTL text support. - **Lightweight** — Pure Rust, no ML models, no external services. Single dependency on `lopdf` for PDF parsing. ## Quick start @@ -107,6 +109,50 @@ cargo run --bin detect-pdf -- document.pdf cargo run --bin detect-pdf -- document.pdf --json ``` +## Architecture + +``` +PDF bytes + │ + ├─► detector → PdfType (TextBased / Scanned / ImageBased / Mixed) + │ + └─► extractor + ├─ fonts → font widths, encodings + ├─ content_stream → walk PDF operators → TextItems + PdfRects + ├─ xobjects → Form XObject text, image placeholders + ├─ links → hyperlinks, AcroForm fields + └─ layout → column detection → line grouping → reading order + │ + ├─► tables + │ ├─ detect_rects → rectangle-based tables (union-find) + │ ├─ detect_heuristic → alignment-based tables + │ ├─ grid → column/row assignment → cells + │ └─ format → cells → Markdown table + │ + └─► markdown + ├─ analysis → font stats, heading tiers + ├─ preprocess → merge headings, drop caps + ├─ convert → line loop + table/image insertion + ├─ classify → captions, lists, code + └─ postprocess → cleanup → final Markdown +``` + +### Project structure + +``` +src/ + lib.rs — Public API, re-exports + types.rs — Shared types: TextItem, TextLine, PdfRect, ItemType + text_utils.rs — Character/text helpers (CJK, RTL, ligatures, bold/italic) + detector.rs — Fast PDF type detection without full document load + glyph_names.rs — Adobe Glyph List → Unicode mapping + tounicode.rs — ToUnicode CMap parsing for CID-encoded text + extractor/ — Text extraction pipeline + tables/ — Table detection and formatting + markdown/ — Markdown conversion and structure detection + bin/ — CLI tools and debug utilities +``` + ## How classification works 1. Parse the xref table and page tree (no full object load) @@ -141,8 +187,9 @@ This detects 300+ page PDFs in milliseconds. The result includes `pages_needing_ | `detect_pdf_type_mem_with_config(bytes, config)` | Classification from bytes with custom config | | `extract_text(path)` | Plain text extraction | | `extract_text_with_positions(path)` | Text with X/Y coordinates and font info | -| `to_markdown(path, options)` | Convert directly to Markdown | +| `to_markdown(text, options)` | Convert plain text to Markdown | | `to_markdown_from_items(items, options)` | Markdown from pre-extracted `TextItem`s | +| `to_markdown_from_items_with_rects(items, options, rects)` | Markdown with rectangle-based table detection | ### Types @@ -163,18 +210,31 @@ The converter handles: | Element | How it's detected | |---|---| -| Headings (H1-H4) | Font size ratios relative to body text | +| Headings (H1-H4) | Font size tiers relative to body text, with 0.5pt clustering | +| Bold/italic | Font name patterns (Bold, Italic, Oblique) | | Bullet lists | `*`, `-`, `*`, `○`, `●`, `◦` prefixes | | Numbered lists | `1.`, `1)`, `(1)` patterns | | Letter lists | `a.`, `a)`, `(a)` patterns | | Code blocks | Monospace fonts (Courier, Consolas, Monaco, Menlo, Fira Code, JetBrains Mono) and keyword detection | -| Tables | Position clustering for column/row boundaries | -| Footnotes | Superscript numbers with corresponding text | +| Tables | Rectangle-based detection from PDF drawing ops + heuristic detection from text alignment | +| Financial tables | Token splitting for consolidated numeric values | +| Captions | "Figure", "Table", "Source:" prefix detection | | Sub/superscript | Font size and Y-offset relative to baseline | | URLs | Converted to Markdown links | | Hyphenation | Rejoins words broken across lines | | Page numbers | Filtered from output | | Drop caps | Large initial letters merged with following text | +| Dot leaders | TOC-style dots collapsed to " ... " | + +## Debug tools + +```bash +cargo run --bin debug_spaces -- file.pdf # Text items with x/y/width per page +cargo run --bin dump_ops -- file.pdf # Raw PDF content stream operators +cargo run --bin debug_ygaps -- file.pdf # Y-gap analysis between lines +cargo run --bin debug_fonts -- file.pdf # Font information +cargo run --bin debug_order -- file.pdf # Reading order visualization +``` ## Use case: smart PDF routing diff --git a/src/extractor.rs b/src/extractor.rs deleted file mode 100644 index 8ac94bd..0000000 --- a/src/extractor.rs +++ /dev/null @@ -1,3998 +0,0 @@ -//! Text extraction from PDF using lopdf -//! -//! This module extracts text with position information for structure detection. - -use crate::glyph_names::glyph_to_char; -use crate::tounicode::FontCMaps; -use crate::PdfError; -use lopdf::{Document, Encoding, Object, ObjectId}; -use std::collections::{HashMap, HashSet}; -use std::path::Path; - -/// Font encoding map: maps byte codes to Unicode characters -type FontEncodingMap = HashMap; - -/// All font encodings for a page -type PageFontEncodings = HashMap; - -/// Font width information extracted from PDF font dictionaries -#[derive(Debug, Clone)] -#[allow(dead_code)] -struct FontWidthInfo { - /// Glyph widths: maps character code to width in font units - widths: HashMap, - /// Default width for glyphs not in the widths table - default_width: u16, - /// Width of the space character (code 32) if known - space_width: u16, - /// Whether this is a CID font (2-byte character codes) - is_cid: bool, - /// Scale factor to convert font units to text space units. - /// For Type1/TrueType: 0.001 (widths in 1000ths of em) - /// For Type3: FontMatrix[0] (e.g., 0.00048828125 for 2048-unit grid) - units_scale: f32, - /// Writing mode: 0 = horizontal (default), 1 = vertical - wmode: u8, -} - -/// All font width info for a page, keyed by font resource name -type PageFontWidths = HashMap; - -/// Resolve a PDF object reference to an array -fn resolve_array<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Vec> { - match obj { - Object::Array(arr) => Some(arr), - Object::Reference(r) => { - if let Ok(Object::Array(arr)) = doc.get_object(*r) { - Some(arr) - } else { - None - } - } - _ => None, - } -} - -/// Resolve a PDF object reference to a dictionary -fn resolve_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a lopdf::Dictionary> { - match obj { - Object::Dictionary(d) => Some(d), - Object::Reference(r) => doc.get_dictionary(*r).ok(), - _ => None, - } -} - -/// Build font width info for all fonts on a page -fn build_font_widths( - doc: &Document, - fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, -) -> PageFontWidths { - let mut widths = PageFontWidths::new(); - - for (font_name, font_dict) in fonts { - let resource_name = String::from_utf8_lossy(font_name).to_string(); - if let Some(info) = parse_font_widths(doc, font_dict) { - widths.insert(resource_name, info); - } - } - - widths -} - -/// Parse font widths from a font dictionary, dispatching by Subtype -fn parse_font_widths(doc: &Document, font_dict: &lopdf::Dictionary) -> Option { - // Get the font subtype - let subtype = font_dict.get(b"Subtype").ok()?; - let subtype_name = subtype.as_name().ok()?; - - match subtype_name { - b"Type0" => parse_type0_widths(doc, font_dict), - b"Type1" | b"TrueType" | b"MMType1" | b"Type3" => parse_simple_font_widths(doc, font_dict), - _ => None, - } -} - -/// Parse widths for simple fonts (Type1, TrueType, MMType1, Type3) -/// Reads FirstChar, LastChar, and Widths array. -/// For Type3 fonts, reads FontMatrix to determine the correct units_scale. -fn parse_simple_font_widths( - doc: &Document, - font_dict: &lopdf::Dictionary, -) -> Option { - let first_char = font_dict.get(b"FirstChar").ok().and_then(|o| match o { - Object::Integer(n) => Some(*n as u16), - Object::Reference(r) => doc.get_object(*r).ok().and_then(|o| { - if let Object::Integer(n) = o { - Some(*n as u16) - } else { - None - } - }), - _ => None, - })?; - - let last_char = font_dict.get(b"LastChar").ok().and_then(|o| match o { - Object::Integer(n) => Some(*n as u16), - Object::Reference(r) => doc.get_object(*r).ok().and_then(|o| { - if let Object::Integer(n) = o { - Some(*n as u16) - } else { - None - } - }), - _ => None, - })?; - - let widths_obj = font_dict.get(b"Widths").ok()?; - let widths_array = resolve_array(doc, widths_obj)?; - - let mut widths = HashMap::new(); - let mut space_width: u16 = 0; - - for (i, w_obj) in widths_array.iter().enumerate() { - let code = first_char + i as u16; - if code > last_char { - break; - } - let w = match w_obj { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - Object::Reference(r) => { - if let Ok(obj) = doc.get_object(*r) { - match obj { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => continue, - } - } else { - continue; - } - } - _ => continue, - }; - if code == 32 { - space_width = w; - } - widths.insert(code, w); - } - - // Determine units_scale: for Type3 fonts, use FontMatrix[0]; for others, use 1/1000 - let units_scale = if let Ok(fm) = font_dict.get(b"FontMatrix") { - if let Some(arr) = resolve_array(doc, fm) { - if !arr.is_empty() { - match &arr[0] { - Object::Real(r) => r.abs(), - Object::Integer(i) => (*i as f32).abs(), - _ => 0.001, - } - } else { - 0.001 - } - } else { - 0.001 - } - } else { - 0.001 // Standard 1000-unit system - }; - - // If space width wasn't found in the table, estimate from font metrics. - // The default of 250 is calibrated for standard 1000-unit fonts (units_scale=0.001). - // For Type3 fonts with different coordinate systems, use average glyph width instead. - if space_width == 0 { - if !widths.is_empty() && (units_scale - 0.001).abs() > 0.0005 { - // Non-standard scale: estimate space as ~45% of average glyph width - let sum: u32 = widths.values().map(|&w| w as u32).sum(); - let avg = sum as f32 / widths.len() as f32; - space_width = (avg * 0.45).max(1.0) as u16; - } else { - space_width = 250; - } - } - - Some(FontWidthInfo { - widths, - default_width: 0, - space_width, - is_cid: false, - units_scale, - wmode: 0, - }) -} - -/// Parse widths for Type0 (composite/CID) fonts -/// Reads DescendantFonts → CIDFont → W array and DW value -fn parse_type0_widths(doc: &Document, font_dict: &lopdf::Dictionary) -> Option { - let desc_fonts_obj = font_dict.get(b"DescendantFonts").ok()?; - let desc_fonts = resolve_array(doc, desc_fonts_obj)?; - - if desc_fonts.is_empty() { - return None; - } - - // Get the first descendant font dictionary - let cid_font_dict = resolve_dict(doc, &desc_fonts[0])?; - - // Get DW (default width) - let default_width = cid_font_dict - .get(b"DW") - .ok() - .and_then(|o| match o { - Object::Integer(n) => Some(*n as u16), - Object::Real(n) => Some(*n as u16), - _ => None, - }) - .unwrap_or(1000); - - let mut widths = HashMap::new(); - - // Parse W array if present - if let Ok(w_obj) = cid_font_dict.get(b"W") { - if let Some(w_array) = resolve_array(doc, w_obj) { - parse_cid_w_array(doc, w_array, &mut widths); - } - } - - // Try to determine space width (CID 32 or CID 3 are common for space) - let space_width = widths - .get(&32) - .or_else(|| widths.get(&3)) - .copied() - .unwrap_or(if default_width > 0 { - default_width / 4 - } else { - 250 - }); - - let wmode = font_dict - .get(b"WMode") - .ok() - .and_then(|o| match o { - Object::Integer(n) => Some(*n as u8), - _ => None, - }) - .unwrap_or(0); - - Some(FontWidthInfo { - widths, - default_width, - space_width, - is_cid: true, - units_scale: 0.001, // CID fonts use standard 1000-unit system - wmode, - }) -} - -/// Parse a CID W array into widths map -/// Format: [c [w1 w2 ...]] (consecutive from c) or [c_first c_last w] (range with same width) -fn parse_cid_w_array(doc: &Document, w_array: &[Object], widths: &mut HashMap) { - let mut i = 0; - while i < w_array.len() { - let start_cid = match &w_array[i] { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => { - i += 1; - continue; - } - }; - i += 1; - if i >= w_array.len() { - break; - } - - // Check if next element is an array (consecutive widths) or integer (range) - match &w_array[i] { - Object::Array(arr) => { - // [c [w1 w2 ...]] — consecutive widths starting at c - for (j, w_obj) in arr.iter().enumerate() { - let w = match w_obj { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => continue, - }; - widths.insert(start_cid + j as u16, w); - } - i += 1; - } - Object::Reference(r) => { - // Could be a reference to an array - if let Ok(Object::Array(arr)) = doc.get_object(*r) { - for (j, w_obj) in arr.iter().enumerate() { - let w = match w_obj { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => continue, - }; - widths.insert(start_cid + j as u16, w); - } - i += 1; - } else { - // Treat as c_first c_last w - i += 1; // skip this - } - } - Object::Integer(end_cid) => { - // [c_first c_last w] — range with uniform width - let end = *end_cid as u16; - i += 1; - if i >= w_array.len() { - break; - } - let w = match &w_array[i] { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => { - i += 1; - continue; - } - }; - for cid in start_cid..=end { - widths.insert(cid, w); - } - i += 1; - } - Object::Real(end_cid) => { - let end = *end_cid as u16; - i += 1; - if i >= w_array.len() { - break; - } - let w = match &w_array[i] { - Object::Integer(n) => *n as u16, - Object::Real(n) => *n as u16, - _ => { - i += 1; - continue; - } - }; - for cid in start_cid..=end { - widths.insert(cid, w); - } - i += 1; - } - _ => { - i += 1; - } - } - } -} - -/// Compute the width of a string in text space units, -/// given raw bytes and font width info. -/// Returns width in text space units (font_units * units_scale * font_size). -fn compute_string_width_ts(bytes: &[u8], font_info: &FontWidthInfo, font_size: f32) -> f32 { - let mut total: f32 = 0.0; - if font_info.is_cid { - // 2-byte (big-endian) character codes - let mut j = 0; - while j + 1 < bytes.len() { - let cid = u16::from_be_bytes([bytes[j], bytes[j + 1]]); - let w = font_info - .widths - .get(&cid) - .copied() - .unwrap_or(font_info.default_width); - total += w as f32; - j += 2; - } - } else { - // 1-byte character codes - for &b in bytes { - let code = b as u16; - let w = font_info - .widths - .get(&code) - .copied() - .unwrap_or(font_info.default_width); - total += w as f32; - } - } - // Convert from font units to text space using the font's scale factor - total * font_info.units_scale * font_size -} - -/// Extract raw bytes from a PDF operand (String object) -fn get_operand_bytes(obj: &Object) -> Option<&[u8]> { - if let Object::String(bytes, _) = obj { - Some(bytes) - } else { - None - } -} - -/// Build encoding maps for all fonts on a page -fn build_font_encodings( - doc: &Document, - fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, -) -> PageFontEncodings { - let mut encodings = PageFontEncodings::new(); - - for (font_name, font_dict) in fonts { - let resource_name = String::from_utf8_lossy(font_name).to_string(); - - if let Some(encoding_map) = parse_font_encoding(doc, font_dict) { - encodings.insert(resource_name, encoding_map); - } - } - - encodings -} - -/// Parse font encoding from a font dictionary -fn parse_font_encoding(doc: &Document, font_dict: &lopdf::Dictionary) -> Option { - let encoding_obj = font_dict.get(b"Encoding").ok()?; - - // Encoding can be a name or a dictionary - match encoding_obj { - Object::Name(_name) => { - // Standard encoding name (e.g., MacRomanEncoding, WinAnsiEncoding) - // For standard encodings, we can use the standard tables - // But we still need to check for Differences - None // Let lopdf handle standard encodings - } - Object::Reference(obj_ref) => { - // Reference to encoding dictionary - if let Ok(enc_dict) = doc.get_dictionary(*obj_ref) { - parse_encoding_dictionary(doc, enc_dict) - } else { - None - } - } - Object::Dictionary(enc_dict) => parse_encoding_dictionary(doc, enc_dict), - _ => None, - } -} - -/// Parse an encoding dictionary with Differences array -fn parse_encoding_dictionary( - doc: &Document, - enc_dict: &lopdf::Dictionary, -) -> Option { - let differences = enc_dict.get(b"Differences").ok()?; - - let diff_array = match differences { - Object::Array(arr) => arr.clone(), - Object::Reference(obj_ref) => { - if let Ok(Object::Array(arr)) = doc.get_object(*obj_ref) { - arr.clone() - } else { - return None; - } - } - _ => return None, - }; - - let mut encoding_map = FontEncodingMap::new(); - let mut current_code: u8 = 0; - - for item in diff_array { - match item { - Object::Integer(n) => { - // This sets the starting code for subsequent glyph names - current_code = n as u8; - } - Object::Name(name) => { - // Map current code to glyph name -> Unicode - let glyph_name = String::from_utf8_lossy(&name).to_string(); - if let Some(ch) = glyph_to_char(&glyph_name) { - encoding_map.insert(current_code, ch); - } - current_code = current_code.wrapping_add(1); - } - _ => {} - } - } - - if encoding_map.is_empty() { - None - } else { - Some(encoding_map) - } -} - -/// Type of content item -#[derive(Debug, Clone, PartialEq, Default)] -pub enum ItemType { - /// Regular text content - #[default] - Text, - /// Image placeholder - Image, - /// Hyperlink (with URL) - Link(String), - /// Form field (name: value) - FormField, -} - -/// A rectangle from a PDF `re` operator (cell boundary, border, etc.) -#[derive(Debug, Clone)] -pub struct PdfRect { - pub x: f32, - pub y: f32, - pub width: f32, - pub height: f32, - pub page: u32, -} - -/// A text item with position information -#[derive(Debug, Clone)] -pub struct TextItem { - /// The text content - pub text: String, - /// X position on page - pub x: f32, - /// Y position on page (PDF coordinates, origin at bottom-left) - pub y: f32, - /// Width of text - pub width: f32, - /// Height (approximated from font size) - pub height: f32, - /// Font name - pub font: String, - /// Font size - pub font_size: f32, - /// Page number (1-indexed) - pub page: u32, - /// Whether the font is bold - pub is_bold: bool, - /// Whether the font is italic - pub is_italic: bool, - /// Type of item (text, image, link) - pub item_type: ItemType, -} - -/// A line of text (grouped text items) -#[derive(Debug, Clone)] -pub struct TextLine { - pub items: Vec, - pub y: f32, - pub page: u32, -} - -impl TextLine { - pub fn text(&self) -> String { - self.text_with_formatting(false, false) - } - - /// Get text with optional bold/italic markdown formatting - pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String { - if !format_bold && !format_italic { - return self.text_plain(); - } - - let mut result = String::new(); - let mut current_bold = false; - let mut current_italic = false; - - for (i, item) in self.items.iter().enumerate() { - let text = item.text.as_str(); - let text_trimmed = text.trim(); - - // Skip empty items - if text_trimmed.is_empty() { - continue; - } - - // Determine spacing - let needs_space = if i == 0 || result.is_empty() { - false - } else { - let prev_item = &self.items[i - 1]; - self.needs_space_between(prev_item, item, &result) - }; - - // Preserve leading whitespace from the item text. - // Items like " means any person" have a leading space that indicates - // a word boundary. needs_space_between returns false for these (because - // space_already_exists), but we still need to emit the space since - // we push text_trimmed below (which strips it). - let has_leading_space = text.starts_with(' '); - - // Check for style changes - let item_bold = format_bold && item.is_bold; - let item_italic = format_italic && item.is_italic; - - // Close previous styles if they change - if current_italic && !item_italic { - result.push('*'); - current_italic = false; - } - if current_bold && !item_bold { - result.push_str("**"); - current_bold = false; - } - - // Add space: either from spacing logic or preserved from item text - if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) { - result.push(' '); - } - - // Open new styles - if item_bold && !current_bold { - result.push_str("**"); - current_bold = true; - } - if item_italic && !current_italic { - result.push('*'); - current_italic = true; - } - - result.push_str(text_trimmed); - } - - // Close any remaining open styles - if current_italic { - result.push('*'); - } - if current_bold { - result.push_str("**"); - } - - result - } - - /// Get plain text without formatting - fn text_plain(&self) -> String { - let mut result = String::new(); - for (i, item) in self.items.iter().enumerate() { - let text = item.text.as_str(); - if i == 0 { - result.push_str(text); - } else { - let prev_item = &self.items[i - 1]; - if self.needs_space_between(prev_item, item, &result) { - result.push(' '); - } - result.push_str(text); - } - } - result - } - - /// Determine if a space is needed between two items - fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool { - let text = item.text.as_str(); - - // Don't add space before/after hyphens for hyphenated words - let prev_ends_with_hyphen = result.ends_with('-'); - let curr_is_hyphen = text.trim() == "-"; - let curr_starts_with_hyphen = text.starts_with('-'); - - // Detect subscript/superscript: smaller font size and/or Y offset - let font_ratio = item.font_size / prev_item.font_size; - let reverse_font_ratio = prev_item.font_size / item.font_size; - let y_diff = (item.y - prev_item.y).abs(); - - let is_sub_super = font_ratio < 0.85 && y_diff > 1.0; - let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; - - // Use position-based spacing detection - let should_join = should_join_items(prev_item, item); - - // Check if space already exists - let prev_ends_with_space = result.ends_with(' '); - let curr_starts_with_space = text.starts_with(' '); - let space_already_exists = prev_ends_with_space || curr_starts_with_space; - - // Add space unless one of these conditions applies - !(prev_ends_with_hyphen - || curr_is_hyphen - || curr_starts_with_hyphen - || is_sub_super - || was_sub_super - || should_join - || space_already_exists) - } -} - -/// Determine if two adjacent text items should be joined without a space -/// based on their physical positions on the page and character case. -/// Uses a hybrid approach: position-based with case-aware thresholds. -/// CID fonts emit one word per text operator with gaps ≈ 0 between words. -/// Non-CID (Type1/TrueType) fonts emit phrases or fragments. -fn is_cid_font(font: &str) -> bool { - font.starts_with("C2_") || font.starts_with("C0_") -} - -fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { - // If either text explicitly has leading/trailing spaces, respect them - if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') { - return false; - } - - // Get the last character of previous and first character of current - let prev_last = prev_item.text.trim_end().chars().last(); - let curr_first = curr_item.text.trim_start().chars().next(); - - // Always join if current starts with punctuation that typically follows without space - // e.g., "www" + ".com" → "www.com", not "www .com" - if let Some(c) = curr_first { - if matches!(c, '.' | ',' | ';' | '!' | '?' | ')' | ']' | '}' | '\'') { - return true; - } - } - - // After colons, add space if followed by alphanumeric (typical label:value pattern) - // e.g., "Clave:" + "T9N2I6" → "Clave: T9N2I6" - if let (Some(p), Some(c)) = (prev_last, curr_first) { - if p == ':' && c.is_alphanumeric() { - return false; - } - } - - // When we have accurate width from font metrics, use a tight threshold - if prev_item.width > 0.0 { - let gap = if prev_item.x <= curr_item.x { - // LTR: prev is left of curr - curr_item.x - (prev_item.x + prev_item.width) - } else { - // RTL: prev is right of curr - prev_item.x - (curr_item.x + curr_item.width) - }; - let font_size = prev_item.font_size; - - // Never join across column-scale gaps - if gap > font_size * 3.0 { - return false; - } - - // CID fonts (C2_*, C0_*) emit one word per text operator with gaps ≈ 0 - // between words. Detect these and add spaces. Only applies to CID fonts — - // non-CID fonts (Type1/TrueType) emit phrases or fragments with small gaps - // from positioning imprecision and should NOT trigger this. - // Skip for CJK text — CJK languages don't use spaces between words. - let prev_chars = prev_item.text.trim().chars().count(); - let curr_chars = curr_item.text.trim().chars().count(); - let prev_last_char = prev_item.text.trim().chars().last(); - let curr_first_char = curr_item.text.trim().chars().next(); - let is_cjk = - prev_last_char.is_some_and(is_cjk_char) || curr_first_char.is_some_and(is_cjk_char); - - if !is_cjk && gap >= 0.0 && gap < font_size * 0.01 && is_cid_font(&prev_item.font) { - let prev_word_count = prev_item.text.split_whitespace().count(); - - if prev_word_count >= 3 { - // Multi-word phrase from a line-level CID operator — likely mid-word boundary - return gap < font_size * 0.15; - } - - // CID font: each text operator is a separate word. Always add space. - return false; - } - - // Numeric continuity: digits, commas, periods, and percent signs that - // are positioned close together are almost always a single number. - // e.g., "34,20" + "8" → "34,208", "+13." + "0" + "%" → "+13.0%" - // Use a generous threshold since word spaces in numbers are rare. - if let (Some(p), Some(c)) = (prev_last, curr_first) { - let prev_is_numeric = p.is_ascii_digit() || p == ',' || p == '.'; - let curr_is_numeric = c.is_ascii_digit() || c == '%' || c == '.'; - if prev_is_numeric && curr_is_numeric { - return gap < font_size * 0.3; - } - // Sign characters (+/-) followed by digits - if (p == '+' || p == '-') && c.is_ascii_digit() { - return gap < font_size * 0.3; - } - } - - // Single-character fragment joined to a multi-character item: use a - // moderately generous threshold to rejoin split words like "b" + "illion" - // or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words. - if (prev_chars == 1) != (curr_chars == 1) { - return gap < font_size * 0.20; - } - - // Both single-char: per-glyph positioning (character-by-character rendering). - // Intra-word gaps are ≈ 0, word boundaries are ≈ 0.15× font_size. - // For numeric chars (digits within "100,000"), use generous threshold. - // For alphabetic, use tight threshold (0.10) to reliably detect word - // boundaries in per-character PDFs like SEC filings. - if prev_chars == 1 && curr_chars == 1 { - if let (Some(p), Some(c)) = (prev_last, curr_first) { - let p_numeric = p.is_ascii_digit() || matches!(p, ',' | '.' | '%' | '+' | '-'); - let c_numeric = c.is_ascii_digit() || matches!(c, ',' | '.' | '%'); - if p_numeric && c_numeric { - return gap < font_size * 0.25; - } - } - return gap < font_size * 0.10; - } - - // With accurate widths, a gap < 15% of font size means glyphs are - // adjacent (same word). Anything larger is a deliberate space. - // For multi-char items with a lowercase→lowercase junction, use a - // slightly wider threshold (0.18) to avoid mid-word space injection - // with imprecise CID font metrics (e.g. "enterta"+"inment"). - // All-caps or mixed-case junctions keep the tighter 0.15 threshold - // to preserve word boundaries (e.g. "LCOE"+"WITH"). - if prev_item.text.trim().chars().count() >= 2 && curr_item.text.trim().chars().count() >= 2 - { - let prev_ends_lower = prev_item - .text - .trim() - .chars() - .last() - .is_some_and(|c| c.is_lowercase()); - let curr_starts_lower = curr_item - .text - .trim() - .chars() - .next() - .is_some_and(|c| c.is_lowercase()); - if prev_ends_lower && curr_starts_lower { - return gap < font_size * 0.18; - } - } - return gap < font_size * 0.15; - } - - // Fallback: estimate width from font size heuristics - let char_width = prev_item.font_size * 0.45; - - let prev_text_len = prev_item.text.chars().count() as f32; - let estimated_prev_width = prev_text_len * char_width; - - // Calculate expected end position of previous item - let prev_end_x = prev_item.x + estimated_prev_width; - - // Calculate gap between items - let gap = curr_item.x - prev_end_x; - - // Never join across column-scale gaps (fallback path) - if gap > char_width * 6.0 { - return false; - } - - // CJK text: always join adjacent items — CJK languages don't use spaces between words. - // The Latin case-based heuristics below would incorrectly insert spaces within CJK words. - let is_cjk = prev_last.is_some_and(is_cjk_char) || curr_first.is_some_and(is_cjk_char); - if is_cjk { - return gap < char_width * 0.8; - } - - // Use different thresholds based on character case - // Same-case sequences (ALL CAPS or all lowercase) are more likely to be - // word fragments that got split. Mixed case suggests word boundaries. - match (prev_last, curr_first) { - (Some(p), Some(c)) if p.is_alphabetic() && c.is_alphabetic() => { - let same_case = - (p.is_uppercase() && c.is_uppercase()) || (p.is_lowercase() && c.is_lowercase()); - if same_case { - // Same case: use generous threshold (likely same word fragment) - // e.g., "CONST" + "ANCIA" → "CONSTANCIA" - gap < char_width * 0.8 - } else if p.is_lowercase() && c.is_uppercase() { - // Lowercase to uppercase transition (e.g., "presente" → "CONSTANCIA") - // This is typically a word boundary. In Spanish/English, words don't - // transition from lowercase to uppercase mid-word. - // Always add a space for this case, regardless of position. - false - } else { - // Uppercase to lowercase (e.g., "REGISTRO" → "para") - // Use stricter threshold (likely word boundary) - gap < char_width * 0.3 - } - } - _ => { - // Non-alphabetic: use moderate threshold - gap < char_width * 0.5 - } - } -} - -/// Extract text from PDF file as plain string -pub fn extract_text>(path: P) -> Result { - crate::validate_pdf_file(&path)?; - let doc = Document::load(path)?; - extract_text_from_doc(&doc) -} - -/// Extract text from PDF memory buffer -pub fn extract_text_mem(buffer: &[u8]) -> Result { - crate::validate_pdf_bytes(buffer)?; - let doc = Document::load_mem(buffer)?; - extract_text_from_doc(&doc) -} - -/// Extract text from loaded document -fn extract_text_from_doc(doc: &Document) -> Result { - let pages = doc.get_pages(); - let page_nums: Vec = pages.keys().cloned().collect(); - - doc.extract_text(&page_nums) - .map_err(|e| PdfError::Parse(e.to_string())) -} - -/// Extract text with position information from PDF file -pub fn extract_text_with_positions>(path: P) -> Result, PdfError> { - extract_text_with_positions_pages(path, None) -} - -/// Extract text with positions from a file, limited to specific pages. -/// -/// `page_filter` is an optional set of 1-indexed page numbers to process. -/// When `None`, all pages are processed. -pub fn extract_text_with_positions_pages>( - path: P, - page_filter: Option<&HashSet>, -) -> Result, PdfError> { - let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?; - Ok(items) -} - -/// Extract text with positions and rectangles from a file. -pub(crate) fn extract_text_with_positions_and_rects>( - path: P, - page_filter: Option<&HashSet>, -) -> Result<(Vec, Vec), PdfError> { - // Read the raw PDF bytes for ToUnicode extraction - let pdf_bytes = std::fs::read(path.as_ref())?; - crate::validate_pdf_bytes(&pdf_bytes)?; - let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes); - - let doc = Document::load_mem(&pdf_bytes)?; - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) -} - -/// Extract text with positions from memory buffer -pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result, PdfError> { - extract_text_with_positions_mem_pages(buffer, None) -} - -/// Extract text with positions from memory buffer, limited to specific pages. -pub fn extract_text_with_positions_mem_pages( - buffer: &[u8], - page_filter: Option<&HashSet>, -) -> Result, PdfError> { - let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?; - Ok(items) -} - -/// Extract text with positions and rectangles from memory buffer. -pub(crate) fn extract_text_with_positions_mem_and_rects( - buffer: &[u8], - page_filter: Option<&HashSet>, -) -> Result<(Vec, Vec), PdfError> { - crate::validate_pdf_bytes(buffer)?; - // Extract ToUnicode CMaps from raw PDF bytes - let font_cmaps = FontCMaps::from_pdf_bytes(buffer); - - let doc = Document::load_mem(buffer)?; - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) -} - -/// Extract positioned text and rectangles from loaded document -fn extract_positioned_text_from_doc( - doc: &Document, - font_cmaps: &FontCMaps, - page_filter: Option<&HashSet>, -) -> Result<(Vec, Vec), PdfError> { - // If raw byte scanning found no CMaps, populate from the document model. - // This handles PDFs with compressed object streams where raw scanning fails. - let mut font_cmaps_owned; - let font_cmaps = if font_cmaps.by_obj_num.is_empty() { - font_cmaps_owned = font_cmaps.clone(); - populate_cmaps_from_doc(doc, &mut font_cmaps_owned); - &font_cmaps_owned - } else { - font_cmaps - }; - - let pages = doc.get_pages(); - let mut all_items = Vec::new(); - let mut all_rects = Vec::new(); - - // Build page ObjectId → page number map for form field extraction - let page_id_to_num: HashMap = - pages.iter().map(|(num, &id)| (id, *num)).collect(); - - for (page_num, &page_id) in pages.iter() { - if let Some(filter) = page_filter { - if !filter.contains(page_num) { - continue; - } - } - let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; - all_items.extend(items); - all_rects.extend(rects); - - // Extract hyperlinks from page annotations - let links = extract_page_links(doc, page_id, *page_num); - all_items.extend(links); - } - - // Extract AcroForm field values - let form_items = extract_form_fields(doc, &page_id_to_num); - all_items.extend(form_items); - - Ok((all_items, all_rects)) -} - -/// Populate FontCMaps from the lopdf document model for ToUnicode streams -/// that weren't found by raw byte scanning (e.g. in compressed object streams). -fn populate_cmaps_from_doc(doc: &Document, font_cmaps: &mut FontCMaps) { - use crate::tounicode::ToUnicodeCMap; - - for (_page_num, &page_id) in doc.get_pages().iter() { - let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); - for (font_name, font_dict) in &fonts { - if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") { - if let Ok(obj_ref) = tounicode_ref.as_reference() { - let obj_num = obj_ref.0; - if font_cmaps.by_obj_num.contains_key(&obj_num) { - continue; - } - // Try to get the stream content via lopdf - if let Ok(stream) = doc.get_object(obj_ref) { - if let Ok(stream) = stream.as_stream() { - if let Ok(data) = stream.decompressed_content() { - if let Some(cmap) = ToUnicodeCMap::parse(&data) { - let resource_name = - String::from_utf8_lossy(font_name).to_string(); - let base_name = font_dict - .get(b"BaseFont") - .ok() - .and_then(|o| o.as_name().ok()) - .map(|n| String::from_utf8_lossy(n).to_string()); - - // Store by object number - font_cmaps.by_obj_num.insert(obj_num, cmap.clone()); - // Store by resource name - font_cmaps - .by_name - .insert(resource_name.clone(), cmap.clone()); - if let Some(base) = base_name { - let unique_key = format!("{}_{}", base, obj_num); - font_cmaps.by_name.insert(unique_key, cmap.clone()); - font_cmaps.by_name.insert(base, cmap); - } - } - } - } - } - } - } - } - } -} - -/// Multiply two 2D transformation matrices -/// Matrix format: [a, b, c, d, e, f] representing: -/// | a b 0 | -/// | c d 0 | -/// | e f 1 | -fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] { - [ - m1[0] * m2[0] + m1[1] * m2[2], - m1[0] * m2[1] + m1[1] * m2[3], - m1[2] * m2[0] + m1[3] * m2[2], - m1[2] * m2[1] + m1[3] * m2[3], - m1[4] * m2[0] + m1[5] * m2[2] + m2[4], - m1[4] * m2[1] + m1[5] * m2[3] + m2[5], - ] -} - -/// Extract text items and rectangles from a single page -fn extract_page_text_items( - doc: &Document, - page_id: ObjectId, - page_num: u32, - font_cmaps: &FontCMaps, -) -> Result<(Vec, Vec), PdfError> { - use lopdf::content::Content; - - let mut items = Vec::new(); - let mut rects: Vec = Vec::new(); - - // Get fonts for encoding - let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); - - // Build font encoding maps from Differences arrays - let font_encodings = build_font_encodings(doc, &fonts); - - // Build font width info for accurate text positioning - let font_widths = build_font_widths(doc, &fonts); - - // Build maps of font resource names to their base font names and ToUnicode object refs - let mut font_base_names: std::collections::HashMap = - std::collections::HashMap::new(); - let mut font_tounicode_refs: std::collections::HashMap = - std::collections::HashMap::new(); - for (font_name, font_dict) in &fonts { - let resource_name = String::from_utf8_lossy(font_name).to_string(); - if let Ok(base_font) = font_dict.get(b"BaseFont") { - if let Ok(name) = base_font.as_name() { - let base_name = String::from_utf8_lossy(name).to_string(); - font_base_names.insert(resource_name.clone(), base_name); - } - } - // Track ToUnicode object reference - if let Ok(tounicode) = font_dict.get(b"ToUnicode") { - if let Ok(obj_ref) = tounicode.as_reference() { - font_tounicode_refs.insert(resource_name, obj_ref.0); - } - } - } - - // Cache font encodings from lopdf (once per font, not per text operand). - // This avoids re-parsing ToUnicode CMap streams for every Tj/TJ operator. - let mut encoding_cache: HashMap> = HashMap::new(); - for (font_name, font_dict) in &fonts { - let name = String::from_utf8_lossy(font_name).to_string(); - if let Ok(enc) = font_dict.get_font_encoding(doc) { - encoding_cache.insert(name, enc); - } - } - - // Get XObjects (images) from page resources - let xobjects = get_page_xobjects(doc, page_id); - - // Get content - let content_data = doc - .get_page_content(page_id) - .map_err(|e| PdfError::Parse(e.to_string()))?; - - let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; - - // Graphics state tracking - let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix - let mut fill_is_white = false; // Fill color is white (invisible text) - let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible - let mut gstate_stack: Vec<([f32; 6], bool, i32)> = Vec::new(); - - // Text state tracking - let mut current_font = String::new(); - let mut current_font_size: f32 = 12.0; - let mut text_leading: f32 = 0.0; // TL parameter (in text-space units) - let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; - let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; - let mut in_text_block = false; - - // Marked content (ActualText) tracking - let mut marked_content_stack: Vec> = Vec::new(); - let mut suppress_glyph_extraction = false; - let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry - - for op in &content.operations { - match op.operator.as_str() { - "q" => { - // Save graphics state - gstate_stack.push((ctm, fill_is_white, text_rendering_mode)); - } - "Q" => { - // Restore graphics state - if let Some((saved_ctm, saved_fill, saved_tr)) = gstate_stack.pop() { - ctm = saved_ctm; - fill_is_white = saved_fill; - text_rendering_mode = saved_tr; - } - } - "cm" => { - // Concatenate matrix to CTM - if op.operands.len() >= 6 { - let new_matrix = [ - get_number(&op.operands[0]).unwrap_or(1.0), - get_number(&op.operands[1]).unwrap_or(0.0), - get_number(&op.operands[2]).unwrap_or(0.0), - get_number(&op.operands[3]).unwrap_or(1.0), - get_number(&op.operands[4]).unwrap_or(0.0), - get_number(&op.operands[5]).unwrap_or(0.0), - ]; - ctm = multiply_matrices(&new_matrix, &ctm); - } - } - "g" => { - // Set grayscale fill color (1.0 = white) - if let Some(gray) = op.operands.first().and_then(get_number) { - fill_is_white = gray > 0.95; - } - } - "rg" => { - // Set RGB fill color - if op.operands.len() >= 3 { - let r = get_number(&op.operands[0]).unwrap_or(0.0); - let g = get_number(&op.operands[1]).unwrap_or(0.0); - let b = get_number(&op.operands[2]).unwrap_or(0.0); - fill_is_white = r > 0.95 && g > 0.95 && b > 0.95; - } - } - "k" => { - // Set CMYK fill color (0,0,0,0 = white) - if op.operands.len() >= 4 { - let c = get_number(&op.operands[0]).unwrap_or(1.0); - let m = get_number(&op.operands[1]).unwrap_or(1.0); - let y = get_number(&op.operands[2]).unwrap_or(1.0); - let k = get_number(&op.operands[3]).unwrap_or(1.0); - fill_is_white = c < 0.05 && m < 0.05 && y < 0.05 && k < 0.05; - } - } - "BT" => { - // Begin text block - in_text_block = true; - text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; - line_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; - text_rendering_mode = 0; - } - "ET" => { - // End text block - in_text_block = false; - } - "Tf" => { - // Set font and size - if op.operands.len() >= 2 { - if let Ok(name) = op.operands[0].as_name() { - current_font = String::from_utf8_lossy(name).to_string(); - } - if let Ok(size) = op.operands[1].as_f32() { - current_font_size = size; - } else if let Ok(size) = op.operands[1].as_i64() { - current_font_size = size as f32; - } - } - } - "TL" => { - // Set text leading (used by T*, ', and " operators) - if let Some(tl) = op.operands.first().and_then(get_number) { - text_leading = tl; - } - } - "Tr" => { - // Set text rendering mode (3 = invisible / OCR overlay) - if let Some(mode) = op.operands.first().and_then(get_number) { - text_rendering_mode = mode as i32; - } - } - "Td" | "TD" => { - // Move text position: TLM = T(tx,ty) × TLM; Tm = TLM - // tx,ty are in text space — must be scaled by the text line matrix - if op.operands.len() >= 2 { - let tx = get_number(&op.operands[0]).unwrap_or(0.0); - let ty = get_number(&op.operands[1]).unwrap_or(0.0); - line_matrix[4] += tx * line_matrix[0] + ty * line_matrix[2]; - line_matrix[5] += tx * line_matrix[1] + ty * line_matrix[3]; - text_matrix = line_matrix; - if op.operator == "TD" { - text_leading = -ty; - } - } - } - "Tm" => { - // Set text matrix - if op.operands.len() >= 6 { - for (i, operand) in op.operands.iter().take(6).enumerate() { - text_matrix[i] = - get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 }); - } - line_matrix = text_matrix; - } - } - "T*" => { - // Move to start of next line: equivalent to 0 -TL Td - let tl = if text_leading != 0.0 { - text_leading - } else { - current_font_size * 1.2 - }; - line_matrix[4] += (-tl) * line_matrix[2]; // Usually 0 for non-rotated text - line_matrix[5] += (-tl) * line_matrix[3]; - text_matrix = line_matrix; - } - "Tj" => { - // Show text string - if in_text_block && !op.operands.is_empty() { - // Advance text matrix regardless of visibility - let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| { - get_operand_bytes(&op.operands[0]) - .map(|raw| compute_string_width_ts(raw, fi, current_font_size)) - }); - // ActualText: suppress glyph extraction, just advance text matrix - if suppress_glyph_extraction { - if let Some(w_ts) = w_ts_opt { - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - } - continue; - } - // Skip invisible (white/Tr=3) text but still advance text matrix - if fill_is_white || text_rendering_mode == 3 { - if let Some(w_ts) = w_ts_opt { - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - } - continue; - } - if let Some(text) = extract_text_from_operand( - &op.operands[0], - ¤t_font, - font_cmaps, - &font_base_names, - &font_tounicode_refs, - &font_encodings, - &encoding_cache, - ) { - let combined = multiply_matrices(&text_matrix, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - let width = if let Some(w_ts) = w_ts_opt { - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - (w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])).abs() - } else { - 0.0 - }; - // Only create text item for non-whitespace; whitespace - // still advances the text matrix above so gap detection works - if !text.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&text), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - } - } - "TJ" => { - // Show text with positioning — split at column-sized gaps - if in_text_block && !op.operands.is_empty() { - if let Ok(array) = op.operands[0].as_array() { - let font_info = font_widths.get(¤t_font); - let is_invisible = - fill_is_white || text_rendering_mode == 3 || suppress_glyph_extraction; - - // Compute space threshold based on font metrics when available - let space_threshold = if let Some(font_info) = font_info { - let space_em = font_info.space_width as f32 * font_info.units_scale; - let threshold = space_em * 1000.0 * 0.4; - threshold.max(80.0) - } else { - 120.0 - }; - let column_gap_threshold = space_threshold * 4.0; - - // Track sub-items for column-gap splitting: - // (text, start_width_ts, end_width_ts) - let mut sub_items: Vec<(String, f32, f32)> = Vec::new(); - let mut current_text = String::new(); - let mut sub_start_width_ts: f32 = 0.0; - let mut total_width_ts: f32 = 0.0; - for element in array { - match element { - Object::Integer(n) => { - let n_val = *n as f32; - let displacement = -n_val / 1000.0 * current_font_size; - if !is_invisible - && n_val < -column_gap_threshold - && !current_text.is_empty() - { - // Column gap: flush current segment - sub_items.push(( - std::mem::take(&mut current_text), - sub_start_width_ts, - total_width_ts, - )); - total_width_ts += displacement; - sub_start_width_ts = total_width_ts; - } else { - total_width_ts += displacement; - if !is_invisible - && n_val < -space_threshold - && !current_text.is_empty() - && !current_text.ends_with(' ') - { - current_text.push(' '); - } - } - continue; - } - Object::Real(n) => { - let n_val = *n; - let displacement = -n_val / 1000.0 * current_font_size; - if !is_invisible - && n_val < -column_gap_threshold - && !current_text.is_empty() - { - sub_items.push(( - std::mem::take(&mut current_text), - sub_start_width_ts, - total_width_ts, - )); - total_width_ts += displacement; - sub_start_width_ts = total_width_ts; - } else { - total_width_ts += displacement; - if !is_invisible - && n_val < -space_threshold - && !current_text.is_empty() - && !current_text.ends_with(' ') - { - current_text.push(' '); - } - } - continue; - } - _ => {} - } - if let Some(fi) = font_info { - if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); - } - } - if !is_invisible { - if let Some(text) = extract_text_from_operand( - element, - ¤t_font, - font_cmaps, - &font_base_names, - &font_tounicode_refs, - &font_encodings, - &encoding_cache, - ) { - current_text.push_str(&text); - } - } - } - // Flush remaining text - if !is_invisible && !current_text.trim().is_empty() { - sub_items.push((current_text, sub_start_width_ts, total_width_ts)); - } - // Emit one TextItem per sub-item - if !sub_items.is_empty() { - let combined = multiply_matrices(&text_matrix, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]; - for (text, start_w, end_w) in &sub_items { - let offset_tm = [ - text_matrix[0], - text_matrix[1], - text_matrix[2], - text_matrix[3], - text_matrix[4] + start_w * text_matrix[0], - text_matrix[5] + start_w * text_matrix[1], - ]; - let combined = multiply_matrices(&offset_tm, &ctm); - let (x, y) = (combined[4], combined[5]); - let width = if font_info.is_some() { - ((end_w - start_w) * scale_x).abs() - } else { - 0.0 - }; - items.push(TextItem { - text: expand_ligatures(text), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - // Always advance text matrix by total width - if font_info.is_some() { - text_matrix[4] += total_width_ts * text_matrix[0]; - text_matrix[5] += total_width_ts * text_matrix[1]; - } - } - } - } - "'" => { - // Move to next line and show text (equivalent to T* then Tj) - let tl = if text_leading != 0.0 { - text_leading - } else { - current_font_size * 1.2 - }; - line_matrix[4] += (-tl) * line_matrix[2]; - line_matrix[5] += (-tl) * line_matrix[3]; - text_matrix = line_matrix; - if !(fill_is_white - || text_rendering_mode == 3 - || suppress_glyph_extraction - || op.operands.is_empty()) - { - if let Some(text) = extract_text_from_operand( - &op.operands[0], - ¤t_font, - font_cmaps, - &font_base_names, - &font_tounicode_refs, - &font_encodings, - &encoding_cache, - ) { - if !text.trim().is_empty() { - let combined = multiply_matrices(&text_matrix, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&text), - x, - y, - width: 0.0, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - } - } - "Do" => { - // XObject invocation - could be an image or form - if !op.operands.is_empty() { - if let Ok(name) = op.operands[0].as_name() { - let xobj_name = String::from_utf8_lossy(name).to_string(); - - if let Some(xobj_type) = xobjects.get(&xobj_name) { - match xobj_type { - XObjectType::Image => { - // Skip images — text extraction only - } - XObjectType::Form(form_id) => { - // Extract text from Form XObject - let form_items = extract_form_xobject_text( - doc, *form_id, page_num, font_cmaps, &ctm, - ); - items.extend(form_items); - } - } - } - } - } - } - "BMC" => { - // Begin Marked Content (no properties) - marked_content_stack.push(None); - } - "BDC" => { - // Begin Marked Content with properties — extract ActualText - let mut actual_text: Option = None; - if op.operands.len() >= 2 { - let dict = match &op.operands[1] { - Object::Dictionary(d) => Some(d.clone()), - Object::Reference(id) => doc.get_dictionary(*id).ok().cloned(), - _ => None, - }; - if let Some(d) = dict { - if let Ok(val) = d.get(b"ActualText") { - actual_text = match val { - Object::String(bytes, _) => Some(decode_text_string(bytes)), - _ => None, - }; - } - } - } - if actual_text.is_some() { - suppress_glyph_extraction = true; - actual_text_start_tm = Some(text_matrix); - } - marked_content_stack.push(actual_text); - } - "EMC" => { - // End Marked Content — emit ActualText item with correct width - if let Some(Some(at)) = marked_content_stack.pop() { - // Compute width from text matrix advancement during BDC..EMC - if let Some(start_tm) = actual_text_start_tm.take() { - let combined = multiply_matrices(&start_tm, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - // Width in device space from text matrix delta - let delta_ts = text_matrix[4] - start_tm[4]; - let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; - let width = (delta_ts * scale_x).abs(); - if !at.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: at, - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); - } - } - "re" => { - // Rectangle operator: collect for table-grid detection - if op.operands.len() >= 4 { - let rx = get_number(&op.operands[0]).unwrap_or(0.0); - let ry = get_number(&op.operands[1]).unwrap_or(0.0); - let rw = get_number(&op.operands[2]).unwrap_or(0.0); - let rh = get_number(&op.operands[3]).unwrap_or(0.0); - // Transform origin to device space - let x_dev = rx * ctm[0] + ry * ctm[2] + ctm[4]; - let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5]; - let w_dev = rw * ctm[0]; - let h_dev = rh * ctm[3]; - rects.push(PdfRect { - x: x_dev, - y: y_dev, - width: w_dev, - height: h_dev, - page: page_num, - }); - } - } - _ => {} - } - } - - let items = merge_text_items(items); - Ok((items, rects)) -} - -/// Merge adjacent single-character TextItems into words. -/// -/// Per-character PDFs (e.g. SEC filings) produce hundreds of single-char items. -/// This merges items on the same line that are close together into words, -/// inserting spaces at word boundaries. -fn merge_text_items(items: Vec) -> Vec { - if items.is_empty() { - return items; - } - - // Group items by (page, Y position) with 5pt tolerance - let y_tolerance = 5.0; - let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new(); - - for item in &items { - let found = line_groups - .iter_mut() - .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance); - if let Some((_, _, group)) = found { - group.push(item); - } else { - line_groups.push((item.page, item.y, vec![item])); - } - } - - // Sort each group by X position (direction-aware) - for (_, _, group) in &mut line_groups { - let rtl = is_rtl_text(group.iter().map(|i| &i.text)); - if rtl { - group.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); - } else { - group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); - } - } - - // Sort groups by page then Y descending (top of page first) - line_groups.sort_by(|a, b| { - a.0.cmp(&b.0) - .then_with(|| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)) - }); - - let mut merged = Vec::new(); - - for (_, _, group) in &line_groups { - let mut i = 0; - while i < group.len() { - let first = group[i]; - let mut text = first.text.clone(); - let mut end_x = first.x + first.width; - let x_gap_max = first.font_size * 0.5; - - let mut j = i + 1; - while j < group.len() { - let next = group[j]; - // Must be similar font size (within 20%) - if (next.font_size - first.font_size).abs() > first.font_size * 0.20 { - break; - } - let gap = next.x - end_x; - if gap > x_gap_max { - break; - } - if gap < -first.font_size * 0.5 { - break; - } - // Insert space at word boundaries - if gap > first.font_size * 0.08 { - text.push(' '); - } - text.push_str(&next.text); - end_x = next.x + next.width; - j += 1; - } - - merged.push(TextItem { - text, - x: first.x, - y: first.y, - width: end_x - first.x, - height: first.height, - font: first.font.clone(), - font_size: first.font_size, - page: first.page, - is_bold: first.is_bold, - is_italic: first.is_italic, - item_type: first.item_type.clone(), - }); - - i = j; - } - } - - merged -} - -/// Helper to get f32 from Object -fn get_number(obj: &Object) -> Option { - match obj { - Object::Integer(i) => Some(*i as f32), - Object::Real(r) => Some(*r), - _ => None, - } -} - -/// Get XObject names that are images from page resources -/// XObject info - either Image or Form -#[derive(Debug)] -enum XObjectType { - Image, - Form(ObjectId), -} - -/// Get XObjects from page resources, categorized by type -fn get_page_xobjects( - doc: &Document, - page_id: ObjectId, -) -> std::collections::HashMap { - let mut xobject_types = std::collections::HashMap::new(); - - // Try to get the page dictionary - if let Ok(page_dict) = doc.get_dictionary(page_id) { - // Get Resources dictionary - let resources = if let Ok(res_ref) = page_dict.get(b"Resources") { - if let Ok(obj_ref) = res_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - res_ref.as_dict().ok() - } - } else { - None - }; - - if let Some(resources) = resources { - // Get XObject dictionary from Resources - if let Ok(xobjects_ref) = resources.get(b"XObject") { - let xobjects = if let Ok(obj_ref) = xobjects_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - xobjects_ref.as_dict().ok() - }; - - if let Some(xobjects) = xobjects { - for (name, value) in xobjects.iter() { - let name_str = String::from_utf8_lossy(name).to_string(); - - // Check XObject subtype - if let Ok(obj_ref) = value.as_reference() { - if let Ok(Object::Stream(stream)) = doc.get_object(obj_ref) { - if let Ok(subtype) = stream.dict.get(b"Subtype") { - if let Ok(subtype_name) = subtype.as_name() { - if subtype_name == b"Image" { - xobject_types.insert(name_str, XObjectType::Image); - } else if subtype_name == b"Form" { - xobject_types - .insert(name_str, XObjectType::Form(obj_ref)); - } - } - } - } - } - } - } - } - } - } - - xobject_types -} - -/// Extract text items from a Form XObject -fn extract_form_xobject_text( - doc: &Document, - form_id: ObjectId, - page_num: u32, - font_cmaps: &FontCMaps, - parent_ctm: &[f32; 6], -) -> Vec { - use lopdf::content::Content; - - let mut items = Vec::new(); - - // Get the Form XObject stream - let Ok(Object::Stream(stream)) = doc.get_object(form_id) else { - return items; - }; - - // Decompress the content stream - let Ok(content_data) = stream.decompressed_content() else { - return items; - }; - - // Decode the content stream - let Ok(content) = Content::decode(&content_data) else { - return items; - }; - - // Get fonts from the Form's Resources - let form_fonts = get_form_fonts(doc, &stream.dict); - let font_encodings = build_font_encodings(doc, &form_fonts); - - // Build font width info for the form - let font_widths = build_font_widths(doc, &form_fonts); - - // Build font base names and ToUnicode refs for the form - let mut font_base_names: std::collections::HashMap = - std::collections::HashMap::new(); - let mut font_tounicode_refs: std::collections::HashMap = - std::collections::HashMap::new(); - - for (font_name, font_dict) in &form_fonts { - let resource_name = String::from_utf8_lossy(font_name).to_string(); - if let Ok(base_font) = font_dict.get(b"BaseFont") { - if let Ok(name) = base_font.as_name() { - let base_name = String::from_utf8_lossy(name).to_string(); - font_base_names.insert(resource_name.clone(), base_name); - } - } - if let Ok(tounicode) = font_dict.get(b"ToUnicode") { - if let Ok(obj_ref) = tounicode.as_reference() { - font_tounicode_refs.insert(resource_name, obj_ref.0); - } - } - } - - // Cache font encodings for form fonts - let mut encoding_cache: HashMap> = HashMap::new(); - for (font_name, font_dict) in &form_fonts { - let name = String::from_utf8_lossy(font_name).to_string(); - if let Ok(enc) = font_dict.get_font_encoding(doc) { - encoding_cache.insert(name, enc); - } - } - - // Process the content stream - let mut current_font = String::new(); - let mut current_font_size: f32 = 12.0; - let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; - let mut in_text_block = false; - let mut fill_is_white = false; - - for op in &content.operations { - match op.operator.as_str() { - "BT" => { - in_text_block = true; - text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; - } - "ET" => { - in_text_block = false; - } - "Tf" => { - if op.operands.len() >= 2 { - if let Ok(name) = op.operands[0].as_name() { - current_font = String::from_utf8_lossy(name).to_string(); - } - current_font_size = get_number(&op.operands[1]).unwrap_or(12.0); - } - } - "Td" | "TD" => { - if op.operands.len() >= 2 { - let tx = get_number(&op.operands[0]).unwrap_or(0.0); - let ty = get_number(&op.operands[1]).unwrap_or(0.0); - text_matrix[4] += tx * text_matrix[0] + ty * text_matrix[2]; - text_matrix[5] += tx * text_matrix[1] + ty * text_matrix[3]; - } - } - "Tm" => { - if op.operands.len() >= 6 { - for (i, operand) in op.operands.iter().take(6).enumerate() { - text_matrix[i] = - get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 }); - } - } - } - "g" => { - if let Some(gray) = op.operands.first().and_then(get_number) { - fill_is_white = gray > 0.95; - } - } - "rg" => { - if op.operands.len() >= 3 { - let r = get_number(&op.operands[0]).unwrap_or(0.0); - let g = get_number(&op.operands[1]).unwrap_or(0.0); - let b = get_number(&op.operands[2]).unwrap_or(0.0); - fill_is_white = r > 0.95 && g > 0.95 && b > 0.95; - } - } - "k" => { - if op.operands.len() >= 4 { - let c = get_number(&op.operands[0]).unwrap_or(1.0); - let m = get_number(&op.operands[1]).unwrap_or(1.0); - let y = get_number(&op.operands[2]).unwrap_or(1.0); - let k = get_number(&op.operands[3]).unwrap_or(1.0); - fill_is_white = c < 0.05 && m < 0.05 && y < 0.05 && k < 0.05; - } - } - "Tj" => { - if in_text_block && !op.operands.is_empty() { - if fill_is_white { - if let Some(font_info) = font_widths.get(¤t_font) { - if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { - let w_ts = compute_string_width_ts( - raw_bytes, - font_info, - current_font_size, - ); - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - } - } - continue; - } - if let Some(text) = extract_text_from_operand( - &op.operands[0], - ¤t_font, - font_cmaps, - &font_base_names, - &font_tounicode_refs, - &font_encodings, - &encoding_cache, - ) { - let combined = multiply_matrices(&text_matrix, parent_ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - let width = if let Some(font_info) = font_widths.get(¤t_font) { - if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { - let w_ts = compute_string_width_ts( - raw_bytes, - font_info, - current_font_size, - ); - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - (w_ts - * (text_matrix[0] * parent_ctm[0] - + text_matrix[1] * parent_ctm[2])) - .abs() - } else { - 0.0 - } - } else { - 0.0 - }; - // Only create text item for non-whitespace; whitespace - // still advances the text matrix above so gap detection works - if !text.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&text), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - } - } - "TJ" => { - // Show text with positioning — split at column-sized gaps - if in_text_block && !op.operands.is_empty() { - if let Ok(array) = op.operands[0].as_array() { - let font_info = font_widths.get(¤t_font); - - let space_threshold = if let Some(fi) = font_info { - let space_em = fi.space_width as f32 * fi.units_scale; - let threshold = space_em * 1000.0 * 0.4; - threshold.max(80.0) - } else { - 120.0 - }; - let column_gap_threshold = space_threshold * 4.0; - - let mut sub_items: Vec<(String, f32, f32)> = Vec::new(); - let mut current_text = String::new(); - let mut sub_start_width_ts: f32 = 0.0; - let mut total_width_ts: f32 = 0.0; - for element in array { - match element { - Object::Integer(n) => { - let n_val = *n as f32; - let displacement = -n_val / 1000.0 * current_font_size; - if !fill_is_white - && n_val < -column_gap_threshold - && !current_text.is_empty() - { - sub_items.push(( - std::mem::take(&mut current_text), - sub_start_width_ts, - total_width_ts, - )); - total_width_ts += displacement; - sub_start_width_ts = total_width_ts; - } else { - total_width_ts += displacement; - if !fill_is_white - && n_val < -space_threshold - && !current_text.is_empty() - && !current_text.ends_with(' ') - { - current_text.push(' '); - } - } - continue; - } - Object::Real(n) => { - let n_val = *n; - let displacement = -n_val / 1000.0 * current_font_size; - if !fill_is_white - && n_val < -column_gap_threshold - && !current_text.is_empty() - { - sub_items.push(( - std::mem::take(&mut current_text), - sub_start_width_ts, - total_width_ts, - )); - total_width_ts += displacement; - sub_start_width_ts = total_width_ts; - } else { - total_width_ts += displacement; - if !fill_is_white - && n_val < -space_threshold - && !current_text.is_empty() - && !current_text.ends_with(' ') - { - current_text.push(' '); - } - } - continue; - } - _ => {} - } - if let Some(fi) = font_info { - if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); - } - } - if !fill_is_white { - if let Some(text) = extract_text_from_operand( - element, - ¤t_font, - font_cmaps, - &font_base_names, - &font_tounicode_refs, - &font_encodings, - &encoding_cache, - ) { - current_text.push_str(&text); - } - } - } - if !fill_is_white && !current_text.trim().is_empty() { - sub_items.push((current_text, sub_start_width_ts, total_width_ts)); - } - if !sub_items.is_empty() { - let combined = multiply_matrices(&text_matrix, parent_ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - let scale_x = - text_matrix[0] * parent_ctm[0] + text_matrix[1] * parent_ctm[2]; - for (text, start_w, end_w) in &sub_items { - let offset_tm = [ - text_matrix[0], - text_matrix[1], - text_matrix[2], - text_matrix[3], - text_matrix[4] + start_w * text_matrix[0], - text_matrix[5] + start_w * text_matrix[1], - ]; - let combined_mat = multiply_matrices(&offset_tm, parent_ctm); - let (x, y) = (combined_mat[4], combined_mat[5]); - let width = if font_info.is_some() { - ((end_w - start_w) * scale_x).abs() - } else { - 0.0 - }; - items.push(TextItem { - text: expand_ligatures(text), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); - } - } - // Always advance text matrix - if font_info.is_some() { - text_matrix[4] += total_width_ts * text_matrix[0]; - text_matrix[5] += total_width_ts * text_matrix[1]; - } - } - } - } - _ => {} - } - } - - items -} - -/// Get fonts from a Form XObject's Resources -fn get_form_fonts<'a>( - doc: &'a Document, - form_dict: &lopdf::Dictionary, -) -> std::collections::BTreeMap, &'a lopdf::Dictionary> { - let mut fonts = std::collections::BTreeMap::new(); - - // Get Resources from Form dictionary - let resources = if let Ok(res_ref) = form_dict.get(b"Resources") { - if let Ok(obj_ref) = res_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - res_ref.as_dict().ok() - } - } else { - return fonts; - }; - - let Some(resources) = resources else { - return fonts; - }; - - // Get Font dictionary - let font_dict = if let Ok(font_ref) = resources.get(b"Font") { - if let Ok(obj_ref) = font_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - font_ref.as_dict().ok() - } - } else { - return fonts; - }; - - let Some(font_dict) = font_dict else { - return fonts; - }; - - // Collect fonts - for (name, value) in font_dict.iter() { - if let Ok(obj_ref) = value.as_reference() { - if let Ok(dict) = doc.get_dictionary(obj_ref) { - fonts.insert(name.clone(), dict); - } - } - } - - fonts -} - -/// Extract hyperlinks from page annotations -pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> Vec { - let mut links = Vec::new(); - - // Try to get the page dictionary - if let Ok(page_dict) = doc.get_dictionary(page_id) { - // Get Annots array - let annots = if let Ok(annots_ref) = page_dict.get(b"Annots") { - if let Ok(obj_ref) = annots_ref.as_reference() { - doc.get_object(obj_ref) - .ok() - .and_then(|o| o.as_array().ok().cloned()) - } else { - annots_ref.as_array().ok().cloned() - } - } else { - None - }; - - if let Some(annots) = annots { - for annot_ref in annots { - // Get annotation dictionary - let annot_dict = if let Ok(obj_ref) = annot_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - annot_ref.as_dict().ok() - }; - - if let Some(annot_dict) = annot_dict { - // Check if this is a Link annotation - if let Ok(subtype) = annot_dict.get(b"Subtype") { - if let Ok(subtype_name) = subtype.as_name() { - if subtype_name != b"Link" { - continue; - } - } - } - - // Get the Rect (position) - let rect = if let Ok(rect_obj) = annot_dict.get(b"Rect") { - if let Ok(rect_array) = rect_obj.as_array() { - if rect_array.len() >= 4 { - let x1 = get_number(&rect_array[0]).unwrap_or(0.0); - let y1 = get_number(&rect_array[1]).unwrap_or(0.0); - let x2 = get_number(&rect_array[2]).unwrap_or(0.0); - let y2 = get_number(&rect_array[3]).unwrap_or(0.0); - Some((x1, y1, x2 - x1, y2 - y1)) - } else { - None - } - } else { - None - } - } else { - None - }; - - // Get the action (A dictionary) or Dest - let uri = extract_link_uri(doc, annot_dict); - - if let (Some((x, y, width, height)), Some(url)) = (rect, uri) { - links.push(TextItem { - text: url.clone(), - x, - y, - width, - height, - font: String::new(), - font_size: 0.0, - page: page_num, - is_bold: false, - is_italic: false, - item_type: ItemType::Link(url), - }); - } - } - } - } - } - - links -} - -/// Extract URI from a link annotation -fn extract_link_uri(doc: &Document, annot_dict: &lopdf::Dictionary) -> Option { - // Try to get the A (Action) dictionary - if let Ok(action_ref) = annot_dict.get(b"A") { - let action_dict = if let Ok(obj_ref) = action_ref.as_reference() { - doc.get_dictionary(obj_ref).ok() - } else { - action_ref.as_dict().ok() - }; - - if let Some(action_dict) = action_dict { - // Check for URI action - if let Ok(uri_obj) = action_dict.get(b"URI") { - if let Ok(uri_str) = uri_obj.as_str() { - return Some(String::from_utf8_lossy(uri_str).to_string()); - } - } - } - } - - // Try Dest (named destination) - less common for external links - // We'll skip this for now as it requires looking up named destinations - - None -} - -/// Extract form field values from AcroForm dictionary. -/// Returns TextItems positioned at each field's Rect so they flow into the markdown pipeline. -fn extract_form_fields(doc: &Document, page_map: &HashMap) -> Vec { - let mut items = Vec::new(); - - // Navigate: trailer -> /Root -> /AcroForm -> /Fields - let root = match doc.trailer.get(b"Root") { - Ok(root_ref) => match root_ref.as_reference() { - Ok(r) => match doc.get_dictionary(r) { - Ok(d) => d, - Err(_) => return items, - }, - Err(_) => return items, - }, - Err(_) => return items, - }; - - let acroform = match root.get(b"AcroForm") { - Ok(obj) => match resolve_dict(doc, obj) { - Some(d) => d, - None => return items, - }, - Err(_) => return items, - }; - - let fields = match acroform.get(b"Fields") { - Ok(obj) => match resolve_array(doc, obj) { - Some(arr) => arr.clone(), - None => return items, - }, - Err(_) => return items, - }; - - for field_obj in &fields { - if let Ok(field_ref) = field_obj.as_reference() { - walk_form_fields(doc, field_ref, None, "", page_map, &mut items); - } - } - - items -} - -/// Recursively walk the form field tree, extracting leaf field values. -fn walk_form_fields( - doc: &Document, - field_id: ObjectId, - parent_ft: Option<&[u8]>, - parent_name: &str, - page_map: &HashMap, - items: &mut Vec, -) { - let field_dict = match doc.get_dictionary(field_id) { - Ok(d) => d, - Err(_) => return, - }; - - // Build fully qualified field name - let local_name = field_dict - .get(b"T") - .ok() - .and_then(|o| o.as_str().ok()) - .map(|s| String::from_utf8_lossy(s).to_string()) - .unwrap_or_default(); - - let full_name = if parent_name.is_empty() { - local_name.clone() - } else if local_name.is_empty() { - parent_name.to_string() - } else { - format!("{}.{}", parent_name, local_name) - }; - - // Determine field type (may be inherited from parent) - let ft = field_dict - .get(b"FT") - .ok() - .and_then(|o| o.as_name().ok()) - .or(parent_ft); - - // Check for /Kids — if present, recurse into children - if let Ok(kids_obj) = field_dict.get(b"Kids") { - if let Some(kids) = resolve_array(doc, kids_obj) { - let kids = kids.clone(); - for kid in &kids { - if let Ok(kid_ref) = kid.as_reference() { - walk_form_fields(doc, kid_ref, ft, &full_name, page_map, items); - } - } - return; - } - } - - // Leaf field — extract value - let ft = match ft { - Some(ft) => ft, - None => return, - }; - - // Skip signature fields - if ft == b"Sig" { - return; - } - - // Get field value - let value = match field_dict.get(b"V") { - Ok(v) => v, - Err(_) => return, - }; - - let value_str = match ft { - b"Tx" | b"Ch" => { - // Text or Choice field — value is a string or array of strings - match value { - Object::String(s, _) => { - let s = String::from_utf8_lossy(s).to_string(); - if s.is_empty() { - return; - } - s - } - Object::Array(arr) => { - let parts: Vec = arr - .iter() - .filter_map(|o| { - if let Object::String(s, _) = o { - Some(String::from_utf8_lossy(s).to_string()) - } else { - None - } - }) - .collect(); - if parts.is_empty() { - return; - } - parts.join(", ") - } - _ => return, - } - } - b"Btn" => { - // Checkbox/radio — value is a name - match value.as_name() { - Ok(name) if name == b"Off" => return, - Ok(name) => { - let name_str = String::from_utf8_lossy(name).to_string(); - if name_str == "Yes" || name_str == "1" { - "Yes".to_string() - } else { - name_str - } - } - Err(_) => return, - } - } - _ => return, - }; - - // Get Rect for positioning - let (x, y, width, height) = match field_dict.get(b"Rect") { - Ok(rect_obj) => match rect_obj.as_array() { - Ok(rect_array) if rect_array.len() >= 4 => { - let x1 = get_number(&rect_array[0]).unwrap_or(0.0); - let y1 = get_number(&rect_array[1]).unwrap_or(0.0); - let x2 = get_number(&rect_array[2]).unwrap_or(0.0); - let y2 = get_number(&rect_array[3]).unwrap_or(0.0); - (x1, y1.min(y2), (x2 - x1).abs(), (y2 - y1).abs()) - } - _ => (0.0, 0.0, 0.0, 0.0), - }, - Err(_) => (0.0, 0.0, 0.0, 0.0), - }; - - // Determine page number from /P reference - let page_num = field_dict - .get(b"P") - .ok() - .and_then(|o| o.as_reference().ok()) - .and_then(|p| page_map.get(&p).copied()) - .unwrap_or(1); - - let text = if full_name.is_empty() { - value_str - } else { - format!("{}: {}", full_name, value_str) - }; - - items.push(TextItem { - text, - x, - y, - width, - height, - font: String::new(), - font_size: 0.0, - page: page_num, - is_bold: false, - is_italic: false, - item_type: ItemType::FormField, - }); -} - -/// Compute effective font size from base size and text matrix -/// Text matrix is [a, b, c, d, tx, ty] where a,d are scale factors -fn effective_font_size(base_size: f32, text_matrix: &[f32; 6]) -> f32 { - // The scale factor is typically the magnitude of the transformation - // For most PDFs, text_matrix[0] (a) is the horizontal scale - // and text_matrix[3] (d) is the vertical scale - let scale_x = (text_matrix[0].powi(2) + text_matrix[1].powi(2)).sqrt(); - let scale_y = (text_matrix[2].powi(2) + text_matrix[3].powi(2)).sqrt(); - // Use the larger of the two scales (usually they're equal for non-rotated text) - let scale = scale_x.max(scale_y); - base_size * scale -} - -/// Check if a character is CJK (Chinese, Japanese, Korean). -/// CJK languages don't use spaces between words, so word-boundary -/// heuristics should not apply when CJK characters are involved. -pub(crate) fn is_cjk_char(c: char) -> bool { - matches!(c, - '\u{1100}'..='\u{11FF}' // Hangul Jamo - | '\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation - | '\u{3040}'..='\u{309F}' // Hiragana - | '\u{30A0}'..='\u{30FF}' // Katakana - | '\u{3130}'..='\u{318F}' // Hangul Compatibility Jamo - | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs - | '\u{AC00}'..='\u{D7AF}' // Hangul Syllables - | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs - | '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms - ) -} - -pub(crate) fn is_rtl_char(c: char) -> bool { - matches!(c, - '\u{0590}'..='\u{05FF}' // Hebrew - | '\u{0600}'..='\u{06FF}' // Arabic - | '\u{0700}'..='\u{074F}' // Syriac - | '\u{0750}'..='\u{077F}' // Arabic Supplement - | '\u{0780}'..='\u{07BF}' // Thaana - | '\u{07C0}'..='\u{07FF}' // NKo - | '\u{0800}'..='\u{083F}' // Samaritan - | '\u{0840}'..='\u{085F}' // Mandaic - | '\u{08A0}'..='\u{08FF}' // Arabic Extended-A - | '\u{FB1D}'..='\u{FB4F}' // Hebrew Presentation Forms - | '\u{FB50}'..='\u{FDFF}' // Arabic Presentation Forms-A - | '\u{FE70}'..='\u{FEFF}' // Arabic Presentation Forms-B - ) -} - -pub(crate) fn is_rtl_text(texts: I) -> bool -where - I: Iterator, - S: AsRef, -{ - let (mut rtl, mut ltr) = (0u32, 0u32); - for t in texts { - for c in t.as_ref().chars() { - if is_rtl_char(c) { - rtl += 1; - } else if c.is_alphabetic() && !is_cjk_char(c) { - ltr += 1; - } - } - } - rtl > 0 && rtl > ltr -} - -fn sort_line_items(items: &mut [TextItem]) { - let rtl = is_rtl_text(items.iter().map(|i| &i.text)); - if rtl { - items.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); - } else { - items.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); - } -} - -/// Detect if a font name indicates bold style -/// Common patterns: "Bold", "Bd", "Black", "Heavy", "Demi", "Semi" (semi-bold) -pub fn is_bold_font(font_name: &str) -> bool { - let lower = font_name.to_lowercase(); - - // Check for common bold indicators - // Note: Need to be careful with "Oblique" not matching "Obl" + false positive for bold - lower.contains("bold") - || lower.contains("-bd") - || lower.contains("_bd") - || lower.contains("black") - || lower.contains("heavy") - || lower.contains("demibold") - || lower.contains("semibold") - || lower.contains("demi-bold") - || lower.contains("semi-bold") - || lower.contains("extrabold") - || lower.contains("ultrabold") - || lower.contains("medium") && !lower.contains("mediumitalic") // Some fonts use Medium for semi-bold -} - -/// Detect if a font name indicates italic/oblique style -/// Common patterns: "Italic", "It", "Oblique", "Obl", "Slant", "Inclined" -pub fn is_italic_font(font_name: &str) -> bool { - let lower = font_name.to_lowercase(); - - // Check for common italic indicators - lower.contains("italic") - || lower.contains("oblique") - || lower.contains("-it") - || lower.contains("_it") - || lower.contains("slant") - || lower.contains("inclined") - || lower.contains("kursiv") // German for italic -} - -/// Extract text from a text operand, handling encoding -#[allow(clippy::too_many_arguments)] -fn extract_text_from_operand( - obj: &Object, - current_font: &str, - font_cmaps: &FontCMaps, - font_base_names: &std::collections::HashMap, - font_tounicode_refs: &std::collections::HashMap, - font_encodings: &PageFontEncodings, - encoding_cache: &HashMap>, -) -> Option { - if let Object::String(bytes, _) = obj { - // First, try to look up CMap by ToUnicode object reference (most reliable) - // This handles cases where multiple fonts have the same BaseFont but different ToUnicode - if let Some(&obj_num) = font_tounicode_refs.get(current_font) { - if let Some(cmap) = font_cmaps.get_by_obj(obj_num) { - let decoded = cmap.decode_cids(bytes); - if !decoded.is_empty() { - return Some(decoded); - } - } - } - - // Fall back to base name lookup with object number - if let (Some(base_name), Some(&obj_num)) = ( - font_base_names.get(current_font), - font_tounicode_refs.get(current_font), - ) { - if let Some(cmap) = font_cmaps.get_with_obj(base_name, obj_num) { - let decoded = cmap.decode_cids(bytes); - if !decoded.is_empty() { - return Some(decoded); - } - } - } - - // Try base name only (legacy fallback) - if let Some(base_name) = font_base_names.get(current_font) { - if let Some(cmap) = font_cmaps.get(base_name) { - let decoded = cmap.decode_cids(bytes); - if !decoded.is_empty() { - return Some(decoded); - } - } - } - - // Also try looking up by resource name directly - if let Some(cmap) = font_cmaps.get(current_font) { - let decoded = cmap.decode_cids(bytes); - if !decoded.is_empty() { - return Some(decoded); - } - } - - // Try our custom encoding map from Differences arrays. - // The Differences array overrides specific codes in a base encoding (typically - // WinAnsiEncoding). We must combine Differences entries with the base encoding - // rather than using filter_map which silently drops unmapped bytes. - if let Some(encoding_map) = font_encodings.get(current_font) { - let has_diff_match = bytes.iter().any(|b| encoding_map.contains_key(b)); - if has_diff_match { - let decoded: String = bytes - .iter() - .filter_map(|&b| { - if let Some(&ch) = encoding_map.get(&b) { - Some(ch) - } else if b >= 0x20 { - // Base encoding fallback for printable bytes. - // For codes 0x20-0x7E this matches all standard PDF encodings. - Some(b as char) - } else { - None // Skip unmapped control characters - } - }) - .collect(); - if !decoded.is_empty() { - return Some(decoded); - } - } - } - - // Try to decode using cached font encoding from lopdf - if let Some(encoding) = encoding_cache.get(current_font) { - if let Ok(text) = Document::decode_text(encoding, bytes) { - return Some(text); - } - } - - // Fallback: try UTF-16BE then Latin-1 - if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { - let utf16: Vec = bytes[2..] - .chunks_exact(2) - .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) - .collect(); - return Some(String::from_utf16_lossy(&utf16)); - } - - // Latin-1 fallback - Some(bytes.iter().map(|&b| b as char).collect()) - } else { - None - } -} - -/// Decode a PDF text string (ActualText, etc.) that may be UTF-16BE (BOM \xFE\xFF) -/// or PDFDocEncoding (Latin-1 superset). -fn decode_text_string(bytes: &[u8]) -> String { - if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { - // UTF-16BE with BOM - let utf16: Vec = bytes[2..] - .chunks_exact(2) - .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) - .collect(); - String::from_utf16_lossy(&utf16) - } else { - // PDFDocEncoding — identical to Latin-1 for the byte range we care about - bytes.iter().map(|&b| b as char).collect() - } -} - -/// Expand Unicode ligature characters to their component characters. -/// This makes extracted text more searchable and semantically correct. -fn expand_ligatures(text: &str) -> String { - // Strip null bytes and other control characters (except newline/tab) - let text = if text - .bytes() - .any(|b| b < 0x20 && b != b'\n' && b != b'\r' && b != b'\t') - { - text.chars() - .filter(|&c| c >= ' ' || c == '\n' || c == '\r' || c == '\t') - .collect::() - } else { - text.to_string() - }; - - let mut result = String::with_capacity(text.len()); - for ch in text.chars() { - match ch { - '\u{FB00}' => result.push_str("ff"), - '\u{FB01}' => result.push_str("fi"), - '\u{FB02}' => result.push_str("fl"), - '\u{FB03}' => result.push_str("ffi"), - '\u{FB04}' => result.push_str("ffl"), - '\u{FB05}' | '\u{FB06}' => result.push_str("st"), - _ => result.push(ch), - } - } - result -} - -/// Estimate the width of a text item, falling back to a character-count heuristic when width is 0. -fn effective_width(item: &TextItem) -> f32 { - if item.width > 0.0 { - item.width - } else { - item.text.chars().count() as f32 * item.font_size * 0.5 - } -} - -/// Represents a column region on a page -#[derive(Debug, Clone)] -pub(crate) struct ColumnRegion { - pub(crate) x_min: f32, - pub(crate) x_max: f32, -} - -/// Detect column boundaries on a page using a horizontal projection profile. -/// -/// Builds an occupancy histogram across the page width and finds empty valleys -/// (gutters) where no text exists. Validates valleys with vertical consistency -/// checks to avoid false positives. -pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec { - const BIN_WIDTH: f32 = 2.0; - const MIN_GUTTER_WIDTH: f32 = 8.0; - const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30; - const MIN_ITEMS_PER_COLUMN: usize = 10; - const NOISE_FRACTION: f32 = 0.15; - - // Get items for this page - let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect(); - - if page_items.is_empty() { - return vec![]; - } - - // Find page bounds - let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min); - let x_max = page_items - .iter() - .map(|i| i.x + effective_width(i)) - .fold(f32::NEG_INFINITY, f32::max); - - let page_width = x_max - x_min; - if page_width < 200.0 { - return vec![ColumnRegion { x_min, x_max }]; - } - - if page_items.len() < 20 { - return vec![ColumnRegion { x_min, x_max }]; - } - - // Build occupancy histogram. - // Exclude items wider than 60% of page width — these are spanning items - // (titles, full-width paragraphs) that would fill the gutter and prevent - // detection of partial-page column layouts (e.g. two-column abstracts on - // a page that also has single-column introduction text). - let wide_threshold = page_width * 0.6; - let num_bins = ((page_width / BIN_WIDTH).ceil() as usize).max(1); - let mut histogram = vec![0u32; num_bins]; - - for item in &page_items { - let w = effective_width(item); - if w > wide_threshold { - continue; - } - let left = ((item.x - x_min) / BIN_WIDTH).floor() as usize; - let right = (((item.x + w) - x_min) / BIN_WIDTH).ceil() as usize; - let left = left.min(num_bins); - let right = right.min(num_bins); - for count in histogram.iter_mut().take(right).skip(left) { - *count += 1; - } - } - - // Find the noise threshold: bins with count <= max_count * NOISE_FRACTION are "empty" - let max_count = *histogram.iter().max().unwrap_or(&0); - let noise_threshold = (max_count as f32 * NOISE_FRACTION) as u32; - - // Find empty valleys (consecutive runs of low-count bins) - // Each valley is stored as (start_bin, end_bin) - let mut valleys: Vec<(usize, usize)> = Vec::new(); - let mut valley_start: Option = None; - - for (i, &count) in histogram.iter().enumerate() { - if count <= noise_threshold { - if valley_start.is_none() { - valley_start = Some(i); - } - } else if let Some(start) = valley_start { - valleys.push((start, i)); - valley_start = None; - } - } - // Close any valley that extends to the end - if let Some(start) = valley_start { - valleys.push((start, num_bins)); - } - - // Filter valleys: must be wide enough and not at page margins - let margin_threshold = page_width * 0.05; - let valleys: Vec<(usize, usize)> = valleys - .into_iter() - .filter(|&(start, end)| { - let width_pts = (end - start) as f32 * BIN_WIDTH; - if width_pts < MIN_GUTTER_WIDTH { - return false; - } - // Valley center must not be within 5% of page edges - let center_pts = ((start + end) as f32 / 2.0) * BIN_WIDTH; - center_pts > margin_threshold && center_pts < (page_width - margin_threshold) - }) - .collect(); - - if valleys.is_empty() { - return vec![ColumnRegion { x_min, x_max }]; - } - - // Compute Y range of the page - let y_min = page_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); - let y_max = page_items - .iter() - .map(|i| i.y) - .fold(f32::NEG_INFINITY, f32::max); - let y_range = y_max - y_min; - - // Validate each valley with vertical consistency - let mut valid_valleys: Vec<(usize, usize)> = Vec::new(); - for &(start, end) in &valleys { - let gutter_left = x_min + start as f32 * BIN_WIDTH; - let gutter_right = x_min + end as f32 * BIN_WIDTH; - let gutter_center = (gutter_left + gutter_right) / 2.0; - - // Collect items on each side of the gutter - let left_items: Vec<&&TextItem> = page_items - .iter() - .filter(|i| i.x + effective_width(i) <= gutter_center) - .collect(); - let right_items: Vec<&&TextItem> = - page_items.iter().filter(|i| i.x >= gutter_center).collect(); - - if left_items.len() < MIN_ITEMS_PER_COLUMN || right_items.len() < MIN_ITEMS_PER_COLUMN { - continue; - } - - // Check vertical overlap - if y_range > 0.0 { - let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); - let left_y_max = left_items - .iter() - .map(|i| i.y) - .fold(f32::NEG_INFINITY, f32::max); - let right_y_min = right_items - .iter() - .map(|i| i.y) - .fold(f32::INFINITY, f32::min); - let right_y_max = right_items - .iter() - .map(|i| i.y) - .fold(f32::NEG_INFINITY, f32::max); - - let overlap_min = left_y_min.max(right_y_min); - let overlap_max = left_y_max.min(right_y_max); - let overlap = (overlap_max - overlap_min).max(0.0); - - if overlap / y_range < MIN_VERTICAL_SPAN_RATIO { - continue; - } - } - - valid_valleys.push((start, end)); - } - - if valid_valleys.is_empty() { - return vec![ColumnRegion { x_min, x_max }]; - } - - // Limit to at most 3 gutters (4 columns) — keep the widest if more found - if valid_valleys.len() > 3 { - valid_valleys.sort_by(|a, b| { - let wa = (a.1 - a.0) as f32; - let wb = (b.1 - b.0) as f32; - wb.partial_cmp(&wa).unwrap_or(std::cmp::Ordering::Equal) - }); - valid_valleys.truncate(3); - // Re-sort by position (left to right) - valid_valleys.sort_by_key(|v| v.0); - } - - // Build column regions from gutter boundaries - let mut columns = Vec::new(); - let mut col_start = x_min; - for &(start, end) in &valid_valleys { - let gutter_center = x_min + ((start + end) as f32 / 2.0) * BIN_WIDTH; - columns.push(ColumnRegion { - x_min: col_start, - x_max: gutter_center, - }); - col_start = gutter_center; - } - columns.push(ColumnRegion { - x_min: col_start, - x_max, - }); - - columns -} - -/// Determines if a text item spans across multiple column regions (e.g. full-width headers/titles). -fn spans_multiple_columns(item: &TextItem, columns: &[ColumnRegion]) -> bool { - let w = effective_width(item); - let item_right = item.x + w; - let overlap_count = columns - .iter() - .filter(|col| { - let overlap_start = item.x.max(col.x_min); - let overlap_end = item_right.min(col.x_max); - let overlap = (overlap_end - overlap_start).max(0.0); - overlap > (col.x_max - col.x_min) * 0.10 || overlap > 20.0 - }) - .count(); - overlap_count >= 2 -} - -/// Check if a text item is likely a page number -fn is_page_number(item: &TextItem) -> bool { - let text = item.text.trim(); - - // Must be 1-4 digits only - if text.is_empty() || text.len() > 4 { - return false; - } - if !text.chars().all(|c| c.is_ascii_digit()) { - return false; - } - - // Must be at top or bottom of page. - // US Letter = 792pt, A4 = 841pt. Page numbers are typically in the - // top ~5% or bottom ~12% of the page. - item.y > 720.0 || item.y < 100.0 -} - -/// Group text items into lines, with multi-column support -/// Detect newspaper-style columns: independent text flows that should be read -/// sequentially (all of col1, then col2) rather than Y-interleaved. -fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { - if per_column_lines.len() < 2 { - return false; - } - - // Each column must independently have substantial content - let min_lines = per_column_lines.iter().map(|c| c.len()).min().unwrap_or(0); - if min_lines < 15 { - return false; - } - - // Check Y-collision: count lines in the smallest column that have a - // Y-match in any other column. High collision with many lines = newspaper. - let y_tol = 3.0; - let (smallest_idx, _) = per_column_lines - .iter() - .enumerate() - .min_by_key(|(_, c)| c.len()) - .unwrap(); - - let smallest = &per_column_lines[smallest_idx]; - let mut collisions = 0u32; - for line in smallest { - for (ci, col) in per_column_lines.iter().enumerate() { - if ci == smallest_idx { - continue; - } - if col.iter().any(|ol| (ol.y - line.y).abs() < y_tol) { - collisions += 1; - break; - } - } - } - - let ratio = collisions as f32 / smallest.len() as f32; - ratio > 0.5 -} - -/// Split column lines into a core cluster and stragglers. -/// The core is the largest group of consecutive lines separated by normal -/// line spacing. Lines in other groups (header remnants, per-word items from -/// full-width lines) are returned as stragglers. -fn split_column_stragglers(lines: Vec) -> (Vec, Vec) { - if lines.len() < 3 { - return (lines, Vec::new()); - } - - // Lines are sorted Y descending (top-first). Compute gaps. - let mut gaps: Vec = Vec::new(); - for i in 0..lines.len() - 1 { - gaps.push(lines[i].y - lines[i + 1].y); - } - - // Median gap = typical line spacing - let mut sorted_gaps = gaps.clone(); - sorted_gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let median_gap = sorted_gaps[sorted_gaps.len() / 2]; - - // A gap > 3× median (min 30pt) indicates a break between content clusters - let threshold = (median_gap * 3.0).max(30.0); - - // Find all split points - let mut split_indices: Vec = Vec::new(); - for (i, &gap) in gaps.iter().enumerate() { - if gap > threshold { - split_indices.push(i); - } - } - - if split_indices.is_empty() { - return (lines, Vec::new()); - } - - // Build segments: (start_line_idx, end_line_idx_exclusive) - let mut segments: Vec<(usize, usize)> = Vec::new(); - let mut start = 0usize; - for &si in &split_indices { - segments.push((start, si + 1)); - start = si + 1; - } - segments.push((start, lines.len())); - - // Find the largest segment (the core cluster) - let (core_seg, _) = segments - .iter() - .enumerate() - .max_by_key(|(_, (s, e))| e - s) - .unwrap(); - - let (cs, ce) = segments[core_seg]; - let mut core = Vec::with_capacity(ce - cs); - let mut stragglers = Vec::new(); - for (i, line) in lines.into_iter().enumerate() { - if i >= cs && i < ce { - core.push(line); - } else { - stragglers.push(line); - } - } - - (core, stragglers) -} - -pub fn group_into_lines(items: Vec) -> Vec { - if items.is_empty() { - return Vec::new(); - } - - // Filter out page numbers (standalone numbers at top/bottom of page) - let items: Vec = items - .into_iter() - .filter(|item| !is_page_number(item)) - .collect(); - - // Get unique pages - let mut pages: Vec = items.iter().map(|i| i.page).collect(); - pages.sort(); - pages.dedup(); - - let mut all_lines = Vec::new(); - - for page in pages { - let page_items: Vec = items.iter().filter(|i| i.page == page).cloned().collect(); - - // Detect columns for this page - let columns = detect_columns(&page_items, page); - - if columns.len() <= 1 { - // Single column - use simple sorting - let lines = group_single_column(page_items); - all_lines.extend(lines); - } else { - // Multi-column - separate spanning items from column items - let mut spanning_items: Vec = Vec::new(); - let mut column_items: Vec = Vec::new(); - - for item in &page_items { - if spans_multiple_columns(item, &columns) { - spanning_items.push(item.clone()); - } else { - column_items.push(item.clone()); - } - } - - // Process each column's items independently, preserving column identity. - // Assign each item to the column with greatest horizontal overlap - // (instead of center-point) to avoid gutter mis-assignment. - let mut col_buckets: Vec> = vec![Vec::new(); columns.len()]; - for item in &column_items { - let item_left = item.x; - let item_right = item.x + effective_width(item); - let mut best_col = 0; - let mut best_overlap = f32::NEG_INFINITY; - for (ci, col) in columns.iter().enumerate() { - let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); - if overlap > best_overlap { - best_overlap = overlap; - best_col = ci; - } - } - col_buckets[best_col].push(item.clone()); - } - - let mut per_column_lines: Vec> = Vec::new(); - for col_items in col_buckets { - let lines = group_single_column(col_items); - per_column_lines.push(lines); - } - - // Process spanning items as their own group - let spanning_lines = group_single_column(spanning_items); - - if is_newspaper_layout(&per_column_lines) { - // Newspaper: columns are independent text flows. - // 1. Split each column into its densest cluster (core) and stragglers - // 2. Use core columns to determine the above/below threshold - // 3. Emit: above items → core columns sequentially → below items - let mut core_columns: Vec> = Vec::new(); - let mut col_stragglers: Vec> = Vec::new(); - for col in per_column_lines { - let (core, stragglers) = split_column_stragglers(col); - core_columns.push(core); - col_stragglers.push(stragglers); - } - - // col_top = min of max Y across core columns - let col_top = core_columns - .iter() - .filter(|c| !c.is_empty()) - .map(|c| c.iter().map(|l| l.y).fold(f32::NEG_INFINITY, f32::max)) - .fold(f32::INFINITY, f32::min); - let margin = 5.0; - - let mut above: Vec = Vec::new(); - let mut below_spanning: Vec = Vec::new(); - - // Spanning items: above or below the column region - for line in spanning_lines { - if line.y > col_top + margin { - above.push(line); - } else { - below_spanning.push(line); - } - } - - // Column stragglers above col_top go to "above"; - // below col_top they stay with their column to avoid - // re-interleaving when sorted by Y. - let mut col_below: Vec> = vec![Vec::new(); core_columns.len()]; - for (ci, stragglers) in col_stragglers.into_iter().enumerate() { - for line in stragglers { - if line.y > col_top + margin { - above.push(line); - } else { - col_below[ci].push(line); - } - } - } - - above.sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); - below_spanning - .sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); - - all_lines.extend(above); - for col in core_columns { - all_lines.extend(col); - } - for cb in col_below { - all_lines.extend(cb); - } - all_lines.extend(below_spanning); - } else { - // Tabular: Y-interleaved merge — rows at the same Y from - // different columns form a single logical line. - let mut all_page_lines: Vec = Vec::new(); - all_page_lines.extend(spanning_lines); - for col_lines in per_column_lines { - all_page_lines.extend(col_lines); - } - - // Sort by Y descending (top-first), then by X for same-Y lines - all_page_lines.sort_by(|a, b| { - b.y.partial_cmp(&a.y) - .unwrap_or(std::cmp::Ordering::Equal) - .then( - a.items - .first() - .map(|i| i.x) - .unwrap_or(0.0) - .partial_cmp(&b.items.first().map(|i| i.x).unwrap_or(0.0)) - .unwrap_or(std::cmp::Ordering::Equal), - ) - }); - - // Merge lines at the same Y (within tolerance) into single lines - let y_tol = 3.0; - let mut merged: Vec = Vec::new(); - for line in all_page_lines { - if let Some(last) = merged.last_mut() { - if last.page == line.page && (last.y - line.y).abs() < y_tol { - last.items.extend(line.items); - sort_line_items(&mut last.items); - continue; - } - } - merged.push(line); - } - - all_lines.extend(merged); - } - } - } - - all_lines -} - -/// Determine if Y-sorting should be used instead of stream order. -/// Returns true if the stream order appears chaotic (items jump around in Y position). -fn should_use_y_sorting(items: &[TextItem]) -> bool { - if items.len() < 5 { - return false; // Not enough items to judge - } - - // Sample Y positions from stream order - let y_positions: Vec = items.iter().map(|i| i.y).collect(); - - // Count "order violations" - cases where Y increases (going up) when it should decrease - // In proper reading order, Y should generally decrease (top to bottom) - let mut large_jumps_up = 0; - let mut large_jumps_down = 0; - let jump_threshold = 50.0; // Significant Y jump - - for window in y_positions.windows(2) { - let delta = window[1] - window[0]; - if delta > jump_threshold { - large_jumps_up += 1; // Y increased significantly (jumped up on page) - } else if delta < -jump_threshold { - large_jumps_down += 1; // Y decreased significantly (normal reading direction) - } - } - - // If there are many upward jumps relative to downward jumps, order is chaotic - // A well-ordered document should have mostly downward progression - let total_jumps = large_jumps_up + large_jumps_down; - if total_jumps < 3 { - return false; // Not enough jumps to judge - } - - // If more than 40% of large jumps are upward, use Y-sorting - let chaos_ratio = large_jumps_up as f32 / total_jumps as f32; - chaos_ratio > 0.4 -} - -/// Group items from a single column into lines -/// Uses heuristics to decide between PDF stream order and Y-position sorting. -fn group_single_column(items: Vec) -> Vec { - if items.is_empty() { - return Vec::new(); - } - - // Decide whether to use stream order or Y-sorting - let use_y_sorting = should_use_y_sorting(&items); - - let items = if use_y_sorting { - // Sort by Y descending (top to bottom in PDF coords) - let mut sorted = items; - sorted.sort_by(|a, b| { - b.y.partial_cmp(&a.y) - .unwrap_or(std::cmp::Ordering::Equal) - .then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)) - }); - sorted - } else { - items - }; - - // Group items into lines - let mut lines: Vec = Vec::new(); - let y_tolerance = 3.0; - - for item in items { - // Only check the most recent line for merging - let should_merge = lines.last().is_some_and(|last_line| { - if last_line.page != item.page { - return false; - } - let y_diff = (last_line.y - item.y).abs(); - if y_diff >= y_tolerance { - return false; - } - // Check if this looks like a new line despite similar Y: - // If items are at the same X position (left margin) but different Y, - // they're vertically stacked lines, not the same line - let has_y_change = y_diff > 0.5; - if has_y_change { - if let Some(first_item) = last_line.items.first() { - let at_same_x = (item.x - first_item.x).abs() < 5.0; - // If at same X (left margin) with Y change, it's likely a new line - if at_same_x { - return false; - } - // If new item starts significantly to the left with Y change, - // it's a new line (not just out-of-order items on same line) - if let Some(last_item) = last_line.items.last() { - if item.x < last_item.x - 10.0 { - return false; - } - } - } - } - true - }); - - if should_merge { - // Add to the most recent line - lines.last_mut().unwrap().items.push(item); - } else { - // Create new line - let y = item.y; - let page = item.page; - lines.push(TextLine { - items: vec![item], - y, - page, - }); - } - } - - // Sort items within each line by X position (direction-aware) - for line in &mut lines { - sort_line_items(&mut line.items); - } - - lines -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_group_into_lines() { - let items = vec![ - TextItem { - text: "Hello".into(), - x: 100.0, - y: 700.0, - width: 50.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "World".into(), - x: 160.0, - y: 700.0, - width: 50.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "Next line".into(), - x: 100.0, - y: 680.0, - width: 80.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 2); - assert_eq!(lines[0].text(), "Hello World"); - assert_eq!(lines[1].text(), "Next line"); - } - - #[test] - fn test_bold_italic_detection() { - // Test bold detection - assert!(is_bold_font("Arial-Bold")); - assert!(is_bold_font("TimesNewRoman-Bold")); - assert!(is_bold_font("Helvetica-BoldOblique")); - assert!(is_bold_font("ABCDEF+ArialMT-Bold")); - assert!(is_bold_font("NotoSans-Black")); - assert!(is_bold_font("Roboto-SemiBold")); - assert!(!is_bold_font("Arial")); - assert!(!is_bold_font("TimesNewRoman-Italic")); - - // Test italic detection - assert!(is_italic_font("Arial-Italic")); - assert!(is_italic_font("TimesNewRoman-Italic")); - assert!(is_italic_font("Helvetica-Oblique")); - assert!(is_italic_font("ABCDEF+ArialMT-Italic")); - assert!(is_italic_font("Helvetica-BoldOblique")); - assert!(!is_italic_font("Arial")); - assert!(!is_italic_font("TimesNewRoman-Bold")); - - // Test bold-italic detection - assert!(is_bold_font("Arial-BoldItalic")); - assert!(is_italic_font("Arial-BoldItalic")); - assert!(is_bold_font("Helvetica-BoldOblique")); - assert!(is_italic_font("Helvetica-BoldOblique")); - } - - #[test] - fn test_word_level_items_get_spaces() { - // Simulate CID font per-word items touching with gap=0 - let items = vec![ - TextItem { - text: "the".into(), - x: 100.0, - y: 500.0, - width: 19.5, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "Prague".into(), - x: 119.5, - y: 500.0, - width: 42.0, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "Rules".into(), - x: 161.5, - y: 500.0, - width: 35.0, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].text(), "the Prague Rules"); - } - - #[test] - fn test_single_char_items_still_join() { - // Per-glyph positioning: single chars should join into words - let items = vec![ - TextItem { - text: "N".into(), - x: 100.0, - y: 500.0, - width: 8.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "A".into(), - x: 108.0, - y: 500.0, - width: 8.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "V".into(), - x: 116.0, - y: 500.0, - width: 8.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].text(), "NAV"); - } - - #[test] - fn test_per_glyph_word_boundaries() { - // Per-character PDF rendering (e.g. SEC filings): each glyph is a - // separate TextItem. Intra-word gaps are ≈ 0, word gaps ≈ 2.0 at - // font_size 13.3 (ratio 0.15). Must detect word boundaries correctly. - fn char_item(ch: &str, x: f32, width: f32) -> TextItem { - TextItem { - text: ch.into(), - x, - y: 719.3, - width, - height: 13.3, - font: "F4".into(), - font_size: 13.3, - page: 1, - is_bold: true, - is_italic: false, - item_type: ItemType::Text, - } - } - - // "Item 2" — gap of 2.0 between 'm' and '2' at font_size 13.3 - let items = vec![ - char_item("I", 24.3, 3.1), - char_item("t", 27.5, 2.7), - char_item("e", 30.1, 3.5), - char_item("m", 33.7, 6.7), - char_item("2", 42.3, 4.0), // gap = 42.3 - 40.4 = 1.9 - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].text(), "Item 2"); - } - - #[test] - fn test_per_glyph_words_not_merged() { - // Verify multiple words from per-character rendering get spaces between them - fn char_item(ch: &str, x: f32, width: f32) -> TextItem { - TextItem { - text: ch.into(), - x, - y: 705.5, - width, - height: 13.3, - font: "F5".into(), - font_size: 13.3, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - } - } - - // "of the" — three words, each with ~2px word gaps - let items = vec![ - char_item("o", 100.0, 4.0), - char_item("f", 104.0, 2.7), - // word gap: 108.7 → 110.7 (gap = 4.0) - char_item("t", 110.7, 2.7), - char_item("h", 113.4, 4.4), - char_item("e", 117.8, 3.5), - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].text(), "of the"); - } - - #[test] - fn test_cjk_items_join_without_spaces() { - // Japanese text items touching at gap=0 should join without spaces - let items = vec![ - TextItem { - text: "である".into(), - x: 100.0, - y: 500.0, - width: 24.0, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "履行義務".into(), - x: 124.0, - y: 500.0, - width: 32.0, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "を識別す".into(), - x: 156.0, - y: 500.0, - width: 32.0, - height: 12.0, - font: "C2_0".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - - let lines = group_into_lines(items); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].text(), "である履行義務を識別す"); - } - - fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem { - TextItem { - text: text.into(), - x, - y, - width, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - } - } - - #[test] - fn test_detect_two_columns() { - let mut items = Vec::new(); - // Left column at x=72, right column at x=350, gutter ~278-350 - for i in 0..30 { - let y = 700.0 - (i as f32) * 14.0; - items.push(make_item("Left text here", 72.0, y, 200.0)); - items.push(make_item("Right text here", 350.0, y, 200.0)); - } - let cols = detect_columns(&items, 1); - assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols); - assert!(cols[0].x_min < cols[1].x_min); - } - - #[test] - fn test_detect_three_columns() { - let mut items = Vec::new(); - // Three columns at x=50, x=220, x=390 - for i in 0..30 { - let y = 700.0 - (i as f32) * 14.0; - items.push(make_item("Col one", 50.0, y, 140.0)); - items.push(make_item("Col two", 220.0, y, 140.0)); - items.push(make_item("Col three", 390.0, y, 140.0)); - } - let cols = detect_columns(&items, 1); - assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols); - } - - #[test] - fn test_width_bleed_tolerance() { - let mut items = Vec::new(); - // Two columns with a clear gutter - for i in 0..30 { - let y = 700.0 - (i as f32) * 14.0; - items.push(make_item("Left text", 72.0, y, 200.0)); - items.push(make_item("Right text", 350.0, y, 200.0)); - } - // Add a few items that bleed across the gutter - for i in 0..3 { - let y = 700.0 - (i as f32) * 14.0; - items.push(make_item("wide", 72.0, y, 320.0)); - } - let cols = detect_columns(&items, 1); - assert!( - cols.len() >= 2, - "Width bleed should not prevent column detection, got {:?}", - cols - ); - } - - #[test] - fn test_single_column_no_false_split() { - let mut items = Vec::new(); - // Single column: items spanning full width - for i in 0..30 { - let y = 700.0 - (i as f32) * 14.0; - items.push(make_item( - "This is a full-width paragraph of text", - 72.0, - y, - 468.0, - )); - } - let cols = detect_columns(&items, 1); - assert!( - cols.len() <= 1, - "Full-width text should not be split into columns, got {:?}", - cols - ); - } - - #[test] - fn test_is_rtl_char() { - // Hebrew alef - assert!(is_rtl_char('\u{05D0}')); - // Arabic alif - assert!(is_rtl_char('\u{0627}')); - // Latin 'A' is not RTL - assert!(!is_rtl_char('A')); - // CJK is not RTL - assert!(!is_rtl_char('\u{4E00}')); - } - - #[test] - fn test_is_rtl_text() { - // Majority Hebrew with digits → RTL - assert!(is_rtl_text(["\u{05E9}\u{05DC}\u{05D5}\u{05DD} 123"].iter())); - // Majority Latin → not RTL - assert!(!is_rtl_text(["Hello world"].iter())); - // Empty → not RTL - assert!(!is_rtl_text(std::iter::empty::<&str>())); - } - - #[test] - fn test_rtl_line_sorting() { - let mut items = vec![ - TextItem { - text: "\u{05D0}".into(), // alef at x=100 - x: 100.0, - y: 700.0, - width: 10.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "\u{05D1}".into(), // bet at x=200 (rightmost) - x: 200.0, - y: 700.0, - width: 10.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - sort_line_items(&mut items); - // RTL: rightmost (higher X) comes first - assert_eq!(items[0].x, 200.0); - assert_eq!(items[1].x, 100.0); - } - - #[test] - fn test_ltr_unaffected() { - let mut items = vec![ - TextItem { - text: "Hello".into(), - x: 100.0, - y: 700.0, - width: 50.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - TextItem { - text: "World".into(), - x: 200.0, - y: 700.0, - width: 50.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page: 1, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }, - ]; - sort_line_items(&mut items); - // LTR: leftmost comes first - assert_eq!(items[0].x, 100.0); - assert_eq!(items[1].x, 200.0); - } - - #[test] - fn test_hangul_is_cjk() { - // Hangul Jamo - assert!(is_cjk_char('\u{1100}')); - // Hangul Compatibility Jamo - assert!(is_cjk_char('\u{3131}')); - // Hangul Syllable '가' - assert!(is_cjk_char('\u{AC00}')); - // Latin is not CJK - assert!(!is_cjk_char('A')); - } - - #[test] - fn test_newspaper_layout_detection() { - // Two dense columns (>15 lines each) with matching Y positions → newspaper - let make_line = |y: f32, x: f32, page: u32| TextLine { - y, - page, - items: vec![TextItem { - text: "text".into(), - x, - y, - width: 100.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }], - }; - - let col1: Vec = (0..20) - .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) - .collect(); - let col2: Vec = (0..20) - .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) - .collect(); - - assert!(is_newspaper_layout(&[col1, col2])); - } - - #[test] - fn test_tabular_layout_detection() { - // Sparse columns (<15 lines) → tabular, not newspaper - let make_line = |y: f32, x: f32, page: u32| TextLine { - y, - page, - items: vec![TextItem { - text: "text".into(), - x, - y, - width: 100.0, - height: 12.0, - font: "F1".into(), - font_size: 12.0, - page, - is_bold: false, - is_italic: false, - item_type: ItemType::Text, - }], - }; - - let col1: Vec = (0..5) - .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) - .collect(); - let col2: Vec = (0..5) - .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) - .collect(); - - assert!(!is_newspaper_layout(&[col1, col2])); - } -} diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs new file mode 100644 index 0000000..312412e --- /dev/null +++ b/src/extractor/content_stream.rs @@ -0,0 +1,608 @@ +//! PDF content-stream operator state machine. +//! +//! Walks the page's content stream, tracking the graphics state and text +//! matrix, and emits `TextItem`s and `PdfRect`s. + +use crate::text_utils::{ + decode_text_string, effective_font_size, expand_ligatures, is_bold_font, is_italic_font, +}; +use crate::tounicode::FontCMaps; +use crate::types::{ItemType, PdfRect, TextItem}; +use crate::PdfError; +use lopdf::{Document, Encoding, Object, ObjectId}; +use std::collections::HashMap; + +use super::fonts::{ + build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand, + get_operand_bytes, +}; +use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; +use super::{get_number, multiply_matrices}; + +pub(crate) fn extract_page_text_items( + doc: &Document, + page_id: ObjectId, + page_num: u32, + font_cmaps: &FontCMaps, +) -> Result<(Vec, Vec), PdfError> { + use lopdf::content::Content; + + let mut items = Vec::new(); + let mut rects: Vec = Vec::new(); + + // Get fonts for encoding + let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); + + // Build font encoding maps from Differences arrays + let font_encodings = build_font_encodings(doc, &fonts); + + // Build font width info for accurate text positioning + let font_widths = build_font_widths(doc, &fonts); + + // Build maps of font resource names to their base font names and ToUnicode object refs + let mut font_base_names: std::collections::HashMap = + std::collections::HashMap::new(); + let mut font_tounicode_refs: std::collections::HashMap = + std::collections::HashMap::new(); + for (font_name, font_dict) in &fonts { + let resource_name = String::from_utf8_lossy(font_name).to_string(); + if let Ok(base_font) = font_dict.get(b"BaseFont") { + if let Ok(name) = base_font.as_name() { + let base_name = String::from_utf8_lossy(name).to_string(); + font_base_names.insert(resource_name.clone(), base_name); + } + } + // Track ToUnicode object reference + if let Ok(tounicode) = font_dict.get(b"ToUnicode") { + if let Ok(obj_ref) = tounicode.as_reference() { + font_tounicode_refs.insert(resource_name, obj_ref.0); + } + } + } + + // Cache font encodings from lopdf (once per font, not per text operand). + // This avoids re-parsing ToUnicode CMap streams for every Tj/TJ operator. + let mut encoding_cache: HashMap> = HashMap::new(); + for (font_name, font_dict) in &fonts { + let name = String::from_utf8_lossy(font_name).to_string(); + if let Ok(enc) = font_dict.get_font_encoding(doc) { + encoding_cache.insert(name, enc); + } + } + + // Get XObjects (images) from page resources + let xobjects = get_page_xobjects(doc, page_id); + + // Get content + let content_data = doc + .get_page_content(page_id) + .map_err(|e| PdfError::Parse(e.to_string()))?; + + let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; + + // Graphics state tracking + let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix + let mut fill_is_white = false; // Fill color is white (invisible text) + let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible + let mut gstate_stack: Vec<([f32; 6], bool, i32)> = Vec::new(); + + // Text state tracking + let mut current_font = String::new(); + let mut current_font_size: f32 = 12.0; + let mut text_leading: f32 = 0.0; // TL parameter (in text-space units) + let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut in_text_block = false; + + // Marked content (ActualText) tracking + let mut marked_content_stack: Vec> = Vec::new(); + let mut suppress_glyph_extraction = false; + let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry + + for op in &content.operations { + match op.operator.as_str() { + "q" => { + // Save graphics state + gstate_stack.push((ctm, fill_is_white, text_rendering_mode)); + } + "Q" => { + // Restore graphics state + if let Some((saved_ctm, saved_fill, saved_tr)) = gstate_stack.pop() { + ctm = saved_ctm; + fill_is_white = saved_fill; + text_rendering_mode = saved_tr; + } + } + "cm" => { + // Concatenate matrix to CTM + if op.operands.len() >= 6 { + let new_matrix = [ + get_number(&op.operands[0]).unwrap_or(1.0), + get_number(&op.operands[1]).unwrap_or(0.0), + get_number(&op.operands[2]).unwrap_or(0.0), + get_number(&op.operands[3]).unwrap_or(1.0), + get_number(&op.operands[4]).unwrap_or(0.0), + get_number(&op.operands[5]).unwrap_or(0.0), + ]; + ctm = multiply_matrices(&new_matrix, &ctm); + } + } + "g" => { + // Set grayscale fill color (1.0 = white) + if let Some(gray) = op.operands.first().and_then(get_number) { + fill_is_white = gray > 0.95; + } + } + "rg" => { + // Set RGB fill color + if op.operands.len() >= 3 { + let r = get_number(&op.operands[0]).unwrap_or(0.0); + let g = get_number(&op.operands[1]).unwrap_or(0.0); + let b = get_number(&op.operands[2]).unwrap_or(0.0); + fill_is_white = r > 0.95 && g > 0.95 && b > 0.95; + } + } + "k" => { + // Set CMYK fill color (0,0,0,0 = white) + if op.operands.len() >= 4 { + let c = get_number(&op.operands[0]).unwrap_or(1.0); + let m = get_number(&op.operands[1]).unwrap_or(1.0); + let y = get_number(&op.operands[2]).unwrap_or(1.0); + let k = get_number(&op.operands[3]).unwrap_or(1.0); + fill_is_white = c < 0.05 && m < 0.05 && y < 0.05 && k < 0.05; + } + } + "BT" => { + // Begin text block + in_text_block = true; + text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + line_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + text_rendering_mode = 0; + } + "ET" => { + // End text block + in_text_block = false; + } + "Tf" => { + // Set font and size + if op.operands.len() >= 2 { + if let Ok(name) = op.operands[0].as_name() { + current_font = String::from_utf8_lossy(name).to_string(); + } + if let Ok(size) = op.operands[1].as_f32() { + current_font_size = size; + } else if let Ok(size) = op.operands[1].as_i64() { + current_font_size = size as f32; + } + } + } + "TL" => { + // Set text leading (used by T*, ', and " operators) + if let Some(tl) = op.operands.first().and_then(get_number) { + text_leading = tl; + } + } + "Tr" => { + // Set text rendering mode (3 = invisible / OCR overlay) + if let Some(mode) = op.operands.first().and_then(get_number) { + text_rendering_mode = mode as i32; + } + } + "Td" | "TD" => { + // Move text position: TLM = T(tx,ty) × TLM; Tm = TLM + // tx,ty are in text space — must be scaled by the text line matrix + if op.operands.len() >= 2 { + let tx = get_number(&op.operands[0]).unwrap_or(0.0); + let ty = get_number(&op.operands[1]).unwrap_or(0.0); + line_matrix[4] += tx * line_matrix[0] + ty * line_matrix[2]; + line_matrix[5] += tx * line_matrix[1] + ty * line_matrix[3]; + text_matrix = line_matrix; + if op.operator == "TD" { + text_leading = -ty; + } + } + } + "Tm" => { + // Set text matrix + if op.operands.len() >= 6 { + for (i, operand) in op.operands.iter().take(6).enumerate() { + text_matrix[i] = + get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 }); + } + line_matrix = text_matrix; + } + } + "T*" => { + // Move to start of next line: equivalent to 0 -TL Td + let tl = if text_leading != 0.0 { + text_leading + } else { + current_font_size * 1.2 + }; + line_matrix[4] += (-tl) * line_matrix[2]; // Usually 0 for non-rotated text + line_matrix[5] += (-tl) * line_matrix[3]; + text_matrix = line_matrix; + } + "Tj" => { + // Show text string + if in_text_block && !op.operands.is_empty() { + // Advance text matrix regardless of visibility + let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| { + get_operand_bytes(&op.operands[0]) + .map(|raw| compute_string_width_ts(raw, fi, current_font_size)) + }); + // ActualText: suppress glyph extraction, just advance text matrix + if suppress_glyph_extraction { + if let Some(w_ts) = w_ts_opt { + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + } + continue; + } + // Skip invisible (white/Tr=3) text but still advance text matrix + if fill_is_white || text_rendering_mode == 3 { + if let Some(w_ts) = w_ts_opt { + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + } + continue; + } + if let Some(text) = extract_text_from_operand( + &op.operands[0], + ¤t_font, + font_cmaps, + &font_base_names, + &font_tounicode_refs, + &font_encodings, + &encoding_cache, + ) { + let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + let width = if let Some(w_ts) = w_ts_opt { + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + (w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])).abs() + } else { + 0.0 + }; + // Only create text item for non-whitespace; whitespace + // still advances the text matrix above so gap detection works + if !text.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&text), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + } + } + "TJ" => { + // Show text with positioning — split at column-sized gaps + if in_text_block && !op.operands.is_empty() { + if let Ok(array) = op.operands[0].as_array() { + let font_info = font_widths.get(¤t_font); + let is_invisible = + fill_is_white || text_rendering_mode == 3 || suppress_glyph_extraction; + + // Compute space threshold based on font metrics when available + let space_threshold = if let Some(font_info) = font_info { + let space_em = font_info.space_width as f32 * font_info.units_scale; + let threshold = space_em * 1000.0 * 0.4; + threshold.max(80.0) + } else { + 120.0 + }; + let column_gap_threshold = space_threshold * 4.0; + + // Track sub-items for column-gap splitting: + // (text, start_width_ts, end_width_ts) + let mut sub_items: Vec<(String, f32, f32)> = Vec::new(); + let mut current_text = String::new(); + let mut sub_start_width_ts: f32 = 0.0; + let mut total_width_ts: f32 = 0.0; + for element in array { + match element { + Object::Integer(n) => { + let n_val = *n as f32; + let displacement = -n_val / 1000.0 * current_font_size; + if !is_invisible + && n_val < -column_gap_threshold + && !current_text.is_empty() + { + // Column gap: flush current segment + sub_items.push(( + std::mem::take(&mut current_text), + sub_start_width_ts, + total_width_ts, + )); + total_width_ts += displacement; + sub_start_width_ts = total_width_ts; + } else { + total_width_ts += displacement; + if !is_invisible + && n_val < -space_threshold + && !current_text.is_empty() + && !current_text.ends_with(' ') + { + current_text.push(' '); + } + } + continue; + } + Object::Real(n) => { + let n_val = *n; + let displacement = -n_val / 1000.0 * current_font_size; + if !is_invisible + && n_val < -column_gap_threshold + && !current_text.is_empty() + { + sub_items.push(( + std::mem::take(&mut current_text), + sub_start_width_ts, + total_width_ts, + )); + total_width_ts += displacement; + sub_start_width_ts = total_width_ts; + } else { + total_width_ts += displacement; + if !is_invisible + && n_val < -space_threshold + && !current_text.is_empty() + && !current_text.ends_with(' ') + { + current_text.push(' '); + } + } + continue; + } + _ => {} + } + if let Some(fi) = font_info { + if let Some(raw_bytes) = get_operand_bytes(element) { + total_width_ts += + compute_string_width_ts(raw_bytes, fi, current_font_size); + } + } + if !is_invisible { + if let Some(text) = extract_text_from_operand( + element, + ¤t_font, + font_cmaps, + &font_base_names, + &font_tounicode_refs, + &font_encodings, + &encoding_cache, + ) { + current_text.push_str(&text); + } + } + } + // Flush remaining text + if !is_invisible && !current_text.trim().is_empty() { + sub_items.push((current_text, sub_start_width_ts, total_width_ts)); + } + // Emit one TextItem per sub-item + if !sub_items.is_empty() { + let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]; + for (text, start_w, end_w) in &sub_items { + let offset_tm = [ + text_matrix[0], + text_matrix[1], + text_matrix[2], + text_matrix[3], + text_matrix[4] + start_w * text_matrix[0], + text_matrix[5] + start_w * text_matrix[1], + ]; + let combined = multiply_matrices(&offset_tm, &ctm); + let (x, y) = (combined[4], combined[5]); + let width = if font_info.is_some() { + ((end_w - start_w) * scale_x).abs() + } else { + 0.0 + }; + items.push(TextItem { + text: expand_ligatures(text), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + // Always advance text matrix by total width + if font_info.is_some() { + text_matrix[4] += total_width_ts * text_matrix[0]; + text_matrix[5] += total_width_ts * text_matrix[1]; + } + } + } + } + "'" => { + // Move to next line and show text (equivalent to T* then Tj) + let tl = if text_leading != 0.0 { + text_leading + } else { + current_font_size * 1.2 + }; + line_matrix[4] += (-tl) * line_matrix[2]; + line_matrix[5] += (-tl) * line_matrix[3]; + text_matrix = line_matrix; + if !(fill_is_white + || text_rendering_mode == 3 + || suppress_glyph_extraction + || op.operands.is_empty()) + { + if let Some(text) = extract_text_from_operand( + &op.operands[0], + ¤t_font, + font_cmaps, + &font_base_names, + &font_tounicode_refs, + &font_encodings, + &encoding_cache, + ) { + if !text.trim().is_empty() { + let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&text), + x, + y, + width: 0.0, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + } + } + "Do" => { + // XObject invocation - could be an image or form + if !op.operands.is_empty() { + if let Ok(name) = op.operands[0].as_name() { + let xobj_name = String::from_utf8_lossy(name).to_string(); + + if let Some(xobj_type) = xobjects.get(&xobj_name) { + match xobj_type { + XObjectType::Image => { + // Skip images — text extraction only + } + XObjectType::Form(form_id) => { + // Extract text from Form XObject + let form_items = extract_form_xobject_text( + doc, *form_id, page_num, font_cmaps, &ctm, + ); + items.extend(form_items); + } + } + } + } + } + } + "BMC" => { + // Begin Marked Content (no properties) + marked_content_stack.push(None); + } + "BDC" => { + // Begin Marked Content with properties — extract ActualText + let mut actual_text: Option = None; + if op.operands.len() >= 2 { + let dict = match &op.operands[1] { + Object::Dictionary(d) => Some(d.clone()), + Object::Reference(id) => doc.get_dictionary(*id).ok().cloned(), + _ => None, + }; + if let Some(d) = dict { + if let Ok(val) = d.get(b"ActualText") { + actual_text = match val { + Object::String(bytes, _) => Some(decode_text_string(bytes)), + _ => None, + }; + } + } + } + if actual_text.is_some() { + suppress_glyph_extraction = true; + actual_text_start_tm = Some(text_matrix); + } + marked_content_stack.push(actual_text); + } + "EMC" => { + // End Marked Content — emit ActualText item with correct width + if let Some(Some(at)) = marked_content_stack.pop() { + // Compute width from text matrix advancement during BDC..EMC + if let Some(start_tm) = actual_text_start_tm.take() { + let combined = multiply_matrices(&start_tm, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + // Width in device space from text matrix delta + let delta_ts = text_matrix[4] - start_tm[4]; + let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; + let width = (delta_ts * scale_x).abs(); + if !at.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: at, + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); + } + } + "re" => { + // Rectangle operator: collect for table-grid detection + if op.operands.len() >= 4 { + let rx = get_number(&op.operands[0]).unwrap_or(0.0); + let ry = get_number(&op.operands[1]).unwrap_or(0.0); + let rw = get_number(&op.operands[2]).unwrap_or(0.0); + let rh = get_number(&op.operands[3]).unwrap_or(0.0); + // Transform origin to device space + let x_dev = rx * ctm[0] + ry * ctm[2] + ctm[4]; + let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5]; + let w_dev = rw * ctm[0]; + let h_dev = rh * ctm[3]; + rects.push(PdfRect { + x: x_dev, + y: y_dev, + width: w_dev, + height: h_dev, + page: page_num, + }); + } + } + _ => {} + } + } + + let items = super::merge_text_items(items); + Ok((items, rects)) +} diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs new file mode 100644 index 0000000..8255864 --- /dev/null +++ b/src/extractor/fonts.rs @@ -0,0 +1,582 @@ +//! Font width parsing, encoding, and text decoding. + +use crate::glyph_names::glyph_to_char; +use crate::tounicode::FontCMaps; +use crate::types::{FontEncodingMap, FontWidthInfo, PageFontEncodings, PageFontWidths}; +use lopdf::{Document, Encoding, Object}; +use std::collections::HashMap; + +/// Resolve a PDF object reference to an array +pub(crate) fn resolve_array<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Vec> { + match obj { + Object::Array(arr) => Some(arr), + Object::Reference(r) => { + if let Ok(Object::Array(arr)) = doc.get_object(*r) { + Some(arr) + } else { + None + } + } + _ => None, + } +} + +/// Resolve a PDF object reference to a dictionary +pub(crate) fn resolve_dict<'a>( + doc: &'a Document, + obj: &'a Object, +) -> Option<&'a lopdf::Dictionary> { + match obj { + Object::Dictionary(d) => Some(d), + Object::Reference(r) => doc.get_dictionary(*r).ok(), + _ => None, + } +} + +/// Build font width info for all fonts on a page +pub(crate) fn build_font_widths( + doc: &Document, + fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, +) -> PageFontWidths { + let mut widths = PageFontWidths::new(); + + for (font_name, font_dict) in fonts { + let resource_name = String::from_utf8_lossy(font_name).to_string(); + if let Some(info) = parse_font_widths(doc, font_dict) { + widths.insert(resource_name, info); + } + } + + widths +} + +/// Parse font widths from a font dictionary, dispatching by Subtype +pub(crate) fn parse_font_widths( + doc: &Document, + font_dict: &lopdf::Dictionary, +) -> Option { + // Get the font subtype + let subtype = font_dict.get(b"Subtype").ok()?; + let subtype_name = subtype.as_name().ok()?; + + match subtype_name { + b"Type0" => parse_type0_widths(doc, font_dict), + b"Type1" | b"TrueType" | b"MMType1" | b"Type3" => parse_simple_font_widths(doc, font_dict), + _ => None, + } +} + +/// Parse widths for simple fonts (Type1, TrueType, MMType1, Type3) +/// Reads FirstChar, LastChar, and Widths array. +/// For Type3 fonts, reads FontMatrix to determine the correct units_scale. +pub(crate) fn parse_simple_font_widths( + doc: &Document, + font_dict: &lopdf::Dictionary, +) -> Option { + let first_char = font_dict.get(b"FirstChar").ok().and_then(|o| match o { + Object::Integer(n) => Some(*n as u16), + Object::Reference(r) => doc.get_object(*r).ok().and_then(|o| { + if let Object::Integer(n) = o { + Some(*n as u16) + } else { + None + } + }), + _ => None, + })?; + + let last_char = font_dict.get(b"LastChar").ok().and_then(|o| match o { + Object::Integer(n) => Some(*n as u16), + Object::Reference(r) => doc.get_object(*r).ok().and_then(|o| { + if let Object::Integer(n) = o { + Some(*n as u16) + } else { + None + } + }), + _ => None, + })?; + + let widths_obj = font_dict.get(b"Widths").ok()?; + let widths_array = resolve_array(doc, widths_obj)?; + + let mut widths = HashMap::new(); + let mut space_width: u16 = 0; + + for (i, w_obj) in widths_array.iter().enumerate() { + let code = first_char + i as u16; + if code > last_char { + break; + } + let w = match w_obj { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + Object::Reference(r) => { + if let Ok(obj) = doc.get_object(*r) { + match obj { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => continue, + } + } else { + continue; + } + } + _ => continue, + }; + if code == 32 { + space_width = w; + } + widths.insert(code, w); + } + + // Determine units_scale: for Type3 fonts, use FontMatrix[0]; for others, use 1/1000 + let units_scale = if let Ok(fm) = font_dict.get(b"FontMatrix") { + if let Some(arr) = resolve_array(doc, fm) { + if !arr.is_empty() { + match &arr[0] { + Object::Real(r) => r.abs(), + Object::Integer(i) => (*i as f32).abs(), + _ => 0.001, + } + } else { + 0.001 + } + } else { + 0.001 + } + } else { + 0.001 // Standard 1000-unit system + }; + + // If space width wasn't found in the table, estimate from font metrics. + // The default of 250 is calibrated for standard 1000-unit fonts (units_scale=0.001). + // For Type3 fonts with different coordinate systems, use average glyph width instead. + if space_width == 0 { + if !widths.is_empty() && (units_scale - 0.001).abs() > 0.0005 { + // Non-standard scale: estimate space as ~45% of average glyph width + let sum: u32 = widths.values().map(|&w| w as u32).sum(); + let avg = sum as f32 / widths.len() as f32; + space_width = (avg * 0.45).max(1.0) as u16; + } else { + space_width = 250; + } + } + + Some(FontWidthInfo { + widths, + default_width: 0, + space_width, + is_cid: false, + units_scale, + wmode: 0, + }) +} + +/// Parse widths for Type0 (composite/CID) fonts +/// Reads DescendantFonts → CIDFont → W array and DW value +pub(crate) fn parse_type0_widths( + doc: &Document, + font_dict: &lopdf::Dictionary, +) -> Option { + let desc_fonts_obj = font_dict.get(b"DescendantFonts").ok()?; + let desc_fonts = resolve_array(doc, desc_fonts_obj)?; + + if desc_fonts.is_empty() { + return None; + } + + // Get the first descendant font dictionary + let cid_font_dict = resolve_dict(doc, &desc_fonts[0])?; + + // Get DW (default width) + let default_width = cid_font_dict + .get(b"DW") + .ok() + .and_then(|o| match o { + Object::Integer(n) => Some(*n as u16), + Object::Real(n) => Some(*n as u16), + _ => None, + }) + .unwrap_or(1000); + + let mut widths = HashMap::new(); + + // Parse W array if present + if let Ok(w_obj) = cid_font_dict.get(b"W") { + if let Some(w_array) = resolve_array(doc, w_obj) { + parse_cid_w_array(doc, w_array, &mut widths); + } + } + + // Try to determine space width (CID 32 or CID 3 are common for space) + let space_width = widths + .get(&32) + .or_else(|| widths.get(&3)) + .copied() + .unwrap_or(if default_width > 0 { + default_width / 4 + } else { + 250 + }); + + let wmode = font_dict + .get(b"WMode") + .ok() + .and_then(|o| match o { + Object::Integer(n) => Some(*n as u8), + _ => None, + }) + .unwrap_or(0); + + Some(FontWidthInfo { + widths, + default_width, + space_width, + is_cid: true, + units_scale: 0.001, // CID fonts use standard 1000-unit system + wmode, + }) +} + +/// Parse a CID W array into widths map +/// Format: [c [w1 w2 ...]] (consecutive from c) or [c_first c_last w] (range with same width) +pub(crate) fn parse_cid_w_array( + doc: &Document, + w_array: &[Object], + widths: &mut HashMap, +) { + let mut i = 0; + while i < w_array.len() { + let start_cid = match &w_array[i] { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => { + i += 1; + continue; + } + }; + i += 1; + if i >= w_array.len() { + break; + } + + // Check if next element is an array (consecutive widths) or integer (range) + match &w_array[i] { + Object::Array(arr) => { + // [c [w1 w2 ...]] — consecutive widths starting at c + for (j, w_obj) in arr.iter().enumerate() { + let w = match w_obj { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => continue, + }; + widths.insert(start_cid + j as u16, w); + } + i += 1; + } + Object::Reference(r) => { + // Could be a reference to an array + if let Ok(Object::Array(arr)) = doc.get_object(*r) { + for (j, w_obj) in arr.iter().enumerate() { + let w = match w_obj { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => continue, + }; + widths.insert(start_cid + j as u16, w); + } + i += 1; + } else { + // Treat as c_first c_last w + i += 1; // skip this + } + } + Object::Integer(end_cid) => { + // [c_first c_last w] — range with uniform width + let end = *end_cid as u16; + i += 1; + if i >= w_array.len() { + break; + } + let w = match &w_array[i] { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => { + i += 1; + continue; + } + }; + for cid in start_cid..=end { + widths.insert(cid, w); + } + i += 1; + } + Object::Real(end_cid) => { + let end = *end_cid as u16; + i += 1; + if i >= w_array.len() { + break; + } + let w = match &w_array[i] { + Object::Integer(n) => *n as u16, + Object::Real(n) => *n as u16, + _ => { + i += 1; + continue; + } + }; + for cid in start_cid..=end { + widths.insert(cid, w); + } + i += 1; + } + _ => { + i += 1; + } + } + } +} + +/// Compute the width of a string in text space units, +/// given raw bytes and font width info. +/// Returns width in text space units (font_units * units_scale * font_size). +pub(crate) fn compute_string_width_ts( + bytes: &[u8], + font_info: &FontWidthInfo, + font_size: f32, +) -> f32 { + let mut total: f32 = 0.0; + if font_info.is_cid { + // 2-byte (big-endian) character codes + let mut j = 0; + while j + 1 < bytes.len() { + let cid = u16::from_be_bytes([bytes[j], bytes[j + 1]]); + let w = font_info + .widths + .get(&cid) + .copied() + .unwrap_or(font_info.default_width); + total += w as f32; + j += 2; + } + } else { + // 1-byte character codes + for &b in bytes { + let code = b as u16; + let w = font_info + .widths + .get(&code) + .copied() + .unwrap_or(font_info.default_width); + total += w as f32; + } + } + // Convert from font units to text space using the font's scale factor + total * font_info.units_scale * font_size +} + +/// Extract raw bytes from a PDF operand (String object) +pub(crate) fn get_operand_bytes(obj: &Object) -> Option<&[u8]> { + if let Object::String(bytes, _) = obj { + Some(bytes) + } else { + None + } +} + +/// Build encoding maps for all fonts on a page +pub(crate) fn build_font_encodings( + doc: &Document, + fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, +) -> PageFontEncodings { + let mut encodings = PageFontEncodings::new(); + + for (font_name, font_dict) in fonts { + let resource_name = String::from_utf8_lossy(font_name).to_string(); + + if let Some(encoding_map) = parse_font_encoding(doc, font_dict) { + encodings.insert(resource_name, encoding_map); + } + } + + encodings +} + +/// Parse font encoding from a font dictionary +pub(crate) fn parse_font_encoding( + doc: &Document, + font_dict: &lopdf::Dictionary, +) -> Option { + let encoding_obj = font_dict.get(b"Encoding").ok()?; + + // Encoding can be a name or a dictionary + match encoding_obj { + Object::Name(_name) => { + // Standard encoding name (e.g., MacRomanEncoding, WinAnsiEncoding) + // For standard encodings, we can use the standard tables + // But we still need to check for Differences + None // Let lopdf handle standard encodings + } + Object::Reference(obj_ref) => { + // Reference to encoding dictionary + if let Ok(enc_dict) = doc.get_dictionary(*obj_ref) { + parse_encoding_dictionary(doc, enc_dict) + } else { + None + } + } + Object::Dictionary(enc_dict) => parse_encoding_dictionary(doc, enc_dict), + _ => None, + } +} + +/// Parse an encoding dictionary with Differences array +pub(crate) fn parse_encoding_dictionary( + doc: &Document, + enc_dict: &lopdf::Dictionary, +) -> Option { + let differences = enc_dict.get(b"Differences").ok()?; + + let diff_array = match differences { + Object::Array(arr) => arr.clone(), + Object::Reference(obj_ref) => { + if let Ok(Object::Array(arr)) = doc.get_object(*obj_ref) { + arr.clone() + } else { + return None; + } + } + _ => return None, + }; + + let mut encoding_map = FontEncodingMap::new(); + let mut current_code: u8 = 0; + + for item in diff_array { + match item { + Object::Integer(n) => { + // This sets the starting code for subsequent glyph names + current_code = n as u8; + } + Object::Name(name) => { + // Map current code to glyph name -> Unicode + let glyph_name = String::from_utf8_lossy(&name).to_string(); + if let Some(ch) = glyph_to_char(&glyph_name) { + encoding_map.insert(current_code, ch); + } + current_code = current_code.wrapping_add(1); + } + _ => {} + } + } + + if encoding_map.is_empty() { + None + } else { + Some(encoding_map) + } +} + +/// Decode text from a PDF string operand using font CMaps, encodings, and fallbacks. +pub(crate) fn extract_text_from_operand( + obj: &Object, + current_font: &str, + font_cmaps: &FontCMaps, + font_base_names: &std::collections::HashMap, + font_tounicode_refs: &std::collections::HashMap, + font_encodings: &PageFontEncodings, + encoding_cache: &HashMap>, +) -> Option { + if let Object::String(bytes, _) = obj { + // First, try to look up CMap by ToUnicode object reference (most reliable) + // This handles cases where multiple fonts have the same BaseFont but different ToUnicode + if let Some(&obj_num) = font_tounicode_refs.get(current_font) { + if let Some(cmap) = font_cmaps.get_by_obj(obj_num) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Fall back to base name lookup with object number + if let (Some(base_name), Some(&obj_num)) = ( + font_base_names.get(current_font), + font_tounicode_refs.get(current_font), + ) { + if let Some(cmap) = font_cmaps.get_with_obj(base_name, obj_num) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Try base name only (legacy fallback) + if let Some(base_name) = font_base_names.get(current_font) { + if let Some(cmap) = font_cmaps.get(base_name) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Also try looking up by resource name directly + if let Some(cmap) = font_cmaps.get(current_font) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + + // Try our custom encoding map from Differences arrays. + // The Differences array overrides specific codes in a base encoding (typically + // WinAnsiEncoding). We must combine Differences entries with the base encoding + // rather than using filter_map which silently drops unmapped bytes. + if let Some(encoding_map) = font_encodings.get(current_font) { + let has_diff_match = bytes.iter().any(|b| encoding_map.contains_key(b)); + if has_diff_match { + let decoded: String = bytes + .iter() + .filter_map(|&b| { + if let Some(&ch) = encoding_map.get(&b) { + Some(ch) + } else if b >= 0x20 { + // Base encoding fallback for printable bytes. + // For codes 0x20-0x7E this matches all standard PDF encodings. + Some(b as char) + } else { + None // Skip unmapped control characters + } + }) + .collect(); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Try to decode using cached font encoding from lopdf + if let Some(encoding) = encoding_cache.get(current_font) { + if let Ok(text) = Document::decode_text(encoding, bytes) { + return Some(text); + } + } + + // Fallback: try UTF-16BE then Latin-1 + if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { + let utf16: Vec = bytes[2..] + .chunks_exact(2) + .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) + .collect(); + return Some(String::from_utf16_lossy(&utf16)); + } + + // Latin-1 fallback + Some(bytes.iter().map(|&b| b as char).collect()) + } else { + None + } +} diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs new file mode 100644 index 0000000..832b7ac --- /dev/null +++ b/src/extractor/layout.rs @@ -0,0 +1,635 @@ +//! Column detection, line grouping, and reading-order layout. + +use crate::text_utils::{effective_width, sort_line_items}; +use crate::types::{TextItem, TextLine}; + +/// Represents a column region on a page +#[derive(Debug, Clone)] +pub(crate) struct ColumnRegion { + pub(crate) x_min: f32, + pub(crate) x_max: f32, +} + +/// Detect column boundaries on a page using a horizontal projection profile. +/// +/// Builds an occupancy histogram across the page width and finds empty valleys +/// (gutters) where no text exists. Validates valleys with vertical consistency +/// checks to avoid false positives. +pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec { + const BIN_WIDTH: f32 = 2.0; + const MIN_GUTTER_WIDTH: f32 = 8.0; + const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30; + const MIN_ITEMS_PER_COLUMN: usize = 10; + const NOISE_FRACTION: f32 = 0.15; + + // Get items for this page + let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect(); + + if page_items.is_empty() { + return vec![]; + } + + // Find page bounds + let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min); + let x_max = page_items + .iter() + .map(|i| i.x + effective_width(i)) + .fold(f32::NEG_INFINITY, f32::max); + + let page_width = x_max - x_min; + if page_width < 200.0 { + return vec![ColumnRegion { x_min, x_max }]; + } + + if page_items.len() < 20 { + return vec![ColumnRegion { x_min, x_max }]; + } + + // Build occupancy histogram. + // Exclude items wider than 60% of page width — these are spanning items + // (titles, full-width paragraphs) that would fill the gutter and prevent + // detection of partial-page column layouts (e.g. two-column abstracts on + // a page that also has single-column introduction text). + let wide_threshold = page_width * 0.6; + let num_bins = ((page_width / BIN_WIDTH).ceil() as usize).max(1); + let mut histogram = vec![0u32; num_bins]; + + for item in &page_items { + let w = effective_width(item); + if w > wide_threshold { + continue; + } + let left = ((item.x - x_min) / BIN_WIDTH).floor() as usize; + let right = (((item.x + w) - x_min) / BIN_WIDTH).ceil() as usize; + let left = left.min(num_bins); + let right = right.min(num_bins); + for count in histogram.iter_mut().take(right).skip(left) { + *count += 1; + } + } + + // Find the noise threshold: bins with count <= max_count * NOISE_FRACTION are "empty" + let max_count = *histogram.iter().max().unwrap_or(&0); + let noise_threshold = (max_count as f32 * NOISE_FRACTION) as u32; + + // Find empty valleys (consecutive runs of low-count bins) + // Each valley is stored as (start_bin, end_bin) + let mut valleys: Vec<(usize, usize)> = Vec::new(); + let mut valley_start: Option = None; + + for (i, &count) in histogram.iter().enumerate() { + if count <= noise_threshold { + if valley_start.is_none() { + valley_start = Some(i); + } + } else if let Some(start) = valley_start { + valleys.push((start, i)); + valley_start = None; + } + } + // Close any valley that extends to the end + if let Some(start) = valley_start { + valleys.push((start, num_bins)); + } + + // Filter valleys: must be wide enough and not at page margins + let margin_threshold = page_width * 0.05; + let valleys: Vec<(usize, usize)> = valleys + .into_iter() + .filter(|&(start, end)| { + let width_pts = (end - start) as f32 * BIN_WIDTH; + if width_pts < MIN_GUTTER_WIDTH { + return false; + } + // Valley center must not be within 5% of page edges + let center_pts = ((start + end) as f32 / 2.0) * BIN_WIDTH; + center_pts > margin_threshold && center_pts < (page_width - margin_threshold) + }) + .collect(); + + if valleys.is_empty() { + return vec![ColumnRegion { x_min, x_max }]; + } + + // Compute Y range of the page + let y_min = page_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); + let y_max = page_items + .iter() + .map(|i| i.y) + .fold(f32::NEG_INFINITY, f32::max); + let y_range = y_max - y_min; + + // Validate each valley with vertical consistency + let mut valid_valleys: Vec<(usize, usize)> = Vec::new(); + for &(start, end) in &valleys { + let gutter_left = x_min + start as f32 * BIN_WIDTH; + let gutter_right = x_min + end as f32 * BIN_WIDTH; + let gutter_center = (gutter_left + gutter_right) / 2.0; + + // Collect items on each side of the gutter + let left_items: Vec<&&TextItem> = page_items + .iter() + .filter(|i| i.x + effective_width(i) <= gutter_center) + .collect(); + let right_items: Vec<&&TextItem> = + page_items.iter().filter(|i| i.x >= gutter_center).collect(); + + if left_items.len() < MIN_ITEMS_PER_COLUMN || right_items.len() < MIN_ITEMS_PER_COLUMN { + continue; + } + + // Check vertical overlap + if y_range > 0.0 { + let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); + let left_y_max = left_items + .iter() + .map(|i| i.y) + .fold(f32::NEG_INFINITY, f32::max); + let right_y_min = right_items + .iter() + .map(|i| i.y) + .fold(f32::INFINITY, f32::min); + let right_y_max = right_items + .iter() + .map(|i| i.y) + .fold(f32::NEG_INFINITY, f32::max); + + let overlap_min = left_y_min.max(right_y_min); + let overlap_max = left_y_max.min(right_y_max); + let overlap = (overlap_max - overlap_min).max(0.0); + + if overlap / y_range < MIN_VERTICAL_SPAN_RATIO { + continue; + } + } + + valid_valleys.push((start, end)); + } + + if valid_valleys.is_empty() { + return vec![ColumnRegion { x_min, x_max }]; + } + + // Limit to at most 3 gutters (4 columns) — keep the widest if more found + if valid_valleys.len() > 3 { + valid_valleys.sort_by(|a, b| { + let wa = (a.1 - a.0) as f32; + let wb = (b.1 - b.0) as f32; + wb.partial_cmp(&wa).unwrap_or(std::cmp::Ordering::Equal) + }); + valid_valleys.truncate(3); + // Re-sort by position (left to right) + valid_valleys.sort_by_key(|v| v.0); + } + + // Build column regions from gutter boundaries + let mut columns = Vec::new(); + let mut col_start = x_min; + for &(start, end) in &valid_valleys { + let gutter_center = x_min + ((start + end) as f32 / 2.0) * BIN_WIDTH; + columns.push(ColumnRegion { + x_min: col_start, + x_max: gutter_center, + }); + col_start = gutter_center; + } + columns.push(ColumnRegion { + x_min: col_start, + x_max, + }); + + columns +} + +/// Determines if a text item spans across multiple column regions (e.g. full-width headers/titles). +fn spans_multiple_columns(item: &TextItem, columns: &[ColumnRegion]) -> bool { + let w = effective_width(item); + let item_right = item.x + w; + let overlap_count = columns + .iter() + .filter(|col| { + let overlap_start = item.x.max(col.x_min); + let overlap_end = item_right.min(col.x_max); + let overlap = (overlap_end - overlap_start).max(0.0); + overlap > (col.x_max - col.x_min) * 0.10 || overlap > 20.0 + }) + .count(); + overlap_count >= 2 +} + +/// Check if a text item is likely a page number +fn is_page_number(item: &TextItem) -> bool { + let text = item.text.trim(); + + // Must be 1-4 digits only + if text.is_empty() || text.len() > 4 { + return false; + } + if !text.chars().all(|c| c.is_ascii_digit()) { + return false; + } + + // Must be at top or bottom of page. + // US Letter = 792pt, A4 = 841pt. Page numbers are typically in the + // top ~5% or bottom ~12% of the page. + item.y > 720.0 || item.y < 100.0 +} + +/// Group text items into lines, with multi-column support +/// Detect newspaper-style columns: independent text flows that should be read +/// sequentially (all of col1, then col2) rather than Y-interleaved. +pub(crate) fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { + if per_column_lines.len() < 2 { + return false; + } + + // Each column must independently have substantial content + let min_lines = per_column_lines.iter().map(|c| c.len()).min().unwrap_or(0); + if min_lines < 15 { + return false; + } + + // Check Y-collision: count lines in the smallest column that have a + // Y-match in any other column. High collision with many lines = newspaper. + let y_tol = 3.0; + let (smallest_idx, _) = per_column_lines + .iter() + .enumerate() + .min_by_key(|(_, c)| c.len()) + .unwrap(); + + let smallest = &per_column_lines[smallest_idx]; + let mut collisions = 0u32; + for line in smallest { + for (ci, col) in per_column_lines.iter().enumerate() { + if ci == smallest_idx { + continue; + } + if col.iter().any(|ol| (ol.y - line.y).abs() < y_tol) { + collisions += 1; + break; + } + } + } + + let ratio = collisions as f32 / smallest.len() as f32; + ratio > 0.5 +} + +/// Split column lines into a core cluster and stragglers. +/// The core is the largest group of consecutive lines separated by normal +/// line spacing. Lines in other groups (header remnants, per-word items from +/// full-width lines) are returned as stragglers. +fn split_column_stragglers(lines: Vec) -> (Vec, Vec) { + if lines.len() < 3 { + return (lines, Vec::new()); + } + + // Lines are sorted Y descending (top-first). Compute gaps. + let mut gaps: Vec = Vec::new(); + for i in 0..lines.len() - 1 { + gaps.push(lines[i].y - lines[i + 1].y); + } + + // Median gap = typical line spacing + let mut sorted_gaps = gaps.clone(); + sorted_gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_gap = sorted_gaps[sorted_gaps.len() / 2]; + + // A gap > 3× median (min 30pt) indicates a break between content clusters + let threshold = (median_gap * 3.0).max(30.0); + + // Find all split points + let mut split_indices: Vec = Vec::new(); + for (i, &gap) in gaps.iter().enumerate() { + if gap > threshold { + split_indices.push(i); + } + } + + if split_indices.is_empty() { + return (lines, Vec::new()); + } + + // Build segments: (start_line_idx, end_line_idx_exclusive) + let mut segments: Vec<(usize, usize)> = Vec::new(); + let mut start = 0usize; + for &si in &split_indices { + segments.push((start, si + 1)); + start = si + 1; + } + segments.push((start, lines.len())); + + // Find the largest segment (the core cluster) + let (core_seg, _) = segments + .iter() + .enumerate() + .max_by_key(|(_, (s, e))| e - s) + .unwrap(); + + let (cs, ce) = segments[core_seg]; + let mut core = Vec::with_capacity(ce - cs); + let mut stragglers = Vec::new(); + for (i, line) in lines.into_iter().enumerate() { + if i >= cs && i < ce { + core.push(line); + } else { + stragglers.push(line); + } + } + + (core, stragglers) +} + +pub fn group_into_lines(items: Vec) -> Vec { + if items.is_empty() { + return Vec::new(); + } + + // Filter out page numbers (standalone numbers at top/bottom of page) + let items: Vec = items + .into_iter() + .filter(|item| !is_page_number(item)) + .collect(); + + // Get unique pages + let mut pages: Vec = items.iter().map(|i| i.page).collect(); + pages.sort(); + pages.dedup(); + + let mut all_lines = Vec::new(); + + for page in pages { + let page_items: Vec = items.iter().filter(|i| i.page == page).cloned().collect(); + + // Detect columns for this page + let columns = detect_columns(&page_items, page); + + if columns.len() <= 1 { + // Single column - use simple sorting + let lines = group_single_column(page_items); + all_lines.extend(lines); + } else { + // Multi-column - separate spanning items from column items + let mut spanning_items: Vec = Vec::new(); + let mut column_items: Vec = Vec::new(); + + for item in &page_items { + if spans_multiple_columns(item, &columns) { + spanning_items.push(item.clone()); + } else { + column_items.push(item.clone()); + } + } + + // Process each column's items independently, preserving column identity. + // Assign each item to the column with greatest horizontal overlap + // (instead of center-point) to avoid gutter mis-assignment. + let mut col_buckets: Vec> = vec![Vec::new(); columns.len()]; + for item in &column_items { + let item_left = item.x; + let item_right = item.x + effective_width(item); + let mut best_col = 0; + let mut best_overlap = f32::NEG_INFINITY; + for (ci, col) in columns.iter().enumerate() { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > best_overlap { + best_overlap = overlap; + best_col = ci; + } + } + col_buckets[best_col].push(item.clone()); + } + + let mut per_column_lines: Vec> = Vec::new(); + for col_items in col_buckets { + let lines = group_single_column(col_items); + per_column_lines.push(lines); + } + + // Process spanning items as their own group + let spanning_lines = group_single_column(spanning_items); + + if is_newspaper_layout(&per_column_lines) { + // Newspaper: columns are independent text flows. + // 1. Split each column into its densest cluster (core) and stragglers + // 2. Use core columns to determine the above/below threshold + // 3. Emit: above items → core columns sequentially → below items + let mut core_columns: Vec> = Vec::new(); + let mut col_stragglers: Vec> = Vec::new(); + for col in per_column_lines { + let (core, stragglers) = split_column_stragglers(col); + core_columns.push(core); + col_stragglers.push(stragglers); + } + + // col_top = min of max Y across core columns + let col_top = core_columns + .iter() + .filter(|c| !c.is_empty()) + .map(|c| c.iter().map(|l| l.y).fold(f32::NEG_INFINITY, f32::max)) + .fold(f32::INFINITY, f32::min); + let margin = 5.0; + + let mut above: Vec = Vec::new(); + let mut below_spanning: Vec = Vec::new(); + + // Spanning items: above or below the column region + for line in spanning_lines { + if line.y > col_top + margin { + above.push(line); + } else { + below_spanning.push(line); + } + } + + // Column stragglers above col_top go to "above"; + // below col_top they stay with their column to avoid + // re-interleaving when sorted by Y. + let mut col_below: Vec> = vec![Vec::new(); core_columns.len()]; + for (ci, stragglers) in col_stragglers.into_iter().enumerate() { + for line in stragglers { + if line.y > col_top + margin { + above.push(line); + } else { + col_below[ci].push(line); + } + } + } + + above.sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + below_spanning + .sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + + all_lines.extend(above); + for col in core_columns { + all_lines.extend(col); + } + for cb in col_below { + all_lines.extend(cb); + } + all_lines.extend(below_spanning); + } else { + // Tabular: Y-interleaved merge — rows at the same Y from + // different columns form a single logical line. + let mut all_page_lines: Vec = Vec::new(); + all_page_lines.extend(spanning_lines); + for col_lines in per_column_lines { + all_page_lines.extend(col_lines); + } + + // Sort by Y descending (top-first), then by X for same-Y lines + all_page_lines.sort_by(|a, b| { + b.y.partial_cmp(&a.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + a.items + .first() + .map(|i| i.x) + .unwrap_or(0.0) + .partial_cmp(&b.items.first().map(|i| i.x).unwrap_or(0.0)) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + + // Merge lines at the same Y (within tolerance) into single lines + let y_tol = 3.0; + let mut merged: Vec = Vec::new(); + for line in all_page_lines { + if let Some(last) = merged.last_mut() { + if last.page == line.page && (last.y - line.y).abs() < y_tol { + last.items.extend(line.items); + sort_line_items(&mut last.items); + continue; + } + } + merged.push(line); + } + + all_lines.extend(merged); + } + } + } + + all_lines +} + +/// Determine if Y-sorting should be used instead of stream order. +/// Returns true if the stream order appears chaotic (items jump around in Y position). +fn should_use_y_sorting(items: &[TextItem]) -> bool { + if items.len() < 5 { + return false; // Not enough items to judge + } + + // Sample Y positions from stream order + let y_positions: Vec = items.iter().map(|i| i.y).collect(); + + // Count "order violations" - cases where Y increases (going up) when it should decrease + // In proper reading order, Y should generally decrease (top to bottom) + let mut large_jumps_up = 0; + let mut large_jumps_down = 0; + let jump_threshold = 50.0; // Significant Y jump + + for window in y_positions.windows(2) { + let delta = window[1] - window[0]; + if delta > jump_threshold { + large_jumps_up += 1; // Y increased significantly (jumped up on page) + } else if delta < -jump_threshold { + large_jumps_down += 1; // Y decreased significantly (normal reading direction) + } + } + + // If there are many upward jumps relative to downward jumps, order is chaotic + // A well-ordered document should have mostly downward progression + let total_jumps = large_jumps_up + large_jumps_down; + if total_jumps < 3 { + return false; // Not enough jumps to judge + } + + // If more than 40% of large jumps are upward, use Y-sorting + let chaos_ratio = large_jumps_up as f32 / total_jumps as f32; + chaos_ratio > 0.4 +} + +/// Group items from a single column into lines +/// Uses heuristics to decide between PDF stream order and Y-position sorting. +fn group_single_column(items: Vec) -> Vec { + if items.is_empty() { + return Vec::new(); + } + + // Decide whether to use stream order or Y-sorting + let use_y_sorting = should_use_y_sorting(&items); + + let items = if use_y_sorting { + // Sort by Y descending (top to bottom in PDF coords) + let mut sorted = items; + sorted.sort_by(|a, b| { + b.y.partial_cmp(&a.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)) + }); + sorted + } else { + items + }; + + // Group items into lines + let mut lines: Vec = Vec::new(); + let y_tolerance = 3.0; + + for item in items { + // Only check the most recent line for merging + let should_merge = lines.last().is_some_and(|last_line| { + if last_line.page != item.page { + return false; + } + let y_diff = (last_line.y - item.y).abs(); + if y_diff >= y_tolerance { + return false; + } + // Check if this looks like a new line despite similar Y: + // If items are at the same X position (left margin) but different Y, + // they're vertically stacked lines, not the same line + let has_y_change = y_diff > 0.5; + if has_y_change { + if let Some(first_item) = last_line.items.first() { + let at_same_x = (item.x - first_item.x).abs() < 5.0; + // If at same X (left margin) with Y change, it's likely a new line + if at_same_x { + return false; + } + // If new item starts significantly to the left with Y change, + // it's a new line (not just out-of-order items on same line) + if let Some(last_item) = last_line.items.last() { + if item.x < last_item.x - 10.0 { + return false; + } + } + } + } + true + }); + + if should_merge { + // Add to the most recent line + lines.last_mut().unwrap().items.push(item); + } else { + // Create new line + let y = item.y; + let page = item.page; + lines.push(TextLine { + items: vec![item], + y, + page, + }); + } + } + + // Sort items within each line by X position (direction-aware) + for line in &mut lines { + sort_line_items(&mut line.items); + } + + lines +} diff --git a/src/extractor/links.rs b/src/extractor/links.rs new file mode 100644 index 0000000..f7e9090 --- /dev/null +++ b/src/extractor/links.rs @@ -0,0 +1,320 @@ +//! Hyperlink and AcroForm field extraction. + +use crate::types::{ItemType, TextItem}; +use lopdf::{Document, Object, ObjectId}; +use std::collections::HashMap; + +use super::fonts::{resolve_array, resolve_dict}; +use super::get_number; + +pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> Vec { + let mut links = Vec::new(); + + // Try to get the page dictionary + if let Ok(page_dict) = doc.get_dictionary(page_id) { + // Get Annots array + let annots = if let Ok(annots_ref) = page_dict.get(b"Annots") { + if let Ok(obj_ref) = annots_ref.as_reference() { + doc.get_object(obj_ref) + .ok() + .and_then(|o| o.as_array().ok().cloned()) + } else { + annots_ref.as_array().ok().cloned() + } + } else { + None + }; + + if let Some(annots) = annots { + for annot_ref in annots { + // Get annotation dictionary + let annot_dict = if let Ok(obj_ref) = annot_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + annot_ref.as_dict().ok() + }; + + if let Some(annot_dict) = annot_dict { + // Check if this is a Link annotation + if let Ok(subtype) = annot_dict.get(b"Subtype") { + if let Ok(subtype_name) = subtype.as_name() { + if subtype_name != b"Link" { + continue; + } + } + } + + // Get the Rect (position) + let rect = if let Ok(rect_obj) = annot_dict.get(b"Rect") { + if let Ok(rect_array) = rect_obj.as_array() { + if rect_array.len() >= 4 { + let x1 = get_number(&rect_array[0]).unwrap_or(0.0); + let y1 = get_number(&rect_array[1]).unwrap_or(0.0); + let x2 = get_number(&rect_array[2]).unwrap_or(0.0); + let y2 = get_number(&rect_array[3]).unwrap_or(0.0); + Some((x1, y1, x2 - x1, y2 - y1)) + } else { + None + } + } else { + None + } + } else { + None + }; + + // Get the action (A dictionary) or Dest + let uri = extract_link_uri(doc, annot_dict); + + if let (Some((x, y, width, height)), Some(url)) = (rect, uri) { + links.push(TextItem { + text: url.clone(), + x, + y, + width, + height, + font: String::new(), + font_size: 0.0, + page: page_num, + is_bold: false, + is_italic: false, + item_type: ItemType::Link(url), + }); + } + } + } + } + } + + links +} + +/// Extract URI from a link annotation +pub(crate) fn extract_link_uri(doc: &Document, annot_dict: &lopdf::Dictionary) -> Option { + // Try to get the A (Action) dictionary + if let Ok(action_ref) = annot_dict.get(b"A") { + let action_dict = if let Ok(obj_ref) = action_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + action_ref.as_dict().ok() + }; + + if let Some(action_dict) = action_dict { + // Check for URI action + if let Ok(uri_obj) = action_dict.get(b"URI") { + if let Ok(uri_str) = uri_obj.as_str() { + return Some(String::from_utf8_lossy(uri_str).to_string()); + } + } + } + } + + // Try Dest (named destination) - less common for external links + // We'll skip this for now as it requires looking up named destinations + + None +} + +/// Extract form field values from AcroForm dictionary. +/// Returns TextItems positioned at each field's Rect so they flow into the markdown pipeline. +pub(crate) fn extract_form_fields( + doc: &Document, + page_map: &HashMap, +) -> Vec { + let mut items = Vec::new(); + + // Navigate: trailer -> /Root -> /AcroForm -> /Fields + let root = match doc.trailer.get(b"Root") { + Ok(root_ref) => match root_ref.as_reference() { + Ok(r) => match doc.get_dictionary(r) { + Ok(d) => d, + Err(_) => return items, + }, + Err(_) => return items, + }, + Err(_) => return items, + }; + + let acroform = match root.get(b"AcroForm") { + Ok(obj) => match resolve_dict(doc, obj) { + Some(d) => d, + None => return items, + }, + Err(_) => return items, + }; + + let fields = match acroform.get(b"Fields") { + Ok(obj) => match resolve_array(doc, obj) { + Some(arr) => arr.clone(), + None => return items, + }, + Err(_) => return items, + }; + + for field_obj in &fields { + if let Ok(field_ref) = field_obj.as_reference() { + walk_form_fields(doc, field_ref, None, "", page_map, &mut items); + } + } + + items +} + +/// Recursively walk the form field tree, extracting leaf field values. +pub(crate) fn walk_form_fields( + doc: &Document, + field_id: ObjectId, + parent_ft: Option<&[u8]>, + parent_name: &str, + page_map: &HashMap, + items: &mut Vec, +) { + let field_dict = match doc.get_dictionary(field_id) { + Ok(d) => d, + Err(_) => return, + }; + + // Build fully qualified field name + let local_name = field_dict + .get(b"T") + .ok() + .and_then(|o| o.as_str().ok()) + .map(|s| String::from_utf8_lossy(s).to_string()) + .unwrap_or_default(); + + let full_name = if parent_name.is_empty() { + local_name.clone() + } else if local_name.is_empty() { + parent_name.to_string() + } else { + format!("{}.{}", parent_name, local_name) + }; + + // Determine field type (may be inherited from parent) + let ft = field_dict + .get(b"FT") + .ok() + .and_then(|o| o.as_name().ok()) + .or(parent_ft); + + // Check for /Kids — if present, recurse into children + if let Ok(kids_obj) = field_dict.get(b"Kids") { + if let Some(kids) = resolve_array(doc, kids_obj) { + let kids = kids.clone(); + for kid in &kids { + if let Ok(kid_ref) = kid.as_reference() { + walk_form_fields(doc, kid_ref, ft, &full_name, page_map, items); + } + } + return; + } + } + + // Leaf field — extract value + let ft = match ft { + Some(ft) => ft, + None => return, + }; + + // Skip signature fields + if ft == b"Sig" { + return; + } + + // Get field value + let value = match field_dict.get(b"V") { + Ok(v) => v, + Err(_) => return, + }; + + let value_str = match ft { + b"Tx" | b"Ch" => { + // Text or Choice field — value is a string or array of strings + match value { + Object::String(s, _) => { + let s = String::from_utf8_lossy(s).to_string(); + if s.is_empty() { + return; + } + s + } + Object::Array(arr) => { + let parts: Vec = arr + .iter() + .filter_map(|o| { + if let Object::String(s, _) = o { + Some(String::from_utf8_lossy(s).to_string()) + } else { + None + } + }) + .collect(); + if parts.is_empty() { + return; + } + parts.join(", ") + } + _ => return, + } + } + b"Btn" => { + // Checkbox/radio — value is a name + match value.as_name() { + Ok(name) if name == b"Off" => return, + Ok(name) => { + let name_str = String::from_utf8_lossy(name).to_string(); + if name_str == "Yes" || name_str == "1" { + "Yes".to_string() + } else { + name_str + } + } + Err(_) => return, + } + } + _ => return, + }; + + // Get Rect for positioning + let (x, y, width, height) = match field_dict.get(b"Rect") { + Ok(rect_obj) => match rect_obj.as_array() { + Ok(rect_array) if rect_array.len() >= 4 => { + let x1 = get_number(&rect_array[0]).unwrap_or(0.0); + let y1 = get_number(&rect_array[1]).unwrap_or(0.0); + let x2 = get_number(&rect_array[2]).unwrap_or(0.0); + let y2 = get_number(&rect_array[3]).unwrap_or(0.0); + (x1, y1.min(y2), (x2 - x1).abs(), (y2 - y1).abs()) + } + _ => (0.0, 0.0, 0.0, 0.0), + }, + Err(_) => (0.0, 0.0, 0.0, 0.0), + }; + + // Determine page number from /P reference + let page_num = field_dict + .get(b"P") + .ok() + .and_then(|o| o.as_reference().ok()) + .and_then(|p| page_map.get(&p).copied()) + .unwrap_or(1); + + let text = if full_name.is_empty() { + value_str + } else { + format!("{}: {}", full_name, value_str) + }; + + items.push(TextItem { + text, + x, + y, + width, + height, + font: String::new(), + font_size: 0.0, + page: page_num, + is_bold: false, + is_italic: false, + item_type: ItemType::FormField, + }); +} diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs new file mode 100644 index 0000000..c109245 --- /dev/null +++ b/src/extractor/mod.rs @@ -0,0 +1,893 @@ +//! Text extraction from PDF using lopdf +//! +//! This module extracts text with position information for structure detection. + +mod content_stream; +mod fonts; +mod layout; +mod links; +mod xobjects; + +use crate::text_utils::is_rtl_text; +use crate::tounicode::FontCMaps; +use crate::types::{PdfRect, TextItem}; +use crate::PdfError; +use lopdf::{Document, Object, ObjectId}; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use content_stream::extract_page_text_items; +use links::{extract_form_fields, extract_page_links}; + +// Re-export public types so existing `crate::extractor::X` paths keep working. +pub use crate::text_utils::{is_bold_font, is_italic_font}; +pub use crate::types::{ItemType, TextLine}; +pub use layout::group_into_lines; + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/// Extract text from PDF file as plain string +pub fn extract_text>(path: P) -> Result { + crate::validate_pdf_file(&path)?; + let doc = Document::load(path)?; + extract_text_from_doc(&doc) +} + +/// Extract text from PDF memory buffer +pub fn extract_text_mem(buffer: &[u8]) -> Result { + crate::validate_pdf_bytes(buffer)?; + let doc = Document::load_mem(buffer)?; + extract_text_from_doc(&doc) +} + +/// Extract text from loaded document +fn extract_text_from_doc(doc: &Document) -> Result { + let pages = doc.get_pages(); + let page_nums: Vec = pages.keys().cloned().collect(); + + doc.extract_text(&page_nums) + .map_err(|e| PdfError::Parse(e.to_string())) +} + +/// Extract text with position information from PDF file +pub fn extract_text_with_positions>(path: P) -> Result, PdfError> { + extract_text_with_positions_pages(path, None) +} + +/// Extract text with positions from a file, limited to specific pages. +/// +/// `page_filter` is an optional set of 1-indexed page numbers to process. +/// When `None`, all pages are processed. +pub fn extract_text_with_positions_pages>( + path: P, + page_filter: Option<&HashSet>, +) -> Result, PdfError> { + let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?; + Ok(items) +} + +/// Extract text with positions and rectangles from a file. +pub(crate) fn extract_text_with_positions_and_rects>( + path: P, + page_filter: Option<&HashSet>, +) -> Result<(Vec, Vec), PdfError> { + // Read the raw PDF bytes for ToUnicode extraction + let pdf_bytes = std::fs::read(path.as_ref())?; + crate::validate_pdf_bytes(&pdf_bytes)?; + let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes); + + let doc = Document::load_mem(&pdf_bytes)?; + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) +} + +/// Extract text with positions from memory buffer +pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result, PdfError> { + extract_text_with_positions_mem_pages(buffer, None) +} + +/// Extract text with positions from memory buffer, limited to specific pages. +pub fn extract_text_with_positions_mem_pages( + buffer: &[u8], + page_filter: Option<&HashSet>, +) -> Result, PdfError> { + let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?; + Ok(items) +} + +/// Extract text with positions and rectangles from memory buffer. +pub(crate) fn extract_text_with_positions_mem_and_rects( + buffer: &[u8], + page_filter: Option<&HashSet>, +) -> Result<(Vec, Vec), PdfError> { + crate::validate_pdf_bytes(buffer)?; + // Extract ToUnicode CMaps from raw PDF bytes + let font_cmaps = FontCMaps::from_pdf_bytes(buffer); + + let doc = Document::load_mem(buffer)?; + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) +} + +// --------------------------------------------------------------------------- +// Orchestration +// --------------------------------------------------------------------------- + +/// Extract positioned text and rectangles from loaded document +fn extract_positioned_text_from_doc( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, +) -> Result<(Vec, Vec), PdfError> { + // If raw byte scanning found no CMaps, populate from the document model. + // This handles PDFs with compressed object streams where raw scanning fails. + let mut font_cmaps_owned; + let font_cmaps = if font_cmaps.by_obj_num.is_empty() { + font_cmaps_owned = font_cmaps.clone(); + populate_cmaps_from_doc(doc, &mut font_cmaps_owned); + &font_cmaps_owned + } else { + font_cmaps + }; + + let pages = doc.get_pages(); + let mut all_items = Vec::new(); + let mut all_rects = Vec::new(); + + // Build page ObjectId → page number map for form field extraction + let page_id_to_num: HashMap = + pages.iter().map(|(num, &id)| (id, *num)).collect(); + + for (page_num, &page_id) in pages.iter() { + if let Some(filter) = page_filter { + if !filter.contains(page_num) { + continue; + } + } + let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + all_items.extend(items); + all_rects.extend(rects); + + // Extract hyperlinks from page annotations + let links = extract_page_links(doc, page_id, *page_num); + all_items.extend(links); + } + + // Extract AcroForm field values + let form_items = extract_form_fields(doc, &page_id_to_num); + all_items.extend(form_items); + + Ok((all_items, all_rects)) +} + +/// Populate FontCMaps from the lopdf document model for ToUnicode streams +/// that weren't found by raw byte scanning (e.g. in compressed object streams). +fn populate_cmaps_from_doc(doc: &Document, font_cmaps: &mut FontCMaps) { + use crate::tounicode::ToUnicodeCMap; + + for (_page_num, &page_id) in doc.get_pages().iter() { + let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); + for (font_name, font_dict) in &fonts { + if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") { + if let Ok(obj_ref) = tounicode_ref.as_reference() { + let obj_num = obj_ref.0; + if font_cmaps.by_obj_num.contains_key(&obj_num) { + continue; + } + // Try to get the stream content via lopdf + if let Ok(stream) = doc.get_object(obj_ref) { + if let Ok(stream) = stream.as_stream() { + if let Ok(data) = stream.decompressed_content() { + if let Some(cmap) = ToUnicodeCMap::parse(&data) { + let resource_name = + String::from_utf8_lossy(font_name).to_string(); + let base_name = font_dict + .get(b"BaseFont") + .ok() + .and_then(|o| o.as_name().ok()) + .map(|n| String::from_utf8_lossy(n).to_string()); + + // Store by object number + font_cmaps.by_obj_num.insert(obj_num, cmap.clone()); + // Store by resource name + font_cmaps + .by_name + .insert(resource_name.clone(), cmap.clone()); + if let Some(base) = base_name { + let unique_key = format!("{}_{}", base, obj_num); + font_cmaps.by_name.insert(unique_key, cmap.clone()); + font_cmaps.by_name.insert(base, cmap); + } + } + } + } + } + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Shared helpers (used by submodules via `super::`) +// --------------------------------------------------------------------------- + +/// Multiply two 2D transformation matrices +/// Matrix format: [a, b, c, d, e, f] representing: +/// | a b 0 | +/// | c d 0 | +/// | e f 1 | +pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] { + [ + m1[0] * m2[0] + m1[1] * m2[2], + m1[0] * m2[1] + m1[1] * m2[3], + m1[2] * m2[0] + m1[3] * m2[2], + m1[2] * m2[1] + m1[3] * m2[3], + m1[4] * m2[0] + m1[5] * m2[2] + m2[4], + m1[4] * m2[1] + m1[5] * m2[3] + m2[5], + ] +} + +/// Merge adjacent text items on the same line into single items. +/// +/// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each +/// group by X, then merges consecutive items that share a similar font size +/// and are close horizontally. +pub(crate) fn merge_text_items(items: Vec) -> Vec { + if items.is_empty() { + return items; + } + + // Group items by (page, Y position) with 5pt tolerance + let y_tolerance = 5.0; + let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new(); + + for item in &items { + let found = line_groups + .iter_mut() + .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance); + if let Some((_, _, group)) = found { + group.push(item); + } else { + line_groups.push((item.page, item.y, vec![item])); + } + } + + // Sort each group by X position (direction-aware) + for (_, _, group) in &mut line_groups { + let rtl = is_rtl_text(group.iter().map(|i| &i.text)); + if rtl { + group.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } + } + + // Sort groups by page then Y descending (top of page first) + line_groups.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)) + }); + + let mut merged = Vec::new(); + + for (_, _, group) in &line_groups { + let mut i = 0; + while i < group.len() { + let first = group[i]; + let mut text = first.text.clone(); + let mut end_x = first.x + first.width; + let x_gap_max = first.font_size * 0.5; + + let mut j = i + 1; + while j < group.len() { + let next = group[j]; + // Must be similar font size (within 20%) + if (next.font_size - first.font_size).abs() > first.font_size * 0.20 { + break; + } + let gap = next.x - end_x; + if gap > x_gap_max { + break; + } + if gap < -first.font_size * 0.5 { + break; + } + // Insert space at word boundaries + if gap > first.font_size * 0.08 { + text.push(' '); + } + text.push_str(&next.text); + end_x = next.x + next.width; + j += 1; + } + + merged.push(TextItem { + text, + x: first.x, + y: first.y, + width: end_x - first.x, + height: first.height, + font: first.font.clone(), + font_size: first.font_size, + page: first.page, + is_bold: first.is_bold, + is_italic: first.is_italic, + item_type: first.item_type.clone(), + }); + + i = j; + } + } + + merged +} + +/// Helper to get f32 from Object +pub(crate) fn get_number(obj: &Object) -> Option { + match obj { + Object::Integer(i) => Some(*i as f32), + Object::Real(r) => Some(*r), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items}; + use crate::types::{ItemType, TextLine}; + use layout::{detect_columns, is_newspaper_layout}; + + #[test] + fn test_group_into_lines() { + let items = vec![ + TextItem { + text: "Hello".into(), + x: 100.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "World".into(), + x: 160.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "Next line".into(), + x: 100.0, + y: 680.0, + width: 80.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].text(), "Hello World"); + assert_eq!(lines[1].text(), "Next line"); + } + + #[test] + fn test_bold_italic_detection() { + // Test bold detection + assert!(is_bold_font("Arial-Bold")); + assert!(is_bold_font("TimesNewRoman-Bold")); + assert!(is_bold_font("Helvetica-BoldOblique")); + assert!(is_bold_font("ABCDEF+ArialMT-Bold")); + assert!(is_bold_font("NotoSans-Black")); + assert!(is_bold_font("Roboto-SemiBold")); + assert!(!is_bold_font("Arial")); + assert!(!is_bold_font("TimesNewRoman-Italic")); + + // Test italic detection + assert!(is_italic_font("Arial-Italic")); + assert!(is_italic_font("TimesNewRoman-Italic")); + assert!(is_italic_font("Helvetica-Oblique")); + assert!(is_italic_font("ABCDEF+ArialMT-Italic")); + assert!(is_italic_font("Helvetica-BoldOblique")); + assert!(!is_italic_font("Arial")); + assert!(!is_italic_font("TimesNewRoman-Bold")); + + // Test bold-italic detection + assert!(is_bold_font("Arial-BoldItalic")); + assert!(is_italic_font("Arial-BoldItalic")); + assert!(is_bold_font("Helvetica-BoldOblique")); + assert!(is_italic_font("Helvetica-BoldOblique")); + } + + #[test] + fn test_word_level_items_get_spaces() { + // Simulate CID font per-word items touching with gap=0 + let items = vec![ + TextItem { + text: "the".into(), + x: 100.0, + y: 500.0, + width: 19.5, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "Prague".into(), + x: 119.5, + y: 500.0, + width: 42.0, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "Rules".into(), + x: 161.5, + y: 500.0, + width: 35.0, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "the Prague Rules"); + } + + #[test] + fn test_single_char_items_still_join() { + // Per-glyph positioning: single chars should join into words + let items = vec![ + TextItem { + text: "N".into(), + x: 100.0, + y: 500.0, + width: 8.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "A".into(), + x: 108.0, + y: 500.0, + width: 8.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "V".into(), + x: 116.0, + y: 500.0, + width: 8.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "NAV"); + } + + #[test] + fn test_per_glyph_word_boundaries() { + // Per-character PDF rendering (e.g. SEC filings): each glyph is a + // separate TextItem. Intra-word gaps are ≈ 0, word gaps ≈ 2.0 at + // font_size 13.3 (ratio 0.15). Must detect word boundaries correctly. + fn char_item(ch: &str, x: f32, width: f32) -> TextItem { + TextItem { + text: ch.into(), + x, + y: 719.3, + width, + height: 13.3, + font: "F4".into(), + font_size: 13.3, + page: 1, + is_bold: true, + is_italic: false, + item_type: ItemType::Text, + } + } + + // "Item 2" — gap of 2.0 between 'm' and '2' at font_size 13.3 + let items = vec![ + char_item("I", 24.3, 3.1), + char_item("t", 27.5, 2.7), + char_item("e", 30.1, 3.5), + char_item("m", 33.7, 6.7), + char_item("2", 42.3, 4.0), // gap = 42.3 - 40.4 = 1.9 + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "Item 2"); + } + + #[test] + fn test_per_glyph_words_not_merged() { + // Verify multiple words from per-character rendering get spaces between them + fn char_item(ch: &str, x: f32, width: f32) -> TextItem { + TextItem { + text: ch.into(), + x, + y: 705.5, + width, + height: 13.3, + font: "F5".into(), + font_size: 13.3, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + // "of the" — three words, each with ~2px word gaps + let items = vec![ + char_item("o", 100.0, 4.0), + char_item("f", 104.0, 2.7), + // word gap: 108.7 → 110.7 (gap = 4.0) + char_item("t", 110.7, 2.7), + char_item("h", 113.4, 4.4), + char_item("e", 117.8, 3.5), + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "of the"); + } + + #[test] + fn test_cjk_items_join_without_spaces() { + // Japanese text items touching at gap=0 should join without spaces + let items = vec![ + TextItem { + text: "である".into(), + x: 100.0, + y: 500.0, + width: 24.0, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "履行義務".into(), + x: 124.0, + y: 500.0, + width: 32.0, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "を識別す".into(), + x: 156.0, + y: 500.0, + width: 32.0, + height: 12.0, + font: "C2_0".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "である履行義務を識別す"); + } + + fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + #[test] + fn test_detect_two_columns() { + let mut items = Vec::new(); + // Left column at x=72, right column at x=350, gutter ~278-350 + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Left text here", 72.0, y, 200.0)); + items.push(make_item("Right text here", 350.0, y, 200.0)); + } + let cols = detect_columns(&items, 1); + assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols); + assert!(cols[0].x_min < cols[1].x_min); + } + + #[test] + fn test_detect_three_columns() { + let mut items = Vec::new(); + // Three columns at x=50, x=220, x=390 + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Col one", 50.0, y, 140.0)); + items.push(make_item("Col two", 220.0, y, 140.0)); + items.push(make_item("Col three", 390.0, y, 140.0)); + } + let cols = detect_columns(&items, 1); + assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols); + } + + #[test] + fn test_width_bleed_tolerance() { + let mut items = Vec::new(); + // Two columns with a clear gutter + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Left text", 72.0, y, 200.0)); + items.push(make_item("Right text", 350.0, y, 200.0)); + } + // Add a few items that bleed across the gutter + for i in 0..3 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("wide", 72.0, y, 320.0)); + } + let cols = detect_columns(&items, 1); + assert!( + cols.len() >= 2, + "Width bleed should not prevent column detection, got {:?}", + cols + ); + } + + #[test] + fn test_single_column_no_false_split() { + let mut items = Vec::new(); + // Single column: items spanning full width + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item( + "This is a full-width paragraph of text", + 72.0, + y, + 468.0, + )); + } + let cols = detect_columns(&items, 1); + assert!( + cols.len() <= 1, + "Full-width text should not be split into columns, got {:?}", + cols + ); + } + + #[test] + fn test_is_rtl_char() { + // Hebrew alef + assert!(is_rtl_char('\u{05D0}')); + // Arabic alif + assert!(is_rtl_char('\u{0627}')); + // Latin 'A' is not RTL + assert!(!is_rtl_char('A')); + // CJK is not RTL + assert!(!is_rtl_char('\u{4E00}')); + } + + #[test] + fn test_is_rtl_text() { + // Majority Hebrew with digits → RTL + assert!(is_rtl_text(["\u{05E9}\u{05DC}\u{05D5}\u{05DD} 123"].iter())); + // Majority Latin → not RTL + assert!(!is_rtl_text(["Hello world"].iter())); + // Empty → not RTL + assert!(!is_rtl_text(std::iter::empty::<&str>())); + } + + #[test] + fn test_rtl_line_sorting() { + let mut items = vec![ + TextItem { + text: "\u{05D0}".into(), // alef at x=100 + x: 100.0, + y: 700.0, + width: 10.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "\u{05D1}".into(), // bet at x=200 (rightmost) + x: 200.0, + y: 700.0, + width: 10.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + sort_line_items(&mut items); + // RTL: rightmost (higher X) comes first + assert_eq!(items[0].x, 200.0); + assert_eq!(items[1].x, 100.0); + } + + #[test] + fn test_ltr_unaffected() { + let mut items = vec![ + TextItem { + text: "Hello".into(), + x: 100.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "World".into(), + x: 200.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + sort_line_items(&mut items); + // LTR: leftmost comes first + assert_eq!(items[0].x, 100.0); + assert_eq!(items[1].x, 200.0); + } + + #[test] + fn test_hangul_is_cjk() { + // Hangul Jamo + assert!(is_cjk_char('\u{1100}')); + // Hangul Compatibility Jamo + assert!(is_cjk_char('\u{3131}')); + // Hangul Syllable '가' + assert!(is_cjk_char('\u{AC00}')); + // Latin is not CJK + assert!(!is_cjk_char('A')); + } + + #[test] + fn test_newspaper_layout_detection() { + // Two dense columns (>15 lines each) with matching Y positions → newspaper + let make_line = |y: f32, x: f32, page: u32| TextLine { + y, + page, + items: vec![TextItem { + text: "text".into(), + x, + y, + width: 100.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }], + }; + + let col1: Vec = (0..20) + .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) + .collect(); + let col2: Vec = (0..20) + .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) + .collect(); + + assert!(is_newspaper_layout(&[col1, col2])); + } + + #[test] + fn test_tabular_layout_detection() { + // Sparse columns (<15 lines) → tabular, not newspaper + let make_line = |y: f32, x: f32, page: u32| TextLine { + y, + page, + items: vec![TextItem { + text: "text".into(), + x, + y, + width: 100.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }], + }; + + let col1: Vec = (0..5) + .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) + .collect(); + let col2: Vec = (0..5) + .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) + .collect(); + + assert!(!is_newspaper_layout(&[col1, col2])); + } +} diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs new file mode 100644 index 0000000..0125683 --- /dev/null +++ b/src/extractor/xobjects.rs @@ -0,0 +1,474 @@ +//! Form XObject and image XObject extraction. + +use crate::text_utils::{effective_font_size, expand_ligatures, is_bold_font, is_italic_font}; +use crate::tounicode::FontCMaps; +use crate::types::{ItemType, TextItem}; +use lopdf::{Document, Encoding, Object, ObjectId}; +use std::collections::HashMap; + +use super::fonts::{ + build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand, + get_operand_bytes, +}; +use super::{get_number, multiply_matrices}; + +pub(crate) enum XObjectType { + Image, + Form(ObjectId), +} + +/// Get XObjects from page resources, categorized by type +pub(crate) fn get_page_xobjects( + doc: &Document, + page_id: ObjectId, +) -> std::collections::HashMap { + let mut xobject_types = std::collections::HashMap::new(); + + // Try to get the page dictionary + if let Ok(page_dict) = doc.get_dictionary(page_id) { + // Get Resources dictionary + let resources = if let Ok(res_ref) = page_dict.get(b"Resources") { + if let Ok(obj_ref) = res_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + res_ref.as_dict().ok() + } + } else { + None + }; + + if let Some(resources) = resources { + // Get XObject dictionary from Resources + if let Ok(xobjects_ref) = resources.get(b"XObject") { + let xobjects = if let Ok(obj_ref) = xobjects_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + xobjects_ref.as_dict().ok() + }; + + if let Some(xobjects) = xobjects { + for (name, value) in xobjects.iter() { + let name_str = String::from_utf8_lossy(name).to_string(); + + // Check XObject subtype + if let Ok(obj_ref) = value.as_reference() { + if let Ok(Object::Stream(stream)) = doc.get_object(obj_ref) { + if let Ok(subtype) = stream.dict.get(b"Subtype") { + if let Ok(subtype_name) = subtype.as_name() { + if subtype_name == b"Image" { + xobject_types.insert(name_str, XObjectType::Image); + } else if subtype_name == b"Form" { + xobject_types + .insert(name_str, XObjectType::Form(obj_ref)); + } + } + } + } + } + } + } + } + } + } + + xobject_types +} + +/// Extract text items from a Form XObject +pub(crate) fn extract_form_xobject_text( + doc: &Document, + form_id: ObjectId, + page_num: u32, + font_cmaps: &FontCMaps, + parent_ctm: &[f32; 6], +) -> Vec { + use lopdf::content::Content; + + let mut items = Vec::new(); + + // Get the Form XObject stream + let Ok(Object::Stream(stream)) = doc.get_object(form_id) else { + return items; + }; + + // Decompress the content stream + let Ok(content_data) = stream.decompressed_content() else { + return items; + }; + + // Decode the content stream + let Ok(content) = Content::decode(&content_data) else { + return items; + }; + + // Get fonts from the Form's Resources + let form_fonts = get_form_fonts(doc, &stream.dict); + let font_encodings = build_font_encodings(doc, &form_fonts); + + // Build font width info for the form + let font_widths = build_font_widths(doc, &form_fonts); + + // Build font base names and ToUnicode refs for the form + let mut font_base_names: std::collections::HashMap = + std::collections::HashMap::new(); + let mut font_tounicode_refs: std::collections::HashMap = + std::collections::HashMap::new(); + + for (font_name, font_dict) in &form_fonts { + let resource_name = String::from_utf8_lossy(font_name).to_string(); + if let Ok(base_font) = font_dict.get(b"BaseFont") { + if let Ok(name) = base_font.as_name() { + let base_name = String::from_utf8_lossy(name).to_string(); + font_base_names.insert(resource_name.clone(), base_name); + } + } + if let Ok(tounicode) = font_dict.get(b"ToUnicode") { + if let Ok(obj_ref) = tounicode.as_reference() { + font_tounicode_refs.insert(resource_name, obj_ref.0); + } + } + } + + // Cache font encodings for form fonts + let mut encoding_cache: HashMap> = HashMap::new(); + for (font_name, font_dict) in &form_fonts { + let name = String::from_utf8_lossy(font_name).to_string(); + if let Ok(enc) = font_dict.get_font_encoding(doc) { + encoding_cache.insert(name, enc); + } + } + + // Process the content stream + let mut current_font = String::new(); + let mut current_font_size: f32 = 12.0; + let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut in_text_block = false; + let mut fill_is_white = false; + + for op in &content.operations { + match op.operator.as_str() { + "BT" => { + in_text_block = true; + text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + } + "ET" => { + in_text_block = false; + } + "Tf" => { + if op.operands.len() >= 2 { + if let Ok(name) = op.operands[0].as_name() { + current_font = String::from_utf8_lossy(name).to_string(); + } + current_font_size = get_number(&op.operands[1]).unwrap_or(12.0); + } + } + "Td" | "TD" => { + if op.operands.len() >= 2 { + let tx = get_number(&op.operands[0]).unwrap_or(0.0); + let ty = get_number(&op.operands[1]).unwrap_or(0.0); + text_matrix[4] += tx * text_matrix[0] + ty * text_matrix[2]; + text_matrix[5] += tx * text_matrix[1] + ty * text_matrix[3]; + } + } + "Tm" => { + if op.operands.len() >= 6 { + for (i, operand) in op.operands.iter().take(6).enumerate() { + text_matrix[i] = + get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 }); + } + } + } + "g" => { + if let Some(gray) = op.operands.first().and_then(get_number) { + fill_is_white = gray > 0.95; + } + } + "rg" => { + if op.operands.len() >= 3 { + let r = get_number(&op.operands[0]).unwrap_or(0.0); + let g = get_number(&op.operands[1]).unwrap_or(0.0); + let b = get_number(&op.operands[2]).unwrap_or(0.0); + fill_is_white = r > 0.95 && g > 0.95 && b > 0.95; + } + } + "k" => { + if op.operands.len() >= 4 { + let c = get_number(&op.operands[0]).unwrap_or(1.0); + let m = get_number(&op.operands[1]).unwrap_or(1.0); + let y = get_number(&op.operands[2]).unwrap_or(1.0); + let k = get_number(&op.operands[3]).unwrap_or(1.0); + fill_is_white = c < 0.05 && m < 0.05 && y < 0.05 && k < 0.05; + } + } + "Tj" => { + if in_text_block && !op.operands.is_empty() { + if fill_is_white { + if let Some(font_info) = font_widths.get(¤t_font) { + if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { + let w_ts = compute_string_width_ts( + raw_bytes, + font_info, + current_font_size, + ); + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + } + } + continue; + } + if let Some(text) = extract_text_from_operand( + &op.operands[0], + ¤t_font, + font_cmaps, + &font_base_names, + &font_tounicode_refs, + &font_encodings, + &encoding_cache, + ) { + let combined = multiply_matrices(&text_matrix, parent_ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + let width = if let Some(font_info) = font_widths.get(¤t_font) { + if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { + let w_ts = compute_string_width_ts( + raw_bytes, + font_info, + current_font_size, + ); + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + (w_ts + * (text_matrix[0] * parent_ctm[0] + + text_matrix[1] * parent_ctm[2])) + .abs() + } else { + 0.0 + } + } else { + 0.0 + }; + // Only create text item for non-whitespace; whitespace + // still advances the text matrix above so gap detection works + if !text.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&text), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + } + } + "TJ" => { + // Show text with positioning — split at column-sized gaps + if in_text_block && !op.operands.is_empty() { + if let Ok(array) = op.operands[0].as_array() { + let font_info = font_widths.get(¤t_font); + + let space_threshold = if let Some(fi) = font_info { + let space_em = fi.space_width as f32 * fi.units_scale; + let threshold = space_em * 1000.0 * 0.4; + threshold.max(80.0) + } else { + 120.0 + }; + let column_gap_threshold = space_threshold * 4.0; + + let mut sub_items: Vec<(String, f32, f32)> = Vec::new(); + let mut current_text = String::new(); + let mut sub_start_width_ts: f32 = 0.0; + let mut total_width_ts: f32 = 0.0; + for element in array { + match element { + Object::Integer(n) => { + let n_val = *n as f32; + let displacement = -n_val / 1000.0 * current_font_size; + if !fill_is_white + && n_val < -column_gap_threshold + && !current_text.is_empty() + { + sub_items.push(( + std::mem::take(&mut current_text), + sub_start_width_ts, + total_width_ts, + )); + total_width_ts += displacement; + sub_start_width_ts = total_width_ts; + } else { + total_width_ts += displacement; + if !fill_is_white + && n_val < -space_threshold + && !current_text.is_empty() + && !current_text.ends_with(' ') + { + current_text.push(' '); + } + } + continue; + } + Object::Real(n) => { + let n_val = *n; + let displacement = -n_val / 1000.0 * current_font_size; + if !fill_is_white + && n_val < -column_gap_threshold + && !current_text.is_empty() + { + sub_items.push(( + std::mem::take(&mut current_text), + sub_start_width_ts, + total_width_ts, + )); + total_width_ts += displacement; + sub_start_width_ts = total_width_ts; + } else { + total_width_ts += displacement; + if !fill_is_white + && n_val < -space_threshold + && !current_text.is_empty() + && !current_text.ends_with(' ') + { + current_text.push(' '); + } + } + continue; + } + _ => {} + } + if let Some(fi) = font_info { + if let Some(raw_bytes) = get_operand_bytes(element) { + total_width_ts += + compute_string_width_ts(raw_bytes, fi, current_font_size); + } + } + if !fill_is_white { + if let Some(text) = extract_text_from_operand( + element, + ¤t_font, + font_cmaps, + &font_base_names, + &font_tounicode_refs, + &font_encodings, + &encoding_cache, + ) { + current_text.push_str(&text); + } + } + } + if !fill_is_white && !current_text.trim().is_empty() { + sub_items.push((current_text, sub_start_width_ts, total_width_ts)); + } + if !sub_items.is_empty() { + let combined = multiply_matrices(&text_matrix, parent_ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + let scale_x = + text_matrix[0] * parent_ctm[0] + text_matrix[1] * parent_ctm[2]; + for (text, start_w, end_w) in &sub_items { + let offset_tm = [ + text_matrix[0], + text_matrix[1], + text_matrix[2], + text_matrix[3], + text_matrix[4] + start_w * text_matrix[0], + text_matrix[5] + start_w * text_matrix[1], + ]; + let combined_mat = multiply_matrices(&offset_tm, parent_ctm); + let (x, y) = (combined_mat[4], combined_mat[5]); + let width = if font_info.is_some() { + ((end_w - start_w) * scale_x).abs() + } else { + 0.0 + }; + items.push(TextItem { + text: expand_ligatures(text), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + }); + } + } + // Always advance text matrix + if font_info.is_some() { + text_matrix[4] += total_width_ts * text_matrix[0]; + text_matrix[5] += total_width_ts * text_matrix[1]; + } + } + } + } + _ => {} + } + } + + items +} + +/// Get fonts from a Form XObject's Resources +pub(crate) fn get_form_fonts<'a>( + doc: &'a Document, + form_dict: &lopdf::Dictionary, +) -> std::collections::BTreeMap, &'a lopdf::Dictionary> { + let mut fonts = std::collections::BTreeMap::new(); + + // Get Resources from Form dictionary + let resources = if let Ok(res_ref) = form_dict.get(b"Resources") { + if let Ok(obj_ref) = res_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + res_ref.as_dict().ok() + } + } else { + return fonts; + }; + + let Some(resources) = resources else { + return fonts; + }; + + // Get Font dictionary + let font_dict = if let Ok(font_ref) = resources.get(b"Font") { + if let Ok(obj_ref) = font_ref.as_reference() { + doc.get_dictionary(obj_ref).ok() + } else { + font_ref.as_dict().ok() + } + } else { + return fonts; + }; + + let Some(font_dict) = font_dict else { + return fonts; + }; + + // Collect fonts + for (name, value) in font_dict.iter() { + if let Ok(obj_ref) = value.as_reference() { + if let Ok(dict) = doc.get_dictionary(obj_ref) { + fonts.insert(name.clone(), dict); + } + } + } + + fonts +} diff --git a/src/lib.rs b/src/lib.rs index f7aec57..4dea8d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,18 +10,19 @@ pub mod extractor; pub mod glyph_names; pub mod markdown; pub mod tables; +pub mod text_utils; pub mod tounicode; +pub mod types; pub use detector::{ detect_pdf_type, detect_pdf_type_mem, detect_pdf_type_mem_with_config, detect_pdf_type_with_config, DetectionConfig, PdfType, PdfTypeResult, ScanStrategy, }; -pub use extractor::{ - extract_text, extract_text_with_positions, extract_text_with_positions_pages, PdfRect, TextItem, -}; +pub use extractor::{extract_text, extract_text_with_positions, extract_text_with_positions_pages}; pub use markdown::{ to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions, }; +pub use types::{PdfRect, TextItem}; use std::path::Path; diff --git a/src/markdown.rs b/src/markdown.rs deleted file mode 100644 index 9608c5f..0000000 --- a/src/markdown.rs +++ /dev/null @@ -1,1805 +0,0 @@ -//! Markdown conversion with structure detection -//! -//! This module converts extracted text to markdown, detecting: -//! - Headers (by font size) -//! - Lists (bullet points, numbered lists) -//! - Code blocks (monospace fonts, indentation) -//! - Paragraphs - -use crate::extractor::{group_into_lines, TextItem, TextLine}; -use std::collections::{HashMap, HashSet}; - -use regex::Regex; - -/// Options for markdown conversion -#[derive(Debug, Clone)] -pub struct MarkdownOptions { - /// Detect headers by font size - pub detect_headers: bool, - /// Detect list items - pub detect_lists: bool, - /// Detect code blocks - pub detect_code: bool, - /// Base font size for comparison - pub base_font_size: Option, - /// Remove standalone page numbers - pub remove_page_numbers: bool, - /// Convert URLs to markdown links - pub format_urls: bool, - /// Fix hyphenation (broken words across lines) - pub fix_hyphenation: bool, - /// Detect and format bold text from font names - pub detect_bold: bool, - /// Detect and format italic text from font names - pub detect_italic: bool, - /// Include image placeholders in output - pub include_images: bool, - /// Include extracted hyperlinks - pub include_links: bool, - /// Insert page break markers () between pages - pub include_page_numbers: bool, -} - -impl Default for MarkdownOptions { - fn default() -> Self { - Self { - detect_headers: true, - detect_lists: true, - detect_code: true, - base_font_size: None, - remove_page_numbers: true, - format_urls: true, - fix_hyphenation: true, - detect_bold: true, - detect_italic: true, - include_images: true, - include_links: true, - include_page_numbers: false, - } - } -} - -/// Convert plain text to markdown (basic conversion) -pub fn to_markdown(text: &str, options: MarkdownOptions) -> String { - let mut output = String::new(); - let mut in_list = false; - let mut in_code_block = false; - - for line in text.lines() { - let trimmed = line.trim(); - - if trimmed.is_empty() { - if in_list { - in_list = false; - } - if in_code_block { - output.push_str("```\n"); - in_code_block = false; - } - output.push('\n'); - continue; - } - - // Detect list items - if options.detect_lists && is_list_item(trimmed) { - let formatted = format_list_item(trimmed); - output.push_str(&formatted); - output.push('\n'); - in_list = true; - continue; - } - - // Detect code blocks (indented lines) - if options.detect_code && is_code_like(trimmed) { - if !in_code_block { - output.push_str("```\n"); - in_code_block = true; - } - output.push_str(trimmed); - output.push('\n'); - continue; - } else if in_code_block { - output.push_str("```\n"); - in_code_block = false; - } - - // Regular paragraph text - output.push_str(trimmed); - output.push('\n'); - } - - if in_code_block { - output.push_str("```\n"); - } - - output -} - -/// Convert positioned text items to markdown with structure detection -pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> String { - to_markdown_from_items_with_rects(items, options, &[]) -} - -/// Convert positioned text items to markdown, using rectangle data for table detection -pub fn to_markdown_from_items_with_rects( - items: Vec, - options: MarkdownOptions, - rects: &[crate::extractor::PdfRect], -) -> String { - use crate::extractor::ItemType; - use crate::tables::{detect_tables, detect_tables_from_rects, table_to_markdown}; - use std::collections::HashSet; - - if items.is_empty() { - return String::new(); - } - - // Separate images and links from text items - let mut images: Vec = Vec::new(); - let mut links: Vec = Vec::new(); - let mut text_items: Vec = Vec::new(); - - for item in items { - match &item.item_type { - ItemType::Image => { - if options.include_images { - images.push(item); - } - } - ItemType::Link(_) => { - if options.include_links { - links.push(item); - } - } - ItemType::Text | ItemType::FormField => { - text_items.push(item); - } - } - } - - // Calculate base font size for table detection - let font_stats = calculate_font_stats_from_items(&text_items); - let base_size = options - .base_font_size - .unwrap_or(font_stats.most_common_size); - - // Detect tables on each page - let mut table_items: HashSet = HashSet::new(); - let mut page_tables: std::collections::HashMap> = - std::collections::HashMap::new(); - - // Store images by page and Y position for insertion - let mut page_images: std::collections::HashMap> = - std::collections::HashMap::new(); - - for img in &images { - // Extract image name from "[Image: Im0]" format - let img_name = img - .text - .strip_prefix("[Image: ") - .and_then(|s| s.strip_suffix(']')) - .unwrap_or(&img.text); - let img_md = format!("![Image: {}](image)\n", img_name); - page_images - .entry(img.page) - .or_default() - .push((img.y, img_md)); - } - - // Pre-group items by page with their global indices (O(n) instead of O(pages*n)) - let mut page_groups: HashMap> = HashMap::new(); - for (global_idx, item) in text_items.iter().enumerate() { - page_groups - .entry(item.page) - .or_default() - .push((global_idx, item)); - } - - let mut pages: Vec = page_groups.keys().copied().collect(); - pages.sort(); - - for page in pages { - let group = page_groups.get(&page).unwrap(); - let page_items: Vec = group.iter().map(|(_, item)| (*item).clone()).collect(); - - // Track which local indices are claimed by rect-based tables - let mut rect_claimed: HashSet = HashSet::new(); - - // Try rectangle-based table detection first - let rect_tables = detect_tables_from_rects(&page_items, rects, page); - for table in &rect_tables { - for &idx in &table.item_indices { - rect_claimed.insert(idx); - if let Some(&(global_idx, _)) = group.get(idx) { - table_items.insert(global_idx); - } - } - let table_y = table.rows.first().copied().unwrap_or(0.0); - let table_md = table_to_markdown(table); - page_tables - .entry(page) - .or_default() - .push((table_y, table_md)); - } - - // Run heuristic detection on unclaimed items only - if rect_claimed.is_empty() { - // No rect tables — run heuristic on all items - let tables = detect_tables(&page_items, base_size, false); - for table in tables { - for &idx in &table.item_indices { - if let Some(&(global_idx, _)) = group.get(idx) { - table_items.insert(global_idx); - } - } - let table_y = table.rows.first().copied().unwrap_or(0.0); - let table_md = table_to_markdown(&table); - page_tables - .entry(page) - .or_default() - .push((table_y, table_md)); - } - } else { - // Rect tables found — run heuristic on unclaimed items - let unclaimed_items: Vec = page_items - .iter() - .enumerate() - .filter(|(idx, _)| !rect_claimed.contains(idx)) - .map(|(_, item)| item.clone()) - .collect(); - if unclaimed_items.len() >= 6 { - let tables = detect_tables(&unclaimed_items, base_size, false); - for table in tables { - // Remap indices from unclaimed-space back to page-space - let unclaimed_map: Vec = page_items - .iter() - .enumerate() - .filter(|(idx, _)| !rect_claimed.contains(idx)) - .map(|(idx, _)| idx) - .collect(); - for &idx in &table.item_indices { - if let Some(&page_idx) = unclaimed_map.get(idx) { - if let Some(&(global_idx, _)) = group.get(page_idx) { - table_items.insert(global_idx); - } - } - } - let table_y = table.rows.first().copied().unwrap_or(0.0); - let table_md = table_to_markdown(&table); - page_tables - .entry(page) - .or_default() - .push((table_y, table_md)); - } - } - } - } - - // Filter out table items and process the rest - let non_table_items: Vec = text_items - .into_iter() - .enumerate() - .filter(|(idx, _)| !table_items.contains(idx)) - .map(|(_, item)| item) - .collect(); - - // Find pages that are table-only (no remaining non-table text) - let table_only_pages: HashSet = { - let pages_with_text: HashSet = non_table_items.iter().map(|i| i.page).collect(); - page_tables - .keys() - .filter(|p| !pages_with_text.contains(p)) - .copied() - .collect() - }; - - // Merge continuation tables across page breaks, but only for table-only pages - merge_continuation_tables(&mut page_tables, &table_only_pages); - - let lines = group_into_lines(non_table_items); - - // Convert to markdown, inserting tables and images at appropriate positions - to_markdown_from_lines_with_tables_and_images(lines, options, page_tables, page_images) -} - -/// Calculate font stats directly from items (before grouping into lines) -fn calculate_font_stats_from_items(items: &[TextItem]) -> FontStats { - let mut size_counts: HashMap = HashMap::new(); - - for item in items { - if item.font_size >= 9.0 { - let size_key = (item.font_size * 10.0) as i32; - *size_counts.entry(size_key).or_insert(0) += 1; - } - } - - let most_common_size = size_counts - .iter() - .max_by_key(|(_, count)| *count) - .map(|(size, _)| *size as f32 / 10.0) - .unwrap_or(12.0); - - FontStats { most_common_size } -} - -/// Merge continuation tables that span across page breaks. -/// -/// When consecutive pages each have exactly one table with the same number of columns -/// AND both pages are table-only (no non-table text), treat them as a single table. -/// We strip their header+separator rows and append their data rows to the first page's -/// table, then remove them from later pages. -fn merge_continuation_tables( - page_tables: &mut std::collections::HashMap>, - table_only_pages: &HashSet, -) { - let mut sorted_pages: Vec = page_tables.keys().copied().collect(); - sorted_pages.sort(); - - if sorted_pages.len() < 2 { - return; - } - - // Find runs of consecutive pages that each have exactly one table with matching columns - let mut i = 0; - while i < sorted_pages.len() { - let first_page = sorted_pages[i]; - let first_tables = match page_tables.get(&first_page) { - Some(t) if t.len() == 1 => t, - _ => { - i += 1; - continue; - } - }; - - // First page must be table-only to start a merge chain - if !table_only_pages.contains(&first_page) { - i += 1; - continue; - } - - let first_col_count = count_table_columns(&first_tables[0].1); - if first_col_count == 0 { - i += 1; - continue; - } - - // Collect continuation pages (must also be table-only) - let mut continuation_pages = Vec::new(); - let mut j = i + 1; - while j < sorted_pages.len() { - let next_page = sorted_pages[j]; - // Must be consecutive page numbers - let prev_page = if continuation_pages.is_empty() { - first_page - } else { - *continuation_pages.last().unwrap() - }; - if next_page != prev_page + 1 { - break; - } - - // Continuation page must be table-only - if !table_only_pages.contains(&next_page) { - break; - } - - let next_tables = match page_tables.get(&next_page) { - Some(t) if t.len() == 1 => t, - _ => break, - }; - - let next_col_count = count_table_columns(&next_tables[0].1); - if next_col_count != first_col_count { - break; - } - - continuation_pages.push(next_page); - j += 1; - } - - if !continuation_pages.is_empty() { - // Collect data rows from continuation pages - let mut extra_rows = String::new(); - for &cont_page in &continuation_pages { - if let Some(tables) = page_tables.get(&cont_page) { - let table_md = &tables[0].1; - // Skip header row (line 1) and separator row (line 2), keep the rest - for (line_idx, line) in table_md.lines().enumerate() { - if line_idx >= 2 { - extra_rows.push_str(line); - extra_rows.push('\n'); - } - } - } - } - - // Append continuation rows to the first page's table - if let Some(tables) = page_tables.get_mut(&first_page) { - tables[0].1.push_str(&extra_rows); - } - - // Remove continuation pages from the map - for &cont_page in &continuation_pages { - page_tables.remove(&cont_page); - } - - // Skip past the merged pages - i = j; - } else { - i += 1; - } - } -} - -/// Count the number of columns in a markdown table by counting `|` in the separator row. -fn count_table_columns(table_md: &str) -> usize { - // The separator row is the second line, containing "| --- | --- |" - if let Some(sep_line) = table_md.lines().nth(1) { - if sep_line.contains("---") { - // Count cells: number of | minus 1 (leading |), but handle edge cases - let pipes = sep_line.chars().filter(|&c| c == '|').count(); - return if pipes >= 2 { pipes - 1 } else { 0 }; - } - } - 0 -} - -/// Flush any remaining tables and images for a given page -fn flush_page_tables_and_images( - page: u32, - page_tables: &std::collections::HashMap>, - page_images: &std::collections::HashMap>, - inserted_tables: &mut HashSet<(u32, usize)>, - inserted_images: &mut HashSet<(u32, usize)>, - output: &mut String, - in_paragraph: &mut bool, -) { - if let Some(tables) = page_tables.get(&page) { - for (idx, (_, table_md)) in tables.iter().enumerate() { - if !inserted_tables.contains(&(page, idx)) { - if *in_paragraph { - output.push_str("\n\n"); - *in_paragraph = false; - } - output.push('\n'); - output.push_str(table_md); - output.push('\n'); - inserted_tables.insert((page, idx)); - } - } - } - if let Some(images) = page_images.get(&page) { - for (idx, (_, image_md)) in images.iter().enumerate() { - if !inserted_images.contains(&(page, idx)) { - if *in_paragraph { - output.push_str("\n\n"); - *in_paragraph = false; - } - output.push('\n'); - output.push_str(image_md); - output.push('\n'); - inserted_images.insert((page, idx)); - } - } - } -} - -/// Convert text lines to markdown, inserting tables and images at appropriate Y positions -fn to_markdown_from_lines_with_tables_and_images( - lines: Vec, - options: MarkdownOptions, - page_tables: std::collections::HashMap>, - page_images: std::collections::HashMap>, -) -> String { - if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { - return String::new(); - } - - // Calculate font statistics - let font_stats = calculate_font_stats(&lines); - let base_size = options - .base_font_size - .unwrap_or(font_stats.most_common_size); - - // Merge drop caps with following text - let lines = merge_drop_caps(lines, base_size); - - // Discover heading tiers for this document - let heading_tiers = compute_heading_tiers(&lines, base_size); - - // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); - - // Compute the typical line spacing for paragraph break detection. - // For double-spaced documents (like legal/government PDFs), the normal - // line spacing can be 2.3x base_size, which would exceed a fixed 1.8x - // threshold and cause every line to be treated as a paragraph break. - let para_threshold = compute_paragraph_threshold(&lines, base_size); - - let mut output = String::new(); - let mut current_page = 0u32; - let mut prev_y = f32::MAX; - let mut in_list = false; - let mut in_paragraph = false; - let mut last_list_x: Option = None; - let mut prev_had_dot_leaders = false; - let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); - let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); - - // Collect all pages that have tables or images (including image-only pages) - let mut all_content_pages: Vec = page_tables - .keys() - .chain(page_images.keys()) - .copied() - .collect(); - all_content_pages.sort(); - all_content_pages.dedup(); - - for line in lines { - // Page break - if line.page != current_page { - // Flush current page's remaining tables and images - if current_page > 0 { - flush_page_tables_and_images( - current_page, - &page_tables, - &page_images, - &mut inserted_tables, - &mut inserted_images, - &mut output, - &mut in_paragraph, - ); - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push_str("\n\n"); - } - - // Flush any intermediate pages (image-only or table-only) between - // current_page and line.page that have no text lines - for &p in &all_content_pages { - if p <= current_page { - continue; - } - if p >= line.page { - break; - } - flush_page_tables_and_images( - p, - &page_tables, - &page_images, - &mut inserted_tables, - &mut inserted_images, - &mut output, - &mut in_paragraph, - ); - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push_str("\n\n"); - } - - current_page = line.page; - prev_y = f32::MAX; - - if options.include_page_numbers { - output.push_str(&format!("\n\n", current_page)); - } - } - - // Check if we should insert a table before this line - if let Some(tables) = page_tables.get(¤t_page) { - for (idx, (table_y, table_md)) in tables.iter().enumerate() { - // Insert table when we pass its Y position - if *table_y > line.y && !inserted_tables.contains(&(current_page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(table_md); - output.push('\n'); - inserted_tables.insert((current_page, idx)); - } - } - } - - // Check if we should insert an image before this line - if let Some(images) = page_images.get(¤t_page) { - for (idx, (image_y, image_md)) in images.iter().enumerate() { - // Insert image when we pass its Y position - if *image_y > line.y && !inserted_images.contains(&(current_page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(image_md); - output.push('\n'); - inserted_images.insert((current_page, idx)); - } - } - } - - // Paragraph break: large forward Y gap (normal) or large backward jump - // (newspaper columns emitted sequentially on the same page). - let y_gap = prev_y - line.y; - let is_para_break = y_gap.abs() > para_threshold; - if is_para_break && in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Don't immediately end list on paragraph break - // Let the continuation check below decide if we're still in a list - prev_y = line.y; - - // Get text with optional bold/italic formatting - let text = line.text_with_formatting(options.detect_bold, options.detect_italic); - let trimmed = text.trim(); - - // Also get plain text for pattern matching (list detection, captions, etc.) - let plain_text = line.text(); - let plain_trimmed = plain_text.trim(); - - if trimmed.is_empty() { - continue; - } - - // Detect figure/table captions and source citations - // These should be on their own line followed by a paragraph break - if is_caption_line(plain_trimmed) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push_str(trimmed); - output.push_str("\n\n"); - continue; - } - - // Detect headers by font size - // Note: Headers typically shouldn't have bold markers since they're already emphasized - // Skip very short text (drop caps/labels) and very long text (body paragraphs) - if options.detect_headers - && plain_trimmed.len() > 3 - && plain_trimmed.split_whitespace().count() <= 15 - { - let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers) - { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let prefix = "#".repeat(header_level); - // Use plain text for headers to avoid redundant formatting - output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); - in_list = false; - continue; - } - } - - // Detect list items - if options.detect_lists && is_list_item(plain_trimmed) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let formatted = format_list_item(trimmed); - output.push_str(&formatted); - output.push('\n'); - in_list = true; - last_list_x = line.items.first().map(|i| i.x); - continue; - } else if in_list { - // Check if this line is a continuation of the previous list item - // Continuations have similar X position and reasonable Y gap - let line_x = line.items.first().map(|i| i.x); - let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) { - // Continuation criteria: - // 1. X is at or past the list text position - // 2. Y gap is not too large (max ~5 line heights) - // 3. Not a new list item - let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0; - let y_ok = y_gap < base_size * 7.0; - x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed) - } else { - false - }; - - if is_continuation { - // Append to previous list item with a space - if output.ends_with('\n') { - output.pop(); - output.push(' '); - } - output.push_str(trimmed); - output.push('\n'); - continue; - } else { - in_list = false; - last_list_x = None; - } - } - - // Detect code blocks by font - if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; - } - } - - // Regular text - join lines within same paragraph with space - let cur_dot_leaders = has_dot_leaders(plain_trimmed); - if in_paragraph { - if cur_dot_leaders || prev_had_dot_leaders { - output.push('\n'); - } else { - output.push(' '); - } - } - output.push_str(trimmed); - in_paragraph = true; - prev_had_dot_leaders = cur_dot_leaders; - } - - // Flush current page and any remaining pages with tables/images - // (handles table-only pages after the last text line, and trailing image-only pages) - flush_page_tables_and_images( - current_page, - &page_tables, - &page_images, - &mut inserted_tables, - &mut inserted_images, - &mut output, - &mut in_paragraph, - ); - for &p in &all_content_pages { - if p <= current_page { - continue; - } - flush_page_tables_and_images( - p, - &page_tables, - &page_images, - &mut inserted_tables, - &mut inserted_images, - &mut output, - &mut in_paragraph, - ); - } - - // Close final paragraph - if in_paragraph { - output.push('\n'); - } - - // Clean up and post-process - clean_markdown(output, &options) -} - -/// Convert text lines to markdown -pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> String { - if lines.is_empty() { - return String::new(); - } - - // Calculate font statistics - let font_stats = calculate_font_stats(&lines); - let base_size = options - .base_font_size - .unwrap_or(font_stats.most_common_size); - - // Merge drop caps with following text - let lines = merge_drop_caps(lines, base_size); - - // Discover heading tiers for this document - let heading_tiers = compute_heading_tiers(&lines, base_size); - - // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); - - // Compute the typical line spacing for paragraph break detection - let para_threshold = compute_paragraph_threshold(&lines, base_size); - - let mut output = String::new(); - let mut current_page = 0u32; - let mut prev_y = f32::MAX; - let mut in_list = false; - let mut in_paragraph = false; - let mut last_list_x: Option = None; - let mut prev_had_dot_leaders = false; - - for line in lines { - // Page break - if line.page != current_page { - if current_page > 0 { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push_str("\n\n"); - } - current_page = line.page; - prev_y = f32::MAX; - in_list = false; - last_list_x = None; - prev_had_dot_leaders = false; - - if options.include_page_numbers { - output.push_str(&format!("\n\n", current_page)); - } - } - - // Paragraph break: large forward Y gap (normal) or large backward jump - // (newspaper columns emitted sequentially on the same page). - let y_gap = prev_y - line.y; - let is_para_break = y_gap.abs() > para_threshold; - if is_para_break && in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Don't immediately end list on paragraph break - // Let the continuation check below decide if we're still in a list - prev_y = line.y; - - // Get text with optional bold/italic formatting - let text = line.text_with_formatting(options.detect_bold, options.detect_italic); - let trimmed = text.trim(); - - // Also get plain text for pattern matching - let plain_text = line.text(); - let plain_trimmed = plain_text.trim(); - - if trimmed.is_empty() { - continue; - } - - // Detect figure/table captions and source citations - // These should be on their own line followed by a paragraph break - if is_caption_line(plain_trimmed) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push_str(trimmed); - output.push_str("\n\n"); - continue; - } - - // Detect headers by font size - // Skip very short text (drop caps/labels) and very long text (body paragraphs) - if options.detect_headers - && plain_trimmed.len() > 3 - && plain_trimmed.split_whitespace().count() <= 15 - { - let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers) - { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let prefix = "#".repeat(header_level); - // Use plain text for headers to avoid redundant formatting - output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); - in_list = false; - continue; - } - } - - // Detect list items - if options.detect_lists && is_list_item(plain_trimmed) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let formatted = format_list_item(trimmed); - output.push_str(&formatted); - output.push('\n'); - in_list = true; - last_list_x = line.items.first().map(|i| i.x); - continue; - } else if in_list { - // Check if this line is a continuation of the previous list item - let line_x = line.items.first().map(|i| i.x); - let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) { - // Continuation criteria: - // 1. X is at or past the list text position - // 2. Y gap is not too large (max ~5 line heights) - // 3. Not a new list item - let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0; - let y_ok = y_gap < base_size * 7.0; - x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed) - } else { - false - }; - - if is_continuation { - // Append to previous list item with a space - if output.ends_with('\n') { - output.pop(); - output.push(' '); - } - output.push_str(trimmed); - output.push('\n'); - continue; - } else { - in_list = false; - last_list_x = None; - } - } - - // Detect code blocks by font - if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; - } - } - - // Regular text - join lines within same paragraph with space - let cur_dot_leaders = has_dot_leaders(plain_trimmed); - if in_paragraph { - if cur_dot_leaders || prev_had_dot_leaders { - output.push('\n'); - } else { - output.push(' '); - } - } - output.push_str(trimmed); - in_paragraph = true; - prev_had_dot_leaders = cur_dot_leaders; - } - - // Close final paragraph - if in_paragraph { - output.push('\n'); - } - - // Clean up and post-process - clean_markdown(output, &options) -} - -/// Merge drop caps with the appropriate line -/// A drop cap is a single large letter at the start of a paragraph -/// Due to PDF coordinate sorting, the drop cap may appear AFTER the line it belongs to -/// Merge consecutive heading lines at the same level into a single line. -/// -/// When a heading wraps across multiple text lines (e.g., "About Glenair, the Mission-Critical" -/// and "Interconnect Company"), each fragment becomes a separate `# Header` in the output. -/// This function detects consecutive lines at the same heading tier on the same page -/// with a small Y gap and merges them into one line. -fn merge_heading_lines( - lines: Vec, - base_size: f32, - heading_tiers: &[f32], -) -> Vec { - if lines.is_empty() { - return lines; - } - - let mut result: Vec = Vec::with_capacity(lines.len()); - - for line in lines { - let line_font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - let line_level = detect_header_level(line_font, base_size, heading_tiers); - - // Check if the previous line is a heading at the same level on the same page - let should_merge = if let (Some(prev), Some(curr_level)) = (result.last(), line_level) { - let prev_font = prev.items.first().map(|i| i.font_size).unwrap_or(base_size); - let prev_level = detect_header_level(prev_font, base_size, heading_tiers); - let same_page = prev.page == line.page; - let same_level = prev_level == Some(curr_level); - let y_gap = prev.y - line.y; - // Merge if gap is within ~2x the font size (normal line wrap spacing) - let close_enough = y_gap > 0.0 && y_gap < line_font * 2.0; - same_page && same_level && close_enough - } else { - false - }; - - if should_merge { - // Append this line's items to the previous line - let prev = result.last_mut().unwrap(); - // Add a space-bearing TextItem to separate the merged text - if let Some(first_item) = line.items.first() { - let mut space_item = first_item.clone(); - space_item.text = format!(" {}", space_item.text.trim_start()); - prev.items.push(space_item); - } - for item in line.items.into_iter().skip(1) { - prev.items.push(item); - } - } else { - result.push(line); - } - } - - result -} - -fn merge_drop_caps(lines: Vec, base_size: f32) -> Vec { - let mut result: Vec = Vec::with_capacity(lines.len()); - - for line in &lines { - let text = line.text(); - let trimmed = text.trim(); - - // Check if this looks like a drop cap: - // 1. Single character (or single char + space) - // 2. Much larger than base font (3x or more) - // 3. The character is uppercase - let is_drop_cap = trimmed.len() <= 2 - && line.items.first().map(|i| i.font_size).unwrap_or(0.0) >= base_size * 2.5 - && trimmed - .chars() - .next() - .map(|c| c.is_uppercase()) - .unwrap_or(false); - - if is_drop_cap { - let drop_char = trimmed.chars().next().unwrap(); - - // Find the first line that starts with lowercase and is at the START of a paragraph - // (i.e., preceded by a header or non-lowercase-starting line) - let mut target_idx: Option = None; - - for (idx, prev_line) in result.iter().enumerate() { - if prev_line.page != line.page { - continue; - } - - let prev_text = prev_line.text(); - let prev_trimmed = prev_text.trim(); - - // Check if this line starts with lowercase - if prev_trimmed - .chars() - .next() - .map(|c| c.is_lowercase()) - .unwrap_or(false) - { - // Check if previous line exists and doesn't start with lowercase - // (meaning this is the start of a paragraph) - let is_para_start = if idx == 0 { - true - } else { - let before = result[idx - 1].text(); - let before_trimmed = before.trim(); - !before_trimmed - .chars() - .next() - .map(|c| c.is_lowercase()) - .unwrap_or(true) - }; - - if is_para_start { - target_idx = Some(idx); - break; - } - } - } - - // Merge with the target line - if let Some(idx) = target_idx { - if let Some(first_item) = result[idx].items.first_mut() { - let prev_text = first_item.text.trim().to_string(); - first_item.text = format!("{}{}", drop_char, prev_text); - } - } - // Don't add the drop cap line itself - continue; - } - - result.push(line.clone()); - } - - result -} - -/// Font statistics for a document -struct FontStats { - most_common_size: f32, -} - -fn calculate_font_stats(lines: &[TextLine]) -> FontStats { - let mut size_counts: HashMap = HashMap::new(); - - for line in lines { - // Count once per line (first item) to give each line equal weight - // Prevents small captions/footnotes from skewing the base - if let Some(first) = line.items.first() { - if first.font_size >= 9.0 { - let size_key = (first.font_size * 10.0) as i32; - *size_counts.entry(size_key).or_insert(0) += 1; - } - } - } - - let most_common_size = size_counts - .iter() - .max_by_key(|(_, count)| *count) - .map(|(size, _)| *size as f32 / 10.0) - .unwrap_or(12.0); - - FontStats { most_common_size } -} - -/// Detect TOC-style lines that contain dot leaders (e.g., "Section Name .... 42"). -/// These lines should never be joined with adjacent lines into a paragraph. -/// Handles both consecutive dots ("....") and spaced dots ("... ..."). -fn has_dot_leaders(text: &str) -> bool { - // Consecutive dots (4+) - if text.contains("....") { - return true; - } - // Spaced dot leaders: "..." followed by whitespace and more dots - // Count occurrences of "..." (3+ dots) — if 2+ groups, it's a dot leader - let mut dot_groups = 0; - let mut consecutive_dots = 0; - for ch in text.chars() { - if ch == '.' { - consecutive_dots += 1; - } else { - if consecutive_dots >= 3 { - dot_groups += 1; - } - consecutive_dots = 0; - } - } - if consecutive_dots >= 3 { - dot_groups += 1; - } - dot_groups >= 2 -} - -/// Compute the Y-gap threshold for paragraph break detection. -/// -/// Instead of using a fixed multiple of base_size (which fails for double-spaced -/// documents), we compute the document's typical (median) line spacing and use -/// a multiplier on that. A gap significantly larger than typical indicates a -/// paragraph break. -/// -/// Fallback: if we can't compute typical spacing, use base_size * 1.8. -fn compute_paragraph_threshold(lines: &[TextLine], base_size: f32) -> f32 { - let fallback = base_size * 1.8; - - // Collect Y gaps between consecutive lines on the same page - let mut gaps: Vec = Vec::new(); - let mut prev_y: Option<(u32, f32)> = None; - - for line in lines { - if let Some((prev_page, py)) = prev_y { - if line.page == prev_page { - let gap = py - line.y; - // Only consider positive gaps within a reasonable range - // (skip huge gaps from page headers/footers) - if gap > 0.0 && gap < base_size * 10.0 { - gaps.push(gap); - } - } - } - prev_y = Some((line.page, line.y)); - } - - if gaps.len() < 5 { - return fallback; - } - - gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - let median = gaps[gaps.len() / 2]; - - // The paragraph threshold should be larger than the typical line spacing. - // Use 1.3x the median gap. This means: - // - Single-spaced (median ~14pt for 12pt font): threshold = 18.2pt - // - Double-spaced (median ~28pt for 12pt font): threshold = 36.4pt - // Also ensure it's at least base_size * 1.5 to avoid false paragraph breaks - // in tightly-spaced documents. - (median * 1.3).max(base_size * 1.5) -} - -/// Discover distinct heading font-size tiers in the document. -/// Returns tiers sorted largest-first (tier 0 = H1, tier 1 = H2, …). -/// Sizes within 0.5pt are clustered into the same tier. Capped at 4 tiers. -fn compute_heading_tiers(lines: &[TextLine], base_size: f32) -> Vec { - let mut heading_sizes: Vec = Vec::new(); - - for line in lines { - if let Some(first) = line.items.first() { - if first.font_size / base_size >= 1.2 { - heading_sizes.push(first.font_size); - } - } - } - - // Sort descending - heading_sizes.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); - - // Cluster sizes within 0.5pt into same tier (use first value as representative) - let mut tiers: Vec = Vec::new(); - for size in heading_sizes { - let already_in_tier = tiers.iter().any(|&t| (t - size).abs() < 0.5); - if !already_in_tier { - tiers.push(size); - } - } - - // Cap at 4 tiers - tiers.truncate(4); - tiers -} - -/// Detect header level from font size using document-specific heading tiers. -/// When tiers are available, maps tier 0→H1, tier 1→H2, etc. -/// Falls back to ratio-based thresholds when no tiers exist. -fn detect_header_level(font_size: f32, base_size: f32, heading_tiers: &[f32]) -> Option { - let ratio = font_size / base_size; - - if ratio < 1.2 { - return None; // Regular text - } - - if !heading_tiers.is_empty() { - // Match font_size to a tier (within 0.5pt tolerance) - for (i, &tier_size) in heading_tiers.iter().enumerate() { - if (font_size - tier_size).abs() < 0.5 { - return Some(i + 1); // tier 0 → H1, tier 1 → H2, etc. - } - } - // No tier match but large ratio — assign level after last tier - if ratio >= 1.5 { - let level = (heading_tiers.len() + 1).min(4); - return Some(level); - } - // No tier match and small ratio — not a heading - return None; - } - - // Fallback: original ratio-based thresholds (no tiers discovered) - if ratio >= 2.0 { - Some(1) - } else if ratio >= 1.5 { - Some(2) - } else if ratio >= 1.25 { - Some(3) - } else { - Some(4) - } -} - -/// Check if text is a figure/table caption or source citation -fn is_caption_line(text: &str) -> bool { - let trimmed = text.trim(); - - // Common caption prefixes in multiple languages - let caption_prefixes = [ - "Figure ", - "Figura ", - "Fig. ", - "Fig ", - "Table ", - "Tabela ", - "Source:", - "Fonte:", - "Source ", - "Fonte ", - "Note:", - "Nota:", - "Chart ", - "Gráfico ", - "Graph ", - "Diagram ", - "Image ", - "Imagem ", - "Photo ", - "Foto ", - ]; - - // Check if line starts with a caption prefix - for prefix in &caption_prefixes { - if trimmed.starts_with(prefix) { - return true; - } - } - - // Check case-insensitive patterns - let lower = trimmed.to_lowercase(); - if lower.starts_with("figure ") || lower.starts_with("table ") || lower.starts_with("source:") { - return true; - } - - false -} - -/// Check if text looks like a list item -fn is_list_item(text: &str) -> bool { - let trimmed = text.trim_start(); - - // Bullet patterns - if trimmed.starts_with("• ") - || trimmed.starts_with("- ") - || trimmed.starts_with("* ") - || trimmed.starts_with("○ ") - || trimmed.starts_with("● ") - || trimmed.starts_with("◦ ") - { - return true; - } - - // Numbered list patterns: "1.", "1)", "(1)", "a.", "a)" - let first_chars: String = trimmed.chars().take(5).collect(); - if first_chars.contains(|c: char| c.is_ascii_digit()) { - // Check for "1.", "1)", "10." - if let Some(idx) = first_chars.find(['.', ')']) { - let prefix = &first_chars[..idx]; - if prefix.chars().all(|c| c.is_ascii_digit()) { - return true; - } - } - } - - // Letter list: "a.", "a)", "(a)" - let mut chars = trimmed.chars(); - if let (Some(first), Some(second)) = (chars.next(), chars.next()) { - if first.is_ascii_alphabetic() && (second == '.' || second == ')') { - return true; - } - if first == '(' && chars.next() == Some(')') { - return true; - } - } - - false -} - -/// Format list item to markdown -fn format_list_item(text: &str) -> String { - let trimmed = text.trim_start(); - - // Convert various bullet styles to markdown - // Note: bullet characters like • are multi-byte in UTF-8, use char indices - for bullet in &['•', '○', '●', '◦'] { - if let Some(rest) = trimmed.strip_prefix(*bullet) { - return format!("- {}", rest.trim_start()); - } - } - - if trimmed.starts_with("- ") || trimmed.starts_with("* ") { - return trimmed.to_string(); - } - - // Keep numbered lists as-is (markdown supports them) - trimmed.to_string() -} - -/// Check if text looks like code -fn is_code_like(text: &str) -> bool { - let trimmed = text.trim(); - - // Code patterns - let code_patterns = [ - // Language keywords - "import ", - "export ", - "from ", - "const ", - "let ", - "var ", - "function ", - "class ", - "def ", - "pub fn ", - "fn ", - "async fn ", - "impl ", - // Syntax patterns - "=> ", - "-> ", - ":: ", - ":= ", - // Common code endings - ]; - - for pattern in &code_patterns { - if trimmed.starts_with(pattern) { - return true; - } - } - - // Check for code-like syntax - let special_chars: usize = trimmed - .chars() - .filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>')) - .count(); - - if special_chars >= 3 && trimmed.len() < 200 { - return true; - } - - // Ends with semicolon or braces - if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') { - return true; - } - - false -} - -/// Check if font name indicates monospace -fn is_monospace_font(font_name: &str) -> bool { - let lower = font_name.to_lowercase(); - let patterns = [ - "courier", - "consolas", - "monaco", - "menlo", - "mono", - "fixed", - "terminal", - "typewriter", - "source code", - "fira code", - "jetbrains", - "inconsolata", - "dejavu sans mono", - "liberation mono", - ]; - - patterns.iter().any(|p| lower.contains(p)) -} - -/// Clean up markdown output with post-processing -fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String { - // Collapse dot leaders (e.g. TOC entries: "Introduction...............................1") - text = collapse_dot_leaders(&text); - - // Fix hyphenation first (before other processing) - if options.fix_hyphenation { - text = fix_hyphenation(&text); - } - - // Remove standalone page numbers - if options.remove_page_numbers { - text = remove_page_numbers(&text); - } - - // Format URLs as markdown links - if options.format_urls { - text = format_urls(&text); - } - - // Remove excessive newlines (more than 2 in a row) - while text.contains("\n\n\n") { - text = text.replace("\n\n\n", "\n\n"); - } - - // Trim leading and trailing whitespace, ensure ends with single newline - text = text.trim().to_string(); - text.push('\n'); - - text -} - -/// Collapse dot leaders (runs of 4+ dots) into " ... " -/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" -fn collapse_dot_leaders(text: &str) -> String { - use once_cell::sync::Lazy; - static DOT_LEADER_RE: Lazy = Lazy::new(|| Regex::new(r"\.{4,}").unwrap()); - - DOT_LEADER_RE.replace_all(text, " ... ").to_string() -} - -/// Fix words broken across lines with spaces before the continuation -/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte" -fn fix_hyphenation(text: &str) -> String { - use once_cell::sync::Lazy; - - // Fix "word - word" patterns that should be "word-word" (compound words) - // But be careful not to break list items (which start with "- ") - static SPACED_HYPHEN_RE: Lazy = Lazy::new(|| { - Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap() - }); - - let result = SPACED_HYPHEN_RE - .replace_all(text, |caps: ®ex::Captures| { - format!("{}-{}", &caps[1], &caps[2]) - }) - .to_string(); - - result -} - -/// Remove standalone page numbers (lines that are just 1-4 digit numbers) -fn remove_page_numbers(text: &str) -> String { - let mut result = Vec::new(); - let lines: Vec<&str> = text.lines().collect(); - - for (i, line) in lines.iter().enumerate() { - let trimmed = line.trim(); - - // Check for page number patterns - if is_page_number_line(trimmed) { - // Check context to determine if this is isolated - let prev_is_break = i > 0 && lines[i - 1].trim() == "---"; - let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---"; - let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty(); - let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty(); - - // Check if it's on its own line (surrounded by empty lines or page breaks) - let is_isolated = (prev_is_break || prev_is_empty || i == 0) - && (next_is_break || next_is_empty || i + 1 == lines.len()); - - // Also remove numbers that appear right before a page break - let before_break = i + 1 < lines.len() - && (lines[i + 1].trim() == "---" - || (i + 2 < lines.len() - && lines[i + 1].trim().is_empty() - && lines[i + 2].trim() == "---")); - - if is_isolated || before_break { - continue; - } - } - - result.push(*line); - } - - result.join("\n") -} - -/// Check if a line looks like a page number -fn is_page_number_line(trimmed: &str) -> bool { - // Empty lines are not page numbers - if trimmed.is_empty() { - return false; - } - - // Pattern 1: Just a number (1-4 digits) - if trimmed.len() <= 4 && trimmed.chars().all(|c| c.is_ascii_digit()) { - return true; - } - - // Pattern 2: "Page X of Y" or "Page X" or "Page of" (placeholder) - let lower = trimmed.to_lowercase(); - if let Some(rest) = lower.strip_prefix("page") { - let rest = rest.trim(); - // "Page of" (empty page numbers) - if rest == "of" || rest.starts_with("of ") { - return true; - } - // "Page X" or "Page X of Y" - if rest - .chars() - .next() - .map(|c| c.is_ascii_digit()) - .unwrap_or(false) - { - return true; - } - // Just "Page" followed by whitespace and maybe "of" - if rest.is_empty() - || rest - .split_whitespace() - .all(|w| w == "of" || w.chars().all(|c| c.is_ascii_digit())) - { - return true; - } - } - - // Pattern 3: "X of Y" where X and Y are numbers - if let Some(of_idx) = trimmed.find(" of ") { - let before = trimmed[..of_idx].trim(); - let after = trimmed[of_idx + 4..].trim(); - if before.chars().all(|c| c.is_ascii_digit()) - && after.chars().all(|c| c.is_ascii_digit()) - && !before.is_empty() - && !after.is_empty() - { - return true; - } - } - - // Pattern 4: "- X -" centered page number - if trimmed.len() >= 3 && trimmed.starts_with('-') && trimmed.ends_with('-') { - let inner = trimmed[1..trimmed.len() - 1].trim(); - if inner.chars().all(|c| c.is_ascii_digit()) && !inner.is_empty() { - return true; - } - } - - false -} - -/// Convert URLs to markdown links -fn format_urls(text: &str) -> String { - use once_cell::sync::Lazy; - - // Match URLs - we'll check context manually to avoid formatting already-linked URLs - static URL_RE: Lazy = - Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap()); - - let mut result = String::with_capacity(text.len()); - let mut last_end = 0; - - for mat in URL_RE.find_iter(text) { - let start = mat.start(); - let url = mat.as_str(); - - // Check if this URL is already in a markdown link by looking at preceding chars - // Use safe character boundary checking for multi-byte UTF-8 - let before = { - let mut check_start = start.saturating_sub(2); - // Find a valid character boundary - while check_start > 0 && !text.is_char_boundary(check_start) { - check_start -= 1; - } - if check_start < start && text.is_char_boundary(start) { - &text[check_start..start] - } else { - "" - } - }; - let already_linked = before.ends_with("](") || before.ends_with("]("); - - // Also check if it's inside square brackets (link text) - // Ensure we're slicing at a valid char boundary - let prefix = if text.is_char_boundary(start) { - &text[..start] - } else { - // Find the nearest valid boundary before start - let mut safe_start = start; - while safe_start > 0 && !text.is_char_boundary(safe_start) { - safe_start -= 1; - } - &text[..safe_start] - }; - let open_brackets = prefix.matches('[').count(); - let close_brackets = prefix.matches(']').count(); - let inside_link_text = open_brackets > close_brackets; - - // Ensure mat boundaries are valid char boundaries - let safe_last_end = if text.is_char_boundary(last_end) { - last_end - } else { - let mut pos = last_end; - while pos < text.len() && !text.is_char_boundary(pos) { - pos += 1; - } - pos - }; - let safe_start = if text.is_char_boundary(start) { - start - } else { - let mut pos = start; - while pos < text.len() && !text.is_char_boundary(pos) { - pos += 1; - } - pos - }; - let safe_end = if text.is_char_boundary(mat.end()) { - mat.end() - } else { - let mut pos = mat.end(); - while pos < text.len() && !text.is_char_boundary(pos) { - pos += 1; - } - pos - }; - - if already_linked || inside_link_text { - // Already formatted, keep as-is - if safe_last_end <= safe_end { - result.push_str(&text[safe_last_end..safe_end]); - } - } else { - // Add text before this URL - if safe_last_end <= safe_start { - result.push_str(&text[safe_last_end..safe_start]); - } - // Format as markdown link - result.push_str(&format!("[{}]({})", url, url)); - } - last_end = safe_end; - } - - // Add remaining text (ensure valid char boundary) - let safe_last_end = if text.is_char_boundary(last_end) { - last_end - } else { - let mut pos = last_end; - while pos < text.len() && !text.is_char_boundary(pos) { - pos += 1; - } - pos - }; - if safe_last_end < text.len() { - result.push_str(&text[safe_last_end..]); - } - result -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_list_item() { - assert!(is_list_item("• Item one")); - assert!(is_list_item("- Item two")); - assert!(is_list_item("* Item three")); - assert!(is_list_item("1. First")); - assert!(is_list_item("2) Second")); - assert!(is_list_item("a. Letter item")); - assert!(!is_list_item("Regular text")); - } - - #[test] - fn test_format_list_item() { - assert_eq!(format_list_item("• Item"), "- Item"); - assert_eq!(format_list_item("- Item"), "- Item"); - assert_eq!(format_list_item("1. First"), "1. First"); - } - - #[test] - fn test_is_code_like() { - assert!(is_code_like("const x = 5;")); - assert!(is_code_like("function foo() {")); - assert!(is_code_like("import React from 'react'")); - assert!(!is_code_like("This is regular text.")); - } - - #[test] - fn test_detect_header_level() { - // With three tiers: 24→H1, 18→H2, 15→H3, 12→None - let tiers = vec![24.0, 18.0, 15.0]; - assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); - - // Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2) - let tiers = vec![15.0]; - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); - - // No tiers (empty): falls back to ratio thresholds - let tiers: Vec = vec![]; - assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); - assert_eq!(detect_header_level(14.5, 12.0, &tiers), Some(4)); - assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); - - // Body text excluded when tiers exist: 13pt (ratio 1.08) → None - let tiers = vec![20.0]; - assert_eq!(detect_header_level(13.0, 12.0, &tiers), None); - } - - #[test] - fn test_to_markdown() { - let text = "• First item\n• Second item\n\nRegular paragraph."; - let md = to_markdown(text, MarkdownOptions::default()); - assert!(md.contains("- First item")); - assert!(md.contains("- Second item")); - } -} diff --git a/src/markdown/analysis.rs b/src/markdown/analysis.rs new file mode 100644 index 0000000..0eea028 --- /dev/null +++ b/src/markdown/analysis.rs @@ -0,0 +1,201 @@ +//! Font statistics, heading detection, and document structure analysis. + +use std::collections::HashMap; + +use crate::types::{TextItem, TextLine}; + +/// Font statistics for a document +pub(crate) struct FontStats { + pub(crate) most_common_size: f32, +} + +/// Calculate font stats directly from items (before grouping into lines) +pub(crate) fn calculate_font_stats_from_items(items: &[TextItem]) -> FontStats { + let mut size_counts: HashMap = HashMap::new(); + + for item in items { + if item.font_size >= 9.0 { + let size_key = (item.font_size * 10.0) as i32; + *size_counts.entry(size_key).or_insert(0) += 1; + } + } + + let most_common_size = size_counts + .iter() + .max_by_key(|(_, count)| *count) + .map(|(size, _)| *size as f32 / 10.0) + .unwrap_or(12.0); + + FontStats { most_common_size } +} + +/// Calculate font stats from grouped lines +pub(crate) fn calculate_font_stats(lines: &[TextLine]) -> FontStats { + let mut size_counts: HashMap = HashMap::new(); + + for line in lines { + // Count once per line (first item) to give each line equal weight + // Prevents small captions/footnotes from skewing the base + if let Some(first) = line.items.first() { + if first.font_size >= 9.0 { + let size_key = (first.font_size * 10.0) as i32; + *size_counts.entry(size_key).or_insert(0) += 1; + } + } + } + + let most_common_size = size_counts + .iter() + .max_by_key(|(_, count)| *count) + .map(|(size, _)| *size as f32 / 10.0) + .unwrap_or(12.0); + + FontStats { most_common_size } +} + +/// Detect TOC-style lines that contain dot leaders (e.g., "Section Name .... 42"). +/// These lines should never be joined with adjacent lines into a paragraph. +/// Handles both consecutive dots ("....") and spaced dots ("... ..."). +pub(crate) fn has_dot_leaders(text: &str) -> bool { + // Consecutive dots (4+) + if text.contains("....") { + return true; + } + // Spaced dot leaders: "..." followed by whitespace and more dots + // Count occurrences of "..." (3+ dots) — if 2+ groups, it's a dot leader + let mut dot_groups = 0; + let mut consecutive_dots = 0; + for ch in text.chars() { + if ch == '.' { + consecutive_dots += 1; + } else { + if consecutive_dots >= 3 { + dot_groups += 1; + } + consecutive_dots = 0; + } + } + if consecutive_dots >= 3 { + dot_groups += 1; + } + dot_groups >= 2 +} + +/// Compute the Y-gap threshold for paragraph break detection. +/// +/// Instead of using a fixed multiple of base_size (which fails for double-spaced +/// documents), we compute the document's typical (median) line spacing and use +/// a multiplier on that. A gap significantly larger than typical indicates a +/// paragraph break. +/// +/// Fallback: if we can't compute typical spacing, use base_size * 1.8. +pub(crate) fn compute_paragraph_threshold(lines: &[TextLine], base_size: f32) -> f32 { + let fallback = base_size * 1.8; + + // Collect Y gaps between consecutive lines on the same page + let mut gaps: Vec = Vec::new(); + let mut prev_y: Option<(u32, f32)> = None; + + for line in lines { + if let Some((prev_page, py)) = prev_y { + if line.page == prev_page { + let gap = py - line.y; + // Only consider positive gaps within a reasonable range + // (skip huge gaps from page headers/footers) + if gap > 0.0 && gap < base_size * 10.0 { + gaps.push(gap); + } + } + } + prev_y = Some((line.page, line.y)); + } + + if gaps.len() < 5 { + return fallback; + } + + gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let median = gaps[gaps.len() / 2]; + + // The paragraph threshold should be larger than the typical line spacing. + // Use 1.3x the median gap. This means: + // - Single-spaced (median ~14pt for 12pt font): threshold = 18.2pt + // - Double-spaced (median ~28pt for 12pt font): threshold = 36.4pt + // Also ensure it's at least base_size * 1.5 to avoid false paragraph breaks + // in tightly-spaced documents. + (median * 1.3).max(base_size * 1.5) +} + +/// Discover distinct heading font-size tiers in the document. +/// Returns tiers sorted largest-first (tier 0 = H1, tier 1 = H2, …). +/// Sizes within 0.5pt are clustered into the same tier. Capped at 4 tiers. +pub(crate) fn compute_heading_tiers(lines: &[TextLine], base_size: f32) -> Vec { + let mut heading_sizes: Vec = Vec::new(); + + for line in lines { + if let Some(first) = line.items.first() { + if first.font_size / base_size >= 1.2 { + heading_sizes.push(first.font_size); + } + } + } + + // Sort descending + heading_sizes.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + // Cluster sizes within 0.5pt into same tier (use first value as representative) + let mut tiers: Vec = Vec::new(); + for size in heading_sizes { + let already_in_tier = tiers.iter().any(|&t| (t - size).abs() < 0.5); + if !already_in_tier { + tiers.push(size); + } + } + + // Cap at 4 tiers + tiers.truncate(4); + tiers +} + +/// Detect header level from font size using document-specific heading tiers. +/// When tiers are available, maps tier 0→H1, tier 1→H2, etc. +/// Falls back to ratio-based thresholds when no tiers exist. +pub(crate) fn detect_header_level( + font_size: f32, + base_size: f32, + heading_tiers: &[f32], +) -> Option { + let ratio = font_size / base_size; + + if ratio < 1.2 { + return None; // Regular text + } + + if !heading_tiers.is_empty() { + // Match font_size to a tier (within 0.5pt tolerance) + for (i, &tier_size) in heading_tiers.iter().enumerate() { + if (font_size - tier_size).abs() < 0.5 { + return Some(i + 1); // tier 0 → H1, tier 1 → H2, etc. + } + } + // No tier match but large ratio — assign level after last tier + if ratio >= 1.5 { + let level = (heading_tiers.len() + 1).min(4); + return Some(level); + } + // No tier match and small ratio — not a heading + return None; + } + + // Fallback: original ratio-based thresholds (no tiers discovered) + if ratio >= 2.0 { + Some(1) + } else if ratio >= 1.5 { + Some(2) + } else if ratio >= 1.25 { + Some(3) + } else { + Some(4) + } +} diff --git a/src/markdown/classify.rs b/src/markdown/classify.rs new file mode 100644 index 0000000..66dce41 --- /dev/null +++ b/src/markdown/classify.rs @@ -0,0 +1,180 @@ +//! Line classification: captions, lists, code detection. + +/// Check if text is a figure/table caption or source citation +pub(crate) fn is_caption_line(text: &str) -> bool { + let trimmed = text.trim(); + + // Common caption prefixes in multiple languages + let caption_prefixes = [ + "Figure ", + "Figura ", + "Fig. ", + "Fig ", + "Table ", + "Tabela ", + "Source:", + "Fonte:", + "Source ", + "Fonte ", + "Note:", + "Nota:", + "Chart ", + "Gráfico ", + "Graph ", + "Diagram ", + "Image ", + "Imagem ", + "Photo ", + "Foto ", + ]; + + // Check if line starts with a caption prefix + for prefix in &caption_prefixes { + if trimmed.starts_with(prefix) { + return true; + } + } + + // Check case-insensitive patterns + let lower = trimmed.to_lowercase(); + if lower.starts_with("figure ") || lower.starts_with("table ") || lower.starts_with("source:") { + return true; + } + + false +} + +/// Check if text looks like a list item +pub(crate) fn is_list_item(text: &str) -> bool { + let trimmed = text.trim_start(); + + // Bullet patterns + if trimmed.starts_with("• ") + || trimmed.starts_with("- ") + || trimmed.starts_with("* ") + || trimmed.starts_with("○ ") + || trimmed.starts_with("● ") + || trimmed.starts_with("◦ ") + { + return true; + } + + // Numbered list patterns: "1.", "1)", "(1)", "a.", "a)" + let first_chars: String = trimmed.chars().take(5).collect(); + if first_chars.contains(|c: char| c.is_ascii_digit()) { + // Check for "1.", "1)", "10." + if let Some(idx) = first_chars.find(['.', ')']) { + let prefix = &first_chars[..idx]; + if prefix.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + // Letter list: "a.", "a)", "(a)" + let mut chars = trimmed.chars(); + if let (Some(first), Some(second)) = (chars.next(), chars.next()) { + if first.is_ascii_alphabetic() && (second == '.' || second == ')') { + return true; + } + if first == '(' && chars.next() == Some(')') { + return true; + } + } + + false +} + +/// Format list item to markdown +pub(crate) fn format_list_item(text: &str) -> String { + let trimmed = text.trim_start(); + + // Convert various bullet styles to markdown + // Note: bullet characters like • are multi-byte in UTF-8, use char indices + for bullet in &['•', '○', '●', '◦'] { + if let Some(rest) = trimmed.strip_prefix(*bullet) { + return format!("- {}", rest.trim_start()); + } + } + + if trimmed.starts_with("- ") || trimmed.starts_with("* ") { + return trimmed.to_string(); + } + + // Keep numbered lists as-is (markdown supports them) + trimmed.to_string() +} + +/// Check if text looks like code +pub(crate) fn is_code_like(text: &str) -> bool { + let trimmed = text.trim(); + + // Code patterns + let code_patterns = [ + // Language keywords + "import ", + "export ", + "from ", + "const ", + "let ", + "var ", + "function ", + "class ", + "def ", + "pub fn ", + "fn ", + "async fn ", + "impl ", + // Syntax patterns + "=> ", + "-> ", + ":: ", + ":= ", + ]; + + for pattern in &code_patterns { + if trimmed.starts_with(pattern) { + return true; + } + } + + // Check for code-like syntax + let special_chars: usize = trimmed + .chars() + .filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>')) + .count(); + + if special_chars >= 3 && trimmed.len() < 200 { + return true; + } + + // Ends with semicolon or braces + if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') { + return true; + } + + false +} + +/// Check if font name indicates monospace +pub(crate) fn is_monospace_font(font_name: &str) -> bool { + let lower = font_name.to_lowercase(); + let patterns = [ + "courier", + "consolas", + "monaco", + "menlo", + "mono", + "fixed", + "terminal", + "typewriter", + "source code", + "fira code", + "jetbrains", + "inconsolata", + "dejavu sans mono", + "liberation mono", + ]; + + patterns.iter().any(|p| lower.contains(p)) +} diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs new file mode 100644 index 0000000..dac55d6 --- /dev/null +++ b/src/markdown/convert.rs @@ -0,0 +1,670 @@ +//! Core line-to-markdown conversion loop with table/image interleaving. + +use std::collections::HashSet; + +use crate::types::TextLine; + +use super::analysis::{ + calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold, detect_header_level, + has_dot_leaders, +}; +use super::classify::{format_list_item, is_caption_line, is_list_item, is_monospace_font}; +use super::postprocess::clean_markdown; +use super::preprocess::{merge_drop_caps, merge_heading_lines}; +use super::MarkdownOptions; + +/// Merge continuation tables that span across page breaks. +/// +/// When consecutive pages each have exactly one table with the same number of columns +/// AND both pages are table-only (no non-table text), treat them as a single table. +/// We strip their header+separator rows and append their data rows to the first page's +/// table, then remove them from later pages. +pub(super) fn merge_continuation_tables( + page_tables: &mut std::collections::HashMap>, + table_only_pages: &HashSet, +) { + let mut sorted_pages: Vec = page_tables.keys().copied().collect(); + sorted_pages.sort(); + + if sorted_pages.len() < 2 { + return; + } + + // Find runs of consecutive pages that each have exactly one table with matching columns + let mut i = 0; + while i < sorted_pages.len() { + let first_page = sorted_pages[i]; + let first_tables = match page_tables.get(&first_page) { + Some(t) if t.len() == 1 => t, + _ => { + i += 1; + continue; + } + }; + + // First page must be table-only to start a merge chain + if !table_only_pages.contains(&first_page) { + i += 1; + continue; + } + + let first_col_count = count_table_columns(&first_tables[0].1); + if first_col_count == 0 { + i += 1; + continue; + } + + // Collect continuation pages (must also be table-only) + let mut continuation_pages = Vec::new(); + let mut j = i + 1; + while j < sorted_pages.len() { + let next_page = sorted_pages[j]; + // Must be consecutive page numbers + let prev_page = if continuation_pages.is_empty() { + first_page + } else { + *continuation_pages.last().unwrap() + }; + if next_page != prev_page + 1 { + break; + } + + // Continuation page must be table-only + if !table_only_pages.contains(&next_page) { + break; + } + + let next_tables = match page_tables.get(&next_page) { + Some(t) if t.len() == 1 => t, + _ => break, + }; + + let next_col_count = count_table_columns(&next_tables[0].1); + if next_col_count != first_col_count { + break; + } + + continuation_pages.push(next_page); + j += 1; + } + + if !continuation_pages.is_empty() { + // Collect data rows from continuation pages + let mut extra_rows = String::new(); + for &cont_page in &continuation_pages { + if let Some(tables) = page_tables.get(&cont_page) { + let table_md = &tables[0].1; + // Skip header row (line 1) and separator row (line 2), keep the rest + for (line_idx, line) in table_md.lines().enumerate() { + if line_idx >= 2 { + extra_rows.push_str(line); + extra_rows.push('\n'); + } + } + } + } + + // Append continuation rows to the first page's table + if let Some(tables) = page_tables.get_mut(&first_page) { + tables[0].1.push_str(&extra_rows); + } + + // Remove continuation pages from the map + for &cont_page in &continuation_pages { + page_tables.remove(&cont_page); + } + + // Skip past the merged pages + i = j; + } else { + i += 1; + } + } +} + +/// Count the number of columns in a markdown table by counting `|` in the separator row. +fn count_table_columns(table_md: &str) -> usize { + // The separator row is the second line, containing "| --- | --- |" + if let Some(sep_line) = table_md.lines().nth(1) { + if sep_line.contains("---") { + // Count cells: number of | minus 1 (leading |), but handle edge cases + let pipes = sep_line.chars().filter(|&c| c == '|').count(); + return if pipes >= 2 { pipes - 1 } else { 0 }; + } + } + 0 +} + +/// Flush any remaining tables and images for a given page +fn flush_page_tables_and_images( + page: u32, + page_tables: &std::collections::HashMap>, + page_images: &std::collections::HashMap>, + inserted_tables: &mut HashSet<(u32, usize)>, + inserted_images: &mut HashSet<(u32, usize)>, + output: &mut String, + in_paragraph: &mut bool, +) { + if let Some(tables) = page_tables.get(&page) { + for (idx, (_, table_md)) in tables.iter().enumerate() { + if !inserted_tables.contains(&(page, idx)) { + if *in_paragraph { + output.push_str("\n\n"); + *in_paragraph = false; + } + output.push('\n'); + output.push_str(table_md); + output.push('\n'); + inserted_tables.insert((page, idx)); + } + } + } + if let Some(images) = page_images.get(&page) { + for (idx, (_, image_md)) in images.iter().enumerate() { + if !inserted_images.contains(&(page, idx)) { + if *in_paragraph { + output.push_str("\n\n"); + *in_paragraph = false; + } + output.push('\n'); + output.push_str(image_md); + output.push('\n'); + inserted_images.insert((page, idx)); + } + } + } +} + +/// Convert text lines to markdown, inserting tables and images at appropriate Y positions +pub(super) fn to_markdown_from_lines_with_tables_and_images( + lines: Vec, + options: MarkdownOptions, + page_tables: std::collections::HashMap>, + page_images: std::collections::HashMap>, +) -> String { + if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { + return String::new(); + } + + // Calculate font statistics + let font_stats = calculate_font_stats(&lines); + let base_size = options + .base_font_size + .unwrap_or(font_stats.most_common_size); + + // Merge drop caps with following text + let lines = merge_drop_caps(lines, base_size); + + // Discover heading tiers for this document + let heading_tiers = compute_heading_tiers(&lines, base_size); + + // Merge consecutive heading lines at the same level (e.g., wrapped titles) + let lines = merge_heading_lines(lines, base_size, &heading_tiers); + + // Compute the typical line spacing for paragraph break detection. + // For double-spaced documents (like legal/government PDFs), the normal + // line spacing can be 2.3x base_size, which would exceed a fixed 1.8x + // threshold and cause every line to be treated as a paragraph break. + let para_threshold = compute_paragraph_threshold(&lines, base_size); + + let mut output = String::new(); + let mut current_page = 0u32; + let mut prev_y = f32::MAX; + let mut in_list = false; + let mut in_paragraph = false; + let mut last_list_x: Option = None; + let mut prev_had_dot_leaders = false; + let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); + let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); + + // Collect all pages that have tables or images (including image-only pages) + let mut all_content_pages: Vec = page_tables + .keys() + .chain(page_images.keys()) + .copied() + .collect(); + all_content_pages.sort(); + all_content_pages.dedup(); + + for line in lines { + // Page break + if line.page != current_page { + // Flush current page's remaining tables and images + if current_page > 0 { + flush_page_tables_and_images( + current_page, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str("\n\n"); + } + + // Flush any intermediate pages (image-only or table-only) between + // current_page and line.page that have no text lines + for &p in &all_content_pages { + if p <= current_page { + continue; + } + if p >= line.page { + break; + } + flush_page_tables_and_images( + p, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str("\n\n"); + } + + current_page = line.page; + prev_y = f32::MAX; + + if options.include_page_numbers { + output.push_str(&format!("\n\n", current_page)); + } + } + + // Check if we should insert a table before this line + if let Some(tables) = page_tables.get(¤t_page) { + for (idx, (table_y, table_md)) in tables.iter().enumerate() { + // Insert table when we pass its Y position + if *table_y > line.y && !inserted_tables.contains(&(current_page, idx)) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push('\n'); + output.push_str(table_md); + output.push('\n'); + inserted_tables.insert((current_page, idx)); + } + } + } + + // Check if we should insert an image before this line + if let Some(images) = page_images.get(¤t_page) { + for (idx, (image_y, image_md)) in images.iter().enumerate() { + // Insert image when we pass its Y position + if *image_y > line.y && !inserted_images.contains(&(current_page, idx)) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push('\n'); + output.push_str(image_md); + output.push('\n'); + inserted_images.insert((current_page, idx)); + } + } + } + + // Paragraph break: large forward Y gap (normal) or large backward jump + // (newspaper columns emitted sequentially on the same page). + let y_gap = prev_y - line.y; + let is_para_break = y_gap.abs() > para_threshold; + if is_para_break && in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + // Don't immediately end list on paragraph break + // Let the continuation check below decide if we're still in a list + prev_y = line.y; + + // Get text with optional bold/italic formatting + let text = line.text_with_formatting(options.detect_bold, options.detect_italic); + let trimmed = text.trim(); + + // Also get plain text for pattern matching (list detection, captions, etc.) + let plain_text = line.text(); + let plain_trimmed = plain_text.trim(); + + if trimmed.is_empty() { + continue; + } + + // Detect figure/table captions and source citations + // These should be on their own line followed by a paragraph break + if is_caption_line(plain_trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(trimmed); + output.push_str("\n\n"); + continue; + } + + // Detect headers by font size + // Note: Headers typically shouldn't have bold markers since they're already emphasized + // Skip very short text (drop caps/labels) and very long text (body paragraphs) + if options.detect_headers + && plain_trimmed.len() > 3 + && plain_trimmed.split_whitespace().count() <= 15 + { + let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + if let Some(header_level) = + detect_header_level(line_font_size, base_size, &heading_tiers) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + let prefix = "#".repeat(header_level); + // Use plain text for headers to avoid redundant formatting + output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); + in_list = false; + continue; + } + } + + // Detect list items + if options.detect_lists && is_list_item(plain_trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + let formatted = format_list_item(trimmed); + output.push_str(&formatted); + output.push('\n'); + in_list = true; + last_list_x = line.items.first().map(|i| i.x); + continue; + } else if in_list { + // Check if this line is a continuation of the previous list item + // Continuations have similar X position and reasonable Y gap + let line_x = line.items.first().map(|i| i.x); + let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) { + // Continuation criteria: + // 1. X is at or past the list text position + // 2. Y gap is not too large (max ~5 line heights) + // 3. Not a new list item + let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0; + let y_ok = y_gap < base_size * 7.0; + x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed) + } else { + false + }; + + if is_continuation { + // Append to previous list item with a space + if output.ends_with('\n') { + output.pop(); + output.push(' '); + } + output.push_str(trimmed); + output.push('\n'); + continue; + } else { + in_list = false; + last_list_x = None; + } + } + + // Detect code blocks by font + if options.detect_code { + let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); + if is_mono { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + // Use plain text for code blocks + output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); + continue; + } + } + + // Regular text - join lines within same paragraph with space + let cur_dot_leaders = has_dot_leaders(plain_trimmed); + if in_paragraph { + if cur_dot_leaders || prev_had_dot_leaders { + output.push('\n'); + } else { + output.push(' '); + } + } + output.push_str(trimmed); + in_paragraph = true; + prev_had_dot_leaders = cur_dot_leaders; + } + + // Flush current page and any remaining pages with tables/images + // (handles table-only pages after the last text line, and trailing image-only pages) + flush_page_tables_and_images( + current_page, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + for &p in &all_content_pages { + if p <= current_page { + continue; + } + flush_page_tables_and_images( + p, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + } + + // Close final paragraph + if in_paragraph { + output.push('\n'); + } + + // Clean up and post-process + clean_markdown(output, &options) +} + +/// Convert text lines to markdown +pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> String { + if lines.is_empty() { + return String::new(); + } + + // Calculate font statistics + let font_stats = calculate_font_stats(&lines); + let base_size = options + .base_font_size + .unwrap_or(font_stats.most_common_size); + + // Merge drop caps with following text + let lines = merge_drop_caps(lines, base_size); + + // Discover heading tiers for this document + let heading_tiers = compute_heading_tiers(&lines, base_size); + + // Merge consecutive heading lines at the same level (e.g., wrapped titles) + let lines = merge_heading_lines(lines, base_size, &heading_tiers); + + // Compute the typical line spacing for paragraph break detection + let para_threshold = compute_paragraph_threshold(&lines, base_size); + + let mut output = String::new(); + let mut current_page = 0u32; + let mut prev_y = f32::MAX; + let mut in_list = false; + let mut in_paragraph = false; + let mut last_list_x: Option = None; + let mut prev_had_dot_leaders = false; + + for line in lines { + // Page break + if line.page != current_page { + if current_page > 0 { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str("\n\n"); + } + current_page = line.page; + prev_y = f32::MAX; + in_list = false; + last_list_x = None; + prev_had_dot_leaders = false; + + if options.include_page_numbers { + output.push_str(&format!("\n\n", current_page)); + } + } + + // Paragraph break: large forward Y gap (normal) or large backward jump + // (newspaper columns emitted sequentially on the same page). + let y_gap = prev_y - line.y; + let is_para_break = y_gap.abs() > para_threshold; + if is_para_break && in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + // Don't immediately end list on paragraph break + // Let the continuation check below decide if we're still in a list + prev_y = line.y; + + // Get text with optional bold/italic formatting + let text = line.text_with_formatting(options.detect_bold, options.detect_italic); + let trimmed = text.trim(); + + // Also get plain text for pattern matching + let plain_text = line.text(); + let plain_trimmed = plain_text.trim(); + + if trimmed.is_empty() { + continue; + } + + // Detect figure/table captions and source citations + // These should be on their own line followed by a paragraph break + if is_caption_line(plain_trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(trimmed); + output.push_str("\n\n"); + continue; + } + + // Detect headers by font size + // Skip very short text (drop caps/labels) and very long text (body paragraphs) + if options.detect_headers + && plain_trimmed.len() > 3 + && plain_trimmed.split_whitespace().count() <= 15 + { + let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + if let Some(header_level) = + detect_header_level(line_font_size, base_size, &heading_tiers) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + let prefix = "#".repeat(header_level); + // Use plain text for headers to avoid redundant formatting + output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); + in_list = false; + continue; + } + } + + // Detect list items + if options.detect_lists && is_list_item(plain_trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + let formatted = format_list_item(trimmed); + output.push_str(&formatted); + output.push('\n'); + in_list = true; + last_list_x = line.items.first().map(|i| i.x); + continue; + } else if in_list { + // Check if this line is a continuation of the previous list item + let line_x = line.items.first().map(|i| i.x); + let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) { + // Continuation criteria: + // 1. X is at or past the list text position + // 2. Y gap is not too large (max ~5 line heights) + // 3. Not a new list item + let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0; + let y_ok = y_gap < base_size * 7.0; + x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed) + } else { + false + }; + + if is_continuation { + // Append to previous list item with a space + if output.ends_with('\n') { + output.pop(); + output.push(' '); + } + output.push_str(trimmed); + output.push('\n'); + continue; + } else { + in_list = false; + last_list_x = None; + } + } + + // Detect code blocks by font + if options.detect_code { + let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); + if is_mono { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + // Use plain text for code blocks + output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); + continue; + } + } + + // Regular text - join lines within same paragraph with space + let cur_dot_leaders = has_dot_leaders(plain_trimmed); + if in_paragraph { + if cur_dot_leaders || prev_had_dot_leaders { + output.push('\n'); + } else { + output.push(' '); + } + } + output.push_str(trimmed); + in_paragraph = true; + prev_had_dot_leaders = cur_dot_leaders; + } + + // Close final paragraph + if in_paragraph { + output.push('\n'); + } + + // Clean up and post-process + clean_markdown(output, &options) +} diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs new file mode 100644 index 0000000..8bbbe70 --- /dev/null +++ b/src/markdown/mod.rs @@ -0,0 +1,382 @@ +//! Markdown conversion with structure detection. +//! +//! Converts extracted text to markdown, detecting: +//! - Headers (by font size) +//! - Lists (bullet points, numbered lists) +//! - Code blocks (monospace fonts, indentation) +//! - Paragraphs + +mod analysis; +mod classify; +mod convert; +mod postprocess; +mod preprocess; + +pub use convert::to_markdown_from_lines; + +use std::collections::{HashMap, HashSet}; + +use crate::extractor::group_into_lines; +use crate::types::TextItem; + +use analysis::calculate_font_stats_from_items; +use classify::{format_list_item, is_code_like, is_list_item}; +use convert::{merge_continuation_tables, to_markdown_from_lines_with_tables_and_images}; + +/// Options for markdown conversion +#[derive(Debug, Clone)] +pub struct MarkdownOptions { + /// Detect headers by font size + pub detect_headers: bool, + /// Detect list items + pub detect_lists: bool, + /// Detect code blocks + pub detect_code: bool, + /// Base font size for comparison + pub base_font_size: Option, + /// Remove standalone page numbers + pub remove_page_numbers: bool, + /// Convert URLs to markdown links + pub format_urls: bool, + /// Fix hyphenation (broken words across lines) + pub fix_hyphenation: bool, + /// Detect and format bold text from font names + pub detect_bold: bool, + /// Detect and format italic text from font names + pub detect_italic: bool, + /// Include image placeholders in output + pub include_images: bool, + /// Include extracted hyperlinks + pub include_links: bool, + /// Insert page break markers () between pages + pub include_page_numbers: bool, +} + +impl Default for MarkdownOptions { + fn default() -> Self { + Self { + detect_headers: true, + detect_lists: true, + detect_code: true, + base_font_size: None, + remove_page_numbers: true, + format_urls: true, + fix_hyphenation: true, + detect_bold: true, + detect_italic: true, + include_images: true, + include_links: true, + include_page_numbers: false, + } + } +} + +/// Convert plain text to markdown (basic conversion) +pub fn to_markdown(text: &str, options: MarkdownOptions) -> String { + let mut output = String::new(); + let mut in_list = false; + let mut in_code_block = false; + + for line in text.lines() { + let trimmed = line.trim(); + + if trimmed.is_empty() { + if in_list { + in_list = false; + } + if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } + output.push('\n'); + continue; + } + + // Detect list items + if options.detect_lists && is_list_item(trimmed) { + let formatted = format_list_item(trimmed); + output.push_str(&formatted); + output.push('\n'); + in_list = true; + continue; + } + + // Detect code blocks (indented lines) + if options.detect_code && is_code_like(trimmed) { + if !in_code_block { + output.push_str("```\n"); + in_code_block = true; + } + output.push_str(trimmed); + output.push('\n'); + continue; + } else if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } + + // Regular paragraph text + output.push_str(trimmed); + output.push('\n'); + } + + if in_code_block { + output.push_str("```\n"); + } + + output +} + +/// Convert positioned text items to markdown with structure detection +pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> String { + to_markdown_from_items_with_rects(items, options, &[]) +} + +/// Convert positioned text items to markdown, using rectangle data for table detection +pub fn to_markdown_from_items_with_rects( + items: Vec, + options: MarkdownOptions, + rects: &[crate::types::PdfRect], +) -> String { + use crate::tables::{detect_tables, detect_tables_from_rects, table_to_markdown}; + use crate::types::ItemType; + + if items.is_empty() { + return String::new(); + } + + // Separate images and links from text items + let mut images: Vec = Vec::new(); + let mut links: Vec = Vec::new(); + let mut text_items: Vec = Vec::new(); + + for item in items { + match &item.item_type { + ItemType::Image => { + if options.include_images { + images.push(item); + } + } + ItemType::Link(_) => { + if options.include_links { + links.push(item); + } + } + ItemType::Text | ItemType::FormField => { + text_items.push(item); + } + } + } + + // Calculate base font size for table detection + let font_stats = calculate_font_stats_from_items(&text_items); + let base_size = options + .base_font_size + .unwrap_or(font_stats.most_common_size); + + // Detect tables on each page + let mut table_items: HashSet = HashSet::new(); + let mut page_tables: HashMap> = HashMap::new(); + + // Store images by page and Y position for insertion + let mut page_images: HashMap> = HashMap::new(); + + for img in &images { + // Extract image name from "[Image: Im0]" format + let img_name = img + .text + .strip_prefix("[Image: ") + .and_then(|s| s.strip_suffix(']')) + .unwrap_or(&img.text); + let img_md = format!("![Image: {}](image)\n", img_name); + page_images + .entry(img.page) + .or_default() + .push((img.y, img_md)); + } + + // Pre-group items by page with their global indices (O(n) instead of O(pages*n)) + let mut page_groups: HashMap> = HashMap::new(); + for (global_idx, item) in text_items.iter().enumerate() { + page_groups + .entry(item.page) + .or_default() + .push((global_idx, item)); + } + + let mut pages: Vec = page_groups.keys().copied().collect(); + pages.sort(); + + for page in pages { + let group = page_groups.get(&page).unwrap(); + let page_items: Vec = group.iter().map(|(_, item)| (*item).clone()).collect(); + + // Track which local indices are claimed by rect-based tables + let mut rect_claimed: HashSet = HashSet::new(); + + // Try rectangle-based table detection first + let rect_tables = detect_tables_from_rects(&page_items, rects, page); + for table in &rect_tables { + for &idx in &table.item_indices { + rect_claimed.insert(idx); + if let Some(&(global_idx, _)) = group.get(idx) { + table_items.insert(global_idx); + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + + // Run heuristic detection on unclaimed items only + if rect_claimed.is_empty() { + // No rect tables — run heuristic on all items + let tables = detect_tables(&page_items, base_size, false); + for table in tables { + for &idx in &table.item_indices { + if let Some(&(global_idx, _)) = group.get(idx) { + table_items.insert(global_idx); + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } else { + // Rect tables found — run heuristic on unclaimed items + let unclaimed_items: Vec = page_items + .iter() + .enumerate() + .filter(|(idx, _)| !rect_claimed.contains(idx)) + .map(|(_, item)| item.clone()) + .collect(); + if unclaimed_items.len() >= 6 { + let tables = detect_tables(&unclaimed_items, base_size, false); + for table in tables { + // Remap indices from unclaimed-space back to page-space + let unclaimed_map: Vec = page_items + .iter() + .enumerate() + .filter(|(idx, _)| !rect_claimed.contains(idx)) + .map(|(idx, _)| idx) + .collect(); + for &idx in &table.item_indices { + if let Some(&page_idx) = unclaimed_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } + } + } + + // Filter out table items and process the rest + let non_table_items: Vec = text_items + .into_iter() + .enumerate() + .filter(|(idx, _)| !table_items.contains(idx)) + .map(|(_, item)| item) + .collect(); + + // Find pages that are table-only (no remaining non-table text) + let table_only_pages: HashSet = { + let pages_with_text: HashSet = non_table_items.iter().map(|i| i.page).collect(); + page_tables + .keys() + .filter(|p| !pages_with_text.contains(p)) + .copied() + .collect() + }; + + // Merge continuation tables across page breaks, but only for table-only pages + merge_continuation_tables(&mut page_tables, &table_only_pages); + + let lines = group_into_lines(non_table_items); + + // Convert to markdown, inserting tables and images at appropriate positions + to_markdown_from_lines_with_tables_and_images(lines, options, page_tables, page_images) +} + +#[cfg(test)] +mod tests { + use super::*; + use analysis::detect_header_level; + use classify::{is_code_like, is_list_item}; + + #[test] + fn test_is_list_item() { + assert!(is_list_item("• Item one")); + assert!(is_list_item("- Item two")); + assert!(is_list_item("* Item three")); + assert!(is_list_item("1. First")); + assert!(is_list_item("2) Second")); + assert!(is_list_item("a. Letter item")); + assert!(!is_list_item("Regular text")); + } + + #[test] + fn test_format_list_item() { + assert_eq!(format_list_item("• Item"), "- Item"); + assert_eq!(format_list_item("- Item"), "- Item"); + assert_eq!(format_list_item("1. First"), "1. First"); + } + + #[test] + fn test_is_code_like() { + assert!(is_code_like("const x = 5;")); + assert!(is_code_like("function foo() {")); + assert!(is_code_like("import React from 'react'")); + assert!(!is_code_like("This is regular text.")); + } + + #[test] + fn test_detect_header_level() { + // With three tiers: 24→H1, 18→H2, 15→H3, 12→None + let tiers = vec![24.0, 18.0, 15.0]; + assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); + assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); + assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); + assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + + // Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2) + let tiers = vec![15.0]; + assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(1)); + assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + + // No tiers (empty): falls back to ratio thresholds + let tiers: Vec = vec![]; + assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); + assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); + assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); + assert_eq!(detect_header_level(14.5, 12.0, &tiers), Some(4)); + assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + + // Body text excluded when tiers exist: 13pt (ratio 1.08) → None + let tiers = vec![20.0]; + assert_eq!(detect_header_level(13.0, 12.0, &tiers), None); + } + + #[test] + fn test_to_markdown() { + let text = "• First item\n• Second item\n\nRegular paragraph."; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("- First item")); + assert!(md.contains("- Second item")); + } +} diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs new file mode 100644 index 0000000..3f34d16 --- /dev/null +++ b/src/markdown/postprocess.rs @@ -0,0 +1,275 @@ +//! Markdown cleanup and post-processing. + +use regex::Regex; + +use super::MarkdownOptions; + +/// Clean up markdown output with post-processing +pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String { + // Collapse dot leaders (e.g. TOC entries: "Introduction...............................1") + text = collapse_dot_leaders(&text); + + // Fix hyphenation first (before other processing) + if options.fix_hyphenation { + text = fix_hyphenation(&text); + } + + // Remove standalone page numbers + if options.remove_page_numbers { + text = remove_page_numbers(&text); + } + + // Format URLs as markdown links + if options.format_urls { + text = format_urls(&text); + } + + // Remove excessive newlines (more than 2 in a row) + while text.contains("\n\n\n") { + text = text.replace("\n\n\n", "\n\n"); + } + + // Trim leading and trailing whitespace, ensure ends with single newline + text = text.trim().to_string(); + text.push('\n'); + + text +} + +/// Collapse dot leaders (runs of 4+ dots) into " ... " +/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" +fn collapse_dot_leaders(text: &str) -> String { + use once_cell::sync::Lazy; + static DOT_LEADER_RE: Lazy = Lazy::new(|| Regex::new(r"\.{4,}").unwrap()); + + DOT_LEADER_RE.replace_all(text, " ... ").to_string() +} + +/// Fix words broken across lines with spaces before the continuation +/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte" +fn fix_hyphenation(text: &str) -> String { + use once_cell::sync::Lazy; + + // Fix "word - word" patterns that should be "word-word" (compound words) + // But be careful not to break list items (which start with "- ") + static SPACED_HYPHEN_RE: Lazy = Lazy::new(|| { + Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap() + }); + + let result = SPACED_HYPHEN_RE + .replace_all(text, |caps: ®ex::Captures| { + format!("{}-{}", &caps[1], &caps[2]) + }) + .to_string(); + + result +} + +/// Remove standalone page numbers (lines that are just 1-4 digit numbers) +fn remove_page_numbers(text: &str) -> String { + let mut result = Vec::new(); + let lines: Vec<&str> = text.lines().collect(); + + for (i, line) in lines.iter().enumerate() { + let trimmed = line.trim(); + + // Check for page number patterns + if is_page_number_line(trimmed) { + // Check context to determine if this is isolated + let prev_is_break = i > 0 && lines[i - 1].trim() == "---"; + let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---"; + let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty(); + let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty(); + + // Check if it's on its own line (surrounded by empty lines or page breaks) + let is_isolated = (prev_is_break || prev_is_empty || i == 0) + && (next_is_break || next_is_empty || i + 1 == lines.len()); + + // Also remove numbers that appear right before a page break + let before_break = i + 1 < lines.len() + && (lines[i + 1].trim() == "---" + || (i + 2 < lines.len() + && lines[i + 1].trim().is_empty() + && lines[i + 2].trim() == "---")); + + if is_isolated || before_break { + continue; + } + } + + result.push(*line); + } + + result.join("\n") +} + +/// Check if a line looks like a page number +fn is_page_number_line(trimmed: &str) -> bool { + // Empty lines are not page numbers + if trimmed.is_empty() { + return false; + } + + // Pattern 1: Just a number (1-4 digits) + if trimmed.len() <= 4 && trimmed.chars().all(|c| c.is_ascii_digit()) { + return true; + } + + // Pattern 2: "Page X of Y" or "Page X" or "Page of" (placeholder) + let lower = trimmed.to_lowercase(); + if let Some(rest) = lower.strip_prefix("page") { + let rest = rest.trim(); + // "Page of" (empty page numbers) + if rest == "of" || rest.starts_with("of ") { + return true; + } + // "Page X" or "Page X of Y" + if rest + .chars() + .next() + .map(|c| c.is_ascii_digit()) + .unwrap_or(false) + { + return true; + } + // Just "Page" followed by whitespace and maybe "of" + if rest.is_empty() + || rest + .split_whitespace() + .all(|w| w == "of" || w.chars().all(|c| c.is_ascii_digit())) + { + return true; + } + } + + // Pattern 3: "X of Y" where X and Y are numbers + if let Some(of_idx) = trimmed.find(" of ") { + let before = trimmed[..of_idx].trim(); + let after = trimmed[of_idx + 4..].trim(); + if before.chars().all(|c| c.is_ascii_digit()) + && after.chars().all(|c| c.is_ascii_digit()) + && !before.is_empty() + && !after.is_empty() + { + return true; + } + } + + // Pattern 4: "- X -" centered page number + if trimmed.len() >= 3 && trimmed.starts_with('-') && trimmed.ends_with('-') { + let inner = trimmed[1..trimmed.len() - 1].trim(); + if inner.chars().all(|c| c.is_ascii_digit()) && !inner.is_empty() { + return true; + } + } + + false +} + +/// Convert URLs to markdown links +fn format_urls(text: &str) -> String { + use once_cell::sync::Lazy; + + // Match URLs - we'll check context manually to avoid formatting already-linked URLs + static URL_RE: Lazy = + Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap()); + + let mut result = String::with_capacity(text.len()); + let mut last_end = 0; + + for mat in URL_RE.find_iter(text) { + let start = mat.start(); + let url = mat.as_str(); + + // Check if this URL is already in a markdown link by looking at preceding chars + // Use safe character boundary checking for multi-byte UTF-8 + let before = { + let mut check_start = start.saturating_sub(2); + // Find a valid character boundary + while check_start > 0 && !text.is_char_boundary(check_start) { + check_start -= 1; + } + if check_start < start && text.is_char_boundary(start) { + &text[check_start..start] + } else { + "" + } + }; + let already_linked = before.ends_with("](") || before.ends_with("]("); + + // Also check if it's inside square brackets (link text) + // Ensure we're slicing at a valid char boundary + let prefix = if text.is_char_boundary(start) { + &text[..start] + } else { + // Find the nearest valid boundary before start + let mut safe_start = start; + while safe_start > 0 && !text.is_char_boundary(safe_start) { + safe_start -= 1; + } + &text[..safe_start] + }; + let open_brackets = prefix.matches('[').count(); + let close_brackets = prefix.matches(']').count(); + let inside_link_text = open_brackets > close_brackets; + + // Ensure mat boundaries are valid char boundaries + let safe_last_end = if text.is_char_boundary(last_end) { + last_end + } else { + let mut pos = last_end; + while pos < text.len() && !text.is_char_boundary(pos) { + pos += 1; + } + pos + }; + let safe_start = if text.is_char_boundary(start) { + start + } else { + let mut pos = start; + while pos < text.len() && !text.is_char_boundary(pos) { + pos += 1; + } + pos + }; + let safe_end = if text.is_char_boundary(mat.end()) { + mat.end() + } else { + let mut pos = mat.end(); + while pos < text.len() && !text.is_char_boundary(pos) { + pos += 1; + } + pos + }; + + if already_linked || inside_link_text { + // Already formatted, keep as-is + if safe_last_end <= safe_end { + result.push_str(&text[safe_last_end..safe_end]); + } + } else { + // Add text before this URL + if safe_last_end <= safe_start { + result.push_str(&text[safe_last_end..safe_start]); + } + // Format as markdown link + result.push_str(&format!("[{}]({})", url, url)); + } + last_end = safe_end; + } + + // Add remaining text (ensure valid char boundary) + let safe_last_end = if text.is_char_boundary(last_end) { + last_end + } else { + let mut pos = last_end; + while pos < text.len() && !text.is_char_boundary(pos) { + pos += 1; + } + pos + }; + if safe_last_end < text.len() { + result.push_str(&text[safe_last_end..]); + } + result +} diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs new file mode 100644 index 0000000..9092c81 --- /dev/null +++ b/src/markdown/preprocess.rs @@ -0,0 +1,142 @@ +//! Line preprocessing: heading merging and drop cap handling. + +use crate::types::TextLine; + +use super::analysis::detect_header_level; + +/// Merge consecutive heading lines at the same level into a single line. +/// +/// When a heading wraps across multiple text lines (e.g., "About Glenair, the Mission-Critical" +/// and "Interconnect Company"), each fragment becomes a separate `# Header` in the output. +/// This function detects consecutive lines at the same heading tier on the same page +/// with a small Y gap and merges them into one line. +pub(crate) fn merge_heading_lines( + lines: Vec, + base_size: f32, + heading_tiers: &[f32], +) -> Vec { + if lines.is_empty() { + return lines; + } + + let mut result: Vec = Vec::with_capacity(lines.len()); + + for line in lines { + let line_font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + let line_level = detect_header_level(line_font, base_size, heading_tiers); + + // Check if the previous line is a heading at the same level on the same page + let should_merge = if let (Some(prev), Some(curr_level)) = (result.last(), line_level) { + let prev_font = prev.items.first().map(|i| i.font_size).unwrap_or(base_size); + let prev_level = detect_header_level(prev_font, base_size, heading_tiers); + let same_page = prev.page == line.page; + let same_level = prev_level == Some(curr_level); + let y_gap = prev.y - line.y; + // Merge if gap is within ~2x the font size (normal line wrap spacing) + let close_enough = y_gap > 0.0 && y_gap < line_font * 2.0; + same_page && same_level && close_enough + } else { + false + }; + + if should_merge { + // Append this line's items to the previous line + let prev = result.last_mut().unwrap(); + // Add a space-bearing TextItem to separate the merged text + if let Some(first_item) = line.items.first() { + let mut space_item = first_item.clone(); + space_item.text = format!(" {}", space_item.text.trim_start()); + prev.items.push(space_item); + } + for item in line.items.into_iter().skip(1) { + prev.items.push(item); + } + } else { + result.push(line); + } + } + + result +} + +/// Merge drop caps with the appropriate line. +/// A drop cap is a single large letter at the start of a paragraph. +/// Due to PDF coordinate sorting, the drop cap may appear AFTER the line it belongs to. +pub(crate) fn merge_drop_caps(lines: Vec, base_size: f32) -> Vec { + let mut result: Vec = Vec::with_capacity(lines.len()); + + for line in &lines { + let text = line.text(); + let trimmed = text.trim(); + + // Check if this looks like a drop cap: + // 1. Single character (or single char + space) + // 2. Much larger than base font (3x or more) + // 3. The character is uppercase + let is_drop_cap = trimmed.len() <= 2 + && line.items.first().map(|i| i.font_size).unwrap_or(0.0) >= base_size * 2.5 + && trimmed + .chars() + .next() + .map(|c| c.is_uppercase()) + .unwrap_or(false); + + if is_drop_cap { + let drop_char = trimmed.chars().next().unwrap(); + + // Find the first line that starts with lowercase and is at the START of a paragraph + // (i.e., preceded by a header or non-lowercase-starting line) + let mut target_idx: Option = None; + + for (idx, prev_line) in result.iter().enumerate() { + if prev_line.page != line.page { + continue; + } + + let prev_text = prev_line.text(); + let prev_trimmed = prev_text.trim(); + + // Check if this line starts with lowercase + if prev_trimmed + .chars() + .next() + .map(|c| c.is_lowercase()) + .unwrap_or(false) + { + // Check if previous line exists and doesn't start with lowercase + // (meaning this is the start of a paragraph) + let is_para_start = if idx == 0 { + true + } else { + let before = result[idx - 1].text(); + let before_trimmed = before.trim(); + !before_trimmed + .chars() + .next() + .map(|c| c.is_lowercase()) + .unwrap_or(true) + }; + + if is_para_start { + target_idx = Some(idx); + break; + } + } + } + + // Merge with the target line + if let Some(idx) = target_idx { + if let Some(first_item) = result[idx].items.first_mut() { + let prev_text = first_item.text.trim().to_string(); + first_item.text = format!("{}{}", drop_char, prev_text); + } + } + // Don't add the drop cap line itself + continue; + } + + result.push(line.clone()); + } + + result +} diff --git a/src/tables.rs b/src/tables.rs deleted file mode 100644 index 4102f7b..0000000 --- a/src/tables.rs +++ /dev/null @@ -1,2418 +0,0 @@ -//! Table detection and formatting -//! -//! Detects tabular data in PDF text items and converts to markdown tables. - -use crate::extractor::{is_rtl_text, PdfRect, TextItem}; - -/// Detection mode controls thresholds for table validation -#[derive(Debug, Clone, Copy, PartialEq)] -enum TableDetectionMode { - /// Existing behavior: items with font size smaller than body text - SmallFont, - /// New: body-font items with stricter structural criteria - BodyFont, -} - -/// A detected table -#[derive(Debug, Clone)] -pub struct Table { - /// Column boundaries (x positions) - pub columns: Vec, - /// Row boundaries (y positions, descending order) - pub rows: Vec, - /// Cell contents indexed by (row, col) - pub cells: Vec>, - /// Items that belong to this table - pub item_indices: Vec, -} - -/// Disjoint-set (union-find) for clustering indices. -struct UnionFind { - parent: Vec, - rank: Vec, -} - -impl UnionFind { - fn new(n: usize) -> Self { - Self { - parent: (0..n).collect(), - rank: vec![0; n], - } - } - - fn find(&mut self, x: usize) -> usize { - if self.parent[x] != x { - self.parent[x] = self.find(self.parent[x]); - } - self.parent[x] - } - - fn union(&mut self, a: usize, b: usize) { - let ra = self.find(a); - let rb = self.find(b); - if ra == rb { - return; - } - if self.rank[ra] < self.rank[rb] { - self.parent[ra] = rb; - } else if self.rank[ra] > self.rank[rb] { - self.parent[rb] = ra; - } else { - self.parent[rb] = ra; - self.rank[ra] += 1; - } - } -} - -/// Check if two rects overlap after expanding each by `tol` on all sides. -fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32), tol: f32) -> bool { - // a and b are (x, y, w, h) where (x,y) is bottom-left corner - let (ax, ay, aw, ah) = *a; - let (bx, by, bw, bh) = *b; - // Expand each rect by tol - let a_left = ax - tol; - let a_right = ax + aw + tol; - let a_bottom = ay - tol; - let a_top = ay + ah + tol; - let b_left = bx - tol; - let b_right = bx + bw + tol; - let b_bottom = by - tol; - let b_top = by + bh + tol; - // AABB overlap: NOT (separated) - !(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom) -} - -/// Cluster rects by spatial overlap using union-find. -/// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned. -fn cluster_rects( - rects: &[(f32, f32, f32, f32)], - tolerance: f32, - min_size: usize, -) -> Vec> { - let n = rects.len(); - let mut uf = UnionFind::new(n); - - for i in 0..n { - for j in (i + 1)..n { - if rects_overlap(&rects[i], &rects[j], tolerance) { - uf.union(i, j); - } - } - } - - // Group indices by root - let mut groups: std::collections::HashMap> = std::collections::HashMap::new(); - for i in 0..n { - groups.entry(uf.find(i)).or_default().push(i); - } - - groups - .into_values() - .filter(|g| g.len() >= min_size) - .collect() -} - -/// Detect tables from explicit rectangle (`re`) operators in the PDF. -/// -/// Many PDFs draw cell borders using `re` (rectangle) operators. Table pages -/// typically have 100-200+ rects while non-table pages have < 30. This function -/// clusters spatially connected rectangles into groups, then identifies grids of -/// cell-sized rectangles within each cluster and assigns text items to cells. -pub fn detect_tables_from_rects(items: &[TextItem], rects: &[PdfRect], page: u32) -> Vec { - // Filter rects on this page; normalize negative widths/heights; skip tiny rects. - let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized - for r in rects { - if r.page != page { - continue; - } - let (mut x, mut y, mut w, mut h) = (r.x, r.y, r.width, r.height); - if w < 0.0 { - x += w; - w = -w; - } - if h < 0.0 { - y += h; - h = -h; - } - // Skip tiny rects (borders, dots, decorations) - if w < 5.0 || h < 5.0 { - continue; - } - page_rects.push((x, y, w, h)); - } - - // Need a reasonable number of cell rects to form a table - if page_rects.len() < 6 { - return vec![]; - } - - // Cluster spatially connected rects into groups - let clusters = cluster_rects(&page_rects, 3.0, 6); - - let mut tables = Vec::new(); - for cluster_indices in &clusters { - let group_rects: Vec<(f32, f32, f32, f32)> = - cluster_indices.iter().map(|&i| page_rects[i]).collect(); - if let Some(table) = detect_table_from_rect_group(items, &group_rects, page) { - tables.push(table); - } - } - - tables -} - -/// Detect a single table from a cluster of spatially connected rects. -/// -/// Contains the grid-detection logic: snap edges, fill-ratio check, -/// assign items to grid, content density validation. -fn detect_table_from_rect_group( - items: &[TextItem], - group_rects: &[(f32, f32, f32, f32)], - page: u32, -) -> Option
{ - // Extract unique X and Y edges from all rects - let mut x_edges: Vec = Vec::new(); - let mut y_edges: Vec = Vec::new(); - for &(x, y, w, h) in group_rects { - x_edges.push(x); - x_edges.push(x + w); - y_edges.push(y); - y_edges.push(y + h); - } - - let x_edges = snap_edges(&x_edges, 2.0); - let y_edges = snap_edges(&y_edges, 2.0); - - if x_edges.len() < 3 || y_edges.len() < 4 { - // Need at least 2 columns (3 edges) and 3 rows (4 edges) - return None; - } - - // Sort column edges left-to-right, row edges top-to-bottom (highest Y first for PDF) - let mut col_edges = x_edges; - col_edges.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let mut row_edges = y_edges; - row_edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); - - let num_cols = col_edges.len() - 1; - let num_rows = row_edges.len() - 1; - - if num_cols < 2 || num_rows < 2 { - return None; - } - - // Reject grids that are too large — real tables rarely exceed 12 columns. - // Form-style PDFs with scattered field boxes produce huge sparse grids. - if num_cols > 12 { - return None; - } - - // Verify that cell-sized rects actually fill the grid - // Count how many grid cells have a matching rect - let mut filled_cells = 0u32; - for row in 0..num_rows { - let y_top = row_edges[row]; - let y_bot = row_edges[row + 1]; - for col in 0..num_cols { - let x_left = col_edges[col]; - let x_right = col_edges[col + 1]; - // Check if any rect approximately covers this cell - let cell_covered = group_rects.iter().any(|&(rx, ry, rw, rh)| { - let tol = 3.0; - rx <= x_left + tol - && (rx + rw) >= x_right - tol - && ry <= y_top + tol - && (ry + rh) >= y_bot - tol - }); - if cell_covered { - filled_cells += 1; - } - } - } - - let total_cells = (num_cols * num_rows) as f32; - let fill_ratio = filled_cells as f32 / total_cells; - - // Require at least 30% of cells to be backed by rects - if fill_ratio < 0.3 { - return None; - } - - // Build table: assign text items to cells - let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); - - // Compute column centers and row centers for the Table struct - let columns: Vec = (0..num_cols) - .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) - .collect(); - let rows: Vec = (0..num_rows) - .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) - .collect(); - - // Skip if no text was assigned - if item_indices.is_empty() { - return None; - } - - // Skip tables with only 1 row of content (header-only) - let non_empty_rows = cells - .iter() - .filter(|row| row.iter().any(|c| !c.trim().is_empty())) - .count(); - if non_empty_rows < 2 { - return None; - } - - // Content density check: reject tables where most cells are empty. - // Real tables have content in most cells; form layouts produce sparse grids. - let non_empty_cells = cells - .iter() - .flat_map(|row| row.iter()) - .filter(|c| !c.trim().is_empty()) - .count(); - let content_ratio = non_empty_cells as f32 / total_cells; - if content_ratio < 0.25 { - return None; - } - - // Reject tables with any completely empty column — indicates a bad grid. - for col in 0..num_cols { - let col_has_content = cells - .iter() - .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); - if !col_has_content { - return None; - } - } - - Some(Table { - columns, - rows, - cells, - item_indices, - }) -} - -/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges. -fn snap_edges(values: &[f32], tolerance: f32) -> Vec { - let mut sorted: Vec = values.to_vec(); - sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - let mut snapped: Vec = Vec::new(); - for &v in &sorted { - if let Some(last) = snapped.last() { - if (v - *last).abs() <= tolerance { - continue; // Skip — too close to previous edge - } - } - snapped.push(v); - } - snapped -} - -/// Assign text items to grid cells defined by column/row edges. -/// -/// Returns `(cells, item_indices)` where `cells[row][col]` is the cell text -/// and `item_indices` lists the original item indices that were consumed. -fn assign_items_to_grid( - items: &[TextItem], - col_edges: &[f32], - row_edges: &[f32], - page: u32, -) -> (Vec>, Vec) { - let num_cols = col_edges.len() - 1; - let num_rows = row_edges.len() - 1; - - // Collect items per cell for proper sorting before joining - let mut cell_items: Vec>> = - vec![vec![Vec::new(); num_cols]; num_rows]; - let mut indices = Vec::new(); - - for (idx, item) in items.iter().enumerate() { - if item.page != page { - continue; - } - // Use item center for assignment - let cx = item.x + item.width / 2.0; - let cy = item.y; - - // Find column: cx must be between col_edges[c] and col_edges[c+1] - let col = (0..num_cols).find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0); - // Find row: cy must be between row_edges[r+1] (bottom) and row_edges[r] (top) - let row = (0..num_rows).find(|&r| cy >= row_edges[r + 1] - 2.0 && cy <= row_edges[r] + 2.0); - - if let (Some(c), Some(r)) = (col, row) { - cell_items[r][c].push((idx, item)); - indices.push(idx); - } - } - - // Build cell strings: sort items within each cell by Y descending then X ascending - let mut cells: Vec> = Vec::with_capacity(num_rows); - for row_items in &mut cell_items { - let mut row_cells = Vec::with_capacity(num_cols); - for col_items in row_items.iter_mut() { - col_items.sort_by(|a, b| { - b.1.y - .partial_cmp(&a.1.y) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| { - a.1.x - .partial_cmp(&b.1.x) - .unwrap_or(std::cmp::Ordering::Equal) - }) - }); - let text: String = col_items - .iter() - .map(|(_, item)| item.text.trim()) - .filter(|t| !t.is_empty()) - .collect::>() - .join(" "); - row_cells.push(text); - } - cells.push(row_cells); - } - - (cells, indices) -} - -/// Check if a whitespace-separated token looks like a financial number. -/// Must contain at least one digit; all chars must be `0-9 , . ( ) - + %`. -fn is_numeric_token(tok: &str) -> bool { - if tok.is_empty() { - return false; - } - let mut has_digit = false; - for c in tok.chars() { - match c { - '0'..='9' => has_digit = true, - ',' | '.' | '(' | ')' | '-' | '+' | '%' => {} - _ => return false, - } - } - has_digit -} - -/// Check for em-dash, en-dash, or minus used as nil marker in financial tables. -fn is_dash_token(tok: &str) -> bool { - matches!(tok, "\u{2014}" | "\u{2013}" | "-" | "\u{2012}") -} - -/// Returns true if text contains 2+ consecutive alphabetic characters. -/// Fast early-exit to reject items like `"Land $ 778,177"`. -fn has_alphabetic_words(text: &str) -> bool { - let mut consecutive = 0u32; - for c in text.chars() { - if c.is_alphabetic() { - consecutive += 1; - if consecutive >= 2 { - return true; - } - } else { - consecutive = 0; - } - } - false -} - -/// Splits text by whitespace, then groups tokens into financial values. -/// - `$` + numeric token → one value (`"$ 5,147,649"`) -/// - standalone numeric token → one value (`"114,167"`) -/// - dash token → one value (`"—"`) -/// - any unrecognized token → return `None` (not a pure-value item) -fn tokenize_financial_values(text: &str) -> Option> { - let tokens: Vec<&str> = text.split_whitespace().collect(); - if tokens.is_empty() { - return None; - } - let mut values = Vec::new(); - let mut i = 0; - while i < tokens.len() { - let tok = tokens[i]; - if tok == "$" { - // Dollar sign followed by a numeric token → one value - if i + 1 < tokens.len() && is_numeric_token(tokens[i + 1]) { - values.push(format!("{} {}", tok, tokens[i + 1])); - i += 2; - } else { - return None; - } - } else if is_numeric_token(tok) || is_dash_token(tok) { - values.push(tok.to_string()); - i += 1; - } else { - return None; - } - } - if values.is_empty() { - None - } else { - Some(values) - } -} - -/// Try to split a consolidated financial item into individual sub-items. -/// Criteria: width > font_size × 20, no alphabetic words, tokenization yields 3+ values. -/// Creates sub-items with evenly-distributed X positions across the original item's span. -fn try_split_financial_item(item: &TextItem) -> Option> { - if item.width <= item.font_size * 20.0 { - return None; - } - let text = &item.text; - if has_alphabetic_words(text) { - return None; - } - let values = tokenize_financial_values(text)?; - if values.len() < 3 { - return None; - } - let n = values.len() as f32; - let spacing = item.width / n; - let sub_width = spacing * 0.9; - let mut sub_items = Vec::with_capacity(values.len()); - for (i, val) in values.iter().enumerate() { - sub_items.push(TextItem { - text: val.clone(), - x: item.x + spacing * i as f32 + spacing * 0.5, - y: item.y, - width: sub_width, - height: item.height, - font: item.font.clone(), - font_size: item.font_size, - page: item.page, - is_bold: item.is_bold, - is_italic: item.is_italic, - item_type: item.item_type.clone(), - }); - } - Some(sub_items) -} - -/// Merge adjacent items on the same line into combined words/phrases. -/// -/// Per-character PDFs render each glyph as a separate TextItem. This creates -/// hundreds of single-char items that confuse column detection. This function -/// merges adjacent items within the same line (similar Y, close X, similar font -/// size) into multi-character items, similar to PyMuPDF's `merge_chars()`. -/// -/// Returns `(merged_items, index_map)` where `index_map[merged_idx]` contains -/// the original item indices that were merged into that item. -fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec>) { - if items.is_empty() { - return (vec![], vec![]); - } - - // Group items by Y position (5pt tolerance for same line) - let y_tolerance = 5.0; - let mut line_groups: Vec<(f32, Vec<(usize, &TextItem)>)> = Vec::new(); - - for (idx, item) in items.iter().enumerate() { - let found = line_groups - .iter_mut() - .find(|(y, _)| (item.y - *y).abs() < y_tolerance); - if let Some((_, group)) = found { - group.push((idx, item)); - } else { - line_groups.push((item.y, vec![(idx, item)])); - } - } - - // Sort each group by X position - for (_, group) in &mut line_groups { - group.sort_by(|a, b| { - a.1.x - .partial_cmp(&b.1.x) - .unwrap_or(std::cmp::Ordering::Equal) - }); - } - - // Sort groups by Y descending (top of page first) - line_groups.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); - - let mut merged_items = Vec::new(); - let mut index_map: Vec> = Vec::new(); - - for (_, group) in &line_groups { - let mut i = 0; - while i < group.len() { - let (first_idx, first_item) = group[i]; - let mut text = first_item.text.clone(); - let mut end_x = first_item.x + first_item.width; - let mut indices = vec![first_idx]; - let x_gap_max = first_item.font_size * 0.5; - - let mut j = i + 1; - while j < group.len() { - let (next_idx, next_item) = group[j]; - - // Must be similar font size (within 20%) - if (next_item.font_size - first_item.font_size).abs() > first_item.font_size * 0.20 - { - break; - } - - let gap = next_item.x - end_x; - // Stop if gap exceeds threshold (inter-column gap) - if gap > x_gap_max { - break; - } - // Stop on large overlap (different column overlapping) - if gap < -first_item.font_size * 0.5 { - break; - } - - // Insert space at word boundaries: within a word characters - // touch (gap ≈ 0), between words there's a visible gap. - if gap > first_item.font_size * 0.08 { - text.push(' '); - } - text.push_str(&next_item.text); - end_x = next_item.x + next_item.width; - indices.push(next_idx); - j += 1; - } - - merged_items.push(TextItem { - text, - x: first_item.x, - y: first_item.y, - width: end_x - first_item.x, - height: first_item.height, - font: first_item.font.clone(), - font_size: first_item.font_size, - page: first_item.page, - is_bold: first_item.is_bold, - is_italic: first_item.is_italic, - item_type: first_item.item_type.clone(), - }); - index_map.push(indices); - - i = j; - } - } - - (merged_items, index_map) -} - -/// Iterates all items, expanding qualifying consolidated financial items. -/// Returns `(expanded_items, index_map)` where `index_map[expanded_idx] = original_idx`. -fn expand_consolidated_items(items: &[TextItem]) -> (Vec, Vec) { - let mut expanded = Vec::with_capacity(items.len()); - let mut index_map = Vec::with_capacity(items.len()); - for (orig_idx, item) in items.iter().enumerate() { - if let Some(sub_items) = try_split_financial_item(item) { - for sub in sub_items { - expanded.push(sub); - index_map.push(orig_idx); - } - } else { - expanded.push(item.clone()); - index_map.push(orig_idx); - } - } - (expanded, index_map) -} - -/// Detect tables in a set of text items from a single page -pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bool) -> Vec
{ - if items.len() < 6 { - return vec![]; - } - - // Step 1: Merge adjacent single-char items into words (handles per-character PDFs) - let (merged_items, merge_map) = merge_adjacent_items(items); - - // Step 2: Expand consolidated financial items (e.g. "$ 1,234 $ 5,678" → sub-items) - let (expanded_items, expand_map) = expand_consolidated_items(&merged_items); - let items = &expanded_items[..]; // shadow parameter — all detection uses processed items - - let mut tables = Vec::new(); - let mut claimed_indices = std::collections::HashSet::new(); - - // === Pass 1: Small-font tables (existing behavior) === - let table_font_threshold = base_font_size * 0.90; - - let table_candidates: Vec<(usize, &TextItem)> = items - .iter() - .enumerate() - .filter(|(_, item)| item.font_size <= table_font_threshold && item.font_size >= 6.0) - .collect(); - - if table_candidates.len() >= 6 { - let regions = find_table_regions(&table_candidates); - - for (y_min, y_max) in regions { - let region_items: Vec<(usize, &TextItem)> = table_candidates - .iter() - .filter(|(_, item)| item.y >= y_min && item.y <= y_max) - .cloned() - .collect(); - - if region_items.len() < 6 { - continue; - } - - if let Some(mut table) = - detect_table_in_region(®ion_items, TableDetectionMode::SmallFont) - { - // Try to recover body-font header row above the small-font table - recover_header_row(&mut table, items, table_font_threshold); - for &idx in &table.item_indices { - claimed_indices.insert(idx); - } - tables.push(table); - } - } - } - - // === Pass 2: Body-font tables (stricter criteria) === - // Skip on multi-column pages where body-font detection causes false positives - if !skip_body_font { - let body_font_low = base_font_size * 0.85; - let body_font_high = base_font_size * 1.05; - - let body_candidates: Vec<(usize, &TextItem)> = items - .iter() - .enumerate() - .filter(|(idx, item)| { - !claimed_indices.contains(idx) - && item.font_size >= body_font_low - && item.font_size <= body_font_high - && item.font_size >= 6.0 - }) - .collect(); - - if body_candidates.len() >= 9 { - let regions = find_table_regions_strict(&body_candidates); - - for (y_min, y_max, x_min, x_max) in regions { - let region_items: Vec<(usize, &TextItem)> = body_candidates - .iter() - .filter(|(_, item)| { - item.y >= y_min && item.y <= y_max && item.x >= x_min && item.x <= x_max - }) - .cloned() - .collect(); - - if region_items.len() < 9 { - continue; - } - - if let Some(table) = - detect_table_in_region(®ion_items, TableDetectionMode::BodyFont) - { - tables.push(table); - } - } - } - } - - // Map indices back: expanded → merged → original - for table in &mut tables { - let original_indices: std::collections::HashSet = table - .item_indices - .iter() - .flat_map(|&exp_idx| { - let merged_idx = expand_map[exp_idx]; - merge_map[merged_idx].iter().copied() - }) - .collect(); - table.item_indices = original_indices.into_iter().collect(); - table.item_indices.sort_unstable(); - } - - tables -} - -/// Find Y-regions that likely contain tables -fn find_table_regions(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { - if items.is_empty() { - return vec![]; - } - - let mut y_positions: Vec = items.iter().map(|(_, i)| i.y).collect(); - y_positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - // Find clusters of Y positions (table regions) - let mut regions = Vec::new(); - let gap_threshold = 30.0; // Smaller gap threshold to separate header from content - - let mut region_start = y_positions[0]; - let mut region_end = y_positions[0]; - let mut region_count = 1; - - for &y in &y_positions[1..] { - if y - region_end > gap_threshold { - // End current region if it has enough items - if region_count >= 4 { - regions.push((region_start - 5.0, region_end + 5.0)); - } - region_start = y; - region_end = y; - region_count = 1; - } else { - region_end = y; - region_count += 1; - } - } - - // Don't forget last region - if region_count >= 4 { - regions.push((region_start - 5.0, region_end + 5.0)); - } - - regions -} - -/// Find Y-regions for body-font table candidates using strict structural criteria. -/// Requires rows with 3+ distinct X-position clusters to qualify, and verifies -/// that column positions are consistent across rows (tables have fixed columns, -/// paragraph text has varying word positions). -fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32, f32)> { - if items.is_empty() { - return vec![]; - } - - // Step 1: Group items by Y position (8pt tolerance for same row) - let mut row_groups: Vec<(f32, Vec)> = Vec::new(); - for (_, item) in items { - let mut found = false; - for (center, x_positions) in row_groups.iter_mut() { - if (item.y - *center).abs() < 8.0 { - x_positions.push(item.x); - found = true; - break; - } - } - if !found { - row_groups.push((item.y, vec![item.x])); - } - } - - // Step 2: Filter to rows with 3+ distinct X-position clusters (20pt tolerance) - // Collect cluster start positions for cross-row alignment analysis - let mut qualifying_rows: Vec<(f32, Vec)> = Vec::new(); // (y, cluster_starts) - for (y, x_positions) in &row_groups { - let mut sorted_xs = x_positions.clone(); - sorted_xs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - if sorted_xs.is_empty() { - continue; - } - - let mut cluster_starts: Vec = vec![sorted_xs[0]]; - let mut last_x = sorted_xs[0]; - for &x in &sorted_xs[1..] { - if x - last_x > 20.0 { - cluster_starts.push(x); - last_x = x; - } - } - - if cluster_starts.len() >= 3 { - qualifying_rows.push((*y, cluster_starts)); - } - } - - if qualifying_rows.len() < 3 { - return vec![]; - } - - // Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap) - qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); - - let mut candidate_regions: Vec)>> = Vec::new(); - let mut current_region: Vec<&(f32, Vec)> = vec![&qualifying_rows[0]]; - - for row in qualifying_rows.iter().skip(1) { - let prev_y = current_region.last().unwrap().0; - if row.0 - prev_y > 25.0 { - if current_region.len() >= 3 { - candidate_regions.push(current_region); - } - current_region = vec![row]; - } else { - current_region.push(row); - } - } - if current_region.len() >= 3 { - candidate_regions.push(current_region); - } - - // Step 4: Cross-row column alignment check per region - // Real tables have consistent column X positions across rows (high pairwise score). - // Paragraph text has varying word positions line-to-line (low pairwise score). - let mut regions = Vec::new(); - for region_rows in &candidate_regions { - let num_rows = region_rows.len(); - let mut total_score = 0.0f32; - let mut pair_count = 0u32; - let tolerance = 10.0f32; - - for i in 0..num_rows { - for j in (i + 1)..num_rows { - let centers_a = ®ion_rows[i].1; - let centers_b = ®ion_rows[j].1; - - let matches_a = centers_a - .iter() - .filter(|&&a| centers_b.iter().any(|&b| (a - b).abs() < tolerance)) - .count(); - let matches_b = centers_b - .iter() - .filter(|&&b| centers_a.iter().any(|&a| (a - b).abs() < tolerance)) - .count(); - - let max_len = centers_a.len().max(centers_b.len()); - if max_len > 0 { - total_score += (matches_a + matches_b) as f32 / (2 * max_len) as f32; - pair_count += 1; - } - } - } - - let avg_score = if pair_count > 0 { - total_score / pair_count as f32 - } else { - 0.0 - }; - if avg_score >= 0.5 { - let y_min = region_rows.first().unwrap().0; - let y_max = region_rows.last().unwrap().0; - // Compute X bounds from qualifying row cluster positions - let x_min = region_rows - .iter() - .flat_map(|(_, clusters)| clusters.iter()) - .cloned() - .fold(f32::INFINITY, f32::min); - let x_max = region_rows - .iter() - .flat_map(|(_, clusters)| clusters.iter()) - .cloned() - .fold(f32::NEG_INFINITY, f32::max); - regions.push((y_min - 5.0, y_max + 5.0, x_min - 15.0, x_max + 50.0)); - } - } - - regions -} - -/// Detect a table within a specific region -fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Option
{ - // Find column boundaries - let columns = find_column_boundaries(items, mode); - let min_cols = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; - if columns.len() < min_cols || columns.len() > 15 { - return None; - } - - // Find row boundaries - let rows = find_row_boundaries(items); - let min_rows = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; - if rows.len() < min_rows { - return None; - } - - // Verify this looks like a table: multiple items should align to columns - let col_alignment = check_column_alignment(items, &columns, mode); - let min_alignment = match mode { - TableDetectionMode::SmallFont => 0.5, - TableDetectionMode::BodyFont => 0.7, - }; - if col_alignment < min_alignment { - return None; - } - - // Build the table grid - first collect items per cell, then join properly - let mut cell_items: Vec>> = - vec![vec![Vec::new(); columns.len()]; rows.len()]; - let mut item_indices = Vec::new(); - - for (idx, item) in items { - let col = find_column_index(&columns, item.x); - let row = find_row_index(&rows, item.y); - - if let (Some(col), Some(row)) = (col, row) { - cell_items[row][col].push(item); - item_indices.push(*idx); - } - } - - // Detect form header rows and exclude their items - // We need to do this BEFORE finalizing item_indices - let (first_table_row, excluded_items) = find_first_table_row(&cell_items, &rows, items); - - // Remove excluded items from item_indices - let item_indices: Vec = item_indices - .into_iter() - .filter(|idx| !excluded_items.contains(idx)) - .collect(); - - // If we excluded rows, adjust the cell_items and rows - let (rows, mut cell_items) = if first_table_row > 0 { - let new_rows = rows[first_table_row..].to_vec(); - let new_cell_items = cell_items[first_table_row..].to_vec(); - (new_rows, new_cell_items) - } else { - (rows, cell_items) - }; - - // Sort items within each cell by X position and join with subscript-aware spacing - let mut cells: Vec> = Vec::with_capacity(rows.len()); - for row_items in &mut cell_items { - let mut row_cells = Vec::with_capacity(columns.len()); - for col_items in row_items.iter_mut() { - // Sort by X position (direction-aware) - let rtl = is_rtl_text(col_items.iter().map(|i| &i.text)); - if rtl { - col_items - .sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); - } else { - col_items - .sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); - } - - // Join items with subscript-aware spacing - let text = join_cell_items(col_items); - row_cells.push(text); - } - cells.push(row_cells); - } - - // Validation 1: most rows should have content in first column - let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count(); - if rows_with_first_col < rows.len() / 2 { - return None; - } - - // Validation 2: real tables have content in MULTIPLE columns, not just first - let rows_with_multi_cols = cells - .iter() - .filter(|row| row.iter().filter(|c| !c.is_empty()).count() >= 2) - .count(); - let multi_col_threshold = match mode { - TableDetectionMode::SmallFont => (rows.len() / 3).max(1), // 33% - TableDetectionMode::BodyFont => (rows.len() / 2).max(1), // 50% - }; - if rows_with_multi_cols < multi_col_threshold { - return None; - } - - // Validation 3: tables shouldn't have too many rows (likely misdetected text) - let max_rows = match mode { - TableDetectionMode::SmallFont => 200, - TableDetectionMode::BodyFont => 200, - }; - if rows.len() > max_rows { - return None; - } - - // Validation 4: average cells per row should be reasonable - let total_filled: usize = cells - .iter() - .map(|row| row.iter().filter(|c| !c.is_empty()).count()) - .sum(); - let avg_cells_per_row = total_filled as f32 / rows.len() as f32; - let min_avg_cells = match mode { - TableDetectionMode::SmallFont => 1.5, - TableDetectionMode::BodyFont => 2.5, - }; - if avg_cells_per_row < min_avg_cells { - return None; - } - - // Validation 5: Check for key-value pair layout (NOT a table) - // Key-value layouts have: mostly 2 filled columns, first column is labels - if is_key_value_layout(&cells) { - return None; - } - - // Validation 6: Check column count consistency - // Real tables have similar column counts across rows - if !has_consistent_columns(&cells) { - return None; - } - - // Validation 7: Tables should have some numeric/data content - // (not just text labels) - if !has_table_like_content(&cells, mode) { - return None; - } - - // Validation 8: Check for Table of Contents pattern - // TOCs have dots (leader lines) and page numbers, not real table data - if is_table_of_contents(&cells) { - return None; - } - - // Validation 9: Reject paragraph-like content falsely detected as tables. - // Real table cells are short and self-contained. Paragraph text split into - // "cells" produces long sentence fragments. - if is_paragraph_content(&cells) { - return None; - } - - Some(Table { - columns, - rows, - cells, - item_indices, - }) -} - -/// Check if this looks like a key-value pair layout rather than a table -fn is_key_value_layout(cells: &[Vec]) -> bool { - if cells.is_empty() { - return false; - } - - let num_cols = cells[0].len(); - - // Key-value layouts typically have 2-3 effective columns - // where the first column contains labels ending with ":" - let mut label_like_first_col = 0; - let mut rows_with_two_or_less = 0; - - for row in cells { - let filled_count = row.iter().filter(|c| !c.is_empty()).count(); - if filled_count <= 2 { - rows_with_two_or_less += 1; - } - - // Check if first column looks like a label (ends with : or is all caps) - let first = row.first().map(|s| s.trim()).unwrap_or(""); - if first.ends_with(':') - || (first.len() > 3 - && first - .chars() - .all(|c| c.is_uppercase() || c.is_whitespace() || c == '(' || c == ')')) - { - label_like_first_col += 1; - } - } - - // If most rows have only 2 columns filled and first column is label-like - let pct_two_or_less = rows_with_two_or_less as f32 / cells.len() as f32; - let pct_label_like = label_like_first_col as f32 / cells.len() as f32; - - // This is likely a key-value layout if: - // - Most rows have 2 or fewer filled columns - // - First column often looks like labels - // - Total columns detected is 6 or fewer (real tables often have more) - pct_two_or_less > 0.7 && pct_label_like > 0.5 && num_cols <= 6 -} - -/// Check if columns are consistent across rows (real tables have this) -fn has_consistent_columns(cells: &[Vec]) -> bool { - if cells.len() < 3 { - return true; // Not enough rows to judge - } - - // Count filled columns per row - let filled_counts: Vec = cells - .iter() - .map(|row| row.iter().filter(|c| !c.is_empty()).count()) - .collect(); - - // Find the most common filled count - let mut count_freq: std::collections::HashMap = std::collections::HashMap::new(); - for &count in &filled_counts { - *count_freq.entry(count).or_insert(0) += 1; - } - - let most_common_count = count_freq - .iter() - .max_by_key(|(_, freq)| *freq) - .map(|(count, _)| *count) - .unwrap_or(0); - - // At least 40% of rows should have the most common column count (or close to it) - let consistent_rows = filled_counts - .iter() - .filter(|&&c| c >= most_common_count.saturating_sub(2) && c <= most_common_count + 2) - .count(); - - consistent_rows as f32 / cells.len() as f32 > 0.4 -} - -/// Check if the content looks like table data (numbers, short values, specs) -fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bool { - let mut data_like_cells = 0; - let mut total_cells = 0; - - for row in cells.iter().skip(1) { - // Skip header row - for cell in row { - let trimmed = cell.trim(); - if !trimmed.is_empty() { - total_cells += 1; - // Check if it looks like table data - if looks_like_table_data(trimmed) { - data_like_cells += 1; - } - } - } - } - - if total_cells == 0 { - return false; - } - - // Data-like content threshold depends on detection mode - let pct_data = data_like_cells as f32 / total_cells as f32; - let num_cols = cells.first().map(|r| r.len()).unwrap_or(0); - - let min_pct = match mode { - TableDetectionMode::SmallFont => 0.2, - TableDetectionMode::BodyFont => 0.3, - }; - - // For SmallFont, bypass content check for wide tables (5+ columns may have text headers). - // For BodyFont, always require data-like content to prevent paragraph false positives. - pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5) -} - -/// Check if a cell value looks like table data -/// Includes: numbers, part numbers, specifications with units, codes -fn looks_like_table_data(s: &str) -> bool { - let s = s.trim(); - if s.is_empty() { - return false; - } - - // Pure numbers - if looks_like_number(s) { - return true; - } - - // Dates: MM/DD/YYYY, DD/MM/YYYY, YYYY-MM-DD, etc. - if s.len() <= 10 - && s.chars().filter(|c| c.is_ascii_digit()).count() >= 4 - && (s.contains('/') || s.contains('-')) - && s.chars() - .all(|c| c.is_ascii_digit() || c == '/' || c == '-') - { - return true; - } - - // Part numbers / model codes (alphanumeric, typically short) - // e.g., "NA555", "NE555", "LM358" - if s.len() <= 10 - && s.chars().all(|c| c.is_alphanumeric()) - && s.chars().any(|c| c.is_ascii_digit()) - { - return true; - } - - // Specifications with units (contains numbers and unit symbols) - // e.g., "–40°C to +105°C", "5V", "200mA", "8-pin" - let has_number = s.chars().any(|c| c.is_ascii_digit()); - let has_unit = s.contains('°') - || s.contains('V') - || s.contains('A') - || s.contains("Hz") - || s.contains("mA") - || s.contains("µ") - || s.contains("pin") - || s.contains("MHz") - || s.contains("kHz"); - if has_number && has_unit { - return true; - } - - // Package designations with parentheses - // e.g., "D (SOIC, 8)", "P (PDIP, 8)" - if s.contains('(') && s.contains(')') && s.chars().any(|c| c.is_ascii_digit()) { - return true; - } - - // Temperature ranges - // e.g., "TA = –40°C to +105°C" - if (s.contains("°C") || s.contains("°F")) && s.contains("to") { - return true; - } - - false -} - -/// Check if a string looks like a number -fn looks_like_number(s: &str) -> bool { - let s = s.trim(); - if s.is_empty() { - return false; - } - - // Handle common number formats: 9.0, 10, 8.6, etc. - s.chars() - .all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+') - && s.chars().any(|c| c.is_ascii_digit()) -} - -/// Check if this looks like a Table of Contents -/// TOCs have characteristic patterns: leader dots, page numbers, section names -fn is_table_of_contents(cells: &[Vec]) -> bool { - if cells.is_empty() { - return false; - } - - let mut dot_cells = 0; - let mut page_number_cells = 0; - let mut total_cells = 0; - - for row in cells { - for cell in row { - let trimmed = cell.trim(); - if trimmed.is_empty() { - continue; - } - total_cells += 1; - - // Check for leader dots (sequences of periods) - // TOCs often have "........" or ". . . ." patterns - let dot_count = trimmed.chars().filter(|&c| c == '.').count(); - let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3; - if is_mostly_dots { - dot_cells += 1; - } - - // Check for standalone page numbers (1-4 digits, possibly with spaces) - let digits_only: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect(); - if digits_only.len() <= 4 - && !digits_only.is_empty() - && digits_only.chars().all(|c| c.is_ascii_digit()) - { - page_number_cells += 1; - } - } - } - - if total_cells == 0 { - return false; - } - - // If a significant portion of cells are dots or page numbers, it's likely a TOC - let dot_ratio = dot_cells as f32 / total_cells as f32; - let page_num_ratio = page_number_cells as f32 / total_cells as f32; - - // TOC typically has >15% dot cells and >10% page number cells - dot_ratio > 0.15 || (dot_ratio > 0.05 && page_num_ratio > 0.15) -} - -/// Check if detected "table" cells are actually paragraph text fragments. -/// -/// Multi-column paragraph text falsely detected as tables produces: -/// - Many empty cells (text doesn't span all columns) -/// - Cells ending with hyphens (word breaks across "columns") -/// - Long sentence fragments or single-word fragments -fn is_paragraph_content(cells: &[Vec]) -> bool { - if cells.is_empty() { - return false; - } - - let num_cols = cells[0].len(); - let total_cells = cells.len() * num_cols; - if total_cells == 0 { - return false; - } - - let filled: Vec<&str> = cells - .iter() - .flat_map(|r| r.iter()) - .map(|c| c.trim()) - .filter(|c| !c.is_empty()) - .collect(); - - let total_filled = filled.len(); - if total_filled < 4 { - return false; - } - - let empty_ratio = 1.0 - (total_filled as f32 / total_cells as f32); - - // Cells ending with a hyphen suggest word breaks across columns. - // Real table cells almost never end with hyphens (except range indicators). - let hyphen_breaks = filled - .iter() - .filter(|c| { - c.ends_with('-') && c.len() > 1 && { - let mut chars = c.chars().rev(); - chars.next(); // skip the '-' - chars.next().is_some_and(|ch| ch.is_alphabetic()) - } - }) - .count(); - let hyphen_ratio = hyphen_breaks as f32 / total_filled as f32; - - // Word-break hyphens are a strong paragraph signal - if hyphen_ratio > 0.03 { - return true; - } - - // High empty ratio with many rows suggests paragraph text spread across a grid - if empty_ratio > 0.55 && cells.len() > 10 { - return true; - } - - // Letter-spaced text (spaces between every character) is never real table data. - // This happens when PDF uses wide character spacing for emphasis/formatting. - // Require at least 9 chars (e.g., "a b c d e") to avoid matching short codes. - let letter_spaced = filled - .iter() - .filter(|c| { - let chars: Vec = c.chars().collect(); - chars.len() >= 9 - && chars.windows(4).all(|w| { - (w[0].is_alphabetic() && w[1] == ' ' && w[2].is_alphabetic() && w[3] == ' ') - || (w[0] == ' ' - && w[1].is_alphabetic() - && w[2] == ' ' - && w[3].is_alphabetic()) - }) - }) - .count(); - if letter_spaced > 0 && letter_spaced as f32 / total_filled as f32 > 0.08 { - return true; - } - - // Long sentence fragments - let long_cells = filled.iter().filter(|c| c.len() > 60).count(); - let long_ratio = long_cells as f32 / total_filled as f32; - let avg_len = filled.iter().map(|c| c.len()).sum::() as f32 / total_filled as f32; - - if avg_len > 40.0 && long_ratio > 0.2 { - return true; - } - if long_ratio > 0.3 { - return true; - } - - false -} - -/// Check what fraction of items align to detected columns -fn check_column_alignment( - items: &[(usize, &TextItem)], - columns: &[f32], - mode: TableDetectionMode, -) -> f32 { - let tolerance = match mode { - TableDetectionMode::SmallFont => 40.0, - TableDetectionMode::BodyFont => 30.0, - }; - let aligned = items - .iter() - .filter(|(_, item)| columns.iter().any(|&col| (item.x - col).abs() < tolerance)) - .count(); - - aligned as f32 / items.len() as f32 -} - -/// Find column boundaries by clustering X positions -fn find_column_boundaries(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Vec { - let mut x_positions: Vec = items.iter().map(|(_, i)| i.x).collect(); - x_positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - if x_positions.is_empty() { - return vec![]; - } - - // Calculate adaptive threshold based on X-position density - // For dense tables (like grade tables), use smaller threshold - let x_range = x_positions.last().unwrap() - x_positions.first().unwrap(); - let avg_gap = if x_positions.len() > 1 { - x_range / (x_positions.len() - 1) as f32 - } else { - 60.0 - }; - - // Use smaller threshold for dense data, larger for sparse - let cluster_threshold = avg_gap.clamp(25.0, 50.0); - - let mut columns = Vec::new(); - let mut cluster_items: Vec = vec![x_positions[0]]; - - for &x in &x_positions[1..] { - let cluster_center = cluster_items.iter().sum::() / cluster_items.len() as f32; - - if x - cluster_center > cluster_threshold { - // End current cluster - columns.push(cluster_center); - cluster_items = vec![x]; - } else { - cluster_items.push(x); - } - } - - // Don't forget last cluster - if !cluster_items.is_empty() { - columns.push(cluster_items.iter().sum::() / cluster_items.len() as f32); - } - - // Filter columns - each should have multiple items - let min_items_per_col = (items.len() / columns.len().max(1) / 4).max(2); - let columns: Vec = columns - .into_iter() - .filter(|&col_x| { - items - .iter() - .filter(|(_, i)| (i.x - col_x).abs() < cluster_threshold) - .count() - >= min_items_per_col - }) - .collect(); - - // Anti-paragraph safeguard for BodyFont mode: - // Paragraphs concentrate items at the left margin; tables distribute evenly. - // Reject if any single column has >60% of all items. - if mode == TableDetectionMode::BodyFont { - let total_items = items.len(); - for &col_x in &columns { - let count = items - .iter() - .filter(|(_, i)| (i.x - col_x).abs() < cluster_threshold) - .count(); - if count as f32 / total_items as f32 > 0.60 { - return vec![]; - } - } - } - - columns -} - -/// Find row boundaries by clustering Y positions -fn find_row_boundaries(items: &[(usize, &TextItem)]) -> Vec { - let mut y_positions: Vec = items.iter().map(|(_, i)| i.y).collect(); - y_positions.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); // Descending - - if y_positions.is_empty() { - return vec![]; - } - - // Cluster Y positions - items within a fraction of the median font size are same row. - // Using 0.8× median font keeps the threshold between intra-row gaps (~0pt) and - // inter-row gaps (≥1× font size), preventing row merging in uniform-spaced PDFs. - let cluster_threshold = { - let mut font_sizes: Vec = items.iter().map(|(_, i)| i.font_size).collect(); - font_sizes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let median_font = font_sizes[font_sizes.len() / 2]; - (median_font * 0.8).max(4.0) - }; - let mut rows = Vec::new(); - let mut cluster_items: Vec = vec![y_positions[0]]; - - for &y in &y_positions[1..] { - let cluster_center = cluster_items.iter().sum::() / cluster_items.len() as f32; - - if cluster_center - y >= cluster_threshold { - // End current cluster (note: Y is descending) - rows.push(cluster_center); - cluster_items = vec![y]; - } else { - cluster_items.push(y); - } - } - - if !cluster_items.is_empty() { - rows.push(cluster_items.iter().sum::() / cluster_items.len() as f32); - } - - rows -} - -/// Find which column index an X position belongs to -fn find_column_index(columns: &[f32], x: f32) -> Option { - // Calculate adaptive threshold based on column spacing - let threshold = if columns.len() >= 2 { - let min_gap = columns - .windows(2) - .map(|w| (w[1] - w[0]).abs()) - .fold(f32::INFINITY, f32::min); - (min_gap / 2.0).clamp(25.0, 50.0) - } else { - 50.0 - }; - - columns - .iter() - .enumerate() - .min_by(|(_, a), (_, b)| { - (x - *a) - .abs() - .partial_cmp(&(x - *b).abs()) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .filter(|(_, col_x)| (x - *col_x).abs() < threshold) - .map(|(idx, _)| idx) -} - -/// Find which row index a Y position belongs to -fn find_row_index(rows: &[f32], y: f32) -> Option { - let threshold = 15.0; - rows.iter() - .enumerate() - .min_by(|(_, a), (_, b)| { - (y - *a) - .abs() - .partial_cmp(&(y - *b).abs()) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .filter(|(_, row_y)| (y - *row_y).abs() < threshold) - .map(|(idx, _)| idx) -} - -/// Join cell items with subscript/superscript-aware spacing -/// Same logic as TextLine::text() but for table cells -fn join_cell_items(items: &[&TextItem]) -> String { - let mut result = String::new(); - - for (i, item) in items.iter().enumerate() { - let text = item.text.trim(); - if text.is_empty() { - continue; - } - - if result.is_empty() { - result.push_str(text); - } else { - let prev_item = items[i - 1]; - - // Don't add space before/after hyphens - let prev_ends_with_hyphen = result.ends_with('-'); - let curr_is_hyphen = text == "-"; - let curr_starts_with_hyphen = text.starts_with('-'); - - // Detect subscript/superscript: smaller font size and/or Y offset - let font_ratio = item.font_size / prev_item.font_size; - let reverse_font_ratio = prev_item.font_size / item.font_size; - let y_diff = (item.y - prev_item.y).abs(); - - // Current item is subscript/superscript (smaller than previous) - let is_sub_super = font_ratio < 0.85 && y_diff > 1.0; - // Previous item was subscript/superscript (returning to normal size) - let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; - - if prev_ends_with_hyphen - || curr_is_hyphen - || curr_starts_with_hyphen - || is_sub_super - || was_sub_super - { - result.push_str(text); - } else { - result.push(' '); - result.push_str(text); - } - } - } - - result -} - -/// Recover a header row for small-font tables by looking at body-font items -/// just above the table's first row. -/// -/// PDF tables often have header rows at the body font size while data rows use -/// a smaller font. Pass 1 (SmallFont) excludes the header because of the -/// font-size filter. This function looks upward from the table's first row for -/// body-font items that align with the table's columns, and prepends them. -fn recover_header_row(table: &mut Table, all_items: &[TextItem], small_font_threshold: f32) { - if table.rows.is_empty() || table.columns.is_empty() { - return; - } - - let first_row_y = table.rows[0]; // highest Y (rows are descending) - - // Compute typical row spacing for gap threshold - let row_gap_limit = if table.rows.len() >= 2 { - let avg_spacing = - (table.rows[0] - table.rows[table.rows.len() - 1]) / (table.rows.len() - 1) as f32; - // Allow up to 2x average row spacing for the header gap - (avg_spacing * 2.0).clamp(10.0, 40.0) - } else { - 30.0 - }; - - // Find body-font items just above the first row - let header_candidates: Vec<(usize, &TextItem)> = all_items - .iter() - .enumerate() - .filter(|(_, item)| { - item.font_size > small_font_threshold - && item.y > first_row_y - && item.y <= first_row_y + row_gap_limit - }) - .collect(); - - if header_candidates.is_empty() { - return; - } - - // Group header candidates by Y (cluster within 5pt) - let mut header_y_groups: Vec<(f32, Vec<(usize, &TextItem)>)> = Vec::new(); - let mut sorted_candidates = header_candidates; - sorted_candidates.sort_by(|a, b| { - b.1.y - .partial_cmp(&a.1.y) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - for (idx, item) in &sorted_candidates { - let found = header_y_groups - .iter_mut() - .find(|(y, _)| (item.y - *y).abs() < 5.0); - if let Some((_, group)) = found { - group.push((*idx, item)); - } else { - header_y_groups.push((item.y, vec![(*idx, item)])); - } - } - - // Take the row closest to the table (lowest Y above first_row_y) - // header_y_groups is sorted by descending Y, so take the last one - let (header_y, header_items) = header_y_groups.last().unwrap(); - - // Map header items to table columns - let num_cols = table.columns.len(); - let mut header_cells: Vec = vec![String::new(); num_cols]; - let mut mapped_count = 0; - let mut header_indices = Vec::new(); - - for (idx, item) in header_items { - if let Some(col) = find_column_index(&table.columns, item.x) { - let text = item.text.trim(); - if !text.is_empty() { - if !header_cells[col].is_empty() { - header_cells[col].push(' '); - } - header_cells[col].push_str(text); - mapped_count += 1; - header_indices.push(*idx); - } - } - } - - // Require at least 2 columns populated to look like a real header row - let populated = header_cells.iter().filter(|c| !c.is_empty()).count(); - if populated < 2 || mapped_count < 2 { - return; - } - - // Prepend header row to the table - table.rows.insert(0, *header_y); - table.cells.insert(0, header_cells); - table.item_indices.extend(header_indices); -} - -/// Format a table as markdown -pub fn table_to_markdown(table: &Table) -> String { - if table.cells.is_empty() || table.cells[0].is_empty() { - return String::new(); - } - - // Clean up the table: merge continuation rows, extract footnotes, remove empty rows - let (cleaned_cells, footnotes) = clean_table_cells(&table.cells); - - if cleaned_cells.is_empty() { - return String::new(); - } - - let num_cols = cleaned_cells[0].len(); - let mut output = String::new(); - - // Calculate column widths for alignment - let col_widths: Vec = (0..num_cols) - .map(|col| { - cleaned_cells - .iter() - .map(|row| row.get(col).map(|c| c.len()).unwrap_or(0)) - .max() - .unwrap_or(3) - .max(3) - }) - .collect(); - - // Output each row - for (row_idx, row) in cleaned_cells.iter().enumerate() { - output.push('|'); - for (col_idx, cell) in row.iter().enumerate() { - let width = col_widths[col_idx]; - output.push_str(&format!(" {:width$} |", cell, width = width)); - } - output.push('\n'); - - // Add separator after header row - if row_idx == 0 { - output.push('|'); - for width in &col_widths { - output.push_str(&format!(" {} |", "-".repeat(*width))); - } - output.push('\n'); - } - } - - // Add footnotes below the table - if !footnotes.is_empty() { - output.push('\n'); - for footnote in footnotes { - output.push_str(&footnote); - output.push('\n'); - } - } - - output -} - -/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows -fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { - let mut cleaned: Vec> = Vec::new(); - let mut footnotes: Vec = Vec::new(); - - for row in cells { - // Check if this row is empty - if row.iter().all(|c| c.trim().is_empty()) { - continue; - } - - // Check if this row is a footnote (starts with (1), (2), etc. or just a number reference) - let first_cell = row.first().map(|s| s.trim()).unwrap_or(""); - if is_footnote_row(first_cell) { - // Combine all cells into a single footnote line - let footnote_text: String = row - .iter() - .map(|c| c.trim()) - .filter(|c| !c.is_empty()) - .collect::>() - .join(" "); - footnotes.push(footnote_text); - continue; - } - - // Check if this is a continuation row (first column is empty but others have content) - let is_continuation = first_cell.is_empty() - && row.iter().skip(1).any(|c| !c.trim().is_empty()) - && !cleaned.is_empty(); - - if is_continuation { - // Merge with previous row - if let Some(prev_row) = cleaned.last_mut() { - for (col_idx, cell) in row.iter().enumerate() { - let cell_text = cell.trim(); - if !cell_text.is_empty() && col_idx < prev_row.len() { - if !prev_row[col_idx].is_empty() { - prev_row[col_idx].push(' '); - } - prev_row[col_idx].push_str(cell_text); - } - } - } - } else { - // Regular row - add as new row - cleaned.push(row.iter().map(|c| c.trim().to_string()).collect()); - } - } - - (cleaned, footnotes) -} - -/// Find the first row that looks like actual table data (not form header) -/// Returns (first_table_row_index, set of item indices to exclude) -fn find_first_table_row( - cell_items: &[Vec>], - rows: &[f32], - original_items: &[(usize, &TextItem)], -) -> (usize, std::collections::HashSet) { - let mut excluded_items = std::collections::HashSet::new(); - - // Build string cells for analysis - let cells: Vec> = cell_items - .iter() - .map(|row| row.iter().map(|col| join_cell_items(col)).collect()) - .collect(); - - if cells.is_empty() { - return (0, excluded_items); - } - - // Strategy: Skip leading rows that look like form metadata - // - // Form/metadata rows have: - // 1. Cells ending with ":" (form labels) - // 2. Very sparse fill with document metadata (grade level, year, etc.) - // - // Table rows have: - // 1. Dense fill (headers spanning columns) - // 2. Numeric content (data rows) - // 3. No form label patterns - - let total_cols = cells[0].len(); - let mut first_table_row = 0; - - for (row_idx, row) in cells.iter().enumerate() { - let filled_cells: Vec<&String> = row.iter().filter(|c| !c.trim().is_empty()).collect(); - let filled_count = filled_cells.len(); - let fill_ratio = filled_count as f32 / total_cols as f32; - - // Check for form-like patterns (cells with colons) - // Only treat as form row if most filled cells look form-like, - // or the row is very sparse with any form pattern. - let form_cell_count = filled_cells - .iter() - .filter(|c| { - let text = c.trim(); - (text.ends_with(':') && text.len() > 1) - || (text.contains(": ") && !looks_like_number(text)) - }) - .count(); - let has_form_patterns = - form_cell_count > 0 && (form_cell_count * 2 >= filled_count || fill_ratio < 0.3); - - // Check for numeric content - let numeric_count = filled_cells - .iter() - .filter(|c| looks_like_number(c.trim())) - .count(); - let has_data = numeric_count >= 2; - - // Skip rows with form patterns (regardless of density) - if has_form_patterns { - continue; - } - - // Data rows are definitely table content - if has_data { - first_table_row = row_idx; - break; - } - - // Dense rows without form patterns are likely table headers - if fill_ratio >= 0.4 { - first_table_row = row_idx; - break; - } - - // Very sparse rows at the start are likely metadata - skip them - if fill_ratio < 0.3 { - continue; - } - - // Moderately sparse row without form patterns - could be multi-line header - // Look ahead to decide - if row_idx + 1 < cells.len() { - let next_row = &cells[row_idx + 1]; - let next_filled = next_row.iter().filter(|c| !c.trim().is_empty()).count(); - let next_fill_ratio = next_filled as f32 / total_cols as f32; - let next_has_form = next_row.iter().any(|c| { - let text = c.trim(); - (text.ends_with(':') && text.len() > 1) - || (text.contains(": ") && !looks_like_number(text)) - }); - - // If next row is dense or has data (and no form patterns), this row starts the table - if (next_fill_ratio >= 0.4 - || next_row - .iter() - .filter(|c| looks_like_number(c.trim())) - .count() - >= 2) - && !next_has_form - { - first_table_row = row_idx; - break; - } - } - - // Otherwise skip this sparse row - } - - // Collect item indices from excluded rows - if first_table_row > 0 { - let y_tolerance = 15.0; - for (idx, item) in original_items { - // Check if this item is in one of the excluded rows - for row_y in rows.iter().take(first_table_row) { - if (item.y - *row_y).abs() < y_tolerance { - excluded_items.insert(*idx); - break; - } - } - } - } - - (first_table_row, excluded_items) -} - -/// Check if a cell value indicates a footnote row -fn is_footnote_row(text: &str) -> bool { - let trimmed = text.trim(); - - // Check for common footnote patterns - // (1), (2), etc. - if trimmed.starts_with('(') && trimmed.len() >= 2 { - let inside = &trimmed[1..]; - if let Some(close_idx) = inside.find(')') { - let num_part = &inside[..close_idx]; - if num_part.chars().all(|c| c.is_ascii_digit()) { - return true; - } - } - } - - // 1), 2), etc. - if trimmed.len() >= 2 { - if let Some(paren_idx) = trimmed.find(')') { - let num_part = &trimmed[..paren_idx]; - if !num_part.is_empty() && num_part.chars().all(|c| c.is_ascii_digit()) { - return true; - } - } - } - - // Check for "Note:" or "Notes:" at the start - let lower = trimmed.to_lowercase(); - if lower.starts_with("note:") || lower.starts_with("notes:") { - return true; - } - - false -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem { - TextItem { - text: text.into(), - x, - y, - width: 10.0, - height: font_size, - font: "F1".into(), - font_size, - page: 1, - is_bold: false, - is_italic: false, - item_type: crate::extractor::ItemType::Text, - } - } - - #[test] - fn test_table_detection() { - // Create a more realistic table with numeric data (like grades) - let items = vec![ - // Header row - make_item("Subject", 100.0, 500.0, 8.0), - make_item("Q1", 200.0, 500.0, 8.0), - make_item("Q2", 280.0, 500.0, 8.0), - make_item("Q3", 360.0, 500.0, 8.0), - // Data row 1 - make_item("Math", 100.0, 480.0, 8.0), - make_item("9.0", 200.0, 480.0, 8.0), - make_item("8.5", 280.0, 480.0, 8.0), - make_item("9.5", 360.0, 480.0, 8.0), - // Data row 2 - make_item("Science", 100.0, 460.0, 8.0), - make_item("8.0", 200.0, 460.0, 8.0), - make_item("9.0", 280.0, 460.0, 8.0), - make_item("8.5", 360.0, 460.0, 8.0), - // Data row 3 - make_item("English", 100.0, 440.0, 8.0), - make_item("9.5", 200.0, 440.0, 8.0), - make_item("9.0", 280.0, 440.0, 8.0), - make_item("9.5", 360.0, 440.0, 8.0), - ]; - - let tables = detect_tables(&items, 10.0, false); - assert_eq!(tables.len(), 1); - assert_eq!(tables[0].columns.len(), 4); - assert_eq!(tables[0].rows.len(), 4); - } - - #[test] - fn test_table_to_markdown() { - let table = Table { - columns: vec![100.0, 200.0], - rows: vec![500.0, 480.0], - cells: vec![ - vec!["Header 1".into(), "Header 2".into()], - vec!["Cell 1".into(), "Cell 2".into()], - ], - item_indices: vec![], - }; - - let md = table_to_markdown(&table); - assert!(md.contains("| Header 1")); - assert!(md.contains("| ---")); - assert!(md.contains("| Cell 1")); - } - - #[test] - fn test_body_font_table_detected() { - // 4-column, 4-row table at font_size == base_font_size - // Pass 1 rejects (not small font), Pass 2 should detect - let items = vec![ - // Header row - make_item("Name", 100.0, 500.0, 10.0), - make_item("Price", 200.0, 500.0, 10.0), - make_item("Qty", 300.0, 500.0, 10.0), - make_item("Total", 400.0, 500.0, 10.0), - // Data row 1 - make_item("Widget", 100.0, 480.0, 10.0), - make_item("5.00", 200.0, 480.0, 10.0), - make_item("10", 300.0, 480.0, 10.0), - make_item("50.00", 400.0, 480.0, 10.0), - // Data row 2 - make_item("Gadget", 100.0, 460.0, 10.0), - make_item("12.50", 200.0, 460.0, 10.0), - make_item("4", 300.0, 460.0, 10.0), - make_item("50.00", 400.0, 460.0, 10.0), - // Data row 3 - make_item("Gizmo", 100.0, 440.0, 10.0), - make_item("3.25", 200.0, 440.0, 10.0), - make_item("20", 300.0, 440.0, 10.0), - make_item("65.00", 400.0, 440.0, 10.0), - ]; - - let tables = detect_tables(&items, 10.0, false); - assert_eq!( - tables.len(), - 1, - "Body-font table should be detected by Pass 2" - ); - assert_eq!(tables[0].columns.len(), 4); - assert!(tables[0].rows.len() >= 3); - } - - #[test] - fn test_paragraph_not_falsely_detected() { - // Body-font single-column paragraph text — must return 0 tables - let items = vec![ - make_item( - "This is a paragraph of text that spans the full width", - 72.0, - 500.0, - 10.0, - ), - make_item( - "of the page and should not be detected as a table.", - 72.0, - 485.0, - 10.0, - ), - make_item( - "It continues for several lines with normal body text", - 72.0, - 470.0, - 10.0, - ), - make_item( - "that is left-aligned and has no columnar structure.", - 72.0, - 455.0, - 10.0, - ), - make_item( - "The paragraph keeps going with more content here.", - 72.0, - 440.0, - 10.0, - ), - make_item( - "And it has even more text on this line as well.", - 72.0, - 425.0, - 10.0, - ), - make_item( - "Finally the paragraph concludes with this last line.", - 72.0, - 410.0, - 10.0, - ), - make_item( - "One more line to have enough items for detection.", - 72.0, - 395.0, - 10.0, - ), - make_item( - "And another line of plain paragraph text content.", - 72.0, - 380.0, - 10.0, - ), - make_item( - "Last line of the paragraph ends here for the test.", - 72.0, - 365.0, - 10.0, - ), - ]; - - let tables = detect_tables(&items, 10.0, false); - assert_eq!( - tables.len(), - 0, - "Single-column paragraph must not be detected as table" - ); - } - - #[test] - fn test_word_level_paragraph_not_detected_as_table() { - // Paragraph text with per-word TextItems (as produced by some PDFs). - // Word X positions vary from line to line — NOT a table. - let items = vec![ - // Line 1 - make_item("We", 72.0, 500.0, 10.0), - make_item("would", 95.0, 500.0, 10.0), - make_item("like", 145.0, 500.0, 10.0), - make_item("to", 180.0, 500.0, 10.0), - make_item("thank", 200.0, 500.0, 10.0), - make_item("all", 250.0, 500.0, 10.0), - make_item("the", 278.0, 500.0, 10.0), - make_item("practitioners", 305.0, 500.0, 10.0), - // Line 2 - make_item("and", 72.0, 485.0, 10.0), - make_item("researchers", 105.0, 485.0, 10.0), - make_item("across", 185.0, 485.0, 10.0), - make_item("the", 232.0, 485.0, 10.0), - make_item("University", 260.0, 485.0, 10.0), - make_item("of", 335.0, 485.0, 10.0), - make_item("Leeds", 355.0, 485.0, 10.0), - // Line 3 - make_item("Libraries", 72.0, 470.0, 10.0), - make_item("whose", 142.0, 470.0, 10.0), - make_item("contributions", 190.0, 470.0, 10.0), - make_item("made", 290.0, 470.0, 10.0), - make_item("this", 328.0, 470.0, 10.0), - make_item("report", 360.0, 470.0, 10.0), - // Line 4 - make_item("possible", 72.0, 455.0, 10.0), - make_item("Both", 140.0, 455.0, 10.0), - make_item("constituent", 178.0, 455.0, 10.0), - make_item("studies", 262.0, 455.0, 10.0), - make_item("were", 315.0, 455.0, 10.0), - make_item("approved", 350.0, 455.0, 10.0), - ]; - - let tables = detect_tables(&items, 10.0, false); - assert_eq!( - tables.len(), - 0, - "Word-level paragraph text must not be detected as table" - ); - } - - #[test] - fn test_large_data_table_not_rejected() { - // 50-row table at small font — must not be rejected by row limit - let mut items = Vec::new(); - // Header row - items.push(make_item("Temp", 100.0, 800.0, 8.0)); - items.push(make_item("Pressure", 200.0, 800.0, 8.0)); - items.push(make_item("Volume", 300.0, 800.0, 8.0)); - items.push(make_item("Enthalpy", 400.0, 800.0, 8.0)); - - // 49 data rows - for i in 1..50 { - let y = 800.0 - (i as f32 * 12.0); - items.push(make_item(&format!("{}", -40 + i * 2), 100.0, y, 8.0)); - items.push(make_item( - &format!("{:.1}", 100.0 + i as f32 * 5.0), - 200.0, - y, - 8.0, - )); - items.push(make_item( - &format!("{:.3}", 0.05 + i as f32 * 0.01), - 300.0, - y, - 8.0, - )); - items.push(make_item( - &format!("{:.1}", 150.0 + i as f32 * 2.5), - 400.0, - y, - 8.0, - )); - } - - let tables = detect_tables(&items, 10.0, false); - assert_eq!(tables.len(), 1, "Large data table should not be rejected"); - assert!( - tables[0].rows.len() >= 40, - "Large table should preserve most rows, got {}", - tables[0].rows.len() - ); - } - - #[test] - fn test_uniform_spacing_rows_not_merged() { - // Reproduces the 210603_ROOFING_BIDRESP bug: 8pt font, 10pt line spacing. - // With the old fixed 10.0pt threshold and strict `>`, adjacent rows at exactly - // 10pt apart were merged in pairs, producing garbled output like - // "1 1SC Priority LLC" (two company names joined). - let companies = [ - "SC Priority LLC", - "Craft Roofing Co", - "Alpha Roofing Inc", - "Beta Construction", - "Gamma Builders", - "Delta Roofing", - "Epsilon Contractors", - ]; - - let mut items = Vec::new(); - - // Header row at y=800 - items.push(make_item("No.", 50.0, 800.0, 8.0)); - items.push(make_item("Company", 120.0, 800.0, 8.0)); - items.push(make_item("Bid Amount", 350.0, 800.0, 8.0)); - - // 7 data rows, each 10pt apart (exactly the old threshold) - for (i, company) in companies.iter().enumerate() { - let y = 790.0 - (i as f32 * 10.0); // 10pt uniform spacing - items.push(make_item(&format!("{}", i + 1), 50.0, y, 8.0)); - items.push(make_item(company, 120.0, y, 8.0)); - items.push(make_item(&format!("${},000", 100 + i * 10), 350.0, y, 8.0)); - } - - let tables = detect_tables(&items, 12.0, false); - assert_eq!(tables.len(), 1, "Should detect one table"); - // 1 header + 7 data = 8 rows total; must NOT merge into 4 - assert_eq!( - tables[0].rows.len(), - 8, - "Each company must be on its own row, got {} rows instead of 8", - tables[0].rows.len() - ); - } - - #[test] - fn test_merge_adjacent_items() { - // Simulate per-character rendering: "June 30," as individual glyphs - let items = vec![ - make_char("J", 310.0, 532.0, 13.3, 4.0), - make_char("u", 314.0, 532.0, 13.3, 4.4), - make_char("n", 318.4, 532.0, 13.3, 4.4), - make_char("e", 322.8, 532.0, 13.3, 3.5), - // word gap (2pt) - make_char("3", 328.3, 532.0, 13.3, 4.0), - make_char("0", 332.3, 532.0, 13.3, 4.0), - make_char(",", 336.3, 532.0, 13.3, 2.0), - // large column gap (40pt) - make_char("M", 378.3, 532.0, 13.3, 7.5), - make_char("a", 385.8, 532.0, 13.3, 4.0), - make_char("r", 389.8, 532.0, 13.3, 3.5), - ]; - - let (merged, map) = merge_adjacent_items(&items); - - // "June 30," should merge into one item, "Mar" into another - assert_eq!( - merged.len(), - 2, - "Should produce 2 merged items, got {}", - merged.len() - ); - assert!( - merged[0].text.contains("June") && merged[0].text.contains("30"), - "First merged item should be 'June 30,' but got {:?}", - merged[0].text - ); - assert_eq!(merged[1].text, "Mar"); - - // Index map should track original indices - assert_eq!( - map[0].len(), - 7, - "First merged item should map to 7 original chars" - ); - assert_eq!( - map[1].len(), - 3, - "Second merged item should map to 3 original chars" - ); - } - - #[test] - fn test_per_char_financial_table_detected() { - // Simulates a financial table with per-character header rendering - // and multi-word data items (like SEC filing EBITDA table). - let mut items = Vec::new(); - - // Per-character header row: "Col1" at x≈300, "Col2" at x≈400, "Col3" at x≈500 - for (i, c) in "Col1".chars().enumerate() { - items.push(make_char( - &c.to_string(), - 300.0 + i as f32 * 5.0, - 540.0, - 13.0, - 5.0, - )); - } - for (i, c) in "Col2".chars().enumerate() { - items.push(make_char( - &c.to_string(), - 400.0 + i as f32 * 5.0, - 540.0, - 13.0, - 5.0, - )); - } - for (i, c) in "Col3".chars().enumerate() { - items.push(make_char( - &c.to_string(), - 500.0 + i as f32 * 5.0, - 540.0, - 13.0, - 5.0, - )); - } - - // Data rows with multi-word items (typical extraction output) - let data = [ - ("Revenue", 520.0, "1,000", "2,000", "3,000"), - ("Expenses", 505.0, "500", "800", "1,200"), - ("Net Income", 490.0, "500", "1,200", "1,800"), - ("Taxes", 475.0, "100", "200", "300"), - ]; - - for (label, y, v1, v2, v3) in &data { - items.push(make_item(label, 50.0, *y, 12.0)); - items.push(make_item(v1, 310.0, *y, 12.0)); - items.push(make_item(v2, 410.0, *y, 12.0)); - items.push(make_item(v3, 510.0, *y, 12.0)); - } - - let tables = detect_tables(&items, 13.0, false); - assert!( - !tables.is_empty(), - "Per-character financial table should be detected" - ); - } - - /// Helper to make a single-character TextItem with a specific width - fn make_char(text: &str, x: f32, y: f32, font_size: f32, width: f32) -> TextItem { - TextItem { - text: text.into(), - x, - y, - width, - height: font_size, - font: "F1".into(), - font_size, - page: 1, - is_bold: false, - is_italic: false, - item_type: crate::extractor::ItemType::Text, - } - } -} diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs new file mode 100644 index 0000000..2b09a02 --- /dev/null +++ b/src/tables/detect_heuristic.rs @@ -0,0 +1,1061 @@ +//! Heuristic table detection and validation. + +use crate::text_utils::is_rtl_text; +use crate::types::TextItem; + +use super::financial::try_split_financial_item; +use super::grid::{ + find_column_boundaries, find_column_index, find_row_boundaries, find_row_index, + join_cell_items, recover_header_row, +}; +use super::{Table, TableDetectionMode}; + +/// PDF text is often emitted as one item per glyph. That produces +/// hundreds of single-char items that confuse column detection. This function +/// merges adjacent items within the same line (similar Y, close X, similar font +/// size) into multi-character items, similar to PyMuPDF's `merge_chars()`. +/// +/// Returns `(merged_items, index_map)` where `index_map[merged_idx]` contains +/// the original item indices that were merged into that item. +pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec>) { + if items.is_empty() { + return (vec![], vec![]); + } + + // Group items by Y position (5pt tolerance for same line) + let y_tolerance = 5.0; + let mut line_groups: Vec<(f32, Vec<(usize, &TextItem)>)> = Vec::new(); + + for (idx, item) in items.iter().enumerate() { + let found = line_groups + .iter_mut() + .find(|(y, _)| (item.y - *y).abs() < y_tolerance); + if let Some((_, group)) = found { + group.push((idx, item)); + } else { + line_groups.push((item.y, vec![(idx, item)])); + } + } + + // Sort each group by X position + for (_, group) in &mut line_groups { + group.sort_by(|a, b| { + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } + + // Sort groups by Y descending (top of page first) + line_groups.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); + + let mut merged_items = Vec::new(); + let mut index_map: Vec> = Vec::new(); + + for (_, group) in &line_groups { + let mut i = 0; + while i < group.len() { + let (first_idx, first_item) = group[i]; + let mut text = first_item.text.clone(); + let mut end_x = first_item.x + first_item.width; + let mut indices = vec![first_idx]; + let x_gap_max = first_item.font_size * 0.5; + + let mut j = i + 1; + while j < group.len() { + let (next_idx, next_item) = group[j]; + + // Must be similar font size (within 20%) + if (next_item.font_size - first_item.font_size).abs() > first_item.font_size * 0.20 + { + break; + } + + let gap = next_item.x - end_x; + // Stop if gap exceeds threshold (inter-column gap) + if gap > x_gap_max { + break; + } + // Stop on large overlap (different column overlapping) + if gap < -first_item.font_size * 0.5 { + break; + } + + // Insert space at word boundaries: within a word characters + // touch (gap ≈ 0), between words there's a visible gap. + if gap > first_item.font_size * 0.08 { + text.push(' '); + } + text.push_str(&next_item.text); + end_x = next_item.x + next_item.width; + indices.push(next_idx); + j += 1; + } + + merged_items.push(TextItem { + text, + x: first_item.x, + y: first_item.y, + width: end_x - first_item.x, + height: first_item.height, + font: first_item.font.clone(), + font_size: first_item.font_size, + page: first_item.page, + is_bold: first_item.is_bold, + is_italic: first_item.is_italic, + item_type: first_item.item_type.clone(), + }); + index_map.push(indices); + + i = j; + } + } + + (merged_items, index_map) +} + +/// Iterates all items, expanding qualifying consolidated financial items. +/// Returns `(expanded_items, index_map)` where `index_map[expanded_idx] = original_idx`. +fn expand_consolidated_items(items: &[TextItem]) -> (Vec, Vec) { + let mut expanded = Vec::with_capacity(items.len()); + let mut index_map = Vec::with_capacity(items.len()); + for (orig_idx, item) in items.iter().enumerate() { + if let Some(sub_items) = try_split_financial_item(item) { + for sub in sub_items { + expanded.push(sub); + index_map.push(orig_idx); + } + } else { + expanded.push(item.clone()); + index_map.push(orig_idx); + } + } + (expanded, index_map) +} + +/// Detect tables in a set of text items from a single page +pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bool) -> Vec
{ + if items.len() < 6 { + return vec![]; + } + + // Step 1: Merge adjacent single-char items into words (handles per-character PDFs) + let (merged_items, merge_map) = merge_adjacent_items(items); + + // Step 2: Expand consolidated financial items (e.g. "$ 1,234 $ 5,678" → sub-items) + let (expanded_items, expand_map) = expand_consolidated_items(&merged_items); + let items = &expanded_items[..]; // shadow parameter — all detection uses processed items + + let mut tables = Vec::new(); + let mut claimed_indices = std::collections::HashSet::new(); + + // === Pass 1: Small-font tables (existing behavior) === + let table_font_threshold = base_font_size * 0.90; + + let table_candidates: Vec<(usize, &TextItem)> = items + .iter() + .enumerate() + .filter(|(_, item)| item.font_size <= table_font_threshold && item.font_size >= 6.0) + .collect(); + + if table_candidates.len() >= 6 { + let regions = find_table_regions(&table_candidates); + + for (y_min, y_max) in regions { + let region_items: Vec<(usize, &TextItem)> = table_candidates + .iter() + .filter(|(_, item)| item.y >= y_min && item.y <= y_max) + .cloned() + .collect(); + + if region_items.len() < 6 { + continue; + } + + if let Some(mut table) = + detect_table_in_region(®ion_items, TableDetectionMode::SmallFont) + { + // Try to recover body-font header row above the small-font table + recover_header_row(&mut table, items, table_font_threshold); + for &idx in &table.item_indices { + claimed_indices.insert(idx); + } + tables.push(table); + } + } + } + + // === Pass 2: Body-font tables (stricter criteria) === + // Skip on multi-column pages where body-font detection causes false positives + if !skip_body_font { + let body_font_low = base_font_size * 0.85; + let body_font_high = base_font_size * 1.05; + + let body_candidates: Vec<(usize, &TextItem)> = items + .iter() + .enumerate() + .filter(|(idx, item)| { + !claimed_indices.contains(idx) + && item.font_size >= body_font_low + && item.font_size <= body_font_high + && item.font_size >= 6.0 + }) + .collect(); + + if body_candidates.len() >= 9 { + let regions = find_table_regions_strict(&body_candidates); + + for (y_min, y_max, x_min, x_max) in regions { + let region_items: Vec<(usize, &TextItem)> = body_candidates + .iter() + .filter(|(_, item)| { + item.y >= y_min && item.y <= y_max && item.x >= x_min && item.x <= x_max + }) + .cloned() + .collect(); + + if region_items.len() < 9 { + continue; + } + + if let Some(table) = + detect_table_in_region(®ion_items, TableDetectionMode::BodyFont) + { + tables.push(table); + } + } + } + } + + // Map indices back: expanded → merged → original + for table in &mut tables { + let original_indices: std::collections::HashSet = table + .item_indices + .iter() + .flat_map(|&exp_idx| { + let merged_idx = expand_map[exp_idx]; + merge_map[merged_idx].iter().copied() + }) + .collect(); + table.item_indices = original_indices.into_iter().collect(); + table.item_indices.sort_unstable(); + } + + tables +} + +/// Find Y-regions that likely contain tables +fn find_table_regions(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { + if items.is_empty() { + return vec![]; + } + + let mut y_positions: Vec = items.iter().map(|(_, i)| i.y).collect(); + y_positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + // Find clusters of Y positions (table regions) + let mut regions = Vec::new(); + let gap_threshold = 30.0; // Smaller gap threshold to separate header from content + + let mut region_start = y_positions[0]; + let mut region_end = y_positions[0]; + let mut region_count = 1; + + for &y in &y_positions[1..] { + if y - region_end > gap_threshold { + // End current region if it has enough items + if region_count >= 4 { + regions.push((region_start - 5.0, region_end + 5.0)); + } + region_start = y; + region_end = y; + region_count = 1; + } else { + region_end = y; + region_count += 1; + } + } + + // Don't forget last region + if region_count >= 4 { + regions.push((region_start - 5.0, region_end + 5.0)); + } + + regions +} + +/// Find Y-regions for body-font table candidates using strict structural criteria. +/// Requires rows with 3+ distinct X-position clusters to qualify, and verifies +/// that column positions are consistent across rows (tables have fixed columns, +/// paragraph text has varying word positions). +fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32, f32)> { + if items.is_empty() { + return vec![]; + } + + // Step 1: Group items by Y position (8pt tolerance for same row) + let mut row_groups: Vec<(f32, Vec)> = Vec::new(); + for (_, item) in items { + let mut found = false; + for (center, x_positions) in row_groups.iter_mut() { + if (item.y - *center).abs() < 8.0 { + x_positions.push(item.x); + found = true; + break; + } + } + if !found { + row_groups.push((item.y, vec![item.x])); + } + } + + // Step 2: Filter to rows with 3+ distinct X-position clusters (20pt tolerance) + // Collect cluster start positions for cross-row alignment analysis + let mut qualifying_rows: Vec<(f32, Vec)> = Vec::new(); // (y, cluster_starts) + for (y, x_positions) in &row_groups { + let mut sorted_xs = x_positions.clone(); + sorted_xs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + if sorted_xs.is_empty() { + continue; + } + + let mut cluster_starts: Vec = vec![sorted_xs[0]]; + let mut last_x = sorted_xs[0]; + for &x in &sorted_xs[1..] { + if x - last_x > 20.0 { + cluster_starts.push(x); + last_x = x; + } + } + + if cluster_starts.len() >= 3 { + qualifying_rows.push((*y, cluster_starts)); + } + } + + if qualifying_rows.len() < 3 { + return vec![]; + } + + // Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap) + qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + + let mut candidate_regions: Vec)>> = Vec::new(); + let mut current_region: Vec<&(f32, Vec)> = vec![&qualifying_rows[0]]; + + for row in qualifying_rows.iter().skip(1) { + let prev_y = current_region.last().unwrap().0; + if row.0 - prev_y > 25.0 { + if current_region.len() >= 3 { + candidate_regions.push(current_region); + } + current_region = vec![row]; + } else { + current_region.push(row); + } + } + if current_region.len() >= 3 { + candidate_regions.push(current_region); + } + + // Step 4: Cross-row column alignment check per region + // Real tables have consistent column X positions across rows (high pairwise score). + // Paragraph text has varying word positions line-to-line (low pairwise score). + let mut regions = Vec::new(); + for region_rows in &candidate_regions { + let num_rows = region_rows.len(); + let mut total_score = 0.0f32; + let mut pair_count = 0u32; + let tolerance = 10.0f32; + + for i in 0..num_rows { + for j in (i + 1)..num_rows { + let centers_a = ®ion_rows[i].1; + let centers_b = ®ion_rows[j].1; + + let matches_a = centers_a + .iter() + .filter(|&&a| centers_b.iter().any(|&b| (a - b).abs() < tolerance)) + .count(); + let matches_b = centers_b + .iter() + .filter(|&&b| centers_a.iter().any(|&a| (a - b).abs() < tolerance)) + .count(); + + let max_len = centers_a.len().max(centers_b.len()); + if max_len > 0 { + total_score += (matches_a + matches_b) as f32 / (2 * max_len) as f32; + pair_count += 1; + } + } + } + + let avg_score = if pair_count > 0 { + total_score / pair_count as f32 + } else { + 0.0 + }; + if avg_score >= 0.5 { + let y_min = region_rows.first().unwrap().0; + let y_max = region_rows.last().unwrap().0; + // Compute X bounds from qualifying row cluster positions + let x_min = region_rows + .iter() + .flat_map(|(_, clusters)| clusters.iter()) + .cloned() + .fold(f32::INFINITY, f32::min); + let x_max = region_rows + .iter() + .flat_map(|(_, clusters)| clusters.iter()) + .cloned() + .fold(f32::NEG_INFINITY, f32::max); + regions.push((y_min - 5.0, y_max + 5.0, x_min - 15.0, x_max + 50.0)); + } + } + + regions +} + +/// Detect a table within a specific region +fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Option
{ + // Find column boundaries + let columns = find_column_boundaries(items, mode); + let min_cols = match mode { + TableDetectionMode::SmallFont => 2, + TableDetectionMode::BodyFont => 3, + }; + if columns.len() < min_cols || columns.len() > 15 { + return None; + } + + // Find row boundaries + let rows = find_row_boundaries(items); + let min_rows = match mode { + TableDetectionMode::SmallFont => 2, + TableDetectionMode::BodyFont => 3, + }; + if rows.len() < min_rows { + return None; + } + + // Verify this looks like a table: multiple items should align to columns + let col_alignment = check_column_alignment(items, &columns, mode); + let min_alignment = match mode { + TableDetectionMode::SmallFont => 0.5, + TableDetectionMode::BodyFont => 0.7, + }; + if col_alignment < min_alignment { + return None; + } + + // Build the table grid - first collect items per cell, then join properly + let mut cell_items: Vec>> = + vec![vec![Vec::new(); columns.len()]; rows.len()]; + let mut item_indices = Vec::new(); + + for (idx, item) in items { + let col = find_column_index(&columns, item.x); + let row = find_row_index(&rows, item.y); + + if let (Some(col), Some(row)) = (col, row) { + cell_items[row][col].push(item); + item_indices.push(*idx); + } + } + + // Detect form header rows and exclude their items + // We need to do this BEFORE finalizing item_indices + let (first_table_row, excluded_items) = find_first_table_row(&cell_items, &rows, items); + + // Remove excluded items from item_indices + let item_indices: Vec = item_indices + .into_iter() + .filter(|idx| !excluded_items.contains(idx)) + .collect(); + + // If we excluded rows, adjust the cell_items and rows + let (rows, mut cell_items) = if first_table_row > 0 { + let new_rows = rows[first_table_row..].to_vec(); + let new_cell_items = cell_items[first_table_row..].to_vec(); + (new_rows, new_cell_items) + } else { + (rows, cell_items) + }; + + // Sort items within each cell by X position and join with subscript-aware spacing + let mut cells: Vec> = Vec::with_capacity(rows.len()); + for row_items in &mut cell_items { + let mut row_cells = Vec::with_capacity(columns.len()); + for col_items in row_items.iter_mut() { + // Sort by X position (direction-aware) + let rtl = is_rtl_text(col_items.iter().map(|i| &i.text)); + if rtl { + col_items + .sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + col_items + .sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } + + // Join items with subscript-aware spacing + let text = join_cell_items(col_items); + row_cells.push(text); + } + cells.push(row_cells); + } + + // Validation 1: most rows should have content in first column + let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count(); + if rows_with_first_col < rows.len() / 2 { + return None; + } + + // Validation 2: real tables have content in MULTIPLE columns, not just first + let rows_with_multi_cols = cells + .iter() + .filter(|row| row.iter().filter(|c| !c.is_empty()).count() >= 2) + .count(); + let multi_col_threshold = match mode { + TableDetectionMode::SmallFont => (rows.len() / 3).max(1), // 33% + TableDetectionMode::BodyFont => (rows.len() / 2).max(1), // 50% + }; + if rows_with_multi_cols < multi_col_threshold { + return None; + } + + // Validation 3: tables shouldn't have too many rows (likely misdetected text) + let max_rows = match mode { + TableDetectionMode::SmallFont => 200, + TableDetectionMode::BodyFont => 200, + }; + if rows.len() > max_rows { + return None; + } + + // Validation 4: average cells per row should be reasonable + let total_filled: usize = cells + .iter() + .map(|row| row.iter().filter(|c| !c.is_empty()).count()) + .sum(); + let avg_cells_per_row = total_filled as f32 / rows.len() as f32; + let min_avg_cells = match mode { + TableDetectionMode::SmallFont => 1.5, + TableDetectionMode::BodyFont => 2.5, + }; + if avg_cells_per_row < min_avg_cells { + return None; + } + + // Validation 5: Check for key-value pair layout (NOT a table) + // Key-value layouts have: mostly 2 filled columns, first column is labels + if is_key_value_layout(&cells) { + return None; + } + + // Validation 6: Check column count consistency + // Real tables have similar column counts across rows + if !has_consistent_columns(&cells) { + return None; + } + + // Validation 7: Tables should have some numeric/data content + // (not just text labels) + if !has_table_like_content(&cells, mode) { + return None; + } + + // Validation 8: Check for Table of Contents pattern + // TOCs have dots (leader lines) and page numbers, not real table data + if is_table_of_contents(&cells) { + return None; + } + + // Validation 9: Reject paragraph-like content falsely detected as tables. + // Real table cells are short and self-contained. Paragraph text split into + // "cells" produces long sentence fragments. + if is_paragraph_content(&cells) { + return None; + } + + Some(Table { + columns, + rows, + cells, + item_indices, + }) +} + +/// Check if this looks like a key-value pair layout rather than a table +fn is_key_value_layout(cells: &[Vec]) -> bool { + if cells.is_empty() { + return false; + } + + let num_cols = cells[0].len(); + + // Key-value layouts typically have 2-3 effective columns + // where the first column contains labels ending with ":" + let mut label_like_first_col = 0; + let mut rows_with_two_or_less = 0; + + for row in cells { + let filled_count = row.iter().filter(|c| !c.is_empty()).count(); + if filled_count <= 2 { + rows_with_two_or_less += 1; + } + + // Check if first column looks like a label (ends with : or is all caps) + let first = row.first().map(|s| s.trim()).unwrap_or(""); + if first.ends_with(':') + || (first.len() > 3 + && first + .chars() + .all(|c| c.is_uppercase() || c.is_whitespace() || c == '(' || c == ')')) + { + label_like_first_col += 1; + } + } + + // If most rows have only 2 columns filled and first column is label-like + let pct_two_or_less = rows_with_two_or_less as f32 / cells.len() as f32; + let pct_label_like = label_like_first_col as f32 / cells.len() as f32; + + // This is likely a key-value layout if: + // - Most rows have 2 or fewer filled columns + // - First column often looks like labels + // - Total columns detected is 6 or fewer (real tables often have more) + pct_two_or_less > 0.7 && pct_label_like > 0.5 && num_cols <= 6 +} + +/// Check if columns are consistent across rows (real tables have this) +fn has_consistent_columns(cells: &[Vec]) -> bool { + if cells.len() < 3 { + return true; // Not enough rows to judge + } + + // Count filled columns per row + let filled_counts: Vec = cells + .iter() + .map(|row| row.iter().filter(|c| !c.is_empty()).count()) + .collect(); + + // Find the most common filled count + let mut count_freq: std::collections::HashMap = std::collections::HashMap::new(); + for &count in &filled_counts { + *count_freq.entry(count).or_insert(0) += 1; + } + + let most_common_count = count_freq + .iter() + .max_by_key(|(_, freq)| *freq) + .map(|(count, _)| *count) + .unwrap_or(0); + + // At least 40% of rows should have the most common column count (or close to it) + let consistent_rows = filled_counts + .iter() + .filter(|&&c| c >= most_common_count.saturating_sub(2) && c <= most_common_count + 2) + .count(); + + consistent_rows as f32 / cells.len() as f32 > 0.4 +} + +/// Check if the content looks like table data (numbers, short values, specs) +fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bool { + let mut data_like_cells = 0; + let mut total_cells = 0; + + for row in cells.iter().skip(1) { + // Skip header row + for cell in row { + let trimmed = cell.trim(); + if !trimmed.is_empty() { + total_cells += 1; + // Check if it looks like table data + if looks_like_table_data(trimmed) { + data_like_cells += 1; + } + } + } + } + + if total_cells == 0 { + return false; + } + + // Data-like content threshold depends on detection mode + let pct_data = data_like_cells as f32 / total_cells as f32; + let num_cols = cells.first().map(|r| r.len()).unwrap_or(0); + + let min_pct = match mode { + TableDetectionMode::SmallFont => 0.2, + TableDetectionMode::BodyFont => 0.3, + }; + + // For SmallFont, bypass content check for wide tables (5+ columns may have text headers). + // For BodyFont, always require data-like content to prevent paragraph false positives. + pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5) +} + +/// Check if a cell value looks like table data +/// Includes: numbers, part numbers, specifications with units, codes +fn looks_like_table_data(s: &str) -> bool { + let s = s.trim(); + if s.is_empty() { + return false; + } + + // Pure numbers + if looks_like_number(s) { + return true; + } + + // Dates: MM/DD/YYYY, DD/MM/YYYY, YYYY-MM-DD, etc. + if s.len() <= 10 + && s.chars().filter(|c| c.is_ascii_digit()).count() >= 4 + && (s.contains('/') || s.contains('-')) + && s.chars() + .all(|c| c.is_ascii_digit() || c == '/' || c == '-') + { + return true; + } + + // Part numbers / model codes (alphanumeric, typically short) + // e.g., "NA555", "NE555", "LM358" + if s.len() <= 10 + && s.chars().all(|c| c.is_alphanumeric()) + && s.chars().any(|c| c.is_ascii_digit()) + { + return true; + } + + // Specifications with units (contains numbers and unit symbols) + // e.g., "–40°C to +105°C", "5V", "200mA", "8-pin" + let has_number = s.chars().any(|c| c.is_ascii_digit()); + let has_unit = s.contains('°') + || s.contains('V') + || s.contains('A') + || s.contains("Hz") + || s.contains("mA") + || s.contains("µ") + || s.contains("pin") + || s.contains("MHz") + || s.contains("kHz"); + if has_number && has_unit { + return true; + } + + // Package designations with parentheses + // e.g., "D (SOIC, 8)", "P (PDIP, 8)" + if s.contains('(') && s.contains(')') && s.chars().any(|c| c.is_ascii_digit()) { + return true; + } + + // Temperature ranges + // e.g., "TA = –40°C to +105°C" + if (s.contains("°C") || s.contains("°F")) && s.contains("to") { + return true; + } + + false +} + +/// Check if a string looks like a number +fn looks_like_number(s: &str) -> bool { + let s = s.trim(); + if s.is_empty() { + return false; + } + + // Handle common number formats: 9.0, 10, 8.6, etc. + s.chars() + .all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+') + && s.chars().any(|c| c.is_ascii_digit()) +} + +/// Check if this looks like a Table of Contents +/// TOCs have characteristic patterns: leader dots, page numbers, section names +fn is_table_of_contents(cells: &[Vec]) -> bool { + if cells.is_empty() { + return false; + } + + let mut dot_cells = 0; + let mut page_number_cells = 0; + let mut total_cells = 0; + + for row in cells { + for cell in row { + let trimmed = cell.trim(); + if trimmed.is_empty() { + continue; + } + total_cells += 1; + + // Check for leader dots (sequences of periods) + // TOCs often have "........" or ". . . ." patterns + let dot_count = trimmed.chars().filter(|&c| c == '.').count(); + let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3; + if is_mostly_dots { + dot_cells += 1; + } + + // Check for standalone page numbers (1-4 digits, possibly with spaces) + let digits_only: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect(); + if digits_only.len() <= 4 + && !digits_only.is_empty() + && digits_only.chars().all(|c| c.is_ascii_digit()) + { + page_number_cells += 1; + } + } + } + + if total_cells == 0 { + return false; + } + + // If a significant portion of cells are dots or page numbers, it's likely a TOC + let dot_ratio = dot_cells as f32 / total_cells as f32; + let page_num_ratio = page_number_cells as f32 / total_cells as f32; + + // TOC typically has >15% dot cells and >10% page number cells + dot_ratio > 0.15 || (dot_ratio > 0.05 && page_num_ratio > 0.15) +} + +/// Check if detected "table" cells are actually paragraph text fragments. +/// +/// Multi-column paragraph text falsely detected as tables produces: +/// - Many empty cells (text doesn't span all columns) +/// - Cells ending with hyphens (word breaks across "columns") +/// - Long sentence fragments or single-word fragments +fn is_paragraph_content(cells: &[Vec]) -> bool { + if cells.is_empty() { + return false; + } + + let num_cols = cells[0].len(); + let total_cells = cells.len() * num_cols; + if total_cells == 0 { + return false; + } + + let filled: Vec<&str> = cells + .iter() + .flat_map(|r| r.iter()) + .map(|c| c.trim()) + .filter(|c| !c.is_empty()) + .collect(); + + let total_filled = filled.len(); + if total_filled < 4 { + return false; + } + + let empty_ratio = 1.0 - (total_filled as f32 / total_cells as f32); + + // Cells ending with a hyphen suggest word breaks across columns. + // Real table cells almost never end with hyphens (except range indicators). + let hyphen_breaks = filled + .iter() + .filter(|c| { + c.ends_with('-') && c.len() > 1 && { + let mut chars = c.chars().rev(); + chars.next(); // skip the '-' + chars.next().is_some_and(|ch| ch.is_alphabetic()) + } + }) + .count(); + let hyphen_ratio = hyphen_breaks as f32 / total_filled as f32; + + // Word-break hyphens are a strong paragraph signal + if hyphen_ratio > 0.03 { + return true; + } + + // High empty ratio with many rows suggests paragraph text spread across a grid + if empty_ratio > 0.55 && cells.len() > 10 { + return true; + } + + // Letter-spaced text (spaces between every character) is never real table data. + // This happens when PDF uses wide character spacing for emphasis/formatting. + // Require at least 9 chars (e.g., "a b c d e") to avoid matching short codes. + let letter_spaced = filled + .iter() + .filter(|c| { + let chars: Vec = c.chars().collect(); + chars.len() >= 9 + && chars.windows(4).all(|w| { + (w[0].is_alphabetic() && w[1] == ' ' && w[2].is_alphabetic() && w[3] == ' ') + || (w[0] == ' ' + && w[1].is_alphabetic() + && w[2] == ' ' + && w[3].is_alphabetic()) + }) + }) + .count(); + if letter_spaced > 0 && letter_spaced as f32 / total_filled as f32 > 0.08 { + return true; + } + + // Long sentence fragments + let long_cells = filled.iter().filter(|c| c.len() > 60).count(); + let long_ratio = long_cells as f32 / total_filled as f32; + let avg_len = filled.iter().map(|c| c.len()).sum::() as f32 / total_filled as f32; + + if avg_len > 40.0 && long_ratio > 0.2 { + return true; + } + if long_ratio > 0.3 { + return true; + } + + false +} + +/// Check what fraction of items align to detected columns +fn check_column_alignment( + items: &[(usize, &TextItem)], + columns: &[f32], + mode: TableDetectionMode, +) -> f32 { + let tolerance = match mode { + TableDetectionMode::SmallFont => 40.0, + TableDetectionMode::BodyFont => 30.0, + }; + let aligned = items + .iter() + .filter(|(_, item)| columns.iter().any(|&col| (item.x - col).abs() < tolerance)) + .count(); + + aligned as f32 / items.len() as f32 +} + +/// Find the first row that looks like actual table data (not form header). +/// Returns (first_table_row_index, set of item indices to exclude). +pub(crate) fn find_first_table_row( + cell_items: &[Vec>], + rows: &[f32], + original_items: &[(usize, &TextItem)], +) -> (usize, std::collections::HashSet) { + let mut excluded_items = std::collections::HashSet::new(); + + // Build string cells for analysis + let cells: Vec> = cell_items + .iter() + .map(|row| row.iter().map(|col| join_cell_items(col)).collect()) + .collect(); + + if cells.is_empty() { + return (0, excluded_items); + } + + // Strategy: Skip leading rows that look like form metadata + // + // Form/metadata rows have: + // 1. Cells ending with ":" (form labels) + // 2. Very sparse fill with document metadata (grade level, year, etc.) + // + // Table rows have: + // 1. Dense fill (headers spanning columns) + // 2. Numeric content (data rows) + // 3. No form label patterns + + let total_cols = cells[0].len(); + let mut first_table_row = 0; + + for (row_idx, row) in cells.iter().enumerate() { + let filled_cells: Vec<&String> = row.iter().filter(|c| !c.trim().is_empty()).collect(); + let filled_count = filled_cells.len(); + let fill_ratio = filled_count as f32 / total_cols as f32; + + // Check for form-like patterns (cells with colons) + // Only treat as form row if most filled cells look form-like, + // or the row is very sparse with any form pattern. + let form_cell_count = filled_cells + .iter() + .filter(|c| { + let text = c.trim(); + (text.ends_with(':') && text.len() > 1) + || (text.contains(": ") && !looks_like_number(text)) + }) + .count(); + let has_form_patterns = + form_cell_count > 0 && (form_cell_count * 2 >= filled_count || fill_ratio < 0.3); + + // Check for numeric content + let numeric_count = filled_cells + .iter() + .filter(|c| looks_like_number(c.trim())) + .count(); + let has_data = numeric_count >= 2; + + // Skip rows with form patterns (regardless of density) + if has_form_patterns { + continue; + } + + // Data rows are definitely table content + if has_data { + first_table_row = row_idx; + break; + } + + // Dense rows without form patterns are likely table headers + if fill_ratio >= 0.4 { + first_table_row = row_idx; + break; + } + + // Very sparse rows at the start are likely metadata - skip them + if fill_ratio < 0.3 { + continue; + } + + // Moderately sparse row without form patterns - could be multi-line header + // Look ahead to decide + if row_idx + 1 < cells.len() { + let next_row = &cells[row_idx + 1]; + let next_filled = next_row.iter().filter(|c| !c.trim().is_empty()).count(); + let next_fill_ratio = next_filled as f32 / total_cols as f32; + let next_has_form = next_row.iter().any(|c| { + let text = c.trim(); + (text.ends_with(':') && text.len() > 1) + || (text.contains(": ") && !looks_like_number(text)) + }); + + // If next row is dense or has data (and no form patterns), this row starts the table + if (next_fill_ratio >= 0.4 + || next_row + .iter() + .filter(|c| looks_like_number(c.trim())) + .count() + >= 2) + && !next_has_form + { + first_table_row = row_idx; + break; + } + } + + // Otherwise skip this sparse row + } + + // Collect item indices from excluded rows + if first_table_row > 0 { + let y_tolerance = 15.0; + for (idx, item) in original_items { + // Check if this item is in one of the excluded rows + for row_y in rows.iter().take(first_table_row) { + if (item.y - *row_y).abs() < y_tolerance { + excluded_items.insert(*idx); + break; + } + } + } + } + + (first_table_row, excluded_items) +} diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs new file mode 100644 index 0000000..a817580 --- /dev/null +++ b/src/tables/detect_rects.rs @@ -0,0 +1,357 @@ +//! Rectangle-based table detection using union-find clustering. + +use std::collections::HashMap; + +use crate::types::{PdfRect, TextItem}; + +use super::Table; + +/// Disjoint-set (union-find) for clustering indices. +struct UnionFind { + parent: Vec, + rank: Vec, +} + +impl UnionFind { + fn new(n: usize) -> Self { + Self { + parent: (0..n).collect(), + rank: vec![0; n], + } + } + + fn find(&mut self, x: usize) -> usize { + if self.parent[x] != x { + self.parent[x] = self.find(self.parent[x]); + } + self.parent[x] + } + + fn union(&mut self, a: usize, b: usize) { + let ra = self.find(a); + let rb = self.find(b); + if ra == rb { + return; + } + if self.rank[ra] < self.rank[rb] { + self.parent[ra] = rb; + } else if self.rank[ra] > self.rank[rb] { + self.parent[rb] = ra; + } else { + self.parent[rb] = ra; + self.rank[ra] += 1; + } + } +} + +/// Check if two rects overlap after expanding each by `tol` on all sides. +pub(crate) fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32), tol: f32) -> bool { + // a and b are (x, y, w, h) where (x,y) is bottom-left corner + let (ax, ay, aw, ah) = *a; + let (bx, by, bw, bh) = *b; + // Expand each rect by tol + let a_left = ax - tol; + let a_right = ax + aw + tol; + let a_bottom = ay - tol; + let a_top = ay + ah + tol; + let b_left = bx - tol; + let b_right = bx + bw + tol; + let b_bottom = by - tol; + let b_top = by + bh + tol; + // AABB overlap: NOT (separated) + !(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom) +} + +/// Cluster rects by spatial overlap using union-find. +/// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned. +pub(crate) fn cluster_rects( + rects: &[(f32, f32, f32, f32)], + tolerance: f32, + min_size: usize, +) -> Vec> { + let n = rects.len(); + let mut uf = UnionFind::new(n); + + for i in 0..n { + for j in (i + 1)..n { + if rects_overlap(&rects[i], &rects[j], tolerance) { + uf.union(i, j); + } + } + } + + // Group indices by root + let mut groups: HashMap> = HashMap::new(); + for i in 0..n { + groups.entry(uf.find(i)).or_default().push(i); + } + + groups + .into_values() + .filter(|g| g.len() >= min_size) + .collect() +} + +/// Detect tables from explicit rectangle (`re`) operators in the PDF. +/// +/// Many PDFs draw cell borders using `re` (rectangle) operators. Table pages +/// typically have 100-200+ rects while non-table pages have < 30. This function +/// clusters spatially connected rectangles into groups, then identifies grids of +/// cell-sized rectangles within each cluster and assigns text items to cells. +pub fn detect_tables_from_rects(items: &[TextItem], rects: &[PdfRect], page: u32) -> Vec
{ + // Filter rects on this page; normalize negative widths/heights; skip tiny rects. + let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized + for r in rects { + if r.page != page { + continue; + } + let (mut x, mut y, mut w, mut h) = (r.x, r.y, r.width, r.height); + if w < 0.0 { + x += w; + w = -w; + } + if h < 0.0 { + y += h; + h = -h; + } + // Skip tiny rects (borders, dots, decorations) + if w < 5.0 || h < 5.0 { + continue; + } + page_rects.push((x, y, w, h)); + } + + // Need a reasonable number of cell rects to form a table + if page_rects.len() < 6 { + return vec![]; + } + + // Cluster spatially connected rects into groups + let clusters = cluster_rects(&page_rects, 3.0, 6); + + let mut tables = Vec::new(); + for cluster_indices in &clusters { + let group_rects: Vec<(f32, f32, f32, f32)> = + cluster_indices.iter().map(|&i| page_rects[i]).collect(); + if let Some(table) = detect_table_from_rect_group(items, &group_rects, page) { + tables.push(table); + } + } + + tables +} + +/// Detect a single table from a cluster of spatially connected rects. +/// +/// Contains the grid-detection logic: snap edges, fill-ratio check, +/// assign items to grid, content density validation. +pub(crate) fn detect_table_from_rect_group( + items: &[TextItem], + group_rects: &[(f32, f32, f32, f32)], + page: u32, +) -> Option
{ + // Extract unique X and Y edges from all rects + let mut x_edges: Vec = Vec::new(); + let mut y_edges: Vec = Vec::new(); + for &(x, y, w, h) in group_rects { + x_edges.push(x); + x_edges.push(x + w); + y_edges.push(y); + y_edges.push(y + h); + } + + let x_edges = snap_edges(&x_edges, 2.0); + let y_edges = snap_edges(&y_edges, 2.0); + + if x_edges.len() < 3 || y_edges.len() < 4 { + // Need at least 2 columns (3 edges) and 3 rows (4 edges) + return None; + } + + // Sort column edges left-to-right, row edges top-to-bottom (highest Y first for PDF) + let mut col_edges = x_edges; + col_edges.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let mut row_edges = y_edges; + row_edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + if num_cols < 2 || num_rows < 2 { + return None; + } + + // Reject grids that are too large — real tables rarely exceed 12 columns. + // Form-style PDFs with scattered field boxes produce huge sparse grids. + if num_cols > 12 { + return None; + } + + // Verify that cell-sized rects actually fill the grid + // Count how many grid cells have a matching rect + let mut filled_cells = 0u32; + for row in 0..num_rows { + let y_top = row_edges[row]; + let y_bot = row_edges[row + 1]; + for col in 0..num_cols { + let x_left = col_edges[col]; + let x_right = col_edges[col + 1]; + // Check if any rect approximately covers this cell + let cell_covered = group_rects.iter().any(|&(rx, ry, rw, rh)| { + let tol = 3.0; + rx <= x_left + tol + && (rx + rw) >= x_right - tol + && ry <= y_top + tol + && (ry + rh) >= y_bot - tol + }); + if cell_covered { + filled_cells += 1; + } + } + } + + let total_cells = (num_cols * num_rows) as f32; + let fill_ratio = filled_cells as f32 / total_cells; + + // Require at least 30% of cells to be backed by rects + if fill_ratio < 0.3 { + return None; + } + + // Build table: assign text items to cells + let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + + // Compute column centers and row centers for the Table struct + let columns: Vec = (0..num_cols) + .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) + .collect(); + let rows: Vec = (0..num_rows) + .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) + .collect(); + + // Skip if no text was assigned + if item_indices.is_empty() { + return None; + } + + // Skip tables with only 1 row of content (header-only) + let non_empty_rows = cells + .iter() + .filter(|row| row.iter().any(|c| !c.trim().is_empty())) + .count(); + if non_empty_rows < 2 { + return None; + } + + // Content density check: reject tables where most cells are empty. + // Real tables have content in most cells; form layouts produce sparse grids. + let non_empty_cells = cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let content_ratio = non_empty_cells as f32 / total_cells; + if content_ratio < 0.25 { + return None; + } + + // Reject tables with any completely empty column — indicates a bad grid. + for col in 0..num_cols { + let col_has_content = cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); + if !col_has_content { + return None; + } + } + + Some(Table { + columns, + rows, + cells, + item_indices, + }) +} + +/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges. +pub(crate) fn snap_edges(values: &[f32], tolerance: f32) -> Vec { + let mut sorted: Vec = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let mut snapped: Vec = Vec::new(); + for &v in &sorted { + if let Some(last) = snapped.last() { + if (v - *last).abs() <= tolerance { + continue; // Skip — too close to previous edge + } + } + snapped.push(v); + } + snapped +} + +/// Assign text items to grid cells defined by column/row edges. +/// +/// Returns `(cells, item_indices)` where `cells[row][col]` is the cell text +/// and `item_indices` lists the original item indices that were consumed. +pub(crate) fn assign_items_to_grid( + items: &[TextItem], + col_edges: &[f32], + row_edges: &[f32], + page: u32, +) -> (Vec>, Vec) { + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + // Collect items per cell for proper sorting before joining + let mut cell_items: Vec>> = + vec![vec![Vec::new(); num_cols]; num_rows]; + let mut indices = Vec::new(); + + for (idx, item) in items.iter().enumerate() { + if item.page != page { + continue; + } + // Use item center for assignment + let cx = item.x + item.width / 2.0; + let cy = item.y; + + // Find column: cx must be between col_edges[c] and col_edges[c+1] + let col = (0..num_cols).find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0); + // Find row: cy must be between row_edges[r+1] (bottom) and row_edges[r] (top) + let row = (0..num_rows).find(|&r| cy >= row_edges[r + 1] - 2.0 && cy <= row_edges[r] + 2.0); + + if let (Some(c), Some(r)) = (col, row) { + cell_items[r][c].push((idx, item)); + indices.push(idx); + } + } + + // Build cell strings: sort items within each cell by Y descending then X ascending + let mut cells: Vec> = Vec::with_capacity(num_rows); + for row_items in &mut cell_items { + let mut row_cells = Vec::with_capacity(num_cols); + for col_items in row_items.iter_mut() { + col_items.sort_by(|a, b| { + b.1.y + .partial_cmp(&a.1.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal) + }) + }); + let text: String = col_items + .iter() + .map(|(_, item)| item.text.trim()) + .filter(|t| !t.is_empty()) + .collect::>() + .join(" "); + row_cells.push(text); + } + cells.push(row_cells); + } + + (cells, indices) +} diff --git a/src/tables/financial.rs b/src/tables/financial.rs new file mode 100644 index 0000000..046ba17 --- /dev/null +++ b/src/tables/financial.rs @@ -0,0 +1,115 @@ +//! Financial token splitting for consolidated value items. + +use crate::types::TextItem; + +/// Check if a whitespace-separated token looks like a financial number. +/// Must contain at least one digit; all chars must be `0-9 , . ( ) - + %`. +pub(crate) fn is_numeric_token(tok: &str) -> bool { + if tok.is_empty() { + return false; + } + let mut has_digit = false; + for c in tok.chars() { + match c { + '0'..='9' => has_digit = true, + ',' | '.' | '(' | ')' | '-' | '+' | '%' => {} + _ => return false, + } + } + has_digit +} + +/// Check for em-dash, en-dash, or minus used as nil marker in financial tables. +pub(crate) fn is_dash_token(tok: &str) -> bool { + matches!(tok, "\u{2014}" | "\u{2013}" | "-" | "\u{2012}") +} + +/// Returns true if text contains 2+ consecutive alphabetic characters. +/// Fast early-exit to reject items like `"Land $ 778,177"`. +pub(crate) fn has_alphabetic_words(text: &str) -> bool { + let mut consecutive = 0u32; + for c in text.chars() { + if c.is_alphabetic() { + consecutive += 1; + if consecutive >= 2 { + return true; + } + } else { + consecutive = 0; + } + } + false +} + +/// Splits text by whitespace, then groups tokens into financial values. +/// - `$` + numeric token → one value (`"$ 5,147,649"`) +/// - standalone numeric token → one value (`"114,167"`) +/// - dash token → one value (`"—"`) +/// - any unrecognized token → return `None` (not a pure-value item) +pub(crate) fn tokenize_financial_values(text: &str) -> Option> { + let tokens: Vec<&str> = text.split_whitespace().collect(); + if tokens.is_empty() { + return None; + } + let mut values = Vec::new(); + let mut i = 0; + while i < tokens.len() { + let tok = tokens[i]; + if tok == "$" { + // Dollar sign followed by a numeric token → one value + if i + 1 < tokens.len() && is_numeric_token(tokens[i + 1]) { + values.push(format!("{} {}", tok, tokens[i + 1])); + i += 2; + } else { + return None; + } + } else if is_numeric_token(tok) || is_dash_token(tok) { + values.push(tok.to_string()); + i += 1; + } else { + return None; + } + } + if values.is_empty() { + None + } else { + Some(values) + } +} + +/// Try to split a consolidated financial item into individual sub-items. +/// Criteria: width > font_size × 20, no alphabetic words, tokenization yields 3+ values. +/// Creates sub-items with evenly-distributed X positions across the original item's span. +pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> { + if item.width <= item.font_size * 20.0 { + return None; + } + let text = &item.text; + if has_alphabetic_words(text) { + return None; + } + let values = tokenize_financial_values(text)?; + if values.len() < 3 { + return None; + } + let n = values.len() as f32; + let spacing = item.width / n; + let sub_width = spacing * 0.9; + let mut sub_items = Vec::with_capacity(values.len()); + for (i, val) in values.iter().enumerate() { + sub_items.push(TextItem { + text: val.clone(), + x: item.x + spacing * i as f32 + spacing * 0.5, + y: item.y, + width: sub_width, + height: item.height, + font: item.font.clone(), + font_size: item.font_size, + page: item.page, + is_bold: item.is_bold, + is_italic: item.is_italic, + item_type: item.item_type.clone(), + }); + } + Some(sub_items) +} diff --git a/src/tables/format.rs b/src/tables/format.rs new file mode 100644 index 0000000..ff320c4 --- /dev/null +++ b/src/tables/format.rs @@ -0,0 +1,148 @@ +//! Table-to-markdown formatting and cell cleanup. + +use super::Table; + +pub fn table_to_markdown(table: &Table) -> String { + if table.cells.is_empty() || table.cells[0].is_empty() { + return String::new(); + } + + // Clean up the table: merge continuation rows, extract footnotes, remove empty rows + let (cleaned_cells, footnotes) = clean_table_cells(&table.cells); + + if cleaned_cells.is_empty() { + return String::new(); + } + + let num_cols = cleaned_cells[0].len(); + let mut output = String::new(); + + // Calculate column widths for alignment + let col_widths: Vec = (0..num_cols) + .map(|col| { + cleaned_cells + .iter() + .map(|row| row.get(col).map(|c| c.len()).unwrap_or(0)) + .max() + .unwrap_or(3) + .max(3) + }) + .collect(); + + // Output each row + for (row_idx, row) in cleaned_cells.iter().enumerate() { + output.push('|'); + for (col_idx, cell) in row.iter().enumerate() { + let width = col_widths[col_idx]; + output.push_str(&format!(" {:width$} |", cell, width = width)); + } + output.push('\n'); + + // Add separator after header row + if row_idx == 0 { + output.push('|'); + for width in &col_widths { + output.push_str(&format!(" {} |", "-".repeat(*width))); + } + output.push('\n'); + } + } + + // Add footnotes below the table + if !footnotes.is_empty() { + output.push('\n'); + for footnote in footnotes { + output.push_str(&footnote); + output.push('\n'); + } + } + + output +} + +/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows +fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { + let mut cleaned: Vec> = Vec::new(); + let mut footnotes: Vec = Vec::new(); + + for row in cells { + // Check if this row is empty + if row.iter().all(|c| c.trim().is_empty()) { + continue; + } + + // Check if this row is a footnote (starts with (1), (2), etc. or just a number reference) + let first_cell = row.first().map(|s| s.trim()).unwrap_or(""); + if is_footnote_row(first_cell) { + // Combine all cells into a single footnote line + let footnote_text: String = row + .iter() + .map(|c| c.trim()) + .filter(|c| !c.is_empty()) + .collect::>() + .join(" "); + footnotes.push(footnote_text); + continue; + } + + // Check if this is a continuation row (first column is empty but others have content) + let is_continuation = first_cell.is_empty() + && row.iter().skip(1).any(|c| !c.trim().is_empty()) + && !cleaned.is_empty(); + + if is_continuation { + // Merge with previous row + if let Some(prev_row) = cleaned.last_mut() { + for (col_idx, cell) in row.iter().enumerate() { + let cell_text = cell.trim(); + if !cell_text.is_empty() && col_idx < prev_row.len() { + if !prev_row[col_idx].is_empty() { + prev_row[col_idx].push(' '); + } + prev_row[col_idx].push_str(cell_text); + } + } + } + } else { + // Regular row - add as new row + cleaned.push(row.iter().map(|c| c.trim().to_string()).collect()); + } + } + + (cleaned, footnotes) +} + +/// Check if a cell value indicates a footnote row +fn is_footnote_row(text: &str) -> bool { + let trimmed = text.trim(); + + // Check for common footnote patterns + // (1), (2), etc. + if trimmed.starts_with('(') && trimmed.len() >= 2 { + let inside = &trimmed[1..]; + if let Some(close_idx) = inside.find(')') { + let num_part = &inside[..close_idx]; + if num_part.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + // 1), 2), etc. + if trimmed.len() >= 2 { + if let Some(paren_idx) = trimmed.find(')') { + let num_part = &trimmed[..paren_idx]; + if !num_part.is_empty() && num_part.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + // Check for "Note:" or "Notes:" at the start + let lower = trimmed.to_lowercase(); + if lower.starts_with("note:") || lower.starts_with("notes:") { + return true; + } + + false +} diff --git a/src/tables/grid.rs b/src/tables/grid.rs new file mode 100644 index 0000000..739bc45 --- /dev/null +++ b/src/tables/grid.rs @@ -0,0 +1,308 @@ +//! Column/row boundary detection and cell assignment for heuristic tables. + +use crate::types::TextItem; + +use super::{Table, TableDetectionMode}; + +pub(crate) fn find_column_boundaries( + items: &[(usize, &TextItem)], + mode: TableDetectionMode, +) -> Vec { + let mut x_positions: Vec = items.iter().map(|(_, i)| i.x).collect(); + x_positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + if x_positions.is_empty() { + return vec![]; + } + + // Calculate adaptive threshold based on X-position density + // For dense tables (like grade tables), use smaller threshold + let x_range = x_positions.last().unwrap() - x_positions.first().unwrap(); + let avg_gap = if x_positions.len() > 1 { + x_range / (x_positions.len() - 1) as f32 + } else { + 60.0 + }; + + // Use smaller threshold for dense data, larger for sparse + let cluster_threshold = avg_gap.clamp(25.0, 50.0); + + let mut columns = Vec::new(); + let mut cluster_items: Vec = vec![x_positions[0]]; + + for &x in &x_positions[1..] { + let cluster_center = cluster_items.iter().sum::() / cluster_items.len() as f32; + + if x - cluster_center > cluster_threshold { + // End current cluster + columns.push(cluster_center); + cluster_items = vec![x]; + } else { + cluster_items.push(x); + } + } + + // Don't forget last cluster + if !cluster_items.is_empty() { + columns.push(cluster_items.iter().sum::() / cluster_items.len() as f32); + } + + // Filter columns - each should have multiple items + let min_items_per_col = (items.len() / columns.len().max(1) / 4).max(2); + let columns: Vec = columns + .into_iter() + .filter(|&col_x| { + items + .iter() + .filter(|(_, i)| (i.x - col_x).abs() < cluster_threshold) + .count() + >= min_items_per_col + }) + .collect(); + + // Anti-paragraph safeguard for BodyFont mode: + // Paragraphs concentrate items at the left margin; tables distribute evenly. + // Reject if any single column has >60% of all items. + if mode == TableDetectionMode::BodyFont { + let total_items = items.len(); + for &col_x in &columns { + let count = items + .iter() + .filter(|(_, i)| (i.x - col_x).abs() < cluster_threshold) + .count(); + if count as f32 / total_items as f32 > 0.60 { + return vec![]; + } + } + } + + columns +} + +/// Find row boundaries by clustering Y positions +pub(crate) fn find_row_boundaries(items: &[(usize, &TextItem)]) -> Vec { + let mut y_positions: Vec = items.iter().map(|(_, i)| i.y).collect(); + y_positions.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); // Descending + + if y_positions.is_empty() { + return vec![]; + } + + // Cluster Y positions - items within a fraction of the median font size are same row. + // Using 0.8× median font keeps the threshold between intra-row gaps (~0pt) and + // inter-row gaps (≥1× font size), preventing row merging in uniform-spaced PDFs. + let cluster_threshold = { + let mut font_sizes: Vec = items.iter().map(|(_, i)| i.font_size).collect(); + font_sizes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_font = font_sizes[font_sizes.len() / 2]; + (median_font * 0.8).max(4.0) + }; + let mut rows = Vec::new(); + let mut cluster_items: Vec = vec![y_positions[0]]; + + for &y in &y_positions[1..] { + let cluster_center = cluster_items.iter().sum::() / cluster_items.len() as f32; + + if cluster_center - y >= cluster_threshold { + // End current cluster (note: Y is descending) + rows.push(cluster_center); + cluster_items = vec![y]; + } else { + cluster_items.push(y); + } + } + + if !cluster_items.is_empty() { + rows.push(cluster_items.iter().sum::() / cluster_items.len() as f32); + } + + rows +} + +/// Find which column index an X position belongs to +pub(crate) fn find_column_index(columns: &[f32], x: f32) -> Option { + // Calculate adaptive threshold based on column spacing + let threshold = if columns.len() >= 2 { + let min_gap = columns + .windows(2) + .map(|w| (w[1] - w[0]).abs()) + .fold(f32::INFINITY, f32::min); + (min_gap / 2.0).clamp(25.0, 50.0) + } else { + 50.0 + }; + + columns + .iter() + .enumerate() + .min_by(|(_, a), (_, b)| { + (x - *a) + .abs() + .partial_cmp(&(x - *b).abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .filter(|(_, col_x)| (x - *col_x).abs() < threshold) + .map(|(idx, _)| idx) +} + +/// Find which row index a Y position belongs to +pub(crate) fn find_row_index(rows: &[f32], y: f32) -> Option { + let threshold = 15.0; + rows.iter() + .enumerate() + .min_by(|(_, a), (_, b)| { + (y - *a) + .abs() + .partial_cmp(&(y - *b).abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .filter(|(_, row_y)| (y - *row_y).abs() < threshold) + .map(|(idx, _)| idx) +} + +/// Join cell items with subscript/superscript-aware spacing +/// Same logic as TextLine::text() but for table cells +pub(crate) fn join_cell_items(items: &[&TextItem]) -> String { + let mut result = String::new(); + + for (i, item) in items.iter().enumerate() { + let text = item.text.trim(); + if text.is_empty() { + continue; + } + + if result.is_empty() { + result.push_str(text); + } else { + let prev_item = items[i - 1]; + + // Don't add space before/after hyphens + let prev_ends_with_hyphen = result.ends_with('-'); + let curr_is_hyphen = text == "-"; + let curr_starts_with_hyphen = text.starts_with('-'); + + // Detect subscript/superscript: smaller font size and/or Y offset + let font_ratio = item.font_size / prev_item.font_size; + let reverse_font_ratio = prev_item.font_size / item.font_size; + let y_diff = (item.y - prev_item.y).abs(); + + // Current item is subscript/superscript (smaller than previous) + let is_sub_super = font_ratio < 0.85 && y_diff > 1.0; + // Previous item was subscript/superscript (returning to normal size) + let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; + + if prev_ends_with_hyphen + || curr_is_hyphen + || curr_starts_with_hyphen + || is_sub_super + || was_sub_super + { + result.push_str(text); + } else { + result.push(' '); + result.push_str(text); + } + } + } + + result +} + +/// Recover a header row for small-font tables by looking at body-font items +/// just above the table's first row. +/// +/// PDF tables often have header rows at the body font size while data rows use +/// a smaller font. Pass 1 (SmallFont) excludes the header because of the +/// font-size filter. This function looks upward from the table's first row for +/// body-font items that align with the table's columns, and prepends them. +pub(crate) fn recover_header_row( + table: &mut Table, + all_items: &[TextItem], + small_font_threshold: f32, +) { + if table.rows.is_empty() || table.columns.is_empty() { + return; + } + + let first_row_y = table.rows[0]; // highest Y (rows are descending) + + // Compute typical row spacing for gap threshold + let row_gap_limit = if table.rows.len() >= 2 { + let avg_spacing = + (table.rows[0] - table.rows[table.rows.len() - 1]) / (table.rows.len() - 1) as f32; + // Allow up to 2x average row spacing for the header gap + (avg_spacing * 2.0).clamp(10.0, 40.0) + } else { + 30.0 + }; + + // Find body-font items just above the first row + let header_candidates: Vec<(usize, &TextItem)> = all_items + .iter() + .enumerate() + .filter(|(_, item)| { + item.font_size > small_font_threshold + && item.y > first_row_y + && item.y <= first_row_y + row_gap_limit + }) + .collect(); + + if header_candidates.is_empty() { + return; + } + + // Group header candidates by Y (cluster within 5pt) + let mut header_y_groups: Vec<(f32, Vec<(usize, &TextItem)>)> = Vec::new(); + let mut sorted_candidates = header_candidates; + sorted_candidates.sort_by(|a, b| { + b.1.y + .partial_cmp(&a.1.y) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + for (idx, item) in &sorted_candidates { + let found = header_y_groups + .iter_mut() + .find(|(y, _)| (item.y - *y).abs() < 5.0); + if let Some((_, group)) = found { + group.push((*idx, item)); + } else { + header_y_groups.push((item.y, vec![(*idx, item)])); + } + } + + // Take the row closest to the table (lowest Y above first_row_y) + // header_y_groups is sorted by descending Y, so take the last one + let (header_y, header_items) = header_y_groups.last().unwrap(); + + // Map header items to table columns + let num_cols = table.columns.len(); + let mut header_cells: Vec = vec![String::new(); num_cols]; + let mut mapped_count = 0; + let mut header_indices = Vec::new(); + + for (idx, item) in header_items { + if let Some(col) = find_column_index(&table.columns, item.x) { + let text = item.text.trim(); + if !text.is_empty() { + if !header_cells[col].is_empty() { + header_cells[col].push(' '); + } + header_cells[col].push_str(text); + mapped_count += 1; + header_indices.push(*idx); + } + } + } + + // Require at least 2 columns populated to look like a real header row + let populated = header_cells.iter().filter(|c| !c.is_empty()).count(); + if populated < 2 || mapped_count < 2 { + return; + } + + // Prepend header row to the table + table.rows.insert(0, *header_y); + table.cells.insert(0, header_cells); + table.item_indices.extend(header_indices); +} diff --git a/src/tables/mod.rs b/src/tables/mod.rs new file mode 100644 index 0000000..06a5f9a --- /dev/null +++ b/src/tables/mod.rs @@ -0,0 +1,452 @@ +//! Table detection and formatting. +//! +//! Detects tabular data in PDF text items and converts to markdown tables. + +mod detect_heuristic; +mod detect_rects; +mod financial; +mod format; +mod grid; + +pub use detect_heuristic::detect_tables; +pub use detect_rects::detect_tables_from_rects; +pub use format::table_to_markdown; + +/// Detection mode controls thresholds for table validation. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum TableDetectionMode { + /// Existing behavior: items with font size smaller than body text + SmallFont, + /// New: body-font items with stricter structural criteria + BodyFont, +} + +/// A detected table. +#[derive(Debug, Clone)] +pub struct Table { + /// Column boundaries (x positions) + pub columns: Vec, + /// Row boundaries (y positions, descending order) + pub rows: Vec, + /// Cell contents indexed by (row, col) + pub cells: Vec>, + /// Items that belong to this table + pub item_indices: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ItemType, TextItem}; + + fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width: 10.0, + height: font_size, + font: "F1".into(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + fn make_char(text: &str, x: f32, y: f32, font_size: f32, width: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: font_size, + font: "F1".into(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + #[test] + fn test_table_detection() { + let items = vec![ + // Header row + make_item("Subject", 100.0, 500.0, 8.0), + make_item("Q1", 200.0, 500.0, 8.0), + make_item("Q2", 280.0, 500.0, 8.0), + make_item("Q3", 360.0, 500.0, 8.0), + // Data row 1 + make_item("Math", 100.0, 480.0, 8.0), + make_item("9.0", 200.0, 480.0, 8.0), + make_item("8.5", 280.0, 480.0, 8.0), + make_item("9.5", 360.0, 480.0, 8.0), + // Data row 2 + make_item("Science", 100.0, 460.0, 8.0), + make_item("8.0", 200.0, 460.0, 8.0), + make_item("9.0", 280.0, 460.0, 8.0), + make_item("8.5", 360.0, 460.0, 8.0), + // Data row 3 + make_item("English", 100.0, 440.0, 8.0), + make_item("9.5", 200.0, 440.0, 8.0), + make_item("9.0", 280.0, 440.0, 8.0), + make_item("9.5", 360.0, 440.0, 8.0), + ]; + + let tables = detect_tables(&items, 10.0, false); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].columns.len(), 4); + assert_eq!(tables[0].rows.len(), 4); + } + + #[test] + fn test_table_to_markdown() { + let table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0], + cells: vec![ + vec!["Header 1".into(), "Header 2".into()], + vec!["Cell 1".into(), "Cell 2".into()], + ], + item_indices: vec![], + }; + + let md = table_to_markdown(&table); + assert!(md.contains("| Header 1")); + assert!(md.contains("| ---")); + assert!(md.contains("| Cell 1")); + } + + #[test] + fn test_body_font_table_detected() { + let items = vec![ + // Header row + make_item("Name", 100.0, 500.0, 10.0), + make_item("Price", 200.0, 500.0, 10.0), + make_item("Qty", 300.0, 500.0, 10.0), + make_item("Total", 400.0, 500.0, 10.0), + // Data row 1 + make_item("Widget", 100.0, 480.0, 10.0), + make_item("5.00", 200.0, 480.0, 10.0), + make_item("10", 300.0, 480.0, 10.0), + make_item("50.00", 400.0, 480.0, 10.0), + // Data row 2 + make_item("Gadget", 100.0, 460.0, 10.0), + make_item("12.50", 200.0, 460.0, 10.0), + make_item("4", 300.0, 460.0, 10.0), + make_item("50.00", 400.0, 460.0, 10.0), + // Data row 3 + make_item("Gizmo", 100.0, 440.0, 10.0), + make_item("3.25", 200.0, 440.0, 10.0), + make_item("20", 300.0, 440.0, 10.0), + make_item("65.00", 400.0, 440.0, 10.0), + ]; + + let tables = detect_tables(&items, 10.0, false); + assert_eq!( + tables.len(), + 1, + "Body-font table should be detected by Pass 2" + ); + assert_eq!(tables[0].columns.len(), 4); + assert!(tables[0].rows.len() >= 3); + } + + #[test] + fn test_paragraph_not_falsely_detected() { + let items = vec![ + make_item( + "This is a paragraph of text that spans the full width", + 72.0, + 500.0, + 10.0, + ), + make_item( + "of the page and should not be detected as a table.", + 72.0, + 485.0, + 10.0, + ), + make_item( + "It continues for several lines with normal body text", + 72.0, + 470.0, + 10.0, + ), + make_item( + "that is left-aligned and has no columnar structure.", + 72.0, + 455.0, + 10.0, + ), + make_item( + "The paragraph keeps going with more content here.", + 72.0, + 440.0, + 10.0, + ), + make_item( + "And it has even more text on this line as well.", + 72.0, + 425.0, + 10.0, + ), + make_item( + "Finally the paragraph concludes with this last line.", + 72.0, + 410.0, + 10.0, + ), + make_item( + "One more line to have enough items for detection.", + 72.0, + 395.0, + 10.0, + ), + make_item( + "And another line of plain paragraph text content.", + 72.0, + 380.0, + 10.0, + ), + make_item( + "Last line of the paragraph ends here for the test.", + 72.0, + 365.0, + 10.0, + ), + ]; + + let tables = detect_tables(&items, 10.0, false); + assert_eq!( + tables.len(), + 0, + "Single-column paragraph must not be detected as table" + ); + } + + #[test] + fn test_word_level_paragraph_not_detected_as_table() { + let items = vec![ + // Line 1 + make_item("We", 72.0, 500.0, 10.0), + make_item("would", 95.0, 500.0, 10.0), + make_item("like", 145.0, 500.0, 10.0), + make_item("to", 180.0, 500.0, 10.0), + make_item("thank", 200.0, 500.0, 10.0), + make_item("all", 250.0, 500.0, 10.0), + make_item("the", 278.0, 500.0, 10.0), + make_item("practitioners", 305.0, 500.0, 10.0), + // Line 2 + make_item("and", 72.0, 485.0, 10.0), + make_item("researchers", 105.0, 485.0, 10.0), + make_item("across", 185.0, 485.0, 10.0), + make_item("the", 232.0, 485.0, 10.0), + make_item("University", 260.0, 485.0, 10.0), + make_item("of", 335.0, 485.0, 10.0), + make_item("Leeds", 355.0, 485.0, 10.0), + // Line 3 + make_item("Libraries", 72.0, 470.0, 10.0), + make_item("whose", 142.0, 470.0, 10.0), + make_item("contributions", 190.0, 470.0, 10.0), + make_item("made", 290.0, 470.0, 10.0), + make_item("this", 328.0, 470.0, 10.0), + make_item("report", 360.0, 470.0, 10.0), + // Line 4 + make_item("possible", 72.0, 455.0, 10.0), + make_item("Both", 140.0, 455.0, 10.0), + make_item("constituent", 178.0, 455.0, 10.0), + make_item("studies", 262.0, 455.0, 10.0), + make_item("were", 315.0, 455.0, 10.0), + make_item("approved", 350.0, 455.0, 10.0), + ]; + + let tables = detect_tables(&items, 10.0, false); + assert_eq!( + tables.len(), + 0, + "Word-level paragraph text must not be detected as table" + ); + } + + #[test] + fn test_large_data_table_not_rejected() { + let mut items = Vec::new(); + // Header row + items.push(make_item("Temp", 100.0, 800.0, 8.0)); + items.push(make_item("Pressure", 200.0, 800.0, 8.0)); + items.push(make_item("Volume", 300.0, 800.0, 8.0)); + items.push(make_item("Enthalpy", 400.0, 800.0, 8.0)); + + // 49 data rows + for i in 1..50 { + let y = 800.0 - (i as f32 * 12.0); + items.push(make_item(&format!("{}", -40 + i * 2), 100.0, y, 8.0)); + items.push(make_item( + &format!("{:.1}", 100.0 + i as f32 * 5.0), + 200.0, + y, + 8.0, + )); + items.push(make_item( + &format!("{:.3}", 0.05 + i as f32 * 0.01), + 300.0, + y, + 8.0, + )); + items.push(make_item( + &format!("{:.1}", 150.0 + i as f32 * 2.5), + 400.0, + y, + 8.0, + )); + } + + let tables = detect_tables(&items, 10.0, false); + assert_eq!(tables.len(), 1, "Large data table should not be rejected"); + assert!( + tables[0].rows.len() >= 40, + "Large table should preserve most rows, got {}", + tables[0].rows.len() + ); + } + + #[test] + fn test_uniform_spacing_rows_not_merged() { + let companies = [ + "SC Priority LLC", + "Craft Roofing Co", + "Alpha Roofing Inc", + "Beta Construction", + "Gamma Builders", + "Delta Roofing", + "Epsilon Contractors", + ]; + + let mut items = Vec::new(); + + // Header row at y=800 + items.push(make_item("No.", 50.0, 800.0, 8.0)); + items.push(make_item("Company", 120.0, 800.0, 8.0)); + items.push(make_item("Bid Amount", 350.0, 800.0, 8.0)); + + // 7 data rows, each 10pt apart (exactly the old threshold) + for (i, company) in companies.iter().enumerate() { + let y = 790.0 - (i as f32 * 10.0); + items.push(make_item(&format!("{}", i + 1), 50.0, y, 8.0)); + items.push(make_item(company, 120.0, y, 8.0)); + items.push(make_item(&format!("${},000", 100 + i * 10), 350.0, y, 8.0)); + } + + let tables = detect_tables(&items, 12.0, false); + assert_eq!(tables.len(), 1, "Should detect one table"); + assert_eq!( + tables[0].rows.len(), + 8, + "Each company must be on its own row, got {} rows instead of 8", + tables[0].rows.len() + ); + } + + #[test] + fn test_merge_adjacent_items() { + let items = vec![ + make_char("J", 310.0, 532.0, 13.3, 4.0), + make_char("u", 314.0, 532.0, 13.3, 4.4), + make_char("n", 318.4, 532.0, 13.3, 4.4), + make_char("e", 322.8, 532.0, 13.3, 3.5), + // word gap (2pt) + make_char("3", 328.3, 532.0, 13.3, 4.0), + make_char("0", 332.3, 532.0, 13.3, 4.0), + make_char(",", 336.3, 532.0, 13.3, 2.0), + // large column gap (40pt) + make_char("M", 378.3, 532.0, 13.3, 7.5), + make_char("a", 385.8, 532.0, 13.3, 4.0), + make_char("r", 389.8, 532.0, 13.3, 3.5), + ]; + + let (merged, map) = detect_heuristic::merge_adjacent_items(&items); + + assert_eq!( + merged.len(), + 2, + "Should produce 2 merged items, got {}", + merged.len() + ); + assert!( + merged[0].text.contains("June") && merged[0].text.contains("30"), + "First merged item should be 'June 30,' but got {:?}", + merged[0].text + ); + assert_eq!(merged[1].text, "Mar"); + + assert_eq!( + map[0].len(), + 7, + "First merged item should map to 7 original chars" + ); + assert_eq!( + map[1].len(), + 3, + "Second merged item should map to 3 original chars" + ); + } + + #[test] + fn test_per_char_financial_table_detected() { + let mut items = Vec::new(); + + // Per-character header row + for (i, c) in "Col1".chars().enumerate() { + items.push(make_char( + &c.to_string(), + 300.0 + i as f32 * 5.0, + 540.0, + 13.0, + 5.0, + )); + } + for (i, c) in "Col2".chars().enumerate() { + items.push(make_char( + &c.to_string(), + 400.0 + i as f32 * 5.0, + 540.0, + 13.0, + 5.0, + )); + } + for (i, c) in "Col3".chars().enumerate() { + items.push(make_char( + &c.to_string(), + 500.0 + i as f32 * 5.0, + 540.0, + 13.0, + 5.0, + )); + } + + // Data rows with multi-word items + let data = [ + ("Revenue", 520.0, "1,000", "2,000", "3,000"), + ("Expenses", 505.0, "500", "800", "1,200"), + ("Net Income", 490.0, "500", "1,200", "1,800"), + ("Taxes", 475.0, "100", "200", "300"), + ]; + + for (label, y, v1, v2, v3) in &data { + items.push(make_item(label, 50.0, *y, 12.0)); + items.push(make_item(v1, 310.0, *y, 12.0)); + items.push(make_item(v2, 410.0, *y, 12.0)); + items.push(make_item(v3, 510.0, *y, 12.0)); + } + + let tables = detect_tables(&items, 13.0, false); + assert!( + !tables.is_empty(), + "Per-character financial table should be detected" + ); + } +} diff --git a/src/text_utils.rs b/src/text_utils.rs new file mode 100644 index 0000000..7bb77ff --- /dev/null +++ b/src/text_utils.rs @@ -0,0 +1,368 @@ +//! Character classification and text utility functions. +//! +//! Pure helpers that operate on characters, strings, or `TextItem` slices. +//! No PDF parsing happens here — these are shared across the extraction +//! and markdown pipelines. + +use crate::types::TextItem; + +/// Check if a character is CJK (Chinese, Japanese, Korean). +/// CJK languages don't use spaces between words, so word-boundary +/// heuristics should not apply when CJK characters are involved. +pub(crate) fn is_cjk_char(c: char) -> bool { + matches!(c, + '\u{1100}'..='\u{11FF}' // Hangul Jamo + | '\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation + | '\u{3040}'..='\u{309F}' // Hiragana + | '\u{30A0}'..='\u{30FF}' // Katakana + | '\u{3130}'..='\u{318F}' // Hangul Compatibility Jamo + | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs + | '\u{AC00}'..='\u{D7AF}' // Hangul Syllables + | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs + | '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms + ) +} + +pub(crate) fn is_rtl_char(c: char) -> bool { + matches!(c, + '\u{0590}'..='\u{05FF}' // Hebrew + | '\u{0600}'..='\u{06FF}' // Arabic + | '\u{0700}'..='\u{074F}' // Syriac + | '\u{0750}'..='\u{077F}' // Arabic Supplement + | '\u{0780}'..='\u{07BF}' // Thaana + | '\u{07C0}'..='\u{07FF}' // NKo + | '\u{0800}'..='\u{083F}' // Samaritan + | '\u{0840}'..='\u{085F}' // Mandaic + | '\u{08A0}'..='\u{08FF}' // Arabic Extended-A + | '\u{FB1D}'..='\u{FB4F}' // Hebrew Presentation Forms + | '\u{FB50}'..='\u{FDFF}' // Arabic Presentation Forms-A + | '\u{FE70}'..='\u{FEFF}' // Arabic Presentation Forms-B + ) +} + +pub(crate) fn is_rtl_text(texts: I) -> bool +where + I: Iterator, + S: AsRef, +{ + let (mut rtl, mut ltr) = (0u32, 0u32); + for t in texts { + for c in t.as_ref().chars() { + if is_rtl_char(c) { + rtl += 1; + } else if c.is_alphabetic() && !is_cjk_char(c) { + ltr += 1; + } + } + } + rtl > 0 && rtl > ltr +} + +pub(crate) fn sort_line_items(items: &mut [TextItem]) { + let rtl = is_rtl_text(items.iter().map(|i| &i.text)); + if rtl { + items.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + items.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } +} + +/// Detect if a font name indicates bold style +/// Common patterns: "Bold", "Bd", "Black", "Heavy", "Demi", "Semi" (semi-bold) +pub fn is_bold_font(font_name: &str) -> bool { + let lower = font_name.to_lowercase(); + + // Check for common bold indicators + // Note: Need to be careful with "Oblique" not matching "Obl" + false positive for bold + lower.contains("bold") + || lower.contains("-bd") + || lower.contains("_bd") + || lower.contains("black") + || lower.contains("heavy") + || lower.contains("demibold") + || lower.contains("semibold") + || lower.contains("demi-bold") + || lower.contains("semi-bold") + || lower.contains("extrabold") + || lower.contains("ultrabold") + || lower.contains("medium") && !lower.contains("mediumitalic") // Some fonts use Medium for semi-bold +} + +/// Detect if a font name indicates italic/oblique style +/// Common patterns: "Italic", "It", "Oblique", "Obl", "Slant", "Inclined" +pub fn is_italic_font(font_name: &str) -> bool { + let lower = font_name.to_lowercase(); + + // Check for common italic indicators + lower.contains("italic") + || lower.contains("oblique") + || lower.contains("-it") + || lower.contains("_it") + || lower.contains("slant") + || lower.contains("inclined") + || lower.contains("kursiv") // German for italic +} + +/// Expand Unicode ligature characters to their component characters. +/// This makes extracted text more searchable and semantically correct. +pub(crate) fn expand_ligatures(text: &str) -> String { + // Strip null bytes and other control characters (except newline/tab) + let text = if text + .bytes() + .any(|b| b < 0x20 && b != b'\n' && b != b'\r' && b != b'\t') + { + text.chars() + .filter(|&c| c >= ' ' || c == '\n' || c == '\r' || c == '\t') + .collect::() + } else { + text.to_string() + }; + + let mut result = String::with_capacity(text.len()); + for ch in text.chars() { + match ch { + '\u{FB00}' => result.push_str("ff"), + '\u{FB01}' => result.push_str("fi"), + '\u{FB02}' => result.push_str("fl"), + '\u{FB03}' => result.push_str("ffi"), + '\u{FB04}' => result.push_str("ffl"), + '\u{FB05}' | '\u{FB06}' => result.push_str("st"), + _ => result.push(ch), + } + } + result +} + +/// Decode a PDF text string (ActualText, etc.) that may be UTF-16BE (BOM \xFE\xFF) +/// or PDFDocEncoding (Latin-1 superset). +pub(crate) fn decode_text_string(bytes: &[u8]) -> String { + if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { + // UTF-16BE with BOM + let utf16: Vec = bytes[2..] + .chunks_exact(2) + .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) + .collect(); + String::from_utf16_lossy(&utf16) + } else { + // PDFDocEncoding — identical to Latin-1 for the byte range we care about + bytes.iter().map(|&b| b as char).collect() + } +} + +/// Compute effective font size from base size and text matrix +/// Text matrix is [a, b, c, d, tx, ty] where a,d are scale factors +pub(crate) fn effective_font_size(base_size: f32, text_matrix: &[f32; 6]) -> f32 { + // The scale factor is typically the magnitude of the transformation + // For most PDFs, text_matrix[0] (a) is the horizontal scale + // and text_matrix[3] (d) is the vertical scale + let scale_x = (text_matrix[0].powi(2) + text_matrix[1].powi(2)).sqrt(); + let scale_y = (text_matrix[2].powi(2) + text_matrix[3].powi(2)).sqrt(); + // Use the larger of the two scales (usually they're equal for non-rotated text) + let scale = scale_x.max(scale_y); + base_size * scale +} + +/// Estimate the width of a text item, falling back to a character-count heuristic when width is 0. +pub(crate) fn effective_width(item: &TextItem) -> f32 { + if item.width > 0.0 { + item.width + } else { + item.text.chars().count() as f32 * item.font_size * 0.5 + } +} + +pub(crate) fn is_cid_font(font: &str) -> bool { + font.starts_with("C2_") || font.starts_with("C0_") +} + +/// Determine if two adjacent text items should be joined without a space +/// based on their physical positions on the page and character case. +/// Uses a hybrid approach: position-based with case-aware thresholds. +/// CID fonts emit one word per text operator with gaps ≈ 0 between words. +/// Non-CID (Type1/TrueType) fonts emit phrases or fragments. +pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { + // If either text explicitly has leading/trailing spaces, respect them + if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') { + return false; + } + + // Get the last character of previous and first character of current + let prev_last = prev_item.text.trim_end().chars().last(); + let curr_first = curr_item.text.trim_start().chars().next(); + + // Always join if current starts with punctuation that typically follows without space + // e.g., "www" + ".com" → "www.com", not "www .com" + if let Some(c) = curr_first { + if matches!(c, '.' | ',' | ';' | '!' | '?' | ')' | ']' | '}' | '\'') { + return true; + } + } + + // After colons, add space if followed by alphanumeric (typical label:value pattern) + // e.g., "Clave:" + "T9N2I6" → "Clave: T9N2I6" + if let (Some(p), Some(c)) = (prev_last, curr_first) { + if p == ':' && c.is_alphanumeric() { + return false; + } + } + + // When we have accurate width from font metrics, use a tight threshold + if prev_item.width > 0.0 { + let gap = if prev_item.x <= curr_item.x { + // LTR: prev is left of curr + curr_item.x - (prev_item.x + prev_item.width) + } else { + // RTL: prev is right of curr + prev_item.x - (curr_item.x + curr_item.width) + }; + let font_size = prev_item.font_size; + + // Never join across column-scale gaps + if gap > font_size * 3.0 { + return false; + } + + // CID fonts (C2_*, C0_*) emit one word per text operator with gaps ≈ 0 + // between words. Detect these and add spaces. Only applies to CID fonts — + // non-CID fonts (Type1/TrueType) emit phrases or fragments with small gaps + // from positioning imprecision and should NOT trigger this. + // Skip for CJK text — CJK languages don't use spaces between words. + let prev_chars = prev_item.text.trim().chars().count(); + let curr_chars = curr_item.text.trim().chars().count(); + let prev_last_char = prev_item.text.trim().chars().last(); + let curr_first_char = curr_item.text.trim().chars().next(); + let is_cjk = + prev_last_char.is_some_and(is_cjk_char) || curr_first_char.is_some_and(is_cjk_char); + + if !is_cjk && gap >= 0.0 && gap < font_size * 0.01 && is_cid_font(&prev_item.font) { + let prev_word_count = prev_item.text.split_whitespace().count(); + + if prev_word_count >= 3 { + // Multi-word phrase from a line-level CID operator — likely mid-word boundary + return gap < font_size * 0.15; + } + + // CID font: each text operator is a separate word. Always add space. + return false; + } + + // Numeric continuity: digits, commas, periods, and percent signs that + // are positioned close together are almost always a single number. + // e.g., "34,20" + "8" → "34,208", "+13." + "0" + "%" → "+13.0%" + // Use a generous threshold since word spaces in numbers are rare. + if let (Some(p), Some(c)) = (prev_last, curr_first) { + let prev_is_numeric = p.is_ascii_digit() || p == ',' || p == '.'; + let curr_is_numeric = c.is_ascii_digit() || c == '%' || c == '.'; + if prev_is_numeric && curr_is_numeric { + return gap < font_size * 0.3; + } + // Sign characters (+/-) followed by digits + if (p == '+' || p == '-') && c.is_ascii_digit() { + return gap < font_size * 0.3; + } + } + + // Single-character fragment joined to a multi-character item: use a + // moderately generous threshold to rejoin split words like "b" + "illion" + // or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words. + if (prev_chars == 1) != (curr_chars == 1) { + return gap < font_size * 0.20; + } + + // Both single-char: per-glyph positioning (character-by-character rendering). + // Intra-word gaps are ≈ 0, word boundaries are ≈ 0.15× font_size. + // For numeric chars (digits within "100,000"), use generous threshold. + // For alphabetic, use tight threshold (0.10) to reliably detect word + // boundaries in per-character PDFs like SEC filings. + if prev_chars == 1 && curr_chars == 1 { + if let (Some(p), Some(c)) = (prev_last, curr_first) { + let p_numeric = p.is_ascii_digit() || matches!(p, ',' | '.' | '%' | '+' | '-'); + let c_numeric = c.is_ascii_digit() || matches!(c, ',' | '.' | '%'); + if p_numeric && c_numeric { + return gap < font_size * 0.25; + } + } + return gap < font_size * 0.10; + } + + // With accurate widths, a gap < 15% of font size means glyphs are + // adjacent (same word). Anything larger is a deliberate space. + // For multi-char items with a lowercase→lowercase junction, use a + // slightly wider threshold (0.18) to avoid mid-word space injection + // with imprecise CID font metrics (e.g. "enterta"+"inment"). + // All-caps or mixed-case junctions keep the tighter 0.15 threshold + // to preserve word boundaries (e.g. "LCOE"+"WITH"). + if prev_item.text.trim().chars().count() >= 2 && curr_item.text.trim().chars().count() >= 2 + { + let prev_ends_lower = prev_item + .text + .trim() + .chars() + .last() + .is_some_and(|c| c.is_lowercase()); + let curr_starts_lower = curr_item + .text + .trim() + .chars() + .next() + .is_some_and(|c| c.is_lowercase()); + if prev_ends_lower && curr_starts_lower { + return gap < font_size * 0.18; + } + } + return gap < font_size * 0.15; + } + + // Fallback: estimate width from font size heuristics + let char_width = prev_item.font_size * 0.45; + + let prev_text_len = prev_item.text.chars().count() as f32; + let estimated_prev_width = prev_text_len * char_width; + + // Calculate expected end position of previous item + let prev_end_x = prev_item.x + estimated_prev_width; + + // Calculate gap between items + let gap = curr_item.x - prev_end_x; + + // Never join across column-scale gaps (fallback path) + if gap > char_width * 6.0 { + return false; + } + + // CJK text: always join adjacent items — CJK languages don't use spaces between words. + // The Latin case-based heuristics below would incorrectly insert spaces within CJK words. + let is_cjk = prev_last.is_some_and(is_cjk_char) || curr_first.is_some_and(is_cjk_char); + if is_cjk { + return gap < char_width * 0.8; + } + + // Use different thresholds based on character case + // Same-case sequences (ALL CAPS or all lowercase) are more likely to be + // word fragments that got split. Mixed case suggests word boundaries. + match (prev_last, curr_first) { + (Some(p), Some(c)) if p.is_alphabetic() && c.is_alphabetic() => { + let same_case = + (p.is_uppercase() && c.is_uppercase()) || (p.is_lowercase() && c.is_lowercase()); + if same_case { + // Same case: use generous threshold (likely same word fragment) + // e.g., "CONST" + "ANCIA" → "CONSTANCIA" + gap < char_width * 0.8 + } else if p.is_lowercase() && c.is_uppercase() { + // Lowercase to uppercase transition (e.g., "presente" → "CONSTANCIA") + // This is typically a word boundary. In Spanish/English, words don't + // transition from lowercase to uppercase mid-word. + // Always add a space for this case, regardless of position. + false + } else { + // Uppercase to lowercase (e.g., "REGISTRO" → "para") + // Use stricter threshold (likely word boundary) + gap < char_width * 0.3 + } + } + _ => { + // Non-alphabetic: use moderate threshold + gap < char_width * 0.5 + } + } +} diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..fe0dd13 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,237 @@ +//! Shared types used across the extraction and markdown pipelines. +//! +//! Centralises `TextItem`, `TextLine`, `PdfRect`, font-width / encoding +//! type aliases, and the `ItemType` enum so that every module can import +//! them from one place. + +use std::collections::HashMap; + +use crate::text_utils::should_join_items; + +// ── Font types (crate-internal) ────────────────────────────────────── + +/// Font encoding map: maps byte codes to Unicode characters +pub(crate) type FontEncodingMap = HashMap; + +/// All font encodings for a page +pub(crate) type PageFontEncodings = HashMap; + +/// Font width information extracted from PDF font dictionaries +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub(crate) struct FontWidthInfo { + /// Glyph widths: maps character code to width in font units + pub(crate) widths: HashMap, + /// Default width for glyphs not in the widths table + pub(crate) default_width: u16, + /// Width of the space character (code 32) if known + pub(crate) space_width: u16, + /// Whether this is a CID font (2-byte character codes) + pub(crate) is_cid: bool, + /// Scale factor to convert font units to text space units. + /// For Type1/TrueType: 0.001 (widths in 1000ths of em) + /// For Type3: FontMatrix[0] (e.g., 0.00048828125 for 2048-unit grid) + pub(crate) units_scale: f32, + /// Writing mode: 0 = horizontal (default), 1 = vertical + pub(crate) wmode: u8, +} + +/// All font width info for a page, keyed by font resource name +pub(crate) type PageFontWidths = HashMap; + +// ── Public types ───────────────────────────────────────────────────── + +/// Type of extracted item +#[derive(Debug, Clone, Default)] +pub enum ItemType { + /// Regular text content + #[default] + Text, + /// Image placeholder + Image, + /// Hyperlink (with URL) + Link(String), + /// Form field (name: value) + FormField, +} + +/// A rectangle from a PDF `re` operator (cell boundary, border, etc.) +#[derive(Debug, Clone)] +pub struct PdfRect { + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, + pub page: u32, +} + +/// A text item with position information +#[derive(Debug, Clone)] +pub struct TextItem { + /// The text content + pub text: String, + /// X position on page + pub x: f32, + /// Y position on page (PDF coordinates, origin at bottom-left) + pub y: f32, + /// Width of text + pub width: f32, + /// Height (approximated from font size) + pub height: f32, + /// Font name + pub font: String, + /// Font size + pub font_size: f32, + /// Page number (1-indexed) + pub page: u32, + /// Whether the font is bold + pub is_bold: bool, + /// Whether the font is italic + pub is_italic: bool, + /// Type of item (text, image, link) + pub item_type: ItemType, +} + +/// A line of text (grouped text items) +#[derive(Debug, Clone)] +pub struct TextLine { + pub items: Vec, + pub y: f32, + pub page: u32, +} + +impl TextLine { + pub fn text(&self) -> String { + self.text_with_formatting(false, false) + } + + /// Get text with optional bold/italic markdown formatting + pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String { + if !format_bold && !format_italic { + return self.text_plain(); + } + + let mut result = String::new(); + let mut current_bold = false; + let mut current_italic = false; + + for (i, item) in self.items.iter().enumerate() { + let text = item.text.as_str(); + let text_trimmed = text.trim(); + + // Skip empty items + if text_trimmed.is_empty() { + continue; + } + + // Determine spacing + let needs_space = if i == 0 || result.is_empty() { + false + } else { + let prev_item = &self.items[i - 1]; + self.needs_space_between(prev_item, item, &result) + }; + + // Preserve leading whitespace from the item text. + // Items like " means any person" have a leading space that indicates + // a word boundary. needs_space_between returns false for these (because + // space_already_exists), but we still need to emit the space since + // we push text_trimmed below (which strips it). + let has_leading_space = text.starts_with(' '); + + // Check for style changes + let item_bold = format_bold && item.is_bold; + let item_italic = format_italic && item.is_italic; + + // Close previous styles if they change + if current_italic && !item_italic { + result.push('*'); + current_italic = false; + } + if current_bold && !item_bold { + result.push_str("**"); + current_bold = false; + } + + // Add space: either from spacing logic or preserved from item text + if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) { + result.push(' '); + } + + // Open new styles + if item_bold && !current_bold { + result.push_str("**"); + current_bold = true; + } + if item_italic && !current_italic { + result.push('*'); + current_italic = true; + } + + result.push_str(text_trimmed); + } + + // Close any remaining open styles + if current_italic { + result.push('*'); + } + if current_bold { + result.push_str("**"); + } + + result + } + + /// Get plain text without formatting + fn text_plain(&self) -> String { + let mut result = String::new(); + for (i, item) in self.items.iter().enumerate() { + let text = item.text.as_str(); + if i == 0 { + result.push_str(text); + } else { + let prev_item = &self.items[i - 1]; + if self.needs_space_between(prev_item, item, &result) { + result.push(' '); + } + result.push_str(text); + } + } + result + } + + /// Determine if a space is needed between two items + fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool { + let text = item.text.as_str(); + + // Don't add space before/after hyphens for hyphenated words + let prev_ends_with_hyphen = result.ends_with('-'); + let curr_is_hyphen = text.trim() == "-"; + let curr_starts_with_hyphen = text.starts_with('-'); + + // Detect subscript/superscript: smaller font size and/or Y offset + let font_ratio = item.font_size / prev_item.font_size; + let reverse_font_ratio = prev_item.font_size / item.font_size; + let y_diff = (item.y - prev_item.y).abs(); + + let is_sub_super = font_ratio < 0.85 && y_diff > 1.0; + let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; + + // Use position-based spacing detection + let should_join = should_join_items(prev_item, item); + + // Check if space already exists + let prev_ends_with_space = result.ends_with(' '); + let curr_starts_with_space = text.starts_with(' '); + let space_already_exists = prev_ends_with_space || curr_starts_with_space; + + // Add space unless one of these conditions applies + !(prev_ends_with_hyphen + || curr_is_hyphen + || curr_starts_with_hyphen + || is_sub_super + || was_sub_super + || should_join + || space_already_exists) + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 9710284..18ccb22 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,7 +1,8 @@ //! Integration tests for pdf-to-markdown library use pdf_inspector::detector::{DetectionConfig, ScanStrategy}; -use pdf_inspector::extractor::{group_into_lines, TextLine}; +use pdf_inspector::extractor::group_into_lines; +use pdf_inspector::types::TextLine; use pdf_inspector::{ detect_pdf_type, extract_text, extract_text_with_positions, to_markdown, MarkdownOptions, PdfError, PdfType, TextItem, @@ -9,7 +10,7 @@ use pdf_inspector::{ // Helper to create test TextItems fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem { - use pdf_inspector::extractor::ItemType; + use pdf_inspector::types::ItemType; TextItem { text: text.to_string(), x,