diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index b2b2dfa..1391b40 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -1,6 +1,8 @@ //! CLI tool for PDF to Markdown conversion -use pdf_inspector::{process_pdf_with_config_pages, DetectionConfig, MarkdownOptions, PdfType}; +use pdf_inspector::{ + process_pdf_with_config_pages, DetectionConfig, LayoutComplexity, MarkdownOptions, PdfType, +}; use std::collections::HashSet; use std::env; use std::fs; @@ -42,6 +44,20 @@ fn parse_page_spec(spec: &str) -> Result, String> { Ok(pages) } +fn print_layout_info(layout: &LayoutComplexity) { + if layout.is_complex { + eprintln!("Layout: COMPLEX"); + if !layout.pages_with_tables.is_empty() { + eprintln!(" Pages with tables: {:?}", layout.pages_with_tables); + } + if !layout.pages_with_columns.is_empty() { + eprintln!(" Pages with columns: {:?}", layout.pages_with_columns); + } + } else { + eprintln!("Layout: simple"); + } +} + fn main() { env_logger::init(); let args: Vec = env::args().collect(); @@ -119,8 +135,20 @@ fn main() { .iter() .map(|p| p.to_string()) .collect(); + let table_pages: Vec = result + .layout + .pages_with_tables + .iter() + .map(|p| p.to_string()) + .collect(); + let col_pages: Vec = result + .layout + .pages_with_columns + .iter() + .map(|p| p.to_string()) + .collect(); println!( - r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"markdown":"{}"}}"#, + r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"markdown":"{}"}}"#, match result.pdf_type { PdfType::TextBased => "text_based", PdfType::Scanned => "scanned", @@ -132,6 +160,9 @@ fn main() { result.processing_time_ms, result.markdown.as_ref().map(|m| m.len()).unwrap_or(0), ocr_pages.join(","), + result.layout.is_complex, + table_pages.join(","), + col_pages.join(","), md_escaped ); } else if raw_output { @@ -159,6 +190,7 @@ fn main() { eprintln!("Type: TEXT-BASED (direct extraction)"); eprintln!("Pages: {}", result.page_count); eprintln!("Processing time: {}ms", result.processing_time_ms); + print_layout_info(&result.layout); if let Some(markdown) = &result.markdown { if let Some(output) = output_file { @@ -194,6 +226,7 @@ fn main() { eprintln!("Type: MIXED (partial text extraction)"); eprintln!("Pages: {}", result.page_count); eprintln!("Processing time: {}ms", result.processing_time_ms); + print_layout_info(&result.layout); if let Some(markdown) = &result.markdown { eprintln!(); diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index ee610d9..a8604df 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -23,6 +23,7 @@ use links::{extract_form_fields, extract_page_links}; // Re-export public types so existing `crate::extractor::X` paths keep working. pub use crate::text_utils::{is_bold_font, is_italic_font}; pub use crate::types::{ItemType, TextLine}; +pub(crate) use layout::detect_columns; pub use layout::group_into_lines; // --------------------------------------------------------------------------- diff --git a/src/lib.rs b/src/lib.rs index 494d07f..e2b2dcb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,7 @@ pub use extractor::{extract_text, extract_text_with_positions, extract_text_with pub use markdown::{ to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions, }; -pub use types::{PdfRect, TextItem}; +pub use types::{LayoutComplexity, PdfRect, TextItem}; use std::path::Path; @@ -46,6 +46,8 @@ pub struct PdfProcessResult { pub title: Option, /// Detection confidence score (0.0 - 1.0) pub confidence: f32, + /// Layout complexity analysis (tables, multi-column detection). + pub layout: LayoutComplexity, } /// Process a PDF file with smart detection and extraction @@ -71,6 +73,7 @@ pub fn process_pdf>(path: P) -> Result { // Step 2: Full extraction with position-aware reading order let (items, rects) = extractor::extract_text_with_positions_and_rects(&path, None)?; + let layout = compute_layout_complexity(&items, &rects); let markdown = to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects); @@ -83,6 +86,7 @@ pub fn process_pdf>(path: P) -> Result { @@ -96,14 +100,24 @@ pub fn process_pdf>(path: P) -> Result { // Try to extract what we can with position-aware reading order - let result = extractor::extract_text_with_positions_and_rects(&path, None).ok(); - let markdown = result.map(|(items, rects)| { - to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects) - }); + let extracted = extractor::extract_text_with_positions_and_rects(&path, None).ok(); + let (markdown, layout) = match extracted { + Some((items, rects)) => { + let layout = compute_layout_complexity(&items, &rects); + let md = to_markdown_from_items_with_rects( + items, + MarkdownOptions::default(), + &rects, + ); + (Some(md), layout) + } + None => (None, LayoutComplexity::default()), + }; PdfProcessResult { pdf_type, @@ -114,6 +128,7 @@ pub fn process_pdf>(path: P) -> Result>( PdfType::TextBased => { let (items, rects) = extractor::extract_text_with_positions_and_rects(&path, page_filter)?; + let layout = compute_layout_complexity(&items, &rects); let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); PdfProcessResult { @@ -166,6 +182,7 @@ pub fn process_pdf_with_config_pages>( pages_needing_ocr, title, confidence, + layout, } } PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { @@ -177,12 +194,20 @@ pub fn process_pdf_with_config_pages>( pages_needing_ocr, title, confidence, + layout: LayoutComplexity::default(), }, PdfType::Mixed => { - let result = extractor::extract_text_with_positions_and_rects(&path, page_filter).ok(); - let markdown = result.map(|(items, rects)| { - to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects) - }); + let extracted = + extractor::extract_text_with_positions_and_rects(&path, page_filter).ok(); + let (markdown, layout) = match extracted { + Some((items, rects)) => { + let layout = compute_layout_complexity(&items, &rects); + let md = + to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects); + (Some(md), layout) + } + None => (None, LayoutComplexity::default()), + }; PdfProcessResult { pdf_type, @@ -193,6 +218,7 @@ pub fn process_pdf_with_config_pages>( pages_needing_ocr, title, confidence, + layout, } } }; @@ -219,6 +245,7 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { // Step 2: Full extraction with position-aware reading order let (items, rects) = extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; + let layout = compute_layout_complexity(&items, &rects); let markdown = to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects); @@ -231,6 +258,7 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { pages_needing_ocr, title, confidence, + layout, } } PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { @@ -242,12 +270,22 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { pages_needing_ocr, title, confidence, + layout: LayoutComplexity::default(), }, PdfType::Mixed => { - let result = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); - let markdown = result.map(|(items, rects)| { - to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects) - }); + let extracted = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); + let (markdown, layout) = match extracted { + Some((items, rects)) => { + let layout = compute_layout_complexity(&items, &rects); + let md = to_markdown_from_items_with_rects( + items, + MarkdownOptions::default(), + &rects, + ); + (Some(md), layout) + } + None => (None, LayoutComplexity::default()), + }; PdfProcessResult { pdf_type, @@ -258,6 +296,7 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { pages_needing_ocr, title, confidence, + layout, } } }; @@ -286,6 +325,7 @@ pub fn process_pdf_mem_with_config( PdfType::TextBased => { let (items, rects) = extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; + let layout = compute_layout_complexity(&items, &rects); let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); PdfProcessResult { @@ -297,6 +337,7 @@ pub fn process_pdf_mem_with_config( pages_needing_ocr, title, confidence, + layout, } } PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { @@ -308,12 +349,19 @@ pub fn process_pdf_mem_with_config( pages_needing_ocr, title, confidence, + layout: LayoutComplexity::default(), }, PdfType::Mixed => { - let result = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); - let markdown = result.map(|(items, rects)| { - to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects) - }); + let extracted = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); + let (markdown, layout) = match extracted { + Some((items, rects)) => { + let layout = compute_layout_complexity(&items, &rects); + let md = + to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects); + (Some(md), layout) + } + None => (None, LayoutComplexity::default()), + }; PdfProcessResult { pdf_type, @@ -324,6 +372,7 @@ pub fn process_pdf_mem_with_config( pages_needing_ocr, title, confidence, + layout, } } }; @@ -331,6 +380,49 @@ pub fn process_pdf_mem_with_config( Ok(result) } +/// Analyse extracted items and rects for layout complexity. +fn compute_layout_complexity( + items: &[types::TextItem], + rects: &[types::PdfRect], +) -> LayoutComplexity { + use std::collections::HashMap; + + // --- Tables: count significant rects per page (w>=5, h>=5), flag pages with >6 --- + let mut rect_counts: HashMap = HashMap::new(); + for r in rects { + if r.width.abs() >= 5.0 && r.height.abs() >= 5.0 { + *rect_counts.entry(r.page).or_default() += 1; + } + } + let mut pages_with_tables: Vec = rect_counts + .into_iter() + .filter(|&(_, count)| count > 6) + .map(|(page, _)| page) + .collect(); + pages_with_tables.sort(); + + // --- Columns: run detect_columns() per page, flag pages with 2+ columns --- + let mut seen_pages: Vec = items.iter().map(|i| i.page).collect(); + seen_pages.sort(); + seen_pages.dedup(); + + let mut pages_with_columns: Vec = Vec::new(); + for page in seen_pages { + let cols = extractor::detect_columns(items, page); + if cols.len() >= 2 { + pages_with_columns.push(page); + } + } + + let is_complex = !pages_with_tables.is_empty() || !pages_with_columns.is_empty(); + + LayoutComplexity { + is_complex, + pages_with_tables, + pages_with_columns, + } +} + #[derive(Debug, thiserror::Error)] pub enum PdfError { #[error("IO error: {0}")] diff --git a/src/types.rs b/src/types.rs index fe0dd13..a268428 100644 --- a/src/types.rs +++ b/src/types.rs @@ -55,6 +55,20 @@ pub enum ItemType { FormField, } +/// Layout complexity analysis result. +/// +/// Callers can use this to decide whether the extracted markdown is reliable +/// or whether the PDF should be routed to an OCR pipeline instead. +#[derive(Debug, Clone, Default)] +pub struct LayoutComplexity { + /// True if any page has tables or multi-column text. + pub is_complex: bool, + /// 1-indexed pages where table borders were detected (rect count > 6). + pub pages_with_tables: Vec, + /// 1-indexed pages where 2+ text columns were detected. + pub pages_with_columns: Vec, +} + /// A rectangle from a PDF `re` operator (cell boundary, border, etc.) #[derive(Debug, Clone)] pub struct PdfRect { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 7933467..d8886ff 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -948,6 +948,7 @@ fn test_pages_needing_ocr_field_accessible() { pages_needing_ocr: vec![1, 3], title: None, confidence: 1.0, + layout: pdf_inspector::LayoutComplexity::default(), }; assert_eq!(process_result.pages_needing_ocr, vec![1, 3]); }