From 7a5af1e9c3c9c132214385e7a5ed6cd789c1cde3 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 19 Feb 2026 10:06:30 -0800 Subject: [PATCH] feat: Add configurable ProcessMode (detect-only / analyze / full) Introduces a ProcessMode enum that controls how far the PDF pipeline runs, enabling fast document triage without paying extraction or markdown conversion costs. Exposed via --detect-only and --analyze flags in both pdf2md and detect-pdf CLIs. Co-Authored-By: Claude Opus 4.6 --- src/bin/detect_pdf.rs | 123 ++++++++++++++++++++++++++-- src/bin/pdf2md.rs | 64 ++++++++++++++- src/extractor/layout.rs | 14 +++- src/extractor/mod.rs | 33 ++++++++ src/lib.rs | 163 ++++++++++++++++++------------------- src/markdown/mod.rs | 4 + src/process_mode.rs | 11 +++ tests/integration_tests.rs | 1 + 8 files changed, 317 insertions(+), 96 deletions(-) create mode 100644 src/process_mode.rs diff --git a/src/bin/detect_pdf.rs b/src/bin/detect_pdf.rs index 811e721..944be50 100644 --- a/src/bin/detect_pdf.rs +++ b/src/bin/detect_pdf.rs @@ -1,6 +1,9 @@ //! CLI tool for detecting PDF type (text-based vs scanned) -use pdf_inspector::{detect_pdf_type, PdfType}; +use pdf_inspector::{ + detect_pdf_type, process_pdf_with_config_pages, DetectionConfig, MarkdownOptions, PdfType, + ProcessMode, +}; use std::env; use std::process; use std::time::Instant; @@ -12,14 +15,123 @@ fn main() { if args.len() < 2 { eprintln!("Usage: {} ", args[0]); eprintln!(" {} --json", args[0]); + eprintln!(" {} --analyze", args[0]); + eprintln!(); + eprintln!("Options:"); + eprintln!(" --json Output result as JSON"); + eprintln!(" --analyze Also run layout analysis (tables, columns)"); process::exit(1); } let pdf_path = &args[1]; - let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false); + let json_output = args.iter().any(|a| a == "--json"); + let analyze = args.iter().any(|a| a == "--analyze"); let start = Instant::now(); + if analyze { + run_analyze(pdf_path, json_output, start); + } else { + run_detect_only(pdf_path, json_output, start); + } +} + +fn pdf_type_str(pdf_type: &PdfType) -> &'static str { + match pdf_type { + PdfType::TextBased => "text_based", + PdfType::Scanned => "scanned", + PdfType::ImageBased => "image_based", + PdfType::Mixed => "mixed", + } +} + +fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) { + let md_options = MarkdownOptions { + process_mode: ProcessMode::Analyze, + ..Default::default() + }; + + match process_pdf_with_config_pages(pdf_path, DetectionConfig::default(), md_options, None) { + Ok(result) => { + let elapsed = start.elapsed(); + + if json_output { + let ocr_pages: Vec = result + .pages_needing_ocr + .iter() + .map(|p| p.to_string()) + .collect(); + let table_pages: Vec = result + .layout + .pages_with_tables + .iter() + .map(|p| p.to_string()) + .collect(); + let col_pages: Vec = result + .layout + .pages_with_columns + .iter() + .map(|p| p.to_string()) + .collect(); + println!( + r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#, + pdf_type_str(&result.pdf_type), + result.page_count, + ocr_pages.join(","), + result.layout.is_complex, + table_pages.join(","), + col_pages.join(","), + elapsed.as_millis() + ); + } else { + println!("PDF Type Detection + Layout Analysis"); + println!("====================================="); + println!("File: {}", pdf_path); + println!(); + println!( + "Type: {}", + match result.pdf_type { + PdfType::TextBased => "TEXT-BASED (extractable text)", + PdfType::Scanned => "SCANNED (OCR needed)", + PdfType::ImageBased => "IMAGE-BASED (mostly images, OCR may help)", + PdfType::Mixed => "MIXED (some text, some images)", + } + ); + println!("Page count: {}", result.page_count); + if !result.pages_needing_ocr.is_empty() { + println!("Pages needing OCR: {:?}", result.pages_needing_ocr); + } + println!(); + if result.layout.is_complex { + println!("Layout: COMPLEX"); + if !result.layout.pages_with_tables.is_empty() { + println!(" Pages with tables: {:?}", result.layout.pages_with_tables); + } + if !result.layout.pages_with_columns.is_empty() { + println!( + " Pages with columns: {:?}", + result.layout.pages_with_columns + ); + } + } else { + println!("Layout: simple"); + } + println!(); + println!("Detection time: {}ms", elapsed.as_millis()); + } + } + Err(e) => { + if json_output { + println!(r#"{{"error":"{}"}}"#, e); + } else { + eprintln!("Error: {}", e); + } + process::exit(1); + } + } +} + +fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) { match detect_pdf_type(pdf_path) { Ok(result) => { let elapsed = start.elapsed(); @@ -32,12 +144,7 @@ fn main() { .collect(); println!( r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"detection_time_ms":{}}}"#, - match result.pdf_type { - PdfType::TextBased => "text_based", - PdfType::Scanned => "scanned", - PdfType::ImageBased => "image_based", - PdfType::Mixed => "mixed", - }, + pdf_type_str(&result.pdf_type), result.page_count, result.pages_sampled, result.pages_with_text, diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index 1391b40..c925ebe 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -2,6 +2,7 @@ use pdf_inspector::{ process_pdf_with_config_pages, DetectionConfig, LayoutComplexity, MarkdownOptions, PdfType, + ProcessMode, }; use std::collections::HashSet; use std::env; @@ -75,6 +76,8 @@ fn main() { eprintln!(" --raw Output only markdown (no headers)"); eprintln!(" --pages Insert page break markers ()"); eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)"); + eprintln!(" --detect-only Only detect PDF type (no extraction)"); + eprintln!(" --analyze Detect + extract + layout analysis (no markdown)"); process::exit(1); } @@ -82,6 +85,8 @@ fn main() { let json_output = args.iter().any(|a| a == "--json"); let raw_output = args.iter().any(|a| a == "--raw"); let page_numbers = args.iter().any(|a| a == "--pages"); + let detect_only = args.iter().any(|a| a == "--detect-only"); + let analyze = args.iter().any(|a| a == "--analyze"); // Parse --select-pages value let page_filter = args @@ -107,8 +112,17 @@ fn main() { .filter(|a| !a.starts_with("--")) .map(|s| s.as_str()); + let process_mode = if detect_only { + ProcessMode::DetectOnly + } else if analyze { + ProcessMode::Analyze + } else { + ProcessMode::Full + }; + let md_options = MarkdownOptions { include_page_numbers: page_numbers, + process_mode, ..Default::default() }; @@ -119,7 +133,55 @@ fn main() { page_filter.as_ref(), ) { Ok(result) => { - if json_output { + if detect_only || analyze { + // Non-full modes: output detection/analysis info + let pdf_type_str = match result.pdf_type { + PdfType::TextBased => "text_based", + PdfType::Scanned => "scanned", + PdfType::ImageBased => "image_based", + PdfType::Mixed => "mixed", + }; + + if json_output { + let ocr_pages: Vec = result + .pages_needing_ocr + .iter() + .map(|p| p.to_string()) + .collect(); + let table_pages: Vec = result + .layout + .pages_with_tables + .iter() + .map(|p| p.to_string()) + .collect(); + let col_pages: Vec = result + .layout + .pages_with_columns + .iter() + .map(|p| p.to_string()) + .collect(); + println!( + r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}]}}"#, + pdf_type_str, + result.page_count, + result.processing_time_ms, + ocr_pages.join(","), + result.layout.is_complex, + table_pages.join(","), + col_pages.join(","), + ); + } else { + eprintln!("Type: {}", pdf_type_str); + eprintln!("Pages: {}", result.page_count); + eprintln!("Processing time: {}ms", result.processing_time_ms); + if !result.pages_needing_ocr.is_empty() { + eprintln!("Pages needing OCR: {:?}", result.pages_needing_ocr); + } + if analyze { + print_layout_info(&result.layout); + } + } + } else if json_output { let md_escaped = result .markdown .as_ref() diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index a1be8eb..15526a4 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -260,9 +260,17 @@ pub(crate) fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { return false; } - // Check Y-collision: count lines in the smallest column that have a - // Y-match in any other column. High collision with many lines = newspaper. - let y_tol = 3.0; + // Dense balanced columns (similar line counts) are newspaper regardless of Y-alignment. + // By this point table items are already removed, so two dense balanced columns + // of remaining text are independent prose flows. + let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0); + let balance_ratio = min_lines as f32 / max_lines as f32; + if balance_ratio > 0.7 { + return true; + } + + // For unbalanced columns, fall back to Y-collision check + let y_tol = 5.0; // was 3.0 — handles government gazette typesetting variance let (smallest_idx, _) = per_column_lines .iter() .enumerate() diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index a8604df..00081af 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -822,6 +822,39 @@ mod tests { assert!(is_newspaper_layout(&[col1, col2])); } + #[test] + fn test_newspaper_layout_misaligned_baselines() { + // Two dense balanced columns with non-aligned Y positions (e.g. government gazettes + // where columns are independently typeset) → should still be newspaper + let make_line = |y: f32, x: f32, page: u32| TextLine { + y, + page, + items: vec![TextItem { + text: "text".into(), + x, + y, + width: 100.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }], + }; + + // Col1 starts at Y=700, col2 starts at Y=685 (15pt offset — no Y-collision) + let col1: Vec = (0..20) + .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) + .collect(); + let col2: Vec = (0..20) + .map(|i| make_line(685.0 - i as f32 * 14.0, 350.0, 1)) + .collect(); + + assert!(is_newspaper_layout(&[col1, col2])); + } + #[test] fn test_tabular_layout_detection() { // Sparse columns (<15 lines) → tabular, not newspaper diff --git a/src/lib.rs b/src/lib.rs index e2b2dcb..e650f99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod detector; pub mod extractor; pub mod glyph_names; pub mod markdown; +pub mod process_mode; pub mod tables; pub mod text_utils; pub mod tounicode; @@ -23,6 +24,7 @@ pub use extractor::{extract_text, extract_text_with_positions, extract_text_with pub use markdown::{ to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions, }; +pub use process_mode::ProcessMode; pub use types::{LayoutComplexity, PdfRect, TextItem}; use std::path::Path; @@ -166,17 +168,41 @@ pub fn process_pdf_with_config_pages>( let title = detection.title; let confidence = detection.confidence; + // DetectOnly: return immediately after detection + if markdown_options.process_mode == ProcessMode::DetectOnly { + return Ok(PdfProcessResult { + pdf_type, + text: None, + markdown: None, + page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + pages_needing_ocr, + title, + confidence, + layout: LayoutComplexity::default(), + }); + } + let result = match pdf_type { PdfType::TextBased => { let (items, rects) = extractor::extract_text_with_positions_and_rects(&path, page_filter)?; let layout = compute_layout_complexity(&items, &rects); - let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); + + let markdown = if markdown_options.process_mode == ProcessMode::Analyze { + None + } else { + Some(to_markdown_from_items_with_rects( + items, + markdown_options, + &rects, + )) + }; PdfProcessResult { pdf_type, text: None, - markdown: Some(markdown), + markdown, page_count, processing_time_ms: start.elapsed().as_millis() as u64, pages_needing_ocr, @@ -202,9 +228,16 @@ pub fn process_pdf_with_config_pages>( let (markdown, layout) = match extracted { Some((items, rects)) => { let layout = compute_layout_complexity(&items, &rects); - let md = - to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects); - (Some(md), layout) + let md = if markdown_options.process_mode == ProcessMode::Analyze { + None + } else { + Some(to_markdown_from_items_with_rects( + items, + markdown_options.clone(), + &rects, + )) + }; + (md, layout) } None => (None, LayoutComplexity::default()), }; @@ -228,80 +261,11 @@ pub fn process_pdf_with_config_pages>( /// Process PDF from memory buffer pub fn process_pdf_mem(buffer: &[u8]) -> Result { - let start = std::time::Instant::now(); - - validate_pdf_bytes(buffer)?; - - // Step 1: Smart detection (fast, no full load) - let detection = detector::detect_pdf_type_mem(buffer)?; - let page_count = detection.page_count; - let pdf_type = detection.pdf_type; - let pages_needing_ocr = detection.pages_needing_ocr; - let title = detection.title; - let confidence = detection.confidence; - - let result = match pdf_type { - PdfType::TextBased => { - // Step 2: Full extraction with position-aware reading order - let (items, rects) = - extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; - let layout = compute_layout_complexity(&items, &rects); - let markdown = - to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects); - - PdfProcessResult { - pdf_type, - text: None, - markdown: Some(markdown), - page_count, - processing_time_ms: start.elapsed().as_millis() as u64, - pages_needing_ocr, - title, - confidence, - layout, - } - } - PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { - pdf_type, - text: None, - markdown: None, - page_count, - processing_time_ms: start.elapsed().as_millis() as u64, - pages_needing_ocr, - title, - confidence, - layout: LayoutComplexity::default(), - }, - PdfType::Mixed => { - let extracted = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); - let (markdown, layout) = match extracted { - Some((items, rects)) => { - let layout = compute_layout_complexity(&items, &rects); - let md = to_markdown_from_items_with_rects( - items, - MarkdownOptions::default(), - &rects, - ); - (Some(md), layout) - } - None => (None, LayoutComplexity::default()), - }; - - PdfProcessResult { - pdf_type, - text: None, - markdown, - page_count, - processing_time_ms: start.elapsed().as_millis() as u64, - pages_needing_ocr, - title, - confidence, - layout, - } - } - }; - - Ok(result) + process_pdf_mem_with_config( + buffer, + DetectionConfig::default(), + MarkdownOptions::default(), + ) } /// Process PDF from memory buffer with custom detection and markdown configuration @@ -321,17 +285,41 @@ pub fn process_pdf_mem_with_config( let title = detection.title; let confidence = detection.confidence; + // DetectOnly: return immediately after detection + if markdown_options.process_mode == ProcessMode::DetectOnly { + return Ok(PdfProcessResult { + pdf_type, + text: None, + markdown: None, + page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + pages_needing_ocr, + title, + confidence, + layout: LayoutComplexity::default(), + }); + } + let result = match pdf_type { PdfType::TextBased => { let (items, rects) = extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; let layout = compute_layout_complexity(&items, &rects); - let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); + + let markdown = if markdown_options.process_mode == ProcessMode::Analyze { + None + } else { + Some(to_markdown_from_items_with_rects( + items, + markdown_options, + &rects, + )) + }; PdfProcessResult { pdf_type, text: None, - markdown: Some(markdown), + markdown, page_count, processing_time_ms: start.elapsed().as_millis() as u64, pages_needing_ocr, @@ -356,9 +344,16 @@ pub fn process_pdf_mem_with_config( let (markdown, layout) = match extracted { Some((items, rects)) => { let layout = compute_layout_complexity(&items, &rects); - let md = - to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects); - (Some(md), layout) + let md = if markdown_options.process_mode == ProcessMode::Analyze { + None + } else { + Some(to_markdown_from_items_with_rects( + items, + markdown_options.clone(), + &rects, + )) + }; + (md, layout) } None => (None, LayoutComplexity::default()), }; diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 5ddabda..432df89 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -17,6 +17,7 @@ pub use convert::to_markdown_from_lines; use std::collections::{HashMap, HashSet}; use crate::extractor::group_into_lines; +use crate::process_mode::ProcessMode; use crate::types::TextItem; use analysis::calculate_font_stats_from_items; @@ -50,6 +51,8 @@ pub struct MarkdownOptions { pub include_links: bool, /// Insert page break markers () between pages pub include_page_numbers: bool, + /// Controls how far the processing pipeline runs. + pub process_mode: ProcessMode, } impl Default for MarkdownOptions { @@ -67,6 +70,7 @@ impl Default for MarkdownOptions { include_images: true, include_links: true, include_page_numbers: false, + process_mode: ProcessMode::default(), } } } diff --git a/src/process_mode.rs b/src/process_mode.rs new file mode 100644 index 0000000..3ff1102 --- /dev/null +++ b/src/process_mode.rs @@ -0,0 +1,11 @@ +/// Controls how far the PDF processing pipeline runs. +#[derive(Debug, Clone, Default, PartialEq)] +pub enum ProcessMode { + /// Only detect PDF type. Very fast — no text extraction. + DetectOnly, + /// Detect type + extract text + compute layout complexity. Skips markdown. + Analyze, + /// Full pipeline: detect, extract, convert to markdown (default). + #[default] + Full, +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index d8886ff..0861d06 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -267,6 +267,7 @@ fn test_markdown_options_custom() { include_images: false, include_links: false, include_page_numbers: false, + ..Default::default() }; assert!(!opts.detect_headers); assert!(opts.detect_lists);