mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add configurable ProcessMode (detect-only / analyze / full)

Browse Source 
Introduces a ProcessMode enum that controls how far the PDF pipeline
runs, enabling fast document triage without paying extraction or
markdown conversion costs. Exposed via --detect-only and --analyze
flags in both pdf2md and detect-pdf CLIs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-19 10:06:30 -08:00
co-authored by Claude Opus 4.6
parent e62444fee0
commit 7a5af1e9c3
8 changed files with 317 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bin/detect_pdf.rs
+115 -8
View File
@@ -1,6 +1,9 @@
//! CLI tool for detecting PDF type (text-based vs scanned)
use pdf_inspector::{detect_pdf_type, PdfType};
use pdf_inspector::{
detect_pdf_type, process_pdf_with_config_pages, DetectionConfig, MarkdownOptions, PdfType,
ProcessMode,
};
use std::env;
use std::process;
use std::time::Instant;
@@ -12,14 +15,123 @@ fn main() {
if args.len() < 2 {
eprintln!("Usage: {} <pdf_file>", args[0]);
eprintln!(" {} <pdf_file> --json", args[0]);
eprintln!(" {} <pdf_file> --analyze", args[0]);
eprintln!();
eprintln!("Options:");
eprintln!(" --json Output result as JSON");
eprintln!(" --analyze Also run layout analysis (tables, columns)");
process::exit(1);
}
let pdf_path = &args[1];
let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false);
let json_output = args.iter().any(|a| a == "--json");
let analyze = args.iter().any(|a| a == "--analyze");
let start = Instant::now();
if analyze {
run_analyze(pdf_path, json_output, start);
} else {
run_detect_only(pdf_path, json_output, start);
}
}
fn pdf_type_str(pdf_type: &PdfType) -> &'static str {
match pdf_type {
PdfType::TextBased => "text_based",
PdfType::Scanned => "scanned",
PdfType::ImageBased => "image_based",
PdfType::Mixed => "mixed",
}
}
fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
let md_options = MarkdownOptions {
process_mode: ProcessMode::Analyze,
..Default::default()
};
match process_pdf_with_config_pages(pdf_path, DetectionConfig::default(), md_options, None) {
Ok(result) => {
let elapsed = start.elapsed();
if json_output {
let ocr_pages: Vec<String> = result
.pages_needing_ocr
.iter()
.map(|p| p.to_string())
.collect();
let table_pages: Vec<String> = result
.layout
.pages_with_tables
.iter()
.map(|p| p.to_string())
.collect();
let col_pages: Vec<String> = result
.layout
.pages_with_columns
.iter()
.map(|p| p.to_string())
.collect();
println!(
r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#,
pdf_type_str(&result.pdf_type),
result.page_count,
ocr_pages.join(","),
result.layout.is_complex,
table_pages.join(","),
col_pages.join(","),
elapsed.as_millis()
);
} else {
println!("PDF Type Detection + Layout Analysis");
println!("=====================================");
println!("File: {}", pdf_path);
println!();
println!(
"Type: {}",
match result.pdf_type {
PdfType::TextBased => "TEXT-BASED (extractable text)",
PdfType::Scanned => "SCANNED (OCR needed)",
PdfType::ImageBased => "IMAGE-BASED (mostly images, OCR may help)",
PdfType::Mixed => "MIXED (some text, some images)",
}
);
println!("Page count: {}", result.page_count);
if !result.pages_needing_ocr.is_empty() {
println!("Pages needing OCR: {:?}", result.pages_needing_ocr);
}
println!();
if result.layout.is_complex {
println!("Layout: COMPLEX");
if !result.layout.pages_with_tables.is_empty() {
println!(" Pages with tables: {:?}", result.layout.pages_with_tables);
}
if !result.layout.pages_with_columns.is_empty() {
println!(
" Pages with columns: {:?}",
result.layout.pages_with_columns
);
}
} else {
println!("Layout: simple");
}
println!();
println!("Detection time: {}ms", elapsed.as_millis());
}
}
Err(e) => {
if json_output {
println!(r#"{{"error":"{}"}}"#, e);
} else {
eprintln!("Error: {}", e);
}
process::exit(1);
}
}
}
fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
match detect_pdf_type(pdf_path) {
Ok(result) => {
let elapsed = start.elapsed();
@@ -32,12 +144,7 @@ fn main() {
.collect();
println!(
r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"detection_time_ms":{}}}"#,
match result.pdf_type {
PdfType::TextBased => "text_based",
PdfType::Scanned => "scanned",
PdfType::ImageBased => "image_based",
PdfType::Mixed => "mixed",
},
pdf_type_str(&result.pdf_type),
result.page_count,
result.pages_sampled,
result.pages_with_text,
src/bin/pdf2md.rs
+63 -1
View File
@@ -2,6 +2,7 @@
use pdf_inspector::{
process_pdf_with_config_pages, DetectionConfig, LayoutComplexity, MarkdownOptions, PdfType,
ProcessMode,
};
use std::collections::HashSet;
use std::env;
@@ -75,6 +76,8 @@ fn main() {
eprintln!(" --raw Output only markdown (no headers)");
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
eprintln!(" --detect-only Only detect PDF type (no extraction)");
eprintln!(" --analyze Detect + extract + layout analysis (no markdown)");
process::exit(1);
}
@@ -82,6 +85,8 @@ fn main() {
let json_output = args.iter().any(|a| a == "--json");
let raw_output = args.iter().any(|a| a == "--raw");
let page_numbers = args.iter().any(|a| a == "--pages");
let detect_only = args.iter().any(|a| a == "--detect-only");
let analyze = args.iter().any(|a| a == "--analyze");
// Parse --select-pages value
let page_filter = args
@@ -107,8 +112,17 @@ fn main() {
.filter(|a| !a.starts_with("--"))
.map(|s| s.as_str());
let process_mode = if detect_only {
ProcessMode::DetectOnly
} else if analyze {
ProcessMode::Analyze
} else {
ProcessMode::Full
};
let md_options = MarkdownOptions {
include_page_numbers: page_numbers,
process_mode,
..Default::default()
};
@@ -119,7 +133,55 @@ fn main() {
page_filter.as_ref(),
) {
Ok(result) => {
if json_output {
if detect_only || analyze {
// Non-full modes: output detection/analysis info
let pdf_type_str = match result.pdf_type {
PdfType::TextBased => "text_based",
PdfType::Scanned => "scanned",
PdfType::ImageBased => "image_based",
PdfType::Mixed => "mixed",
};
if json_output {
let ocr_pages: Vec<String> = result
.pages_needing_ocr
.iter()
.map(|p| p.to_string())
.collect();
let table_pages: Vec<String> = result
.layout
.pages_with_tables
.iter()
.map(|p| p.to_string())
.collect();
let col_pages: Vec<String> = result
.layout
.pages_with_columns
.iter()
.map(|p| p.to_string())
.collect();
println!(
r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}]}}"#,
pdf_type_str,
result.page_count,
result.processing_time_ms,
ocr_pages.join(","),
result.layout.is_complex,
table_pages.join(","),
col_pages.join(","),
);
} else {
eprintln!("Type: {}", pdf_type_str);
eprintln!("Pages: {}", result.page_count);
eprintln!("Processing time: {}ms", result.processing_time_ms);
if !result.pages_needing_ocr.is_empty() {
eprintln!("Pages needing OCR: {:?}", result.pages_needing_ocr);
}
if analyze {
print_layout_info(&result.layout);
}
}
} else if json_output {
let md_escaped = result
.markdown
.as_ref()
src/extractor/layout.rs
+11 -3
View File
@@ -260,9 +260,17 @@ pub(crate) fn is_newspaper_layout(per_column_lines: &[Vec<TextLine>]) -> bool {
return false;
}
// Check Y-collision: count lines in the smallest column that have a
// Y-match in any other column. High collision with many lines = newspaper.
let y_tol = 3.0;
// Dense balanced columns (similar line counts) are newspaper regardless of Y-alignment.
// By this point table items are already removed, so two dense balanced columns
// of remaining text are independent prose flows.
let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0);
let balance_ratio = min_lines as f32 / max_lines as f32;
if balance_ratio > 0.7 {
return true;
}
// For unbalanced columns, fall back to Y-collision check
let y_tol = 5.0; // was 3.0 — handles government gazette typesetting variance
let (smallest_idx, _) = per_column_lines
.iter()
.enumerate()
src/extractor/mod.rs
+33
View File
@@ -822,6 +822,39 @@ mod tests {
assert!(is_newspaper_layout(&[col1, col2]));
}
#[test]
fn test_newspaper_layout_misaligned_baselines() {
// Two dense balanced columns with non-aligned Y positions (e.g. government gazettes
// where columns are independently typeset) → should still be newspaper
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
items: vec![TextItem {
text: "text".into(),
x,
y,
width: 100.0,
height: 12.0,
font: "F1".into(),
font_size: 12.0,
page,
is_bold: false,
is_italic: false,
item_type: ItemType::Text,
}],
};
// Col1 starts at Y=700, col2 starts at Y=685 (15pt offset — no Y-collision)
let col1: Vec<TextLine> = (0..20)
.map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
.collect();
let col2: Vec<TextLine> = (0..20)
.map(|i| make_line(685.0 - i as f32 * 14.0, 350.0, 1))
.collect();
assert!(is_newspaper_layout(&[col1, col2]));
}
#[test]
fn test_tabular_layout_detection() {
// Sparse columns (<15 lines) → tabular, not newspaper
src/lib.rs
+79 -84
View File
@@ -10,6 +10,7 @@ pub mod detector;
pub mod extractor;
pub mod glyph_names;
pub mod markdown;
pub mod process_mode;
pub mod tables;
pub mod text_utils;
pub mod tounicode;
@@ -23,6 +24,7 @@ pub use extractor::{extract_text, extract_text_with_positions, extract_text_with
pub use markdown::{
to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions,
};
pub use process_mode::ProcessMode;
pub use types::{LayoutComplexity, PdfRect, TextItem};
use std::path::Path;
@@ -166,17 +168,41 @@ pub fn process_pdf_with_config_pages<P: AsRef<Path>>(
let title = detection.title;
let confidence = detection.confidence;
// DetectOnly: return immediately after detection
if markdown_options.process_mode == ProcessMode::DetectOnly {
return Ok(PdfProcessResult {
pdf_type,
text: None,
markdown: None,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
title,
confidence,
layout: LayoutComplexity::default(),
});
}
let result = match pdf_type {
PdfType::TextBased => {
let (items, rects) =
extractor::extract_text_with_positions_and_rects(&path, page_filter)?;
let layout = compute_layout_complexity(&items, &rects);
let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects);
let markdown = if markdown_options.process_mode == ProcessMode::Analyze {
None
} else {
Some(to_markdown_from_items_with_rects(
items,
markdown_options,
&rects,
))
};
PdfProcessResult {
pdf_type,
text: None,
markdown: Some(markdown),
markdown,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
@@ -202,9 +228,16 @@ pub fn process_pdf_with_config_pages<P: AsRef<Path>>(
let (markdown, layout) = match extracted {
Some((items, rects)) => {
let layout = compute_layout_complexity(&items, &rects);
let md =
to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects);
(Some(md), layout)
let md = if markdown_options.process_mode == ProcessMode::Analyze {
None
} else {
Some(to_markdown_from_items_with_rects(
items,
markdown_options.clone(),
&rects,
))
};
(md, layout)
}
None => (None, LayoutComplexity::default()),
};
@@ -228,80 +261,11 @@ pub fn process_pdf_with_config_pages<P: AsRef<Path>>(
/// Process PDF from memory buffer
pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
validate_pdf_bytes(buffer)?;
// Step 1: Smart detection (fast, no full load)
let detection = detector::detect_pdf_type_mem(buffer)?;
let page_count = detection.page_count;
let pdf_type = detection.pdf_type;
let pages_needing_ocr = detection.pages_needing_ocr;
let title = detection.title;
let confidence = detection.confidence;
let result = match pdf_type {
PdfType::TextBased => {
// Step 2: Full extraction with position-aware reading order
let (items, rects) =
extractor::extract_text_with_positions_mem_and_rects(buffer, None)?;
let layout = compute_layout_complexity(&items, &rects);
let markdown =
to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects);
PdfProcessResult {
pdf_type,
text: None,
markdown: Some(markdown),
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
title,
confidence,
layout,
}
}
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
pdf_type,
text: None,
markdown: None,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
title,
confidence,
layout: LayoutComplexity::default(),
},
PdfType::Mixed => {
let extracted = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok();
let (markdown, layout) = match extracted {
Some((items, rects)) => {
let layout = compute_layout_complexity(&items, &rects);
let md = to_markdown_from_items_with_rects(
items,
MarkdownOptions::default(),
&rects,
);
(Some(md), layout)
}
None => (None, LayoutComplexity::default()),
};
PdfProcessResult {
pdf_type,
text: None,
markdown,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
title,
confidence,
layout,
}
}
};
Ok(result)
process_pdf_mem_with_config(
buffer,
DetectionConfig::default(),
MarkdownOptions::default(),
)
}
/// Process PDF from memory buffer with custom detection and markdown configuration
@@ -321,17 +285,41 @@ pub fn process_pdf_mem_with_config(
let title = detection.title;
let confidence = detection.confidence;
// DetectOnly: return immediately after detection
if markdown_options.process_mode == ProcessMode::DetectOnly {
return Ok(PdfProcessResult {
pdf_type,
text: None,
markdown: None,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
title,
confidence,
layout: LayoutComplexity::default(),
});
}
let result = match pdf_type {
PdfType::TextBased => {
let (items, rects) =
extractor::extract_text_with_positions_mem_and_rects(buffer, None)?;
let layout = compute_layout_complexity(&items, &rects);
let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects);
let markdown = if markdown_options.process_mode == ProcessMode::Analyze {
None
} else {
Some(to_markdown_from_items_with_rects(
items,
markdown_options,
&rects,
))
};
PdfProcessResult {
pdf_type,
text: None,
markdown: Some(markdown),
markdown,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr,
@@ -356,9 +344,16 @@ pub fn process_pdf_mem_with_config(
let (markdown, layout) = match extracted {
Some((items, rects)) => {
let layout = compute_layout_complexity(&items, &rects);
let md =
to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects);
(Some(md), layout)
let md = if markdown_options.process_mode == ProcessMode::Analyze {
None
} else {
Some(to_markdown_from_items_with_rects(
items,
markdown_options.clone(),
&rects,
))
};
(md, layout)
}
None => (None, LayoutComplexity::default()),
};
src/markdown/mod.rs
+4
View File
@@ -17,6 +17,7 @@ pub use convert::to_markdown_from_lines;
use std::collections::{HashMap, HashSet};
use crate::extractor::group_into_lines;
use crate::process_mode::ProcessMode;
use crate::types::TextItem;
use analysis::calculate_font_stats_from_items;
@@ -50,6 +51,8 @@ pub struct MarkdownOptions {
pub include_links: bool,
/// Insert page break markers (<!-- Page N -->) between pages
pub include_page_numbers: bool,
/// Controls how far the processing pipeline runs.
pub process_mode: ProcessMode,
}
impl Default for MarkdownOptions {
@@ -67,6 +70,7 @@ impl Default for MarkdownOptions {
include_images: true,
include_links: true,
include_page_numbers: false,
process_mode: ProcessMode::default(),
}
}
}
src/process_mode.rs
+11
View File
@@ -0,0 +1,11 @@
/// Controls how far the PDF processing pipeline runs.
#[derive(Debug, Clone, Default, PartialEq)]
pub enum ProcessMode {
/// Only detect PDF type. Very fast — no text extraction.
DetectOnly,
/// Detect type + extract text + compute layout complexity. Skips markdown.
Analyze,
/// Full pipeline: detect, extract, convert to markdown (default).
#[default]
Full,
}