diff --git a/src/bin/detect_pdf.rs b/src/bin/detect_pdf.rs index d2957d9..89e90fb 100644 --- a/src/bin/detect_pdf.rs +++ b/src/bin/detect_pdf.rs @@ -11,6 +11,37 @@ use std::process; use std::time::Instant; /// Escape a string for embedding in a JSON string value. +fn format_detector_ocr_reasons(reasons: &std::collections::BTreeMap>) -> String { + reasons + .iter() + .map(|(page, page_reasons)| { + let reasons_json = page_reasons + .iter() + .map(|reason| format!(r#""{}""#, json_escape(reason))) + .collect::>() + .join(","); + format!(r#"{{"page":{},"reasons":[{}]}}"#, page, reasons_json) + }) + .collect::>() + .join(",") +} + +fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> String { + reasons + .iter() + .map(|entry| { + let reasons_json = entry + .reasons + .iter() + .map(|reason| format!(r#""{}""#, json_escape(reason))) + .collect::>() + .join(","); + format!(r#"{{"page":{},"reasons":[{}]}}"#, entry.page, reasons_json) + }) + .collect::>() + .join(",") +} + fn json_escape(s: &str) -> String { let mut out = String::with_capacity(s.len() + 16); for ch in s.chars() { @@ -117,11 +148,13 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) { .iter() .map(|p| p.to_string()) .collect(); + let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page); println!( - r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#, + r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#, pdf_type_str(&result.pdf_type), result.page_count, ocr_pages.join(","), + ocr_reasons, result.layout.is_complex, table_pages.join(","), col_pages.join(","), @@ -144,6 +177,9 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) { println!("Page count: {}", result.page_count); if !result.pages_needing_ocr.is_empty() { println!("Pages needing OCR: {:?}", result.pages_needing_ocr); + for entry in &result.ocr_reasons_by_page { + println!(" page {}: {}", entry.page, entry.reasons.join(", ")); + } } println!(); if result.layout.is_complex { @@ -184,8 +220,9 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) { .iter() .map(|p| p.to_string()) .collect(); + let ocr_reasons = format_detector_ocr_reasons(&result.ocr_reasons_by_page); println!( - r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"detection_time_ms":{}}}"#, + r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"detection_time_ms":{}}}"#, pdf_type_str(&result.pdf_type), result.page_count, result.pages_sampled, @@ -198,6 +235,7 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) { .unwrap_or_else(|| "null".to_string()), result.ocr_recommended, ocr_pages.join(","), + ocr_reasons, elapsed.as_millis() ); } else { @@ -232,6 +270,9 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) { result.pages_needing_ocr, result.page_count ); } + for (page, reasons) in &result.ocr_reasons_by_page { + println!(" page {}: {}", page, reasons.join(", ")); + } } if let Some(title) = &result.title { println!("Title: {}", title); diff --git a/src/detector.rs b/src/detector.rs index 92fedaa..6f9ea47 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -60,6 +60,10 @@ pub struct PdfTypeResult { /// 1-indexed page numbers that need OCR (image-only or insufficient text). /// Empty for TextBased. All pages for Scanned/ImageBased. Specific pages for Mixed. pub pages_needing_ocr: Vec, + /// Per-page explanation for `pages_needing_ocr`: 1-indexed page → reason + /// codes (`scanned`, `no_text`, `vector_text`, `suspected_garbled_text`). + /// Only contains pages that need OCR. + pub ocr_reasons_by_page: std::collections::BTreeMap>, } /// Configuration for PDF type detection @@ -382,7 +386,12 @@ pub(crate) fn detect_from_document( let analysis = if let Some(cached) = analysis_cache.get(&page_num) { cached.clone() } else if let Some(&page_id) = pages.get(&page_num) { - analyze_page_content(doc, page_id) + // Cache the fresh analysis so the reason-classification pass + // below sees the real signals (vector_text, etc.) instead of + // defaulting to "scanned". + let a = analyze_page_content(doc, page_id); + analysis_cache.insert(page_num, a.clone()); + a } else { continue; }; @@ -429,6 +438,9 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); if analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts { pages_needing_ocr.push(page_num); + // Cache so the reason pass reports suspected_garbled_text + // rather than defaulting to "scanned". + analysis_cache.insert(page_num, analysis); } } } @@ -436,6 +448,19 @@ pub(crate) fn detect_from_document( pages_needing_ocr.sort(); pages_needing_ocr.dedup(); + // Explain each OCR-flagged page. Pages we analyzed get a signal-derived + // reason; pages flagged only by whole-document classification (unsampled + // pages of a Scanned/ImageBased doc) default to `scanned`. + let mut ocr_reasons_by_page: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for &page_num in &pages_needing_ocr { + let reasons = match analysis_cache.get(&page_num) { + Some(analysis) => page_ocr_reasons(analysis), + None => vec![crate::OCR_REASON_SCANNED], + }; + ocr_reasons_by_page.insert(page_num, reasons.into_iter().map(String::from).collect()); + } + // Try to get title from metadata let title = get_document_title(doc); @@ -448,6 +473,7 @@ pub(crate) fn detect_from_document( title, ocr_recommended, pages_needing_ocr, + ocr_reasons_by_page, }) } @@ -487,7 +513,7 @@ fn distribute_pages(n: u32, total: u32) -> Vec { } /// Page content analysis result -#[derive(Clone)] +#[derive(Clone, Default)] struct PageAnalysis { text_operator_count: u32, has_images: bool, @@ -523,6 +549,31 @@ struct PageAnalysis { has_decodable_text_fonts: bool, } +/// Explain *why* a page needs OCR, from its content analysis. Priority: +/// undecodable fonts (`suspected_garbled_text`) and vector-outlined text +/// (`vector_text`) come first because they persist even when a text layer is +/// present; otherwise a page with no extractable text is `scanned` when an +/// image backs it or `no_text` when nothing does. +fn page_ocr_reasons(a: &PageAnalysis) -> Vec<&'static str> { + let mut reasons = Vec::new(); + if a.has_identity_h_no_tounicode || a.has_only_type3_fonts { + reasons.push(crate::OCR_REASON_SUSPECTED_GARBLED_TEXT); + } + if a.has_vector_text { + reasons.push(crate::OCR_REASON_VECTOR_TEXT); + } + if reasons.is_empty() { + let has_extractable_text = a.text_operator_count > 0 && a.unique_text_chars > 0; + if !has_extractable_text && !a.has_images && !a.has_template_image { + reasons.push(crate::OCR_REASON_NO_TEXT); + } else { + // Image-backed with no usable text, or too little text to trust. + reasons.push(crate::OCR_REASON_SCANNED); + } + } + reasons +} + /// Extracted font information from a Resource dictionary entry. /// Stores the properties needed for decodability/identity-h checks /// without holding a reference to the document. @@ -1809,6 +1860,64 @@ fn get_document_title(doc: &Document) -> Option { mod tests { use super::*; + #[test] + fn page_ocr_reasons_classify() { + // Scanned: no text, full-page image. + let scanned = PageAnalysis { + has_template_image: true, + ..Default::default() + }; + assert_eq!(page_ocr_reasons(&scanned), vec![crate::OCR_REASON_SCANNED]); + + // Image-only page (no template flag, but has an image). + let image_only = PageAnalysis { + has_images: true, + ..Default::default() + }; + assert_eq!( + page_ocr_reasons(&image_only), + vec![crate::OCR_REASON_SCANNED] + ); + + // No text, no image → no_text. + let blank = PageAnalysis::default(); + assert_eq!(page_ocr_reasons(&blank), vec![crate::OCR_REASON_NO_TEXT]); + + // Vector-outlined text. + let vector = PageAnalysis { + has_vector_text: true, + ..Default::default() + }; + assert_eq!( + page_ocr_reasons(&vector), + vec![crate::OCR_REASON_VECTOR_TEXT] + ); + + // Undecodable fonts → garbled, and it wins over the fall-through. + let garbled = PageAnalysis { + has_identity_h_no_tounicode: true, + has_images: true, + ..Default::default() + }; + assert_eq!( + page_ocr_reasons(&garbled), + vec![crate::OCR_REASON_SUSPECTED_GARBLED_TEXT] + ); + + // A page with real extractable text and an image is not flagged here + // as scanned/no_text (only reached for pages already needing OCR). + let text_with_image = PageAnalysis { + text_operator_count: 40, + unique_text_chars: 120, + has_images: true, + ..Default::default() + }; + assert_eq!( + page_ocr_reasons(&text_with_image), + vec![crate::OCR_REASON_SCANNED] + ); + } + #[test] fn test_scan_content_operators() { let mut uchars = HashSet::new(); diff --git a/src/lib.rs b/src/lib.rs index 5a9bad2..e473c3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,6 +71,18 @@ use tounicode::FontCMaps; /// broken font decoding or mojibake. pub const OCR_REASON_SUSPECTED_GARBLED_TEXT: &str = "suspected_garbled_text"; +/// OCR reason: the page is a scanned image (a full-page raster / image-only +/// page) with no usable text layer. +pub const OCR_REASON_SCANNED: &str = "scanned"; + +/// OCR reason: the page has no extractable text and no image to OCR — blank, +/// or content the parser cannot reach. +pub const OCR_REASON_NO_TEXT: &str = "no_text"; + +/// OCR reason: the page's text is drawn as vector outlines (path operators) +/// rather than real text operators, so it cannot be extracted as characters. +pub const OCR_REASON_VECTOR_TEXT: &str = "vector_text"; + // ========================================================================= // Result type // ========================================================================= @@ -3465,6 +3477,7 @@ fn process_document( let pages_needing_ocr = detection.pages_needing_ocr; let title = detection.title; let confidence = detection.confidence; + let detection_ocr_reasons = detection.ocr_reasons_by_page; // DetectOnly → return immediately if options.mode == ProcessMode::DetectOnly { @@ -3474,7 +3487,7 @@ fn process_document( page_count, processing_time_ms: start.elapsed().as_millis() as u64, pages_needing_ocr, - ocr_reasons_by_page: Vec::new(), + ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons), title, confidence, layout: LayoutComplexity::default(), @@ -3490,7 +3503,7 @@ fn process_document( page_count, processing_time_ms: start.elapsed().as_millis() as u64, pages_needing_ocr, - ocr_reasons_by_page: Vec::new(), + ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons), title, confidence, layout: LayoutComplexity::default(), @@ -3756,7 +3769,13 @@ fn process_document( page_count, processing_time_ms: start.elapsed().as_millis() as u64, pages_needing_ocr, - ocr_reasons_by_page: page_ocr_reasons_vec(text_quality_reasons_by_page), + ocr_reasons_by_page: { + // Detector reasons (scanned / no_text / vector_text / garbled) merged + // with the markdown-stage garbled detection, deduped per page. + let mut merged = detection_ocr_reasons; + merge_ocr_reasons(&mut merged, text_quality_reasons_by_page); + page_ocr_reasons_vec(merged) + }, title, confidence, layout, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 796af19..c5fd3fe 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1102,6 +1102,7 @@ fn test_pages_needing_ocr_field_accessible() { title: None, ocr_recommended: false, pages_needing_ocr: Vec::new(), + ocr_reasons_by_page: std::collections::BTreeMap::new(), }; assert!(detection_result.pages_needing_ocr.is_empty());