diff --git a/docs/rust-api.md b/docs/rust-api.md index bc93a19..bd79937 100644 --- a/docs/rust-api.md +++ b/docs/rust-api.md @@ -380,6 +380,15 @@ provides the same native-only behavior through the OCR result/provenance shape; `Force` renders every selected page. Learned layout intentionally returns an explicit unsupported error in this lightweight pipeline. +For ambiguous mixed pages, `Auto` privately retains clean native fragments +instead of discarding them when OCR is selected. After recognition it compares +script-agnostic text quality, OCR confidence, character overlap, and material +new coverage. Exact native text wins over a duplicate or weak OCR hypothesis; +complementary image-backed text is fused; and pages where both candidates are +weak recommend the hosted document pipeline. Public native-only extraction +continues to suppress pages marked unreliable, and clean text documents pay no +renderer or model-initialization cost. + In `Auto`, pages routed only for suspicious font encoding or vectorized text first get a bounded positioned-text probe through PDFium. A credible recovered text layer with sufficient geometric page coverage skips rasterization and diff --git a/src/lib.rs b/src/lib.rs index 215798d..2b0ecce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -459,8 +459,15 @@ pub fn extract_pages_markdown_mem( buffer: &[u8], pages: Option<&[u32]>, ) -> Result { - extract_pages_markdown_mem_impl(buffer, pages, None, &MarkdownOptions::default(), false) - .map(|(result, _)| result) + extract_pages_markdown_mem_impl( + buffer, + pages, + None, + &MarkdownOptions::default(), + false, + false, + ) + .map(|(result, _)| result) } #[cfg(all(feature = "ocr", not(target_arch = "wasm32")))] @@ -476,6 +483,7 @@ pub(crate) fn extract_pages_markdown_mem_for_ocr( password, markdown_options, markdown_options.strip_headers_footers, + true, ) } @@ -485,6 +493,7 @@ fn extract_pages_markdown_mem_impl( password: Option<&str>, markdown_options: &MarkdownOptions, strip_repeated_headers_footers: bool, + preserve_ocr_candidates: bool, ) -> Result<(PagesExtractionResult, u32), PdfError> { validate_pdf_bytes(buffer)?; let (doc, page_count) = load_document_from_mem_with_password(buffer, password)?; @@ -663,7 +672,15 @@ fn extract_pages_markdown_mem_impl( results.push(PageMarkdown { page: page_0idx, - markdown: if needs_ocr { String::new() } else { md }, + // The public native extractor continues to suppress unreliable + // text. The OCR orchestrator retains clean partial text + // internally so it can compare/fuse it with OCR before deciding + // what is safe to return. + markdown: if needs_ocr && !preserve_ocr_candidates { + String::new() + } else { + md + }, needs_ocr, ocr_reason, }); diff --git a/src/vision/fusion.rs b/src/vision/fusion.rs index 50e76e4..e4e186d 100644 --- a/src/vision/fusion.rs +++ b/src/vision/fusion.rs @@ -6,6 +6,7 @@ use std::time::Instant; use thiserror::Error; use crate::markdown::{to_markdown_from_items_with_rects_and_page_count, MarkdownOptions}; +use crate::text_quality::{detect_encoding_issues, is_cid_garbage, is_garbage_text}; use crate::types::{ItemType, TextItem}; use crate::PageMarkdown; @@ -91,6 +92,73 @@ pub struct FusedPages { pub ocr_time_ms: u64, } +/// Origin of a trustworthy native-text candidate retained for adaptive OCR. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NativeCandidateOrigin { + /// Text produced by pdf-inspector's normal native extractor. + Extractor, + /// Positioned text independently recovered through PDFium. + Pdfium, +} + +impl NativeCandidateOrigin { + fn description(self) -> &'static str { + match self { + Self::Extractor => "native extraction", + Self::Pdfium => "PDFium native recovery", + } + } +} + +#[derive(Debug, Clone, Copy)] +struct TextCandidateQuality { + alphanumeric_chars: usize, + score: f32, +} + +/// Clean native text retained while an ambiguous page is compared with OCR. +#[derive(Debug, Clone)] +pub(crate) struct NativeFallbackCandidate { + markdown: String, + quality: TextCandidateQuality, + origin: NativeCandidateOrigin, +} + +impl NativeFallbackCandidate { + /// True when an independent native recovery is substantial enough to + /// cancel OCR for recoverable font/vector routing reasons. + pub(crate) fn is_complete_recovery(&self) -> bool { + self.quality.alphanumeric_chars >= 40 && self.quality.score >= 0.68 + } + + pub(crate) fn markdown(&self) -> &str { + &self.markdown + } + + pub(crate) fn is_stronger_than(&self, other: &Self) -> bool { + self.quality.alphanumeric_chars > other.quality.alphanumeric_chars + || (self.quality.alphanumeric_chars == other.quality.alphanumeric_chars + && self.quality.score > other.quality.score) + } +} + +/// Validates and scores native Markdown for possible post-OCR comparison. +/// +/// This intentionally uses script-agnostic evidence. A native candidate only +/// needs to be trustworthy, not necessarily complete: a clean native header +/// can still be fused with an image-backed OCR body. +pub(crate) fn assess_native_candidate( + markdown: String, + origin: NativeCandidateOrigin, +) -> Option { + let quality = assess_text_candidate(&markdown)?; + (quality.alphanumeric_chars >= 8).then_some(NativeFallbackCandidate { + markdown, + quality, + origin, + }) +} + /// Converts positioned OCR spans to Markdown through pdf-inspector's existing /// deterministic geometry, reading-order, table, and Markdown pipeline. /// @@ -116,13 +184,46 @@ pub fn ocr_page_to_markdown( /// OCR replaces pages whose native extraction was already rejected. On clean /// native pages (for example in `Force` mode), normalized duplicate OCR blocks /// are removed and only genuinely additional blocks are appended. Pages that -/// needed OCR but still have no credible local result recommend the hosted +/// needed OCR but still have no credible OCR result recommend the hosted /// document pipeline instead of silently presenting an empty result as final. pub fn fuse_ocr_pages( native_pages: &[PageMarkdown], ocr_run: &OcrRun, document_page_count: u32, options: &OcrFusionOptions, +) -> Result { + fuse_ocr_pages_impl( + native_pages, + ocr_run, + document_page_count, + options, + &BTreeMap::new(), + ) +} + +/// OCR-pipeline fusion with trustworthy partial native candidates. +pub(crate) fn fuse_ocr_pages_adaptive( + native_pages: &[PageMarkdown], + ocr_run: &OcrRun, + document_page_count: u32, + options: &OcrFusionOptions, + native_candidates: &BTreeMap, +) -> Result { + fuse_ocr_pages_impl( + native_pages, + ocr_run, + document_page_count, + options, + native_candidates, + ) +} + +fn fuse_ocr_pages_impl( + native_pages: &[PageMarkdown], + ocr_run: &OcrRun, + document_page_count: u32, + options: &OcrFusionOptions, + native_candidates: &BTreeMap, ) -> Result { validate_options(options)?; @@ -178,16 +279,31 @@ pub fn fuse_ocr_pages( document_page_count, ); let ocr_markdown = preserve_ocr_line_breaks(&ocr_markdown, local); - let (markdown, source) = if native.markdown.trim().is_empty() || native.needs_ocr { - (ocr_markdown, PageContentSource::Ocr) + let (markdown, source, adaptive_recommends_hosted) = if let Some(candidate) = + native_candidates + .get(&page_number) + .filter(|_| native.needs_ocr) + { + let choice = choose_adaptive_content( + candidate, + &ocr_markdown, + local.ocr.mean_confidence, + options.hosted_recommendation_confidence, + ); + warnings.push(choice.warning); + (choice.markdown, choice.source, choice.recommend_hosted) + } else if native.markdown.trim().is_empty() || native.needs_ocr { + (ocr_markdown, PageContentSource::Ocr, false) } else { - merge_native_and_ocr(&native.markdown, &ocr_markdown) + let (markdown, source) = merge_native_and_ocr(&native.markdown, &ocr_markdown); + (markdown, source, false) }; let weak_ocr = local .ocr .mean_confidence .is_none_or(|confidence| confidence < options.hosted_recommendation_confidence); - let recommend_hosted = native.needs_ocr && (markdown.trim().is_empty() || weak_ocr); + let recommend_hosted = native.needs_ocr + && (markdown.trim().is_empty() || weak_ocr || adaptive_recommends_hosted); if native.needs_ocr && markdown.trim().is_empty() { warnings.push("OCR produced no usable text".to_string()); } @@ -244,6 +360,155 @@ pub fn fuse_ocr_pages( }) } +struct AdaptiveContentChoice { + markdown: String, + source: PageContentSource, + warning: String, + recommend_hosted: bool, +} + +fn choose_adaptive_content( + native: &NativeFallbackCandidate, + ocr: &str, + ocr_confidence: Option, + weak_ocr_threshold: f32, +) -> AdaptiveContentChoice { + let ocr_quality = assess_text_candidate(ocr); + let weak_ocr = ocr_confidence.is_none_or(|confidence| confidence < weak_ocr_threshold); + if weak_ocr || ocr_quality.is_none() { + return AdaptiveContentChoice { + markdown: ensure_trailing_newline(native.markdown()), + source: PageContentSource::Native, + warning: format!( + "kept trustworthy {} because OCR was weak or unusable", + native.origin.description() + ), + recommend_hosted: true, + }; + } + + let ocr_quality = ocr_quality.expect("checked above"); + let overlap = content_overlap(native.markdown(), ocr); + let ocr_novel_chars = ocr_quality + .alphanumeric_chars + .saturating_sub(overlap.shared_chars); + let material_novelty = + ocr_novel_chars >= 2 && ocr_novel_chars * 8 >= ocr_quality.alphanumeric_chars.max(1); + let native_substantially_covered = + overlap.shared_chars * 4 >= native.quality.alphanumeric_chars.max(1) * 3; + + if native_substantially_covered && !material_novelty { + return AdaptiveContentChoice { + markdown: ensure_trailing_newline(native.markdown()), + source: PageContentSource::Native, + warning: format!( + "kept trustworthy {} because OCR added no material coverage", + native.origin.description() + ), + recommend_hosted: false, + }; + } + + // A shorter, materially lower-quality OCR hypothesis should not displace + // exact native text even when its mean confidence happens to be high. + if ocr_quality.score + 0.12 < native.quality.score + && ocr_quality.alphanumeric_chars * 10 + <= native.quality.alphanumeric_chars.saturating_mul(11) + { + return AdaptiveContentChoice { + markdown: ensure_trailing_newline(native.markdown()), + source: PageContentSource::Native, + warning: format!( + "kept higher-quality {} after comparing OCR", + native.origin.description() + ), + recommend_hosted: false, + }; + } + + let (markdown, source) = merge_native_and_ocr(native.markdown(), ocr); + let warning = match source { + PageContentSource::Native => format!( + "kept trustworthy {} because OCR duplicated its content", + native.origin.description() + ), + PageContentSource::Fused => format!( + "fused trustworthy {} with complementary OCR", + native.origin.description() + ), + PageContentSource::Ocr => unreachable!("merge never returns OCR-only content"), + }; + AdaptiveContentChoice { + markdown, + source, + warning, + recommend_hosted: false, + } +} + +#[derive(Debug, Clone, Copy)] +struct ContentOverlap { + shared_chars: usize, +} + +fn content_overlap(first: &str, second: &str) -> ContentOverlap { + let mut first_counts = BTreeMap::::new(); + for character in normalized_content_chars(first) { + *first_counts.entry(character).or_insert(0) += 1; + } + let mut second_counts = BTreeMap::::new(); + for character in normalized_content_chars(second) { + *second_counts.entry(character).or_insert(0) += 1; + } + let shared_chars = first_counts + .iter() + .map(|(character, count)| (*count).min(second_counts.get(character).copied().unwrap_or(0))) + .sum(); + ContentOverlap { shared_chars } +} + +fn normalized_content_chars(text: &str) -> impl Iterator + '_ { + text.chars() + .flat_map(char::to_lowercase) + .filter(|character| character.is_alphanumeric()) +} + +fn assess_text_candidate(markdown: &str) -> Option { + if markdown.trim().is_empty() + || is_garbage_text(markdown) + || is_cid_garbage(markdown) + || detect_encoding_issues(markdown) + { + return None; + } + + let alphanumeric_chars = markdown + .chars() + .filter(|character| character.is_alphanumeric()) + .count(); + if alphanumeric_chars == 0 { + return None; + } + let visible_chars = markdown + .chars() + .filter(|character| !character.is_whitespace()) + .count() + .max(1); + let density = alphanumeric_chars as f32 / visible_chars as f32; + let length_score = (alphanumeric_chars as f32 / 160.0).min(1.0); + let nonempty_lines = markdown + .lines() + .filter(|line| !line.trim().is_empty()) + .count() + .max(1); + let line_score = (alphanumeric_chars as f32 / nonempty_lines as f32 / 12.0).min(1.0); + let score = (0.45 + length_score * 0.25 + density * 0.20 + line_score * 0.10).min(1.0); + Some(TextCandidateQuality { + alphanumeric_chars, + score, + }) +} + /// Converts recognized line polygons to ordinary PDF-space text items. fn ocr_text_items(page: &RoutedOcrPage) -> (Vec, usize) { let mut discarded = 0usize; @@ -778,6 +1043,10 @@ mod tests { } } + fn native_candidate(markdown: &str) -> NativeFallbackCandidate { + assess_native_candidate(markdown.to_string(), NativeCandidateOrigin::Extractor).unwrap() + } + #[test] fn scanned_page_uses_geometry_ordered_ocr_and_provenance() { let native = [native(0, "", true)]; @@ -930,6 +1199,133 @@ mod tests { assert_eq!(result.pages[0].provenance.source, PageContentSource::Native); } + #[test] + fn adaptive_fallback_keeps_native_text_when_ocr_is_weak() { + let native = [native(0, "", true)]; + let run = run(vec![routed_page( + 1, + vec![span("Inv0ice total uncertain", 10.0, 0.3)], + Some(0.3), + )]); + let candidates = BTreeMap::from([( + 1, + native_candidate("Invoice total: $420.00\nPayment received\n"), + )]); + + let result = + fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates) + .unwrap(); + + assert_eq!(result.pages[0].provenance.source, PageContentSource::Native); + assert_eq!( + result.pages[0].markdown, + "Invoice total: $420.00\nPayment received\n" + ); + assert!(result.pages[0].provenance.hosted_recommended); + assert!(result.pages[0].provenance.warnings[0].contains("OCR was weak")); + } + + #[test] + fn adaptive_fallback_prefers_exact_native_text_over_duplicate_ocr() { + let native = [native(0, "", true)]; + let run = run(vec![routed_page( + 1, + vec![span("Invoice total 420.00 Payment received", 10.0, 0.98)], + Some(0.98), + )]); + let candidates = BTreeMap::from([( + 1, + native_candidate("Invoice total: $420.00\nPayment received\n"), + )]); + + let result = + fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates) + .unwrap(); + + assert_eq!(result.pages[0].provenance.source, PageContentSource::Native); + assert!(!result.pages[0].provenance.hosted_recommended); + assert_eq!(result.pages[0].markdown.matches("Invoice").count(), 1); + assert!(result.pages[0].provenance.warnings[0].contains("no material coverage")); + } + + #[test] + fn adaptive_fallback_fuses_short_novel_ocr_content() { + let native = [native(0, "", true)]; + let run = run(vec![routed_page( + 1, + vec![span("Status ready 42", 10.0, 0.98)], + Some(0.98), + )]); + let candidates = BTreeMap::from([(1, native_candidate("Status ready\n"))]); + + let result = + fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates) + .unwrap(); + + assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused); + assert!(result.pages[0].markdown.contains("42")); + assert!(!result.pages[0].provenance.hosted_recommended); + } + + #[test] + fn adaptive_fallback_recommends_hosted_for_high_confidence_garbage_ocr() { + let native = [native(0, "", true)]; + let garbage = "a@@b%%c&&d==e~~".repeat(12); + let run = run(vec![routed_page( + 1, + vec![span(&garbage, 10.0, 0.99)], + Some(0.99), + )]); + let candidates = BTreeMap::from([(1, native_candidate("Invoice total 420\n"))]); + + let result = + fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates) + .unwrap(); + + assert_eq!(result.pages[0].provenance.source, PageContentSource::Native); + assert!(result.pages[0].provenance.hosted_recommended); + assert!(!result.pages[0].markdown.contains("@@")); + } + + #[test] + fn adaptive_fallback_fuses_native_header_with_scanned_body() { + let native = [native(0, "", true)]; + let run = run(vec![routed_page( + 1, + vec![ + span("Quarterly account report", 10.0, 0.96), + span("March revenue 420 units", 30.0, 0.96), + span("April revenue 510 units", 50.0, 0.96), + ], + Some(0.96), + )]); + let candidates = BTreeMap::from([(1, native_candidate("# Quarterly account report\n"))]); + + let result = + fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates) + .unwrap(); + + assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused); + assert_eq!(result.pages[0].markdown.matches("Quarterly").count(), 1); + assert!(result.pages[0].markdown.contains("March revenue 420 units")); + assert!(result.pages[0].markdown.contains("April revenue 510 units")); + assert!(result.pages[0].provenance.warnings[0].contains("complementary")); + } + + #[test] + fn native_candidate_scoring_is_multilingual_and_rejects_garbage() { + assert!(assess_native_candidate( + "請求書 合計金額 4200円\n支払済み\n".to_string(), + NativeCandidateOrigin::Extractor, + ) + .is_some()); + assert!(assess_native_candidate( + "a@@b%%c&&d==e~~".repeat(12), + NativeCandidateOrigin::Extractor, + ) + .is_none()); + } + #[test] fn force_mode_appends_only_additional_ocr_blocks() { let native = [native(0, "Native title\n", false)]; diff --git a/src/vision/pipeline.rs b/src/vision/pipeline.rs index d3eb42b..76a7cbf 100644 --- a/src/vision/pipeline.rs +++ b/src/vision/pipeline.rs @@ -15,13 +15,17 @@ use crate::{ OCR_REASON_VECTOR_TEXT, }; +use super::fusion::{ + assess_native_candidate, fuse_ocr_pages_adaptive, NativeCandidateOrigin, + NativeFallbackCandidate, +}; use super::oar::onnx_runtime_library_path; use super::pdfium::PdfiumTextPage; use super::{ - fuse_ocr_pages, route_ocr_pages, run_ocr_pages, FusedPageMarkdown, HttpModelDownloadError, - HttpModelDownloader, ModelAcquireError, ModelStore, ModelStoreError, OarOcrEngine, OarOcrError, - OcrFusionError, OcrFusionOptions, OcrMode, OcrOptions, OcrRoutingError, OcrRun, OcrRunError, - PdfiumRenderer, RenderError, RenderOptions, PP_OCR_V6_SMALL, + route_ocr_pages, run_ocr_pages, FusedPageMarkdown, HttpModelDownloadError, HttpModelDownloader, + ModelAcquireError, ModelStore, ModelStoreError, OarOcrEngine, OarOcrError, OcrFusionError, + OcrFusionOptions, OcrMode, OcrOptions, OcrRoutingError, OcrRun, OcrRunError, PdfiumRenderer, + RenderError, RenderOptions, PP_OCR_V6_SMALL, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -230,22 +234,52 @@ pub fn process_pdf_with_ocr_mem( return Err(OcrPipelineError::InvalidSelectedPage { page: invalid }); } - let mut routed = route_ocr_pages( + let initially_routed = route_ocr_pages( options.ocr.mode, page_count, &native.pages_needing_ocr, selected_pages.as_deref(), )?; + // The OCR extractor retains clean native fragments on pages that still + // require OCR. Remove them from the ordinary native result now so Off mode + // preserves the existing suppression contract, while keeping a private + // candidate for post-OCR comparison and geometry-aware fusion. + let mut native_candidates = BTreeMap::::new(); + for page in &mut native.pages { + if !page.needs_ocr || page.markdown.trim().is_empty() { + continue; + } + let page_number = page.page + 1; + let markdown = std::mem::take(&mut page.markdown); + if may_preserve_extractor_candidate(page_number, &native.ocr_reasons_by_page) { + if let Some(candidate) = + assess_native_candidate(markdown, NativeCandidateOrigin::Extractor) + { + native_candidates.insert(page_number, candidate); + } + } + } + + let mut routed = initially_routed.clone(); let mut renderer = None; let mut recovered_natively = BTreeSet::new(); if options.ocr.mode == OcrMode::Auto { - let recovery_candidates = native_recovery_candidates(&routed, &native.ocr_reasons_by_page); - if !recovery_candidates.is_empty() { + let complete_recovery_candidates = + native_recovery_candidates(&initially_routed, &native.ocr_reasons_by_page); + let mut native_probe_pages: BTreeSet = + complete_recovery_candidates.iter().copied().collect(); + native_probe_pages.extend( + native_candidates + .keys() + .filter(|page| routed.contains(page)) + .copied(), + ); + if !native_probe_pages.is_empty() { let native_renderer = PdfiumRenderer::load()?; let recovered = native_renderer.extract_text_pages( buffer, - &recovery_candidates, + &native_probe_pages.iter().copied().collect::>(), options.password.as_deref(), )?; renderer = Some(native_renderer); @@ -256,19 +290,38 @@ pub fn process_pdf_with_ocr_mem( else { continue; }; - if !is_complete_native_recovery(&markdown) || !native_recovery_covers_page(&page) { - continue; - } - let Some(native_page) = native - .pages - .iter_mut() - .find(|entry| entry.page + 1 == page.page) + let Some(candidate) = + assess_native_candidate(markdown, NativeCandidateOrigin::Pdfium) else { continue; }; - native_page.markdown = markdown; - native_page.needs_ocr = false; - recovered_natively.insert(page.page); + let may_skip_ocr = complete_recovery_candidates.contains(&page.page) + && candidate.is_complete_recovery() + && native_recovery_covers_page(&page); + if may_skip_ocr { + let Some(native_page) = native + .pages + .iter_mut() + .find(|entry| entry.page + 1 == page.page) + else { + continue; + }; + native_page.markdown = candidate.markdown().to_string(); + native_page.needs_ocr = false; + native_candidates.remove(&page.page); + recovered_natively.insert(page.page); + } else { + match native_candidates.entry(page.page) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(candidate); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + if candidate.is_stronger_than(entry.get()) { + entry.insert(candidate); + } + } + } + } } routed.retain(|page| !recovered_natively.contains(page)); } @@ -303,7 +356,13 @@ pub fn process_pdf_with_ocr_mem( .markdown(page_markdown_options) .render_dpi(options.render.dpi) .hosted_recommendation_confidence(options.hosted_recommendation_confidence); - let mut fused = fuse_ocr_pages(&native.pages, &ocr_run, page_count, &fusion_options)?; + let mut fused = fuse_ocr_pages_adaptive( + &native.pages, + &ocr_run, + page_count, + &fusion_options, + &native_candidates, + )?; for page in &mut fused.pages { if recovered_natively.contains(&page.page) { page.provenance @@ -440,6 +499,22 @@ fn native_recovery_candidates(routed: &[u32], reasons: &[PageOcrReasons]) -> Vec .collect() } +fn may_preserve_extractor_candidate(page: u32, reasons: &[PageOcrReasons]) -> bool { + reasons + .iter() + .find(|entry| entry.page == page) + .is_some_and(|entry| { + !entry.reasons.is_empty() + && !entry + .reasons + .iter() + .any(|reason| reason == OCR_REASON_SUSPECTED_GARBLED_TEXT) + && entry.reasons.iter().any(|reason| { + reason == crate::OCR_REASON_SCANNED || reason == OCR_REASON_VECTOR_TEXT + }) + }) +} + fn credible_native_recovery( items: &[crate::TextItem], document_page_count: u32, @@ -469,31 +544,6 @@ fn credible_native_recovery( (!markdown.trim().is_empty()).then_some(markdown) } -fn is_complete_native_recovery(markdown: &str) -> bool { - let alphanumeric_chars = markdown - .chars() - .filter(|character| character.is_alphanumeric()) - .count(); - if alphanumeric_chars < 40 { - return false; - } - let visible_chars = markdown - .chars() - .filter(|character| !character.is_whitespace()) - .count() - .max(1); - let density = alphanumeric_chars as f32 / visible_chars as f32; - let length_score = (alphanumeric_chars as f32 / 160.0).min(1.0); - let nonempty_lines = markdown - .lines() - .filter(|line| !line.trim().is_empty()) - .count() - .max(1); - let line_score = (alphanumeric_chars as f32 / nonempty_lines as f32 / 12.0).min(1.0); - let score = (0.45 + length_score * 0.25 + density * 0.20 + line_score * 0.10).min(1.0); - score >= 0.68 -} - fn native_recovery_covers_page(page: &PdfiumTextPage) -> bool { const VERTICAL_BANDS: f32 = 6.0; @@ -858,6 +908,37 @@ mod tests { assert_eq!(remove_duplicate_table_lines(markdown), markdown); } + #[test] + fn extractor_candidates_require_clean_partial_content_reasons() { + let reasons = [ + PageOcrReasons { + page: 1, + reasons: vec![crate::OCR_REASON_SCANNED.to_string()], + }, + PageOcrReasons { + page: 2, + reasons: vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()], + }, + PageOcrReasons { + page: 3, + reasons: vec![OCR_REASON_VECTOR_TEXT.to_string()], + }, + PageOcrReasons { + page: 4, + reasons: vec![ + crate::OCR_REASON_SCANNED.to_string(), + OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string(), + ], + }, + ]; + + assert!(may_preserve_extractor_candidate(1, &reasons)); + assert!(!may_preserve_extractor_candidate(2, &reasons)); + assert!(may_preserve_extractor_candidate(3, &reasons)); + assert!(!may_preserve_extractor_candidate(4, &reasons)); + assert!(!may_preserve_extractor_candidate(5, &reasons)); + } + #[test] fn off_mode_extracts_native_text_without_runtime_side_effects() { let bytes = std::fs::read("tests/fixtures/thermo-freon12.pdf").unwrap(); @@ -904,9 +985,30 @@ mod tests { let result = process_pdf_with_ocr_mem(&bytes, OcrPdfOptions::new()).unwrap(); assert!(result.pages_routed_to_ocr.is_empty()); + assert!(result.markdown.trim().is_empty()); assert_eq!(result.pages_recommending_hosted, vec![1]); } + #[test] + fn partial_native_scan_text_is_retained_only_inside_ocr_orchestration() { + let bytes = std::fs::read("tests/fixtures/scan_with_native_header_text.pdf").unwrap(); + let public = crate::extract_pages_markdown_mem(&bytes, None).unwrap(); + assert!(public.pages[0].needs_ocr); + assert!(public.pages[0].markdown.trim().is_empty()); + + let (ocr, _) = crate::extract_pages_markdown_mem_for_ocr( + &bytes, + None, + None, + &MarkdownOptions::default(), + ) + .unwrap(); + assert!(ocr.pages[0].needs_ocr); + assert!(ocr.pages[0] + .markdown + .contains("Order Detail Report by Account")); + } + #[test] fn password_is_redacted_and_used_for_native_extraction() { let options = OcrPdfOptions::new().password("secret123"); diff --git a/tests/ocr_tests.rs b/tests/ocr_tests.rs index fd100ee..333f220 100644 --- a/tests/ocr_tests.rs +++ b/tests/ocr_tests.rs @@ -128,7 +128,16 @@ fn complete_ocr_pipeline_routes_and_assembles_a_scanned_fixture() { assert_eq!(result.pages_routed_to_ocr, vec![1]); assert!(!result.markdown.trim().is_empty()); - assert_eq!(result.pages[0].provenance.source, PageContentSource::Ocr); + assert!(result + .markdown + .contains("Order Date Item Code Description Status Unit Cost\n\n03/14/2024")); + assert!(result.markdown.contains("$482,110.40\n\n05/02/2024")); + assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused); + assert!(result.pages[0] + .provenance + .warnings + .iter() + .any(|warning| warning.contains("complementary OCR"))); assert_eq!( result.pages[0].provenance.ocr_model.as_ref().unwrap().name, PP_OCR_V6_SMALL.id