diff --git a/Cargo.toml b/Cargo.toml index 62f7694..bd8d051 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pdf-inspector" -version = "0.1.3" +version = "0.1.4" edition = "2021" autobins = false authors = ["Firecrawl Team"] diff --git a/napi/Cargo.lock b/napi/Cargo.lock index 98de023..b3f734f 100644 --- a/napi/Cargo.lock +++ b/napi/Cargo.lock @@ -830,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdf-inspector" -version = "0.1.3" +version = "0.1.4" dependencies = [ "env_logger", "log", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "pdf-inspector-napi" -version = "0.2.2" +version = "0.2.3" dependencies = [ "napi", "napi-build", diff --git a/napi/Cargo.toml b/napi/Cargo.toml index 466312a..3fec04b 100644 --- a/napi/Cargo.toml +++ b/napi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pdf-inspector-napi" -version = "0.2.2" +version = "0.2.3" edition = "2021" [lib] diff --git a/napi/package.json b/napi/package.json index af49b66..e3ac149 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.9.10", + "version": "1.9.11", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/src/lib.rs b/src/lib.rs index e56ebbd..e341b0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3734,6 +3734,10 @@ struct PageTextQualityEvidence { replacement_chars: usize, replacement_spans: usize, longest_replacement_run: usize, + ascii_letters: usize, + ascii_vowels: usize, + ascii_word_tokens: usize, + ascii_common_word_hits: usize, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -3753,6 +3757,7 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport { let evidence = evidence_by_page.entry(item.page).or_default(); evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count(); + record_ascii_language_evidence(evidence, &item.text); match text_span_decoding_issue_kind(&item.text) { Some(TextSpanIssueKind::Strong) => { @@ -3776,7 +3781,9 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport { if reasons_by_page.contains_key(&page) { continue; } - if page_replacement_evidence_needs_ocr(&evidence) { + if page_replacement_evidence_needs_ocr(&evidence) + || printable_ascii_mojibake_needs_ocr(&evidence) + { add_ocr_reason( &mut reasons_by_page, page, @@ -3879,6 +3886,102 @@ fn replacement_text_stats(text: &str) -> (usize, usize) { (replacement, longest_run) } +fn record_ascii_language_evidence(evidence: &mut PageTextQualityEvidence, text: &str) { + let mut word = String::new(); + + for ch in text.chars() { + if ch.is_ascii_alphabetic() { + evidence.ascii_letters += 1; + if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') { + evidence.ascii_vowels += 1; + } + word.push(ch.to_ascii_lowercase()); + } else { + flush_ascii_word_evidence(evidence, &mut word); + } + } + + flush_ascii_word_evidence(evidence, &mut word); +} + +fn flush_ascii_word_evidence(evidence: &mut PageTextQualityEvidence, word: &mut String) { + const COMMON_WORDS: [&str; 46] = [ + "the", + "and", + "that", + "for", + "with", + "from", + "this", + "are", + "not", + "our", + "was", + "were", + "which", + "will", + "shall", + "has", + "have", + "had", + "its", + "their", + "these", + "those", + "into", + "over", + "under", + "between", + "during", + "page", + "exhibit", + "certificate", + "company", + "agreement", + "securities", + "dated", + "period", + "ending", + "december", + "february", + "registered", + "holders", + "obligations", + "guarantee", + "deferred", + "compensation", + "telephone", + "pursuant", + ]; + + if word.len() >= 3 { + evidence.ascii_word_tokens += 1; + if COMMON_WORDS.contains(&word.as_str()) { + evidence.ascii_common_word_hits += 1; + } + } + word.clear(); +} + +fn printable_ascii_mojibake_needs_ocr(evidence: &PageTextQualityEvidence) -> bool { + // A broken ToUnicode bfrange can still produce entirely printable ASCII, + // so character-validity checks alone cannot catch it. Require a large, + // prose-sized sample and combine two independent language signals to keep + // short labels, identifiers, formulas, and non-Latin pages out of scope. + if evidence.ascii_letters < 600 || evidence.ascii_word_tokens < 80 { + return false; + } + + if evidence.ascii_letters * 2 < evidence.chars { + return false; + } + + let vowel_ratio_bps = evidence.ascii_vowels * 10_000 / evidence.ascii_letters; + let common_word_hit_bps = evidence.ascii_common_word_hits * 10_000 / evidence.ascii_word_tokens; + + vowel_ratio_bps <= 3_000 && common_word_hit_bps <= 300 +} + fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool { if evidence.replacement_chars == 0 || evidence.chars == 0 { return false; @@ -6010,6 +6113,31 @@ mod tests { ); } + #[test] + fn test_text_quality_flags_printable_ascii_mojibake() { + let shifted = "8VceZWZTReVW9VdZXReZdhZeYcVdaVTeeHVcZVd8EcVWVccVUHeT:iYZSZe VScfRcj CZdecfVehYZTYUVWZVdeYVcZXYedWY]UVcdW]X'eVcUVSeYVcVXZdecReRUR]]ed"; + let items = vec![test_text_item_on_page(1, &shifted.repeat(12))]; + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + assert_eq!( + quality.reasons_by_page.get(&1).cloned(), + Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()]) + ); + } + + #[test] + fn test_text_quality_allows_large_printable_ascii_prose() { + let prose = "The company has entered into an agreement with registered holders during the period ending in December. "; + let items = vec![test_text_item_on_page(1, &prose.repeat(20))]; + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + #[test] fn test_text_quality_allows_clean_multilingual_and_latin1_text() { let items = vec![ diff --git a/tests/fixtures/text_simple__att10k.pdf b/tests/fixtures/text_simple__att10k.pdf new file mode 100644 index 0000000..fe7062f Binary files /dev/null and b/tests/fixtures/text_simple__att10k.pdf differ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index a7de497..51ede90 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -9,7 +9,7 @@ use pdf_inspector::{ extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem, extract_text_with_positions, extract_text_with_positions_mem, process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, - PdfType, TextItem, + PdfType, TextItem, OCR_REASON_SUSPECTED_GARBLED_TEXT, }; use std::collections::HashSet; @@ -2942,6 +2942,41 @@ fn test_extract_pages_markdown_gid_pages_need_ocr() { assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed } +#[test] +fn test_extract_pages_markdown_flags_printable_ascii_mojibake() { + let buf = std::fs::read("tests/fixtures/text_simple__att10k.pdf").unwrap(); + + let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap(); + + assert_eq!(result.pages.len(), 1); + assert!(result.pages[0].needs_ocr); + assert_eq!( + result.pages[0].ocr_reason.as_deref(), + Some(OCR_REASON_SUSPECTED_GARBLED_TEXT) + ); + assert!( + result.pages[0].markdown.is_empty(), + "garbled direct text should be suppressed when OCR is needed" + ); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert_eq!(result.ocr_reasons_by_page.len(), 1); + assert_eq!(result.ocr_reasons_by_page[0].page, 1); + assert_eq!( + result.ocr_reasons_by_page[0].reasons, + vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()] + ); + + let full = process_pdf_mem(&buf).unwrap(); + assert_eq!(full.pages_needing_ocr, vec![1]); + assert!(full.has_encoding_issues); + assert_eq!(full.ocr_reasons_by_page.len(), 1); + assert_eq!(full.ocr_reasons_by_page[0].page, 1); + assert_eq!( + full.ocr_reasons_by_page[0].reasons, + vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()] + ); +} + #[test] fn test_extract_pages_markdown_classification_with_tables() { // nexo-price-en.pdf is known to have tables