fix(extractor): flag printable ASCII mojibake for OCR

Detect large prose-like pages whose printable text has implausibly low vowel and common-word rates, preventing broken ToUnicode output from being served as valid extraction. Add the issue fixture regression and bump Rust/N-API/npm patch versions.

Co-authored-by: Cursor <cursoragent@cursor.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 autobins = false
 authors = ["Firecrawl Team"]

napi/Cargo.lock

@@ -830,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
 [[package]]
 name = "pdf-inspector"
-version = "0.1.3"
+version = "0.1.4"
 dependencies = [
  "env_logger",
  "log",
@@ -845,7 +845,7 @@ dependencies = [
 
 [[package]]
 name = "pdf-inspector-napi"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "napi",
  "napi-build",

napi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector-napi"
-version = "0.2.2"
+version = "0.2.3"
 edition = "2021"
 
 [lib]

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.10",
+  "version": "1.9.11",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/lib.rs

@@ -3734,6 +3734,10 @@ struct PageTextQualityEvidence {
     replacement_chars: usize,
     replacement_spans: usize,
     longest_replacement_run: usize,
+    ascii_letters: usize,
+    ascii_vowels: usize,
+    ascii_word_tokens: usize,
+    ascii_common_word_hits: usize,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -3753,6 +3757,7 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
 
         let evidence = evidence_by_page.entry(item.page).or_default();
         evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
+        record_ascii_language_evidence(evidence, &item.text);
 
         match text_span_decoding_issue_kind(&item.text) {
             Some(TextSpanIssueKind::Strong) => {
@@ -3776,7 +3781,9 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
         if reasons_by_page.contains_key(&page) {
             continue;
         }
-        if page_replacement_evidence_needs_ocr(&evidence) {
+        if page_replacement_evidence_needs_ocr(&evidence)
+            || printable_ascii_mojibake_needs_ocr(&evidence)
+        {
             add_ocr_reason(
                 &mut reasons_by_page,
                 page,
@@ -3879,6 +3886,102 @@ fn replacement_text_stats(text: &str) -> (usize, usize) {
     (replacement, longest_run)
 }
 
+fn record_ascii_language_evidence(evidence: &mut PageTextQualityEvidence, text: &str) {
+    let mut word = String::new();
+
+    for ch in text.chars() {
+        if ch.is_ascii_alphabetic() {
+            evidence.ascii_letters += 1;
+            if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
+                evidence.ascii_vowels += 1;
+            }
+            word.push(ch.to_ascii_lowercase());
+        } else {
+            flush_ascii_word_evidence(evidence, &mut word);
+        }
+    }
+
+    flush_ascii_word_evidence(evidence, &mut word);
+}
+
+fn flush_ascii_word_evidence(evidence: &mut PageTextQualityEvidence, word: &mut String) {
+    const COMMON_WORDS: [&str; 46] = [
+        "the",
+        "and",
+        "that",
+        "for",
+        "with",
+        "from",
+        "this",
+        "are",
+        "not",
+        "our",
+        "was",
+        "were",
+        "which",
+        "will",
+        "shall",
+        "has",
+        "have",
+        "had",
+        "its",
+        "their",
+        "these",
+        "those",
+        "into",
+        "over",
+        "under",
+        "between",
+        "during",
+        "page",
+        "exhibit",
+        "certificate",
+        "company",
+        "agreement",
+        "securities",
+        "dated",
+        "period",
+        "ending",
+        "december",
+        "february",
+        "registered",
+        "holders",
+        "obligations",
+        "guarantee",
+        "deferred",
+        "compensation",
+        "telephone",
+        "pursuant",
+    ];
+
+    if word.len() >= 3 {
+        evidence.ascii_word_tokens += 1;
+        if COMMON_WORDS.contains(&word.as_str()) {
+            evidence.ascii_common_word_hits += 1;
+        }
+    }
+    word.clear();
+}
+
+fn printable_ascii_mojibake_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
+    // A broken ToUnicode bfrange can still produce entirely printable ASCII,
+    // so character-validity checks alone cannot catch it. Require a large,
+    // prose-sized sample and combine two independent language signals to keep
+    // short labels, identifiers, formulas, and non-Latin pages out of scope.
+    if evidence.ascii_letters < 600 || evidence.ascii_word_tokens < 80 {
+        return false;
+    }
+
+    if evidence.ascii_letters * 2 < evidence.chars {
+        return false;
+    }
+
+    let vowel_ratio_bps = evidence.ascii_vowels * 10_000 / evidence.ascii_letters;
+    let common_word_hit_bps = evidence.ascii_common_word_hits * 10_000 / evidence.ascii_word_tokens;
+
+    vowel_ratio_bps <= 3_000 && common_word_hit_bps <= 300
+}
+
 fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
     if evidence.replacement_chars == 0 || evidence.chars == 0 {
         return false;
@@ -6010,6 +6113,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_text_quality_flags_printable_ascii_mojibake() {
+        let shifted = "8VceZWZTReVW9VdZXReZdhZeYcVdaVTeeHVcZVd8EcVWVccVUHeT:iYZSZe VScfRcj CZdecfVehYZTYUVWZVdeYVcZXYedWY]UVcdW]X'eVcUVSeYVcVXZdecReRUR]]ed";
+        let items = vec![test_text_item_on_page(1, &shifted.repeat(12))];
+
+        let quality = analyze_text_quality(&items);
+
+        assert_eq!(quality.pages_needing_ocr, vec![1]);
+        assert_eq!(
+            quality.reasons_by_page.get(&1).cloned(),
+            Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
+        );
+    }
+
+    #[test]
+    fn test_text_quality_allows_large_printable_ascii_prose() {
+        let prose = "The company has entered into an agreement with registered holders during the period ending in December. ";
+        let items = vec![test_text_item_on_page(1, &prose.repeat(20))];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+    }
+
     #[test]
     fn test_text_quality_allows_clean_multilingual_and_latin1_text() {
         let items = vec![

tests/integration_tests.rs

@@ -9,7 +9,7 @@ use pdf_inspector::{
     extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
     extract_text_in_regions_mem, extract_text_with_positions, extract_text_with_positions_mem,
     process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
-    PdfType, TextItem,
+    PdfType, TextItem, OCR_REASON_SUSPECTED_GARBLED_TEXT,
 };
 use std::collections::HashSet;
 
@@ -2942,6 +2942,41 @@ fn test_extract_pages_markdown_gid_pages_need_ocr() {
     assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
 }
 
+#[test]
+fn test_extract_pages_markdown_flags_printable_ascii_mojibake() {
+    let buf = std::fs::read("tests/fixtures/text_simple__att10k.pdf").unwrap();
+
+    let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert!(result.pages[0].needs_ocr);
+    assert_eq!(
+        result.pages[0].ocr_reason.as_deref(),
+        Some(OCR_REASON_SUSPECTED_GARBLED_TEXT)
+    );
+    assert!(
+        result.pages[0].markdown.is_empty(),
+        "garbled direct text should be suppressed when OCR is needed"
+    );
+    assert_eq!(result.pages_needing_ocr, vec![1]);
+    assert_eq!(result.ocr_reasons_by_page.len(), 1);
+    assert_eq!(result.ocr_reasons_by_page[0].page, 1);
+    assert_eq!(
+        result.ocr_reasons_by_page[0].reasons,
+        vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()]
+    );
+
+    let full = process_pdf_mem(&buf).unwrap();
+    assert_eq!(full.pages_needing_ocr, vec![1]);
+    assert!(full.has_encoding_issues);
+    assert_eq!(full.ocr_reasons_by_page.len(), 1);
+    assert_eq!(full.ocr_reasons_by_page[0].page, 1);
+    assert_eq!(
+        full.ocr_reasons_by_page[0].reasons,
+        vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()]
+    );
+}
+
 #[test]
 fn test_extract_pages_markdown_classification_with_tables() {
     // nexo-price-en.pdf is known to have tables