review: block table roles from heading promotion too

Add Table/TR/TH/TD/THead/TBody/TFoot to is_non_heading_content. When
table reconstruction falls back and cells reach the line loop as plain
text, a short isolated cell (a TH column header in particular) could be
promoted to a heading. Defensive: no change across either regression
corpus, so pure hardening for the fallback path.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/extractor/mod.rs

@@ -9,7 +9,7 @@ mod links;
 pub(crate) mod underline;
 mod xobjects;
 
-use crate::text_utils::{is_cjk_char, is_rtl_text};
+use crate::text_utils::is_rtl_text;
 use crate::tounicode::FontCMaps;
 use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
 use crate::PdfError;
@@ -527,136 +527,6 @@ fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
     saw_backtrack
 }
 
-/// Detect a tracked (letter-spaced) run of single-glyph items and derive its
-/// run-local space floor.
-///
-/// Display type set with tracking renders one glyph per show op; the merge
-/// loop's fixed thresholds (0.08-0.13 em) then read every letter gap as a
-/// word boundary and emit "H O W" instead of "HOW". Within such a run the
-/// gaps carry the real signal: letter gaps cluster tightly just above the
-/// fixed threshold, word gaps sit clearly higher. Returns (run_end_index,
-/// space_floor) when the run starting at `start` is tracked — spaces are
-/// then inserted only at gaps above the floor (infinity = single word).
-/// Normal text (multi-char items, or single-char runs with sub-threshold
-/// gaps) returns None and keeps the existing behavior.
-/// Han/Kana scripts write without inter-word spaces. Hangul (Korean) DOES
-/// space between words and deliberately stays out of this set — a Korean
-/// tracked run keeps normal word-boundary handling.
-fn is_spaceless_cjk(c: char) -> bool {
-    matches!(c,
-        '\u{3000}'..='\u{303F}'   // CJK Symbols and Punctuation
-        | '\u{3040}'..='\u{309F}' // Hiragana
-        | '\u{30A0}'..='\u{30FF}' // Katakana
-        | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs
-        | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs
-        | '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms
-    )
-}
-
-fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize, f32)> {
-    const MIN_GAPS: usize = 4;
-    let first = group[start];
-    if first.text.trim().chars().count() != 1 {
-        return None;
-    }
-    let fs = first.font_size;
-    if fs <= 0.0 {
-        return None;
-    }
-
-    // Walk the run under the SAME break conditions as the merge loop
-    // (size band, style equality, mergeable gap) so indices stay aligned.
-    let mut gaps: Vec<f32> = Vec::new();
-    let mut end_x = first.x + effective_merge_width(first);
-    let mut end = start;
-    for (offset, next) in group[start + 1..].iter().enumerate() {
-        if next.text.trim().chars().count() != 1 {
-            break;
-        }
-        if (next.font_size - fs).abs() > fs * 0.20 {
-            break;
-        }
-        if next.is_bold != first.is_bold
-            || next.is_italic != first.is_italic
-            || next.is_underline != first.is_underline
-            || next.is_strikeout != first.is_strikeout
-        {
-            break;
-        }
-        let gap = next.x - end_x;
-        if gap > fs * 0.5 || gap < -fs * 0.5 {
-            break;
-        }
-        gaps.push(gap / fs);
-        end_x = next.x + effective_merge_width(next);
-        end = start + 1 + offset;
-    }
-    if gaps.len() < 2 {
-        return None;
-    }
-
-    // Tracked signature: the run's TYPICAL gap clears the fixed space
-    // threshold (0.08) — the merge loop would break almost every letter
-    // pair into "words". Short runs (2-3 gaps: "H O W") demand a stricter
-    // shape — clearly wide, uniform, ALL-CAPS — because a genuine spaced
-    // sequence of single letters ("x y z" variables) has the same gap
-    // count; display tracking is a caps convention.
-    let mut sorted = gaps.clone();
-    sorted.sort_by(|a, b| a.total_cmp(b));
-    let median = sorted[sorted.len() / 2];
-    // Typographic convention gate, both tiers: display tracking is an
-    // all-caps convention, and Han/Kana never space between glyphs. Mixed-
-    // or lowercase Latin runs keep their boundaries because geometry alone
-    // cannot distinguish spaced singles ("A b c d e") from a tracked
-    // title-case word ("B u f f a l o").
-    let run_chars = || {
-        group[start..=end]
-            .iter()
-            .flat_map(|it| it.text.trim().chars())
-    };
-    let spaceless_cjk = run_chars().all(|c| is_spaceless_cjk(c) || !c.is_alphanumeric())
-        && run_chars().any(is_spaceless_cjk);
-    let all_caps = run_chars().all(|c| c.is_uppercase() || is_cjk_char(c) || !c.is_alphabetic());
-    if !(spaceless_cjk || all_caps) {
-        return None;
-    }
-
-    if gaps.len() >= MIN_GAPS {
-        if median <= 0.075 {
-            return None;
-        }
-    } else {
-        let uniform = sorted[sorted.len() - 1] <= sorted[0].max(0.01) * 1.4;
-        if median < 0.09 || !uniform {
-            return None;
-        }
-    }
-
-    // Han/Kana: no inter-glyph spaces, period — a nonuniform gap
-    // distribution (punctuation spacing, justification) must not
-    // manufacture word boundaries.
-    if spaceless_cjk {
-        return Some((end, f32::INFINITY));
-    }
-
-    // Word gaps, if present, form a second mode above the letter-gap
-    // cluster: split at the largest relative jump. Unimodal → one word.
-    let mut best_jump = 1.0f32;
-    let mut floor = f32::INFINITY;
-    for pair in sorted.windows(2) {
-        let (lo, hi) = (pair[0].max(0.01), pair[1].max(0.01));
-        let jump = hi / lo;
-        if jump > best_jump {
-            best_jump = jump;
-            floor = (lo + hi) / 2.0;
-        }
-    }
-    if best_jump < 1.4 {
-        floor = f32::INFINITY;
-    }
-    Some((end, floor * fs))
-}
-
 pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
     if items.is_empty() {
         return items;
@@ -704,14 +574,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
             let mut text = first.text.clone();
             let mut end_x = first.x + effective_merge_width(first);
 
-            // Tracked display text: run-local space floor overrides the
-            // fixed thresholds for this run's junctions (see helper).
-            let tracked = if *preserve_stream_order {
-                None
-            } else {
-                tracked_run_space_floor(group, i)
-            };
-
             let mut j = i + 1;
             while j < group.len() {
                 let next = group[j];
@@ -766,11 +628,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                 let needs_bullet_space = *preserve_stream_order
                     && is_standalone_bullet_text(&text)
                     && !next.text.trim().is_empty();
-                let effective_threshold = match tracked {
+                if needs_bullet_space || gap > threshold {
-                    Some((run_end, floor)) if j <= run_end => floor,
-                    _ => threshold,
-                };
-                if needs_bullet_space || gap > effective_threshold {
                     text.push(' ');
                 }
                 text.push_str(&next.text);
@@ -936,107 +794,6 @@ mod tests {
     use crate::types::{ItemType, PdfLine, TextLine};
     use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
 
-    /// Glyph-per-item run at `fs`=12 with the given inter-glyph gap (pt).
-    fn glyph_run(chars: &str, start_x: f32, glyph_w: f32, gap: f32) -> Vec<TextItem> {
-        let mut x = start_x;
-        let mut out = Vec::new();
-        for c in chars.chars() {
-            out.push(make_merge_item(&c.to_string(), x, glyph_w));
-            x += glyph_w + gap;
-        }
-        out
-    }
-
-    #[test]
-    fn tracked_caps_run_collapses_to_word() {
-        // Display tracking: every letter gap (0.19 em) clears the fixed
-        // space threshold — without the run-local floor this reads "H O W".
-        let items = glyph_run("HOW", 100.0, 10.0, 2.3);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "HOW");
-    }
-
-    #[test]
-    fn tracked_run_keeps_word_gaps_bimodal() {
-        // Letters at 0.19 em, word gaps at 0.42 em (below the 0.5 em item
-        // break): the split must land between the modes. Needs >=4 gaps to
-        // enter the bimodal tier — short runs use the strict uniform gate.
-        let mut items = glyph_run("ITISOK", 100.0, 8.0, 2.3);
-        for i in 2..6 {
-            items[i].x += 2.8; // word gap at T|I
-        }
-        for i in 4..6 {
-            items[i].x += 2.8; // word gap at S|O
-        }
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "IT IS OK");
-    }
-
-    #[test]
-    fn lowercase_spaced_singles_stay_words() {
-        // "x y z" variables: same gap shape but lowercase — the short-run
-        // caps requirement keeps genuine spaced singles apart.
-        let items = glyph_run("xyz", 100.0, 6.0, 2.3);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "x y z");
-    }
-
-    #[test]
-    fn kerned_singles_unaffected() {
-        // Tiny kerning gaps never triggered spaces before and still don't.
-        let items = glyph_run("WORD", 100.0, 8.0, 0.3);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "WORD");
-    }
-
-    #[test]
-    fn long_lowercase_spaced_singles_keep_boundaries() {
-        // Review: a 5+ single-letter lowercase list has the tracked gap
-        // shape at any length — the convention gate must protect it in
-        // the >=4-gap tier too.
-        let items = glyph_run("abcde", 100.0, 6.0, 2.3);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "a b c d e");
-    }
-
-    #[test]
-    fn han_run_with_nonuniform_gaps_never_gains_spaces() {
-        // Review: a bimodal gap distribution (justification, punctuation
-        // spacing) must not manufacture word boundaries in Han text.
-        let mut items = glyph_run("北京时事快报", 100.0, 12.0, 1.4);
-        for item in items.iter_mut().skip(3) {
-            item.x += 3.0; // wide gap after the third glyph
-        }
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "北京时事快报");
-    }
-
-    #[test]
-    fn uppercase_leading_spaced_singles_keep_boundaries() {
-        // "A b c d e" is indistinguishable from a title-case tracked word
-        // without reliable tracking metadata, so preserve its boundaries.
-        let items = glyph_run("Abcde", 100.0, 7.0, 2.3);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "A b c d e");
-    }
-
-    #[test]
-    fn cjk_glyph_run_collapses_without_spaces() {
-        // CJK sets one glyph per item with loose gaps; CJK uses no spaces,
-        // and the non-alphabetic run passes the caps gate.
-        let items = glyph_run("北京时事", 100.0, 12.0, 1.4);
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1);
-        assert_eq!(merged[0].text, "北京时事");
-    }
-
     fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
         TextItem {
             text: text.into(),

src/lib.rs

@@ -39,7 +39,6 @@ pub mod markdown;
 pub mod process_mode;
 pub mod structure_tree;
 pub mod tables;
-mod text_quality;
 pub mod text_utils;
 pub mod tounicode;
 pub mod types;
@@ -61,10 +60,6 @@ pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
 use lopdf::Document;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::path::Path;
-use text_quality::{
-    analyze_text_quality, detect_encoding_issues, is_cid_garbage, is_garbage_text,
-    region_items_have_decoding_issue,
-};
 use tounicode::FontCMaps;
 
 /// OCR reason emitted when the extracted text layer appears garbled due to
@@ -3708,15 +3703,283 @@ fn process_document(
 // Internal helpers
 // =========================================================================
 
+/// Detect broken font encodings in extracted markdown text.
+///
+/// Two heuristics:
+/// 1. **U+FFFD**: Any replacement character indicates decode failures.
+/// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a
+///    word separator due to broken ToUnicode CMaps. Triggers when either:
+///    - More than 50% of `$` are between letters (clear substitution pattern), OR
+///    - More than 20 letter-dollar-letter occurrences (even if some `$` are also
+///      used as trailing/leading separators, 20+ is far beyond normal financial text).
+fn detect_encoding_issues(markdown: &str) -> bool {
+    // Heuristic 1: U+FFFD replacement characters
+    if markdown.contains('\u{FFFD}') {
+        return true;
+    }
+
+    // Heuristic 2: dollar-as-space pattern
+    if has_dollar_as_space_pattern(markdown) {
+        return true;
+    }
+
+    // Heuristic 3: substitution-cipher letter statistics (broken ToUnicode)
+    let mut stats = CipherGarbleStats::default();
+    stats.add_text(markdown);
+    stats.looks_garbled()
+}
+
+fn has_dollar_as_space_pattern(markdown: &str) -> bool {
+    let total_dollars = markdown.matches('$').count();
+    if total_dollars > 10 {
+        let bytes = markdown.as_bytes();
+        let mut letter_dollar_letter = 0usize;
+        for i in 1..bytes.len().saturating_sub(1) {
+            if bytes[i] == b'$'
+                && bytes[i - 1].is_ascii_alphabetic()
+                && bytes[i + 1].is_ascii_alphabetic()
+            {
+                letter_dollar_letter += 1;
+            }
+        }
+        if letter_dollar_letter > 20 || letter_dollar_letter * 2 > total_dollars {
+            return true;
+        }
+    }
+
+    false
+}
+
+/// English letter frequencies (percent, a–z). Used as a natural-language
+/// reference: every Latin-script language in the eval corpus (Swedish,
+/// Finnish, Turkish, German, romaji) scores ≥ 0.80 cosine similarity against
+/// it, while substitution-cipher text scores ~0.53.
+const ENGLISH_LETTER_FREQ: [f64; 26] = [
+    8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, 6.7, 7.5, 1.9, 0.1, 6.0,
+    6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07,
+];
+
+/// Letter statistics for detecting substitution-cipher garbling: broken
+/// ToUnicode CMaps that shift every character by a per-range constant (e.g.
+/// `Certificate` extracted as `8VceZWZTReV`). Such text is 100% printable
+/// ASCII with word-like token lengths, so it defeats `is_garbage_text` and
+/// produces no replacement characters — it needs its own discriminator.
+#[derive(Debug, Default)]
+struct CipherGarbleStats {
+    /// Case-folded ASCII letter histogram.
+    letter_counts: [u32; 26],
+    ascii_letters: usize,
+    ascii_vowels: usize,
+    /// Accented Latin letters (Latin-1 Supplement through Latin Extended-B,
+    /// plus Latin Extended Additional). Count toward Latin dominance only.
+    latin_ext_letters: usize,
+    non_latin_letters: usize,
+    /// Adjacent ASCII-letter pairs, and how many of them switch from
+    /// lowercase straight to uppercase mid-word.
+    letter_bigrams: usize,
+    case_shift_bigrams: usize,
+}
+
+impl CipherGarbleStats {
+    fn add_text(&mut self, text: &str) {
+        let mut prev: Option<char> = None;
+        for ch in text.chars() {
+            if ch.is_ascii_alphabetic() {
+                let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize;
+                self.letter_counts[idx] += 1;
+                self.ascii_letters += 1;
+                if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
+                    self.ascii_vowels += 1;
+                }
+                if let Some(p) = prev {
+                    self.letter_bigrams += 1;
+                    if p.is_ascii_lowercase() && ch.is_ascii_uppercase() {
+                        self.case_shift_bigrams += 1;
+                    }
+                }
+                prev = Some(ch);
+            } else {
+                if ch.is_alphabetic() {
+                    if matches!(ch as u32, 0xC0..=0x24F | 0x1E00..=0x1EFF) {
+                        self.latin_ext_letters += 1;
+                    } else {
+                        self.non_latin_letters += 1;
+                    }
+                }
+                prev = None;
+            }
+        }
+    }
+
+    /// Cosine similarity between the observed letter histogram and English
+    /// letter frequencies. A shifted alphabet permutes the histogram, which
+    /// destroys the similarity regardless of the shift amount.
+    fn english_cosine(&self) -> f64 {
+        if self.ascii_letters == 0 {
+            return 1.0;
+        }
+        let n = self.ascii_letters as f64;
+        let mut dot = 0.0;
+        let mut norm_obs = 0.0;
+        for (count, freq) in self.letter_counts.iter().zip(ENGLISH_LETTER_FREQ) {
+            let p = *count as f64 / n;
+            dot += p * freq;
+            norm_obs += p * p;
+        }
+        let norm_en = ENGLISH_LETTER_FREQ
+            .iter()
+            .map(|f| f * f)
+            .sum::<f64>()
+            .sqrt();
+        dot / (norm_obs.sqrt() * norm_en)
+    }
+
+    /// Cosine similarity between the observed histogram and English
+    /// frequencies after sorting BOTH descending — i.e. comparing the *shape*
+    /// of the frequency profile, ignoring which letter sits where. A
+    /// substitution cipher is a bijection, so it preserves this shape exactly
+    /// (att10k 0.97, arbitrary shifts 0.99) regardless of case or offset.
+    /// Non-linguistic ASCII has a different profile: a small alphabet is far
+    /// steeper (random DNA 0.74, hex dumps 0.81), so the shape diverges.
+    fn english_shape_cosine(&self) -> f64 {
+        if self.ascii_letters == 0 {
+            return 1.0;
+        }
+        let n = self.ascii_letters as f64;
+        let mut obs: [f64; 26] = std::array::from_fn(|i| self.letter_counts[i] as f64 / n);
+        obs.sort_unstable_by(|a, b| b.total_cmp(a));
+        let mut en = ENGLISH_LETTER_FREQ;
+        en.sort_unstable_by(|a, b| b.total_cmp(a));
+
+        let dot: f64 = obs.iter().zip(en).map(|(o, e)| o * e).sum();
+        let norm_obs = obs.iter().map(|o| o * o).sum::<f64>().sqrt();
+        let norm_en = en.iter().map(|e| e * e).sum::<f64>().sqrt();
+        dot / (norm_obs * norm_en)
+    }
+
+    /// Thresholds validated against the 380-document pdf-evals snapshot
+    /// corpus (0 false positives) and the garbled ParseBench `att10k` page
+    /// (vowel ratio 0.245, case-shift rate 0.225, cosine 0.532). Closest
+    /// legitimate document on each axis: vowel ratio 0.264 (circuit
+    /// schematic), case-shift rate 0.021, cosine 0.801.
+    fn looks_garbled(&self) -> bool {
+        // Need a statistically meaningful, Latin-dominant sample.
+        if self.ascii_letters < 200
+            || self.non_latin_letters > self.ascii_letters + self.latin_ext_letters
+        {
+            return false;
+        }
+
+        // Real Latin-script text keeps vowels above ~30% of letters even in
+        // acronym- and part-number-heavy documents; shifted text starves them.
+        let vowel_ratio = self.ascii_vowels as f64 / self.ascii_letters as f64;
+        if vowel_ratio > 0.30 {
+            return false;
+        }
+
+        // Signal 1: lowercase→uppercase transitions inside words. A shifted
+        // lowercase alphabet straddles the ASCII uppercase block ('i'→'Z',
+        // 't'→'e'), so garbled words flip case constantly. Real documents
+        // stay ≤ 0.02 even with camelCase identifiers.
+        let case_shifts = self.letter_bigrams >= 100
+            && self.case_shift_bigrams as f64 >= self.letter_bigrams as f64 * 0.10;
+
+        // Signal 2: the histogram is a permutation of natural language — an
+        // English-like frequency SHAPE (sorted cosine high) but with letters
+        // in the wrong POSITIONS (unsorted cosine low). This is the signature
+        // of a substitution cipher and is case-independent, so it catches
+        // all-lowercase and all-uppercase shifts as well as case-straddling
+        // ones. Genuinely non-linguistic ASCII that is merely "unlike English"
+        // fails one of the two halves: DNA/hex dumps have too steep a profile
+        // (shape cosine < 0.90), while protein sequences, ticker symbols and
+        // base64 are not sufficiently unlike English in position (unsorted
+        // cosine ≥ 0.60) — so none of them are routed to OCR.
+        let permuted_language = self.english_cosine() < 0.60 && self.english_shape_cosine() >= 0.90;
+
+        case_shifts || permuted_language
+    }
+}
+
+#[derive(Debug, Default)]
+struct TextQualityReport {
+    pages_needing_ocr: Vec<u32>,
+    has_encoding_issues: bool,
+    reasons_by_page: BTreeMap<u32, Vec<String>>,
+}
+
+#[derive(Debug, Default)]
+struct PageTextQualityEvidence {
+    chars: usize,
+    replacement_chars: usize,
+    replacement_spans: usize,
+    longest_replacement_run: usize,
+    cipher_garble: CipherGarbleStats,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum TextSpanIssueKind {
+    Replacement,
+    Strong,
+}
+
+fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
+    let mut reasons_by_page = BTreeMap::new();
+    let mut evidence_by_page = BTreeMap::<u32, PageTextQualityEvidence>::new();
+
+    for item in items {
+        if !matches!(item.item_type, crate::types::ItemType::Text) {
+            continue;
+        }
+
+        let evidence = evidence_by_page.entry(item.page).or_default();
+        evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
+        evidence.cipher_garble.add_text(&item.text);
+
+        match text_span_decoding_issue_kind(&item.text) {
+            Some(TextSpanIssueKind::Strong) => {
+                add_ocr_reason(
+                    &mut reasons_by_page,
+                    item.page,
+                    OCR_REASON_SUSPECTED_GARBLED_TEXT,
+                );
+            }
+            Some(TextSpanIssueKind::Replacement) => {
+                let stats = replacement_text_stats(&item.text);
+                evidence.replacement_chars += stats.0;
+                evidence.replacement_spans += 1;
+                evidence.longest_replacement_run = evidence.longest_replacement_run.max(stats.1);
+            }
+            None => {}
+        }
+    }
+
+    for (page, evidence) in evidence_by_page {
+        if reasons_by_page.contains_key(&page) {
+            continue;
+        }
+        if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
+        {
+            add_ocr_reason(
+                &mut reasons_by_page,
+                page,
+                OCR_REASON_SUSPECTED_GARBLED_TEXT,
+            );
+        }
+    }
+
+    let pages_needing_ocr: Vec<u32> = reasons_by_page.keys().copied().collect();
+    TextQualityReport {
+        has_encoding_issues: !pages_needing_ocr.is_empty(),
+        pages_needing_ocr,
+        reasons_by_page,
+    }
+}
+
 fn suspected_garbled_reason() -> String {
     OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()
 }
 
-pub(crate) fn add_ocr_reason(
+fn add_ocr_reason(reasons_by_page: &mut BTreeMap<u32, Vec<String>>, page: u32, reason: &str) {
-    reasons_by_page: &mut BTreeMap<u32, Vec<String>>,
-    page: u32,
-    reason: &str,
-) {
     let reasons = reasons_by_page.entry(page).or_default();
     if !reasons.iter().any(|existing| existing == reason) {
         reasons.push(reason.to_string());
@@ -3748,6 +4011,225 @@ fn page_ocr_reasons_vec(reasons_by_page: BTreeMap<u32, Vec<String>>) -> Vec<Page
         .collect()
 }
 
+fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
+    items.iter().any(|item| {
+        matches!(item.item_type, crate::types::ItemType::Text)
+            && text_span_has_decoding_issue(&item.text)
+    })
+}
+
+fn text_span_has_decoding_issue(text: &str) -> bool {
+    text_span_decoding_issue_kind(text).is_some()
+}
+
+fn text_span_decoding_issue_kind(text: &str) -> Option<TextSpanIssueKind> {
+    let text = text.trim();
+    if text.is_empty() {
+        return None;
+    }
+
+    if has_dollar_as_space_pattern(text)
+        || has_private_use_text_run(text)
+        || is_cid_garbage(text)
+        || has_cid_control_token(text)
+    {
+        return Some(TextSpanIssueKind::Strong);
+    }
+
+    if has_replacement_text_run(text) {
+        return Some(TextSpanIssueKind::Replacement);
+    }
+
+    None
+}
+
+fn replacement_text_stats(text: &str) -> (usize, usize) {
+    let mut replacement = 0usize;
+    let mut current_run = 0usize;
+    let mut longest_run = 0usize;
+
+    for ch in text.chars() {
+        if ch == '\u{FFFD}' {
+            replacement += 1;
+            current_run += 1;
+            longest_run = longest_run.max(current_run);
+        } else {
+            current_run = 0;
+        }
+    }
+
+    (replacement, longest_run)
+}
+
+fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
+    if evidence.replacement_chars == 0 || evidence.chars == 0 {
+        return false;
+    }
+
+    // If the entire page is only a short broken text layer, even a short
+    // replacement run is enough evidence. On otherwise text-heavy pages,
+    // require density so math formulas do not force full-page OCR.
+    if evidence.chars <= 80 && evidence.longest_replacement_run >= 2 {
+        return true;
+    }
+
+    let replacement_density_bps = evidence.replacement_chars * 10_000 / evidence.chars;
+    let enough_bad_text = evidence.replacement_chars >= 12 && replacement_density_bps >= 500;
+    let repeated_bad_spans = evidence.replacement_spans >= 3 && replacement_density_bps >= 250;
+    let long_bad_run = evidence.longest_replacement_run >= 8 && replacement_density_bps >= 250;
+
+    enough_bad_text || repeated_bad_spans || long_bad_run
+}
+
+fn has_replacement_text_run(text: &str) -> bool {
+    let (replacement, longest_run) = replacement_text_stats(text);
+    longest_run >= 2 || replacement >= 3
+}
+
+fn has_private_use_text_run(text: &str) -> bool {
+    let mut total = 0usize;
+    let mut private_use = 0usize;
+    let mut current_run = 0usize;
+    let mut longest_run = 0usize;
+
+    for ch in text.chars() {
+        if ch.is_whitespace() {
+            current_run = 0;
+            continue;
+        }
+        total += 1;
+        if is_private_use_char(ch) {
+            private_use += 1;
+            current_run += 1;
+            longest_run = longest_run.max(current_run);
+        } else {
+            current_run = 0;
+        }
+    }
+
+    if private_use == 0 {
+        return false;
+    }
+
+    longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
+}
+
+fn has_cid_control_token(text: &str) -> bool {
+    text.split_whitespace().any(token_has_cid_control)
+}
+
+fn token_has_cid_control(token: &str) -> bool {
+    let mut total = 0usize;
+    let mut c1_control = 0usize;
+
+    for ch in token.chars() {
+        total += 1;
+        if ('\u{0080}'..='\u{009F}').contains(&ch) {
+            c1_control += 1;
+        }
+    }
+
+    total >= 5 && c1_control >= 2 && c1_control * 20 >= total
+}
+
+fn is_private_use_char(ch: char) -> bool {
+    matches!(
+        ch as u32,
+        0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
+    )
+}
+
+/// Check if extracted text is predominantly garbage (non-alphanumeric).
+///
+/// Broken font encodings produce text like "----1-.-.-.___  --.-. .._ I_---."
+/// where most characters are punctuation/symbols. Real text in any language
+/// has >50% alphanumeric characters.
+fn is_garbage_text(markdown: &str) -> bool {
+    let mut alphanum = 0usize;
+    let mut non_alphanum = 0usize;
+
+    let chars: Vec<char> = markdown.chars().collect();
+    let mut i = 0usize;
+    while i < chars.len() {
+        let ch = chars[i];
+        let mut run_end = i + 1;
+        while run_end < chars.len() && chars[run_end] == ch {
+            run_end += 1;
+        }
+
+        let is_decorative_leader = matches!(ch, '.' | '_' | '·') && run_end - i >= 3;
+        if !is_decorative_leader {
+            for &run_ch in &chars[i..run_end] {
+                if run_ch.is_whitespace() {
+                    continue;
+                }
+                // Skip markdown syntax chars that we add (not from the PDF)
+                if matches!(run_ch, '#' | '*' | '|' | '-' | '\n') {
+                    continue;
+                }
+                if run_ch.is_alphanumeric() {
+                    alphanum += 1;
+                } else {
+                    non_alphanum += 1;
+                }
+            }
+        }
+        i = run_end;
+    }
+
+    let total = alphanum + non_alphanum;
+    total >= 50 && alphanum * 2 < total
+}
+
+/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts.
+///
+/// When CID values don't correspond to Unicode codepoints, the raw bytes often
+/// produce characters in the C1 control range (U+0080–U+009F) or Private Use
+/// Area, mixed with random Latin Extended characters.  Valid text in any
+/// language almost never contains C1 controls.  We also fall back to the
+/// general `is_garbage_text` check for non-alphanumeric-heavy patterns.
+fn is_cid_garbage(text: &str) -> bool {
+    if is_garbage_text(text) {
+        return true;
+    }
+    let mut total = 0usize;
+    let mut c1_control = 0usize;
+    let mut high_latin = 0usize;
+    for ch in text.chars() {
+        if ch.is_whitespace() {
+            continue;
+        }
+        total += 1;
+        // C1 control characters (U+0080–U+009F) — almost never in real text
+        if ch == '·' {
+            continue;
+        }
+        if ('\u{0080}'..='\u{009F}').contains(&ch) {
+            c1_control += 1;
+        }
+        // High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text
+        // but when combined with ASCII in CID passthrough, indicates mojibake
+        // from CID values being misinterpreted as Latin-1 characters.
+        if ('\u{00A0}'..='\u{00FF}').contains(&ch) {
+            high_latin += 1;
+        }
+    }
+    if total < 5 {
+        return false;
+    }
+    // If ≥5% of non-whitespace chars are C1 controls, it's garbage
+    if c1_control >= 2 && c1_control * 20 >= total {
+        return true;
+    }
+    // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
+    // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
+    // where CID values 0x80-0xFF become accented Latin characters).  Keep a
+    // minimum length so short math tokens like "2×()×" do not route a clean
+    // page to OCR.
+    let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
+    total >= 20 && high_latin * 5 >= total * 2 && ascii_letters * 3 < total
+}
+
 /// Detect markdown tables with suspicious structure that suggest the heuristic
 /// missed/mangled rows or columns. Returns true when the caller should treat
 /// the result as `needs_ocr` and fall back to GPU OCR.

src/text_quality.rs

@@ -1,520 +0,0 @@
-//! Text-quality detection: deciding when an extracted text layer is too broken
-//! to serve and a page should fall back to OCR.
-//!
-//! Extraction can produce plausible-looking bytes that are actually garbage —
-//! failed CID→Unicode mappings, broken ToUnicode CMaps, mojibake. These
-//! detectors catch that and let callers set `needs_ocr`. They come in two
-//! layers, sharing the same primitives:
-//!
-//! - **Markdown-level** ([`detect_encoding_issues`], [`is_garbage_text`],
-//!   [`is_cid_garbage`]) run on a page's final markdown string. Used as a
-//!   backstop on the region-extraction and whole-document paths.
-//! - **Item/span-level** ([`analyze_text_quality`],
-//!   [`region_items_have_decoding_issue`]) run on individual `TextItem`s and
-//!   accumulate per-page evidence, so localized garbled spans on an otherwise
-//!   clean page are caught without a single span having to condemn the page.
-//!
-//! Detection classes, roughly by signal:
-//! - **Replacement runs**: U+FFFD clusters ([`has_replacement_text_run`]).
-//! - **Private-use / C1-control runs**: CID passthrough landing in PUA or the
-//!   C1 block ([`has_private_use_text_run`], [`has_cid_control_token`]).
-//! - **Dollar-as-space**: `Word$Word$Word` from broken CMaps
-//!   ([`has_dollar_as_space_pattern`]).
-//! - **Non-alphanumeric dominance**: symbol soup ([`is_garbage_text`]).
-//! - **Substitution-cipher letter statistics**: pure-ASCII output whose letter
-//!   distribution is a permutation of natural language ([`CipherGarbleStats`]).
-
-use crate::types::TextItem;
-use crate::{add_ocr_reason, OCR_REASON_SUSPECTED_GARBLED_TEXT};
-use std::collections::BTreeMap;
-
-/// Detect broken font encodings in extracted markdown text.
-///
-/// Two heuristics:
-/// 1. **U+FFFD**: Any replacement character indicates decode failures.
-/// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a
-///    word separator due to broken ToUnicode CMaps. Triggers when either:
-///    - More than 50% of `$` are between letters (clear substitution pattern), OR
-///    - More than 20 letter-dollar-letter occurrences (even if some `$` are also
-///      used as trailing/leading separators, 20+ is far beyond normal financial text).
-pub(crate) fn detect_encoding_issues(markdown: &str) -> bool {
-    // Heuristic 1: U+FFFD replacement characters
-    if markdown.contains('\u{FFFD}') {
-        return true;
-    }
-
-    // Heuristic 2: dollar-as-space pattern
-    if has_dollar_as_space_pattern(markdown) {
-        return true;
-    }
-
-    // Heuristic 3: substitution-cipher letter statistics (broken ToUnicode)
-    let mut stats = CipherGarbleStats::default();
-    stats.add_text(markdown);
-    stats.looks_garbled()
-}
-
-fn has_dollar_as_space_pattern(markdown: &str) -> bool {
-    let total_dollars = markdown.matches('$').count();
-    if total_dollars > 10 {
-        let bytes = markdown.as_bytes();
-        let mut letter_dollar_letter = 0usize;
-        for i in 1..bytes.len().saturating_sub(1) {
-            if bytes[i] == b'$'
-                && bytes[i - 1].is_ascii_alphabetic()
-                && bytes[i + 1].is_ascii_alphabetic()
-            {
-                letter_dollar_letter += 1;
-            }
-        }
-        if letter_dollar_letter > 20 || letter_dollar_letter * 2 > total_dollars {
-            return true;
-        }
-    }
-
-    false
-}
-
-/// English letter frequencies (percent, a–z). Used as a natural-language
-/// reference: every Latin-script language in the eval corpus (Swedish,
-/// Finnish, Turkish, German, romaji) scores ≥ 0.80 cosine similarity against
-/// it, while substitution-cipher text scores ~0.53.
-const ENGLISH_LETTER_FREQ: [f64; 26] = [
-    8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, 6.7, 7.5, 1.9, 0.1, 6.0,
-    6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07,
-];
-
-/// Letter statistics for detecting substitution-cipher garbling: broken
-/// ToUnicode CMaps that shift every character by a per-range constant (e.g.
-/// `Certificate` extracted as `8VceZWZTReV`). Such text is 100% printable
-/// ASCII with word-like token lengths, so it defeats `is_garbage_text` and
-/// produces no replacement characters — it needs its own discriminator.
-#[derive(Debug, Default)]
-struct CipherGarbleStats {
-    /// Case-folded ASCII letter histogram.
-    letter_counts: [u32; 26],
-    ascii_letters: usize,
-    ascii_vowels: usize,
-    /// Accented Latin letters (Latin-1 Supplement through Latin Extended-B,
-    /// plus Latin Extended Additional). Count toward Latin dominance only.
-    latin_ext_letters: usize,
-    non_latin_letters: usize,
-    /// Adjacent ASCII-letter pairs, and how many of them switch from
-    /// lowercase straight to uppercase mid-word.
-    letter_bigrams: usize,
-    case_shift_bigrams: usize,
-}
-
-impl CipherGarbleStats {
-    fn add_text(&mut self, text: &str) {
-        let mut prev: Option<char> = None;
-        for ch in text.chars() {
-            if ch.is_ascii_alphabetic() {
-                let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize;
-                self.letter_counts[idx] += 1;
-                self.ascii_letters += 1;
-                if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
-                    self.ascii_vowels += 1;
-                }
-                if let Some(p) = prev {
-                    self.letter_bigrams += 1;
-                    if p.is_ascii_lowercase() && ch.is_ascii_uppercase() {
-                        self.case_shift_bigrams += 1;
-                    }
-                }
-                prev = Some(ch);
-            } else {
-                if ch.is_alphabetic() {
-                    if matches!(ch as u32, 0xC0..=0x24F | 0x1E00..=0x1EFF) {
-                        self.latin_ext_letters += 1;
-                    } else {
-                        self.non_latin_letters += 1;
-                    }
-                }
-                prev = None;
-            }
-        }
-    }
-
-    /// Cosine similarity between the observed letter histogram and English
-    /// letter frequencies. A shifted alphabet permutes the histogram, which
-    /// destroys the similarity regardless of the shift amount.
-    fn english_cosine(&self) -> f64 {
-        if self.ascii_letters == 0 {
-            return 1.0;
-        }
-        let n = self.ascii_letters as f64;
-        let mut dot = 0.0;
-        let mut norm_obs = 0.0;
-        for (count, freq) in self.letter_counts.iter().zip(ENGLISH_LETTER_FREQ) {
-            let p = *count as f64 / n;
-            dot += p * freq;
-            norm_obs += p * p;
-        }
-        let norm_en = ENGLISH_LETTER_FREQ
-            .iter()
-            .map(|f| f * f)
-            .sum::<f64>()
-            .sqrt();
-        dot / (norm_obs.sqrt() * norm_en)
-    }
-
-    /// Cosine similarity between the observed histogram and English
-    /// frequencies after sorting BOTH descending — i.e. comparing the *shape*
-    /// of the frequency profile, ignoring which letter sits where. A
-    /// substitution cipher is a bijection, so it preserves this shape exactly
-    /// (att10k 0.97, arbitrary shifts 0.99) regardless of case or offset.
-    /// Non-linguistic ASCII has a different profile: a small alphabet is far
-    /// steeper (random DNA 0.74, hex dumps 0.81), so the shape diverges.
-    fn english_shape_cosine(&self) -> f64 {
-        if self.ascii_letters == 0 {
-            return 1.0;
-        }
-        let n = self.ascii_letters as f64;
-        let mut obs: [f64; 26] = std::array::from_fn(|i| self.letter_counts[i] as f64 / n);
-        obs.sort_unstable_by(|a, b| b.total_cmp(a));
-        let mut en = ENGLISH_LETTER_FREQ;
-        en.sort_unstable_by(|a, b| b.total_cmp(a));
-
-        let dot: f64 = obs.iter().zip(en).map(|(o, e)| o * e).sum();
-        let norm_obs = obs.iter().map(|o| o * o).sum::<f64>().sqrt();
-        let norm_en = en.iter().map(|e| e * e).sum::<f64>().sqrt();
-        dot / (norm_obs * norm_en)
-    }
-
-    /// Thresholds validated against the 380-document pdf-evals snapshot
-    /// corpus (0 false positives) and the garbled ParseBench `att10k` page
-    /// (vowel ratio 0.245, case-shift rate 0.225, cosine 0.532). Closest
-    /// legitimate document on each axis: vowel ratio 0.264 (circuit
-    /// schematic), case-shift rate 0.021, cosine 0.801.
-    fn looks_garbled(&self) -> bool {
-        // Need a statistically meaningful, Latin-dominant sample.
-        if self.ascii_letters < 200
-            || self.non_latin_letters > self.ascii_letters + self.latin_ext_letters
-        {
-            return false;
-        }
-
-        // Real Latin-script text keeps vowels above ~30% of letters even in
-        // acronym- and part-number-heavy documents; shifted text starves them.
-        let vowel_ratio = self.ascii_vowels as f64 / self.ascii_letters as f64;
-        if vowel_ratio > 0.30 {
-            return false;
-        }
-
-        // Signal 1: lowercase→uppercase transitions inside words. A shifted
-        // lowercase alphabet straddles the ASCII uppercase block ('i'→'Z',
-        // 't'→'e'), so garbled words flip case constantly. Real documents
-        // stay ≤ 0.02 even with camelCase identifiers.
-        let case_shifts = self.letter_bigrams >= 100
-            && self.case_shift_bigrams as f64 >= self.letter_bigrams as f64 * 0.10;
-
-        // Signal 2: the histogram is a permutation of natural language — an
-        // English-like frequency SHAPE (sorted cosine high) but with letters
-        // in the wrong POSITIONS (unsorted cosine low). This is the signature
-        // of a substitution cipher and is case-independent, so it catches
-        // all-lowercase and all-uppercase shifts as well as case-straddling
-        // ones. Genuinely non-linguistic ASCII that is merely "unlike English"
-        // fails one of the two halves: DNA/hex dumps have too steep a profile
-        // (shape cosine < 0.90), while protein sequences, ticker symbols and
-        // base64 are not sufficiently unlike English in position (unsorted
-        // cosine ≥ 0.60) — so none of them are routed to OCR.
-        let permuted_language = self.english_cosine() < 0.60 && self.english_shape_cosine() >= 0.90;
-
-        case_shifts || permuted_language
-    }
-}
-
-#[derive(Debug, Default)]
-pub(crate) struct TextQualityReport {
-    pub(crate) pages_needing_ocr: Vec<u32>,
-    pub(crate) has_encoding_issues: bool,
-    pub(crate) reasons_by_page: BTreeMap<u32, Vec<String>>,
-}
-
-#[derive(Debug, Default)]
-struct PageTextQualityEvidence {
-    chars: usize,
-    replacement_chars: usize,
-    replacement_spans: usize,
-    longest_replacement_run: usize,
-    cipher_garble: CipherGarbleStats,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum TextSpanIssueKind {
-    Replacement,
-    Strong,
-}
-
-pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
-    let mut reasons_by_page = BTreeMap::new();
-    let mut evidence_by_page = BTreeMap::<u32, PageTextQualityEvidence>::new();
-
-    for item in items {
-        if !matches!(item.item_type, crate::types::ItemType::Text) {
-            continue;
-        }
-
-        let evidence = evidence_by_page.entry(item.page).or_default();
-        evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
-        evidence.cipher_garble.add_text(&item.text);
-
-        match text_span_decoding_issue_kind(&item.text) {
-            Some(TextSpanIssueKind::Strong) => {
-                add_ocr_reason(
-                    &mut reasons_by_page,
-                    item.page,
-                    OCR_REASON_SUSPECTED_GARBLED_TEXT,
-                );
-            }
-            Some(TextSpanIssueKind::Replacement) => {
-                let stats = replacement_text_stats(&item.text);
-                evidence.replacement_chars += stats.0;
-                evidence.replacement_spans += 1;
-                evidence.longest_replacement_run = evidence.longest_replacement_run.max(stats.1);
-            }
-            None => {}
-        }
-    }
-
-    for (page, evidence) in evidence_by_page {
-        if reasons_by_page.contains_key(&page) {
-            continue;
-        }
-        if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
-        {
-            add_ocr_reason(
-                &mut reasons_by_page,
-                page,
-                OCR_REASON_SUSPECTED_GARBLED_TEXT,
-            );
-        }
-    }
-
-    let pages_needing_ocr: Vec<u32> = reasons_by_page.keys().copied().collect();
-    TextQualityReport {
-        has_encoding_issues: !pages_needing_ocr.is_empty(),
-        pages_needing_ocr,
-        reasons_by_page,
-    }
-}
-
-pub(crate) fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
-    items.iter().any(|item| {
-        matches!(item.item_type, crate::types::ItemType::Text)
-            && text_span_has_decoding_issue(&item.text)
-    })
-}
-
-fn text_span_has_decoding_issue(text: &str) -> bool {
-    text_span_decoding_issue_kind(text).is_some()
-}
-
-fn text_span_decoding_issue_kind(text: &str) -> Option<TextSpanIssueKind> {
-    let text = text.trim();
-    if text.is_empty() {
-        return None;
-    }
-
-    if has_dollar_as_space_pattern(text)
-        || has_private_use_text_run(text)
-        || is_cid_garbage(text)
-        || has_cid_control_token(text)
-    {
-        return Some(TextSpanIssueKind::Strong);
-    }
-
-    if has_replacement_text_run(text) {
-        return Some(TextSpanIssueKind::Replacement);
-    }
-
-    None
-}
-
-fn replacement_text_stats(text: &str) -> (usize, usize) {
-    let mut replacement = 0usize;
-    let mut current_run = 0usize;
-    let mut longest_run = 0usize;
-
-    for ch in text.chars() {
-        if ch == '\u{FFFD}' {
-            replacement += 1;
-            current_run += 1;
-            longest_run = longest_run.max(current_run);
-        } else {
-            current_run = 0;
-        }
-    }
-
-    (replacement, longest_run)
-}
-
-fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
-    if evidence.replacement_chars == 0 || evidence.chars == 0 {
-        return false;
-    }
-
-    // If the entire page is only a short broken text layer, even a short
-    // replacement run is enough evidence. On otherwise text-heavy pages,
-    // require density so math formulas do not force full-page OCR.
-    if evidence.chars <= 80 && evidence.longest_replacement_run >= 2 {
-        return true;
-    }
-
-    let replacement_density_bps = evidence.replacement_chars * 10_000 / evidence.chars;
-    let enough_bad_text = evidence.replacement_chars >= 12 && replacement_density_bps >= 500;
-    let repeated_bad_spans = evidence.replacement_spans >= 3 && replacement_density_bps >= 250;
-    let long_bad_run = evidence.longest_replacement_run >= 8 && replacement_density_bps >= 250;
-
-    enough_bad_text || repeated_bad_spans || long_bad_run
-}
-
-fn has_replacement_text_run(text: &str) -> bool {
-    let (replacement, longest_run) = replacement_text_stats(text);
-    longest_run >= 2 || replacement >= 3
-}
-
-fn has_private_use_text_run(text: &str) -> bool {
-    let mut total = 0usize;
-    let mut private_use = 0usize;
-    let mut current_run = 0usize;
-    let mut longest_run = 0usize;
-
-    for ch in text.chars() {
-        if ch.is_whitespace() {
-            current_run = 0;
-            continue;
-        }
-        total += 1;
-        if is_private_use_char(ch) {
-            private_use += 1;
-            current_run += 1;
-            longest_run = longest_run.max(current_run);
-        } else {
-            current_run = 0;
-        }
-    }
-
-    if private_use == 0 {
-        return false;
-    }
-
-    longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
-}
-
-fn has_cid_control_token(text: &str) -> bool {
-    text.split_whitespace().any(token_has_cid_control)
-}
-
-fn token_has_cid_control(token: &str) -> bool {
-    let mut total = 0usize;
-    let mut c1_control = 0usize;
-
-    for ch in token.chars() {
-        total += 1;
-        if ('\u{0080}'..='\u{009F}').contains(&ch) {
-            c1_control += 1;
-        }
-    }
-
-    total >= 5 && c1_control >= 2 && c1_control * 20 >= total
-}
-
-fn is_private_use_char(ch: char) -> bool {
-    matches!(
-        ch as u32,
-        0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
-    )
-}
-
-/// Check if extracted text is predominantly garbage (non-alphanumeric).
-///
-/// Broken font encodings produce text like "----1-.-.-.___  --.-. .._ I_---."
-/// where most characters are punctuation/symbols. Real text in any language
-/// has >50% alphanumeric characters.
-pub(crate) fn is_garbage_text(markdown: &str) -> bool {
-    let mut alphanum = 0usize;
-    let mut non_alphanum = 0usize;
-
-    let chars: Vec<char> = markdown.chars().collect();
-    let mut i = 0usize;
-    while i < chars.len() {
-        let ch = chars[i];
-        let mut run_end = i + 1;
-        while run_end < chars.len() && chars[run_end] == ch {
-            run_end += 1;
-        }
-
-        let is_decorative_leader = matches!(ch, '.' | '_' | '·') && run_end - i >= 3;
-        if !is_decorative_leader {
-            for &run_ch in &chars[i..run_end] {
-                if run_ch.is_whitespace() {
-                    continue;
-                }
-                // Skip markdown syntax chars that we add (not from the PDF)
-                if matches!(run_ch, '#' | '*' | '|' | '-' | '\n') {
-                    continue;
-                }
-                if run_ch.is_alphanumeric() {
-                    alphanum += 1;
-                } else {
-                    non_alphanum += 1;
-                }
-            }
-        }
-        i = run_end;
-    }
-
-    let total = alphanum + non_alphanum;
-    total >= 50 && alphanum * 2 < total
-}
-
-/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts.
-///
-/// When CID values don't correspond to Unicode codepoints, the raw bytes often
-/// produce characters in the C1 control range (U+0080–U+009F) or Private Use
-/// Area, mixed with random Latin Extended characters.  Valid text in any
-/// language almost never contains C1 controls.  We also fall back to the
-/// general `is_garbage_text` check for non-alphanumeric-heavy patterns.
-pub(crate) fn is_cid_garbage(text: &str) -> bool {
-    if is_garbage_text(text) {
-        return true;
-    }
-    let mut total = 0usize;
-    let mut c1_control = 0usize;
-    let mut high_latin = 0usize;
-    for ch in text.chars() {
-        if ch.is_whitespace() {
-            continue;
-        }
-        total += 1;
-        // C1 control characters (U+0080–U+009F) — almost never in real text
-        if ch == '·' {
-            continue;
-        }
-        if ('\u{0080}'..='\u{009F}').contains(&ch) {
-            c1_control += 1;
-        }
-        // High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text
-        // but when combined with ASCII in CID passthrough, indicates mojibake
-        // from CID values being misinterpreted as Latin-1 characters.
-        if ('\u{00A0}'..='\u{00FF}').contains(&ch) {
-            high_latin += 1;
-        }
-    }
-    if total < 5 {
-        return false;
-    }
-    // If ≥5% of non-whitespace chars are C1 controls, it's garbage
-    if c1_control >= 2 && c1_control * 20 >= total {
-        return true;
-    }
-    // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
-    // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
-    // where CID values 0x80-0xFF become accented Latin characters).  Keep a
-    // minimum length so short math tokens like "2×()×" do not route a clean
-    // page to OCR.
-    let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
-    total >= 20 && high_latin * 5 >= total * 2 && ascii_letters * 3 < total
-}