Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael MartellandClaude Fable 5
|
573bc3cc90 | ||
|
Abimael MartellandClaude Fable 5
|
8951d90958 | ||
|
Abimael MartellandClaude Fable 5
|
2349474432 |
@@ -26,16 +26,16 @@ Evaluated on the [opendataloader-bench](https://github.com/opendataloader-projec
|
||||
|
||||
| Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
|
||||
|---|---|---|---|---|---|
|
||||
| pdf-inspector | 0.83 | 0.88 | 0.66 | 0.74 | 4s |
|
||||
| pdf-inspector | 0.78 | 0.87 | 0.59 | 0.57 | 4s |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
| markitdown | 0.58 | 0.88 | 0.00 | 0.00 | 8s |
|
||||
|
||||
For context, engines that use OCR/ML (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus — pdf-inspector reaches the low end of that range without any OCR, in 4 seconds.
|
||||
For context, engines that use OCR/ML (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus.
|
||||
|
||||
**Where we do well:** Speed (fastest of all engines), the best table detection of any engine shown, and heading detection now on par with opendataloader. Overall lands within 0.01 of opendataloader at roughly 2.5× the speed.
|
||||
**Where we do well:** Speed (fastest of all engines), reading order, table detection vs other direct-text tools.
|
||||
|
||||
**Where we lag:** Reading order still trails opendataloader slightly, and table structure trails OCR-based engines that can see visual layout.
|
||||
**Where we lag:** Heading detection trails opendataloader — many PDFs use bold text at body font size for headings, or headings that are only slightly larger than body text. Table detection trails OCR-based engines that can see visual table structure.
|
||||
|
||||
## Quick start
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.10.1",
|
||||
"version": "1.10.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+2
-245
@@ -9,7 +9,7 @@ mod links;
|
||||
pub(crate) mod underline;
|
||||
mod xobjects;
|
||||
|
||||
use crate::text_utils::{is_cjk_char, is_rtl_text};
|
||||
use crate::text_utils::is_rtl_text;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
|
||||
use crate::PdfError;
|
||||
@@ -527,136 +527,6 @@ fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
|
||||
saw_backtrack
|
||||
}
|
||||
|
||||
/// Detect a tracked (letter-spaced) run of single-glyph items and derive its
|
||||
/// run-local space floor.
|
||||
///
|
||||
/// Display type set with tracking renders one glyph per show op; the merge
|
||||
/// loop's fixed thresholds (0.08-0.13 em) then read every letter gap as a
|
||||
/// word boundary and emit "H O W" instead of "HOW". Within such a run the
|
||||
/// gaps carry the real signal: letter gaps cluster tightly just above the
|
||||
/// fixed threshold, word gaps sit clearly higher. Returns (run_end_index,
|
||||
/// space_floor) when the run starting at `start` is tracked — spaces are
|
||||
/// then inserted only at gaps above the floor (infinity = single word).
|
||||
/// Normal text (multi-char items, or single-char runs with sub-threshold
|
||||
/// gaps) returns None and keeps the existing behavior.
|
||||
/// Han/Kana scripts write without inter-word spaces. Hangul (Korean) DOES
|
||||
/// space between words and deliberately stays out of this set — a Korean
|
||||
/// tracked run keeps normal word-boundary handling.
|
||||
fn is_spaceless_cjk(c: char) -> bool {
|
||||
matches!(c,
|
||||
'\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation
|
||||
| '\u{3040}'..='\u{309F}' // Hiragana
|
||||
| '\u{30A0}'..='\u{30FF}' // Katakana
|
||||
| '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs
|
||||
| '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs
|
||||
| '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms
|
||||
)
|
||||
}
|
||||
|
||||
fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize, f32)> {
|
||||
const MIN_GAPS: usize = 4;
|
||||
let first = group[start];
|
||||
if first.text.trim().chars().count() != 1 {
|
||||
return None;
|
||||
}
|
||||
let fs = first.font_size;
|
||||
if fs <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Walk the run under the SAME break conditions as the merge loop
|
||||
// (size band, style equality, mergeable gap) so indices stay aligned.
|
||||
let mut gaps: Vec<f32> = Vec::new();
|
||||
let mut end_x = first.x + effective_merge_width(first);
|
||||
let mut end = start;
|
||||
for (offset, next) in group[start + 1..].iter().enumerate() {
|
||||
if next.text.trim().chars().count() != 1 {
|
||||
break;
|
||||
}
|
||||
if (next.font_size - fs).abs() > fs * 0.20 {
|
||||
break;
|
||||
}
|
||||
if next.is_bold != first.is_bold
|
||||
|| next.is_italic != first.is_italic
|
||||
|| next.is_underline != first.is_underline
|
||||
|| next.is_strikeout != first.is_strikeout
|
||||
{
|
||||
break;
|
||||
}
|
||||
let gap = next.x - end_x;
|
||||
if gap > fs * 0.5 || gap < -fs * 0.5 {
|
||||
break;
|
||||
}
|
||||
gaps.push(gap / fs);
|
||||
end_x = next.x + effective_merge_width(next);
|
||||
end = start + 1 + offset;
|
||||
}
|
||||
if gaps.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Tracked signature: the run's TYPICAL gap clears the fixed space
|
||||
// threshold (0.08) — the merge loop would break almost every letter
|
||||
// pair into "words". Short runs (2-3 gaps: "H O W") demand a stricter
|
||||
// shape — clearly wide, uniform, ALL-CAPS — because a genuine spaced
|
||||
// sequence of single letters ("x y z" variables) has the same gap
|
||||
// count; display tracking is a caps convention.
|
||||
let mut sorted = gaps.clone();
|
||||
sorted.sort_by(|a, b| a.total_cmp(b));
|
||||
let median = sorted[sorted.len() / 2];
|
||||
// Typographic convention gate, both tiers: display tracking is an
|
||||
// all-caps convention, and Han/Kana never space between glyphs. Mixed-
|
||||
// or lowercase Latin runs keep their boundaries because geometry alone
|
||||
// cannot distinguish spaced singles ("A b c d e") from a tracked
|
||||
// title-case word ("B u f f a l o").
|
||||
let run_chars = || {
|
||||
group[start..=end]
|
||||
.iter()
|
||||
.flat_map(|it| it.text.trim().chars())
|
||||
};
|
||||
let spaceless_cjk = run_chars().all(|c| is_spaceless_cjk(c) || !c.is_alphanumeric())
|
||||
&& run_chars().any(is_spaceless_cjk);
|
||||
let all_caps = run_chars().all(|c| c.is_uppercase() || is_cjk_char(c) || !c.is_alphabetic());
|
||||
if !(spaceless_cjk || all_caps) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if gaps.len() >= MIN_GAPS {
|
||||
if median <= 0.075 {
|
||||
return None;
|
||||
}
|
||||
} else {
|
||||
let uniform = sorted[sorted.len() - 1] <= sorted[0].max(0.01) * 1.4;
|
||||
if median < 0.09 || !uniform {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
// Han/Kana: no inter-glyph spaces, period — a nonuniform gap
|
||||
// distribution (punctuation spacing, justification) must not
|
||||
// manufacture word boundaries.
|
||||
if spaceless_cjk {
|
||||
return Some((end, f32::INFINITY));
|
||||
}
|
||||
|
||||
// Word gaps, if present, form a second mode above the letter-gap
|
||||
// cluster: split at the largest relative jump. Unimodal → one word.
|
||||
let mut best_jump = 1.0f32;
|
||||
let mut floor = f32::INFINITY;
|
||||
for pair in sorted.windows(2) {
|
||||
let (lo, hi) = (pair[0].max(0.01), pair[1].max(0.01));
|
||||
let jump = hi / lo;
|
||||
if jump > best_jump {
|
||||
best_jump = jump;
|
||||
floor = (lo + hi) / 2.0;
|
||||
}
|
||||
}
|
||||
if best_jump < 1.4 {
|
||||
floor = f32::INFINITY;
|
||||
}
|
||||
Some((end, floor * fs))
|
||||
}
|
||||
|
||||
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if items.is_empty() {
|
||||
return items;
|
||||
@@ -704,14 +574,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
let mut text = first.text.clone();
|
||||
let mut end_x = first.x + effective_merge_width(first);
|
||||
|
||||
// Tracked display text: run-local space floor overrides the
|
||||
// fixed thresholds for this run's junctions (see helper).
|
||||
let tracked = if *preserve_stream_order {
|
||||
None
|
||||
} else {
|
||||
tracked_run_space_floor(group, i)
|
||||
};
|
||||
|
||||
let mut j = i + 1;
|
||||
while j < group.len() {
|
||||
let next = group[j];
|
||||
@@ -766,11 +628,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
let needs_bullet_space = *preserve_stream_order
|
||||
&& is_standalone_bullet_text(&text)
|
||||
&& !next.text.trim().is_empty();
|
||||
let effective_threshold = match tracked {
|
||||
Some((run_end, floor)) if j <= run_end => floor,
|
||||
_ => threshold,
|
||||
};
|
||||
if needs_bullet_space || gap > effective_threshold {
|
||||
if needs_bullet_space || gap > threshold {
|
||||
text.push(' ');
|
||||
}
|
||||
text.push_str(&next.text);
|
||||
@@ -936,107 +794,6 @@ mod tests {
|
||||
use crate::types::{ItemType, PdfLine, TextLine};
|
||||
use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
|
||||
|
||||
/// Glyph-per-item run at `fs`=12 with the given inter-glyph gap (pt).
|
||||
fn glyph_run(chars: &str, start_x: f32, glyph_w: f32, gap: f32) -> Vec<TextItem> {
|
||||
let mut x = start_x;
|
||||
let mut out = Vec::new();
|
||||
for c in chars.chars() {
|
||||
out.push(make_merge_item(&c.to_string(), x, glyph_w));
|
||||
x += glyph_w + gap;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_caps_run_collapses_to_word() {
|
||||
// Display tracking: every letter gap (0.19 em) clears the fixed
|
||||
// space threshold — without the run-local floor this reads "H O W".
|
||||
let items = glyph_run("HOW", 100.0, 10.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "HOW");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_run_keeps_word_gaps_bimodal() {
|
||||
// Letters at 0.19 em, word gaps at 0.42 em (below the 0.5 em item
|
||||
// break): the split must land between the modes. Needs >=4 gaps to
|
||||
// enter the bimodal tier — short runs use the strict uniform gate.
|
||||
let mut items = glyph_run("ITISOK", 100.0, 8.0, 2.3);
|
||||
for i in 2..6 {
|
||||
items[i].x += 2.8; // word gap at T|I
|
||||
}
|
||||
for i in 4..6 {
|
||||
items[i].x += 2.8; // word gap at S|O
|
||||
}
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "IT IS OK");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lowercase_spaced_singles_stay_words() {
|
||||
// "x y z" variables: same gap shape but lowercase — the short-run
|
||||
// caps requirement keeps genuine spaced singles apart.
|
||||
let items = glyph_run("xyz", 100.0, 6.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "x y z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kerned_singles_unaffected() {
|
||||
// Tiny kerning gaps never triggered spaces before and still don't.
|
||||
let items = glyph_run("WORD", 100.0, 8.0, 0.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "WORD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_lowercase_spaced_singles_keep_boundaries() {
|
||||
// Review: a 5+ single-letter lowercase list has the tracked gap
|
||||
// shape at any length — the convention gate must protect it in
|
||||
// the >=4-gap tier too.
|
||||
let items = glyph_run("abcde", 100.0, 6.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "a b c d e");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn han_run_with_nonuniform_gaps_never_gains_spaces() {
|
||||
// Review: a bimodal gap distribution (justification, punctuation
|
||||
// spacing) must not manufacture word boundaries in Han text.
|
||||
let mut items = glyph_run("北京时事快报", 100.0, 12.0, 1.4);
|
||||
for item in items.iter_mut().skip(3) {
|
||||
item.x += 3.0; // wide gap after the third glyph
|
||||
}
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "北京时事快报");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uppercase_leading_spaced_singles_keep_boundaries() {
|
||||
// "A b c d e" is indistinguishable from a title-case tracked word
|
||||
// without reliable tracking metadata, so preserve its boundaries.
|
||||
let items = glyph_run("Abcde", 100.0, 7.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "A b c d e");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cjk_glyph_run_collapses_without_spaces() {
|
||||
// CJK sets one glyph per item with loose gaps; CJK uses no spaces,
|
||||
// and the non-alphabetic run passes the caps gate.
|
||||
let items = glyph_run("北京时事", 100.0, 12.0, 1.4);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "北京时事");
|
||||
}
|
||||
|
||||
fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.into(),
|
||||
|
||||
+492
-10
@@ -39,7 +39,6 @@ pub mod markdown;
|
||||
pub mod process_mode;
|
||||
pub mod structure_tree;
|
||||
pub mod tables;
|
||||
mod text_quality;
|
||||
pub mod text_utils;
|
||||
pub mod tounicode;
|
||||
pub mod types;
|
||||
@@ -61,10 +60,6 @@ pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
|
||||
use lopdf::Document;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use text_quality::{
|
||||
analyze_text_quality, detect_encoding_issues, is_cid_garbage, is_garbage_text,
|
||||
region_items_have_decoding_issue,
|
||||
};
|
||||
use tounicode::FontCMaps;
|
||||
|
||||
/// OCR reason emitted when the extracted text layer appears garbled due to
|
||||
@@ -3708,15 +3703,283 @@ fn process_document(
|
||||
// Internal helpers
|
||||
// =========================================================================
|
||||
|
||||
/// Detect broken font encodings in extracted markdown text.
|
||||
///
|
||||
/// Two heuristics:
|
||||
/// 1. **U+FFFD**: Any replacement character indicates decode failures.
|
||||
/// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a
|
||||
/// word separator due to broken ToUnicode CMaps. Triggers when either:
|
||||
/// - More than 50% of `$` are between letters (clear substitution pattern), OR
|
||||
/// - More than 20 letter-dollar-letter occurrences (even if some `$` are also
|
||||
/// used as trailing/leading separators, 20+ is far beyond normal financial text).
|
||||
fn detect_encoding_issues(markdown: &str) -> bool {
|
||||
// Heuristic 1: U+FFFD replacement characters
|
||||
if markdown.contains('\u{FFFD}') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Heuristic 2: dollar-as-space pattern
|
||||
if has_dollar_as_space_pattern(markdown) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Heuristic 3: substitution-cipher letter statistics (broken ToUnicode)
|
||||
let mut stats = CipherGarbleStats::default();
|
||||
stats.add_text(markdown);
|
||||
stats.looks_garbled()
|
||||
}
|
||||
|
||||
fn has_dollar_as_space_pattern(markdown: &str) -> bool {
|
||||
let total_dollars = markdown.matches('$').count();
|
||||
if total_dollars > 10 {
|
||||
let bytes = markdown.as_bytes();
|
||||
let mut letter_dollar_letter = 0usize;
|
||||
for i in 1..bytes.len().saturating_sub(1) {
|
||||
if bytes[i] == b'$'
|
||||
&& bytes[i - 1].is_ascii_alphabetic()
|
||||
&& bytes[i + 1].is_ascii_alphabetic()
|
||||
{
|
||||
letter_dollar_letter += 1;
|
||||
}
|
||||
}
|
||||
if letter_dollar_letter > 20 || letter_dollar_letter * 2 > total_dollars {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// English letter frequencies (percent, a–z). Used as a natural-language
|
||||
/// reference: every Latin-script language in the eval corpus (Swedish,
|
||||
/// Finnish, Turkish, German, romaji) scores ≥ 0.80 cosine similarity against
|
||||
/// it, while substitution-cipher text scores ~0.53.
|
||||
const ENGLISH_LETTER_FREQ: [f64; 26] = [
|
||||
8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, 6.7, 7.5, 1.9, 0.1, 6.0,
|
||||
6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07,
|
||||
];
|
||||
|
||||
/// Letter statistics for detecting substitution-cipher garbling: broken
|
||||
/// ToUnicode CMaps that shift every character by a per-range constant (e.g.
|
||||
/// `Certificate` extracted as `8VceZWZTReV`). Such text is 100% printable
|
||||
/// ASCII with word-like token lengths, so it defeats `is_garbage_text` and
|
||||
/// produces no replacement characters — it needs its own discriminator.
|
||||
#[derive(Debug, Default)]
|
||||
struct CipherGarbleStats {
|
||||
/// Case-folded ASCII letter histogram.
|
||||
letter_counts: [u32; 26],
|
||||
ascii_letters: usize,
|
||||
ascii_vowels: usize,
|
||||
/// Accented Latin letters (Latin-1 Supplement through Latin Extended-B,
|
||||
/// plus Latin Extended Additional). Count toward Latin dominance only.
|
||||
latin_ext_letters: usize,
|
||||
non_latin_letters: usize,
|
||||
/// Adjacent ASCII-letter pairs, and how many of them switch from
|
||||
/// lowercase straight to uppercase mid-word.
|
||||
letter_bigrams: usize,
|
||||
case_shift_bigrams: usize,
|
||||
}
|
||||
|
||||
impl CipherGarbleStats {
|
||||
fn add_text(&mut self, text: &str) {
|
||||
let mut prev: Option<char> = None;
|
||||
for ch in text.chars() {
|
||||
if ch.is_ascii_alphabetic() {
|
||||
let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize;
|
||||
self.letter_counts[idx] += 1;
|
||||
self.ascii_letters += 1;
|
||||
if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
|
||||
self.ascii_vowels += 1;
|
||||
}
|
||||
if let Some(p) = prev {
|
||||
self.letter_bigrams += 1;
|
||||
if p.is_ascii_lowercase() && ch.is_ascii_uppercase() {
|
||||
self.case_shift_bigrams += 1;
|
||||
}
|
||||
}
|
||||
prev = Some(ch);
|
||||
} else {
|
||||
if ch.is_alphabetic() {
|
||||
if matches!(ch as u32, 0xC0..=0x24F | 0x1E00..=0x1EFF) {
|
||||
self.latin_ext_letters += 1;
|
||||
} else {
|
||||
self.non_latin_letters += 1;
|
||||
}
|
||||
}
|
||||
prev = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cosine similarity between the observed letter histogram and English
|
||||
/// letter frequencies. A shifted alphabet permutes the histogram, which
|
||||
/// destroys the similarity regardless of the shift amount.
|
||||
fn english_cosine(&self) -> f64 {
|
||||
if self.ascii_letters == 0 {
|
||||
return 1.0;
|
||||
}
|
||||
let n = self.ascii_letters as f64;
|
||||
let mut dot = 0.0;
|
||||
let mut norm_obs = 0.0;
|
||||
for (count, freq) in self.letter_counts.iter().zip(ENGLISH_LETTER_FREQ) {
|
||||
let p = *count as f64 / n;
|
||||
dot += p * freq;
|
||||
norm_obs += p * p;
|
||||
}
|
||||
let norm_en = ENGLISH_LETTER_FREQ
|
||||
.iter()
|
||||
.map(|f| f * f)
|
||||
.sum::<f64>()
|
||||
.sqrt();
|
||||
dot / (norm_obs.sqrt() * norm_en)
|
||||
}
|
||||
|
||||
/// Cosine similarity between the observed histogram and English
|
||||
/// frequencies after sorting BOTH descending — i.e. comparing the *shape*
|
||||
/// of the frequency profile, ignoring which letter sits where. A
|
||||
/// substitution cipher is a bijection, so it preserves this shape exactly
|
||||
/// (att10k 0.97, arbitrary shifts 0.99) regardless of case or offset.
|
||||
/// Non-linguistic ASCII has a different profile: a small alphabet is far
|
||||
/// steeper (random DNA 0.74, hex dumps 0.81), so the shape diverges.
|
||||
fn english_shape_cosine(&self) -> f64 {
|
||||
if self.ascii_letters == 0 {
|
||||
return 1.0;
|
||||
}
|
||||
let n = self.ascii_letters as f64;
|
||||
let mut obs: [f64; 26] = std::array::from_fn(|i| self.letter_counts[i] as f64 / n);
|
||||
obs.sort_unstable_by(|a, b| b.total_cmp(a));
|
||||
let mut en = ENGLISH_LETTER_FREQ;
|
||||
en.sort_unstable_by(|a, b| b.total_cmp(a));
|
||||
|
||||
let dot: f64 = obs.iter().zip(en).map(|(o, e)| o * e).sum();
|
||||
let norm_obs = obs.iter().map(|o| o * o).sum::<f64>().sqrt();
|
||||
let norm_en = en.iter().map(|e| e * e).sum::<f64>().sqrt();
|
||||
dot / (norm_obs * norm_en)
|
||||
}
|
||||
|
||||
/// Thresholds validated against the 380-document pdf-evals snapshot
|
||||
/// corpus (0 false positives) and the garbled ParseBench `att10k` page
|
||||
/// (vowel ratio 0.245, case-shift rate 0.225, cosine 0.532). Closest
|
||||
/// legitimate document on each axis: vowel ratio 0.264 (circuit
|
||||
/// schematic), case-shift rate 0.021, cosine 0.801.
|
||||
fn looks_garbled(&self) -> bool {
|
||||
// Need a statistically meaningful, Latin-dominant sample.
|
||||
if self.ascii_letters < 200
|
||||
|| self.non_latin_letters > self.ascii_letters + self.latin_ext_letters
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Real Latin-script text keeps vowels above ~30% of letters even in
|
||||
// acronym- and part-number-heavy documents; shifted text starves them.
|
||||
let vowel_ratio = self.ascii_vowels as f64 / self.ascii_letters as f64;
|
||||
if vowel_ratio > 0.30 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Signal 1: lowercase→uppercase transitions inside words. A shifted
|
||||
// lowercase alphabet straddles the ASCII uppercase block ('i'→'Z',
|
||||
// 't'→'e'), so garbled words flip case constantly. Real documents
|
||||
// stay ≤ 0.02 even with camelCase identifiers.
|
||||
let case_shifts = self.letter_bigrams >= 100
|
||||
&& self.case_shift_bigrams as f64 >= self.letter_bigrams as f64 * 0.10;
|
||||
|
||||
// Signal 2: the histogram is a permutation of natural language — an
|
||||
// English-like frequency SHAPE (sorted cosine high) but with letters
|
||||
// in the wrong POSITIONS (unsorted cosine low). This is the signature
|
||||
// of a substitution cipher and is case-independent, so it catches
|
||||
// all-lowercase and all-uppercase shifts as well as case-straddling
|
||||
// ones. Genuinely non-linguistic ASCII that is merely "unlike English"
|
||||
// fails one of the two halves: DNA/hex dumps have too steep a profile
|
||||
// (shape cosine < 0.90), while protein sequences, ticker symbols and
|
||||
// base64 are not sufficiently unlike English in position (unsorted
|
||||
// cosine ≥ 0.60) — so none of them are routed to OCR.
|
||||
let permuted_language = self.english_cosine() < 0.60 && self.english_shape_cosine() >= 0.90;
|
||||
|
||||
case_shifts || permuted_language
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TextQualityReport {
|
||||
pages_needing_ocr: Vec<u32>,
|
||||
has_encoding_issues: bool,
|
||||
reasons_by_page: BTreeMap<u32, Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PageTextQualityEvidence {
|
||||
chars: usize,
|
||||
replacement_chars: usize,
|
||||
replacement_spans: usize,
|
||||
longest_replacement_run: usize,
|
||||
cipher_garble: CipherGarbleStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum TextSpanIssueKind {
|
||||
Replacement,
|
||||
Strong,
|
||||
}
|
||||
|
||||
fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
|
||||
let mut reasons_by_page = BTreeMap::new();
|
||||
let mut evidence_by_page = BTreeMap::<u32, PageTextQualityEvidence>::new();
|
||||
|
||||
for item in items {
|
||||
if !matches!(item.item_type, crate::types::ItemType::Text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let evidence = evidence_by_page.entry(item.page).or_default();
|
||||
evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
|
||||
evidence.cipher_garble.add_text(&item.text);
|
||||
|
||||
match text_span_decoding_issue_kind(&item.text) {
|
||||
Some(TextSpanIssueKind::Strong) => {
|
||||
add_ocr_reason(
|
||||
&mut reasons_by_page,
|
||||
item.page,
|
||||
OCR_REASON_SUSPECTED_GARBLED_TEXT,
|
||||
);
|
||||
}
|
||||
Some(TextSpanIssueKind::Replacement) => {
|
||||
let stats = replacement_text_stats(&item.text);
|
||||
evidence.replacement_chars += stats.0;
|
||||
evidence.replacement_spans += 1;
|
||||
evidence.longest_replacement_run = evidence.longest_replacement_run.max(stats.1);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
|
||||
for (page, evidence) in evidence_by_page {
|
||||
if reasons_by_page.contains_key(&page) {
|
||||
continue;
|
||||
}
|
||||
if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
|
||||
{
|
||||
add_ocr_reason(
|
||||
&mut reasons_by_page,
|
||||
page,
|
||||
OCR_REASON_SUSPECTED_GARBLED_TEXT,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let pages_needing_ocr: Vec<u32> = reasons_by_page.keys().copied().collect();
|
||||
TextQualityReport {
|
||||
has_encoding_issues: !pages_needing_ocr.is_empty(),
|
||||
pages_needing_ocr,
|
||||
reasons_by_page,
|
||||
}
|
||||
}
|
||||
|
||||
fn suspected_garbled_reason() -> String {
|
||||
OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn add_ocr_reason(
|
||||
reasons_by_page: &mut BTreeMap<u32, Vec<String>>,
|
||||
page: u32,
|
||||
reason: &str,
|
||||
) {
|
||||
fn add_ocr_reason(reasons_by_page: &mut BTreeMap<u32, Vec<String>>, page: u32, reason: &str) {
|
||||
let reasons = reasons_by_page.entry(page).or_default();
|
||||
if !reasons.iter().any(|existing| existing == reason) {
|
||||
reasons.push(reason.to_string());
|
||||
@@ -3748,6 +4011,225 @@ fn page_ocr_reasons_vec(reasons_by_page: BTreeMap<u32, Vec<String>>) -> Vec<Page
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
|
||||
items.iter().any(|item| {
|
||||
matches!(item.item_type, crate::types::ItemType::Text)
|
||||
&& text_span_has_decoding_issue(&item.text)
|
||||
})
|
||||
}
|
||||
|
||||
fn text_span_has_decoding_issue(text: &str) -> bool {
|
||||
text_span_decoding_issue_kind(text).is_some()
|
||||
}
|
||||
|
||||
fn text_span_decoding_issue_kind(text: &str) -> Option<TextSpanIssueKind> {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if has_dollar_as_space_pattern(text)
|
||||
|| has_private_use_text_run(text)
|
||||
|| is_cid_garbage(text)
|
||||
|| has_cid_control_token(text)
|
||||
{
|
||||
return Some(TextSpanIssueKind::Strong);
|
||||
}
|
||||
|
||||
if has_replacement_text_run(text) {
|
||||
return Some(TextSpanIssueKind::Replacement);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn replacement_text_stats(text: &str) -> (usize, usize) {
|
||||
let mut replacement = 0usize;
|
||||
let mut current_run = 0usize;
|
||||
let mut longest_run = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch == '\u{FFFD}' {
|
||||
replacement += 1;
|
||||
current_run += 1;
|
||||
longest_run = longest_run.max(current_run);
|
||||
} else {
|
||||
current_run = 0;
|
||||
}
|
||||
}
|
||||
|
||||
(replacement, longest_run)
|
||||
}
|
||||
|
||||
fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
|
||||
if evidence.replacement_chars == 0 || evidence.chars == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If the entire page is only a short broken text layer, even a short
|
||||
// replacement run is enough evidence. On otherwise text-heavy pages,
|
||||
// require density so math formulas do not force full-page OCR.
|
||||
if evidence.chars <= 80 && evidence.longest_replacement_run >= 2 {
|
||||
return true;
|
||||
}
|
||||
|
||||
let replacement_density_bps = evidence.replacement_chars * 10_000 / evidence.chars;
|
||||
let enough_bad_text = evidence.replacement_chars >= 12 && replacement_density_bps >= 500;
|
||||
let repeated_bad_spans = evidence.replacement_spans >= 3 && replacement_density_bps >= 250;
|
||||
let long_bad_run = evidence.longest_replacement_run >= 8 && replacement_density_bps >= 250;
|
||||
|
||||
enough_bad_text || repeated_bad_spans || long_bad_run
|
||||
}
|
||||
|
||||
fn has_replacement_text_run(text: &str) -> bool {
|
||||
let (replacement, longest_run) = replacement_text_stats(text);
|
||||
longest_run >= 2 || replacement >= 3
|
||||
}
|
||||
|
||||
fn has_private_use_text_run(text: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut private_use = 0usize;
|
||||
let mut current_run = 0usize;
|
||||
let mut longest_run = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
current_run = 0;
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
if is_private_use_char(ch) {
|
||||
private_use += 1;
|
||||
current_run += 1;
|
||||
longest_run = longest_run.max(current_run);
|
||||
} else {
|
||||
current_run = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if private_use == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
|
||||
}
|
||||
|
||||
fn has_cid_control_token(text: &str) -> bool {
|
||||
text.split_whitespace().any(token_has_cid_control)
|
||||
}
|
||||
|
||||
fn token_has_cid_control(token: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
|
||||
for ch in token.chars() {
|
||||
total += 1;
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total >= 5 && c1_control >= 2 && c1_control * 20 >= total
|
||||
}
|
||||
|
||||
fn is_private_use_char(ch: char) -> bool {
|
||||
matches!(
|
||||
ch as u32,
|
||||
0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if extracted text is predominantly garbage (non-alphanumeric).
|
||||
///
|
||||
/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---."
|
||||
/// where most characters are punctuation/symbols. Real text in any language
|
||||
/// has >50% alphanumeric characters.
|
||||
fn is_garbage_text(markdown: &str) -> bool {
|
||||
let mut alphanum = 0usize;
|
||||
let mut non_alphanum = 0usize;
|
||||
|
||||
let chars: Vec<char> = markdown.chars().collect();
|
||||
let mut i = 0usize;
|
||||
while i < chars.len() {
|
||||
let ch = chars[i];
|
||||
let mut run_end = i + 1;
|
||||
while run_end < chars.len() && chars[run_end] == ch {
|
||||
run_end += 1;
|
||||
}
|
||||
|
||||
let is_decorative_leader = matches!(ch, '.' | '_' | '·') && run_end - i >= 3;
|
||||
if !is_decorative_leader {
|
||||
for &run_ch in &chars[i..run_end] {
|
||||
if run_ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
// Skip markdown syntax chars that we add (not from the PDF)
|
||||
if matches!(run_ch, '#' | '*' | '|' | '-' | '\n') {
|
||||
continue;
|
||||
}
|
||||
if run_ch.is_alphanumeric() {
|
||||
alphanum += 1;
|
||||
} else {
|
||||
non_alphanum += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
i = run_end;
|
||||
}
|
||||
|
||||
let total = alphanum + non_alphanum;
|
||||
total >= 50 && alphanum * 2 < total
|
||||
}
|
||||
|
||||
/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts.
|
||||
///
|
||||
/// When CID values don't correspond to Unicode codepoints, the raw bytes often
|
||||
/// produce characters in the C1 control range (U+0080–U+009F) or Private Use
|
||||
/// Area, mixed with random Latin Extended characters. Valid text in any
|
||||
/// language almost never contains C1 controls. We also fall back to the
|
||||
/// general `is_garbage_text` check for non-alphanumeric-heavy patterns.
|
||||
fn is_cid_garbage(text: &str) -> bool {
|
||||
if is_garbage_text(text) {
|
||||
return true;
|
||||
}
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
let mut high_latin = 0usize;
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
// C1 control characters (U+0080–U+009F) — almost never in real text
|
||||
if ch == '·' {
|
||||
continue;
|
||||
}
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
// High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text
|
||||
// but when combined with ASCII in CID passthrough, indicates mojibake
|
||||
// from CID values being misinterpreted as Latin-1 characters.
|
||||
if ('\u{00A0}'..='\u{00FF}').contains(&ch) {
|
||||
high_latin += 1;
|
||||
}
|
||||
}
|
||||
if total < 5 {
|
||||
return false;
|
||||
}
|
||||
// If ≥5% of non-whitespace chars are C1 controls, it's garbage
|
||||
if c1_control >= 2 && c1_control * 20 >= total {
|
||||
return true;
|
||||
}
|
||||
// If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
|
||||
// ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
|
||||
// where CID values 0x80-0xFF become accented Latin characters). Keep a
|
||||
// minimum length so short math tokens like "2×()×" do not route a clean
|
||||
// page to OCR.
|
||||
let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
|
||||
total >= 20 && high_latin * 5 >= total * 2 && ascii_letters * 3 < total
|
||||
}
|
||||
|
||||
/// Detect markdown tables with suspicious structure that suggest the heuristic
|
||||
/// missed/mangled rows or columns. Returns true when the caller should treat
|
||||
/// the result as `needs_ocr` and fall back to GPU OCR.
|
||||
|
||||
@@ -1,520 +0,0 @@
|
||||
//! Text-quality detection: deciding when an extracted text layer is too broken
|
||||
//! to serve and a page should fall back to OCR.
|
||||
//!
|
||||
//! Extraction can produce plausible-looking bytes that are actually garbage —
|
||||
//! failed CID→Unicode mappings, broken ToUnicode CMaps, mojibake. These
|
||||
//! detectors catch that and let callers set `needs_ocr`. They come in two
|
||||
//! layers, sharing the same primitives:
|
||||
//!
|
||||
//! - **Markdown-level** ([`detect_encoding_issues`], [`is_garbage_text`],
|
||||
//! [`is_cid_garbage`]) run on a page's final markdown string. Used as a
|
||||
//! backstop on the region-extraction and whole-document paths.
|
||||
//! - **Item/span-level** ([`analyze_text_quality`],
|
||||
//! [`region_items_have_decoding_issue`]) run on individual `TextItem`s and
|
||||
//! accumulate per-page evidence, so localized garbled spans on an otherwise
|
||||
//! clean page are caught without a single span having to condemn the page.
|
||||
//!
|
||||
//! Detection classes, roughly by signal:
|
||||
//! - **Replacement runs**: U+FFFD clusters ([`has_replacement_text_run`]).
|
||||
//! - **Private-use / C1-control runs**: CID passthrough landing in PUA or the
|
||||
//! C1 block ([`has_private_use_text_run`], [`has_cid_control_token`]).
|
||||
//! - **Dollar-as-space**: `Word$Word$Word` from broken CMaps
|
||||
//! ([`has_dollar_as_space_pattern`]).
|
||||
//! - **Non-alphanumeric dominance**: symbol soup ([`is_garbage_text`]).
|
||||
//! - **Substitution-cipher letter statistics**: pure-ASCII output whose letter
|
||||
//! distribution is a permutation of natural language ([`CipherGarbleStats`]).
|
||||
|
||||
use crate::types::TextItem;
|
||||
use crate::{add_ocr_reason, OCR_REASON_SUSPECTED_GARBLED_TEXT};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Detect broken font encodings in extracted markdown text.
|
||||
///
|
||||
/// Two heuristics:
|
||||
/// 1. **U+FFFD**: Any replacement character indicates decode failures.
|
||||
/// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a
|
||||
/// word separator due to broken ToUnicode CMaps. Triggers when either:
|
||||
/// - More than 50% of `$` are between letters (clear substitution pattern), OR
|
||||
/// - More than 20 letter-dollar-letter occurrences (even if some `$` are also
|
||||
/// used as trailing/leading separators, 20+ is far beyond normal financial text).
|
||||
pub(crate) fn detect_encoding_issues(markdown: &str) -> bool {
|
||||
// Heuristic 1: U+FFFD replacement characters
|
||||
if markdown.contains('\u{FFFD}') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Heuristic 2: dollar-as-space pattern
|
||||
if has_dollar_as_space_pattern(markdown) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Heuristic 3: substitution-cipher letter statistics (broken ToUnicode)
|
||||
let mut stats = CipherGarbleStats::default();
|
||||
stats.add_text(markdown);
|
||||
stats.looks_garbled()
|
||||
}
|
||||
|
||||
fn has_dollar_as_space_pattern(markdown: &str) -> bool {
|
||||
let total_dollars = markdown.matches('$').count();
|
||||
if total_dollars > 10 {
|
||||
let bytes = markdown.as_bytes();
|
||||
let mut letter_dollar_letter = 0usize;
|
||||
for i in 1..bytes.len().saturating_sub(1) {
|
||||
if bytes[i] == b'$'
|
||||
&& bytes[i - 1].is_ascii_alphabetic()
|
||||
&& bytes[i + 1].is_ascii_alphabetic()
|
||||
{
|
||||
letter_dollar_letter += 1;
|
||||
}
|
||||
}
|
||||
if letter_dollar_letter > 20 || letter_dollar_letter * 2 > total_dollars {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// English letter frequencies (percent, a–z). Used as a natural-language
|
||||
/// reference: every Latin-script language in the eval corpus (Swedish,
|
||||
/// Finnish, Turkish, German, romaji) scores ≥ 0.80 cosine similarity against
|
||||
/// it, while substitution-cipher text scores ~0.53.
|
||||
const ENGLISH_LETTER_FREQ: [f64; 26] = [
|
||||
8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, 6.7, 7.5, 1.9, 0.1, 6.0,
|
||||
6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07,
|
||||
];
|
||||
|
||||
/// Letter statistics for detecting substitution-cipher garbling: broken
|
||||
/// ToUnicode CMaps that shift every character by a per-range constant (e.g.
|
||||
/// `Certificate` extracted as `8VceZWZTReV`). Such text is 100% printable
|
||||
/// ASCII with word-like token lengths, so it defeats `is_garbage_text` and
|
||||
/// produces no replacement characters — it needs its own discriminator.
|
||||
#[derive(Debug, Default)]
|
||||
struct CipherGarbleStats {
|
||||
/// Case-folded ASCII letter histogram.
|
||||
letter_counts: [u32; 26],
|
||||
ascii_letters: usize,
|
||||
ascii_vowels: usize,
|
||||
/// Accented Latin letters (Latin-1 Supplement through Latin Extended-B,
|
||||
/// plus Latin Extended Additional). Count toward Latin dominance only.
|
||||
latin_ext_letters: usize,
|
||||
non_latin_letters: usize,
|
||||
/// Adjacent ASCII-letter pairs, and how many of them switch from
|
||||
/// lowercase straight to uppercase mid-word.
|
||||
letter_bigrams: usize,
|
||||
case_shift_bigrams: usize,
|
||||
}
|
||||
|
||||
impl CipherGarbleStats {
|
||||
fn add_text(&mut self, text: &str) {
|
||||
let mut prev: Option<char> = None;
|
||||
for ch in text.chars() {
|
||||
if ch.is_ascii_alphabetic() {
|
||||
let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize;
|
||||
self.letter_counts[idx] += 1;
|
||||
self.ascii_letters += 1;
|
||||
if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
|
||||
self.ascii_vowels += 1;
|
||||
}
|
||||
if let Some(p) = prev {
|
||||
self.letter_bigrams += 1;
|
||||
if p.is_ascii_lowercase() && ch.is_ascii_uppercase() {
|
||||
self.case_shift_bigrams += 1;
|
||||
}
|
||||
}
|
||||
prev = Some(ch);
|
||||
} else {
|
||||
if ch.is_alphabetic() {
|
||||
if matches!(ch as u32, 0xC0..=0x24F | 0x1E00..=0x1EFF) {
|
||||
self.latin_ext_letters += 1;
|
||||
} else {
|
||||
self.non_latin_letters += 1;
|
||||
}
|
||||
}
|
||||
prev = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cosine similarity between the observed letter histogram and English
|
||||
/// letter frequencies. A shifted alphabet permutes the histogram, which
|
||||
/// destroys the similarity regardless of the shift amount.
|
||||
fn english_cosine(&self) -> f64 {
|
||||
if self.ascii_letters == 0 {
|
||||
return 1.0;
|
||||
}
|
||||
let n = self.ascii_letters as f64;
|
||||
let mut dot = 0.0;
|
||||
let mut norm_obs = 0.0;
|
||||
for (count, freq) in self.letter_counts.iter().zip(ENGLISH_LETTER_FREQ) {
|
||||
let p = *count as f64 / n;
|
||||
dot += p * freq;
|
||||
norm_obs += p * p;
|
||||
}
|
||||
let norm_en = ENGLISH_LETTER_FREQ
|
||||
.iter()
|
||||
.map(|f| f * f)
|
||||
.sum::<f64>()
|
||||
.sqrt();
|
||||
dot / (norm_obs.sqrt() * norm_en)
|
||||
}
|
||||
|
||||
/// Cosine similarity between the observed histogram and English
|
||||
/// frequencies after sorting BOTH descending — i.e. comparing the *shape*
|
||||
/// of the frequency profile, ignoring which letter sits where. A
|
||||
/// substitution cipher is a bijection, so it preserves this shape exactly
|
||||
/// (att10k 0.97, arbitrary shifts 0.99) regardless of case or offset.
|
||||
/// Non-linguistic ASCII has a different profile: a small alphabet is far
|
||||
/// steeper (random DNA 0.74, hex dumps 0.81), so the shape diverges.
|
||||
fn english_shape_cosine(&self) -> f64 {
|
||||
if self.ascii_letters == 0 {
|
||||
return 1.0;
|
||||
}
|
||||
let n = self.ascii_letters as f64;
|
||||
let mut obs: [f64; 26] = std::array::from_fn(|i| self.letter_counts[i] as f64 / n);
|
||||
obs.sort_unstable_by(|a, b| b.total_cmp(a));
|
||||
let mut en = ENGLISH_LETTER_FREQ;
|
||||
en.sort_unstable_by(|a, b| b.total_cmp(a));
|
||||
|
||||
let dot: f64 = obs.iter().zip(en).map(|(o, e)| o * e).sum();
|
||||
let norm_obs = obs.iter().map(|o| o * o).sum::<f64>().sqrt();
|
||||
let norm_en = en.iter().map(|e| e * e).sum::<f64>().sqrt();
|
||||
dot / (norm_obs * norm_en)
|
||||
}
|
||||
|
||||
/// Thresholds validated against the 380-document pdf-evals snapshot
|
||||
/// corpus (0 false positives) and the garbled ParseBench `att10k` page
|
||||
/// (vowel ratio 0.245, case-shift rate 0.225, cosine 0.532). Closest
|
||||
/// legitimate document on each axis: vowel ratio 0.264 (circuit
|
||||
/// schematic), case-shift rate 0.021, cosine 0.801.
|
||||
fn looks_garbled(&self) -> bool {
|
||||
// Need a statistically meaningful, Latin-dominant sample.
|
||||
if self.ascii_letters < 200
|
||||
|| self.non_latin_letters > self.ascii_letters + self.latin_ext_letters
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Real Latin-script text keeps vowels above ~30% of letters even in
|
||||
// acronym- and part-number-heavy documents; shifted text starves them.
|
||||
let vowel_ratio = self.ascii_vowels as f64 / self.ascii_letters as f64;
|
||||
if vowel_ratio > 0.30 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Signal 1: lowercase→uppercase transitions inside words. A shifted
|
||||
// lowercase alphabet straddles the ASCII uppercase block ('i'→'Z',
|
||||
// 't'→'e'), so garbled words flip case constantly. Real documents
|
||||
// stay ≤ 0.02 even with camelCase identifiers.
|
||||
let case_shifts = self.letter_bigrams >= 100
|
||||
&& self.case_shift_bigrams as f64 >= self.letter_bigrams as f64 * 0.10;
|
||||
|
||||
// Signal 2: the histogram is a permutation of natural language — an
|
||||
// English-like frequency SHAPE (sorted cosine high) but with letters
|
||||
// in the wrong POSITIONS (unsorted cosine low). This is the signature
|
||||
// of a substitution cipher and is case-independent, so it catches
|
||||
// all-lowercase and all-uppercase shifts as well as case-straddling
|
||||
// ones. Genuinely non-linguistic ASCII that is merely "unlike English"
|
||||
// fails one of the two halves: DNA/hex dumps have too steep a profile
|
||||
// (shape cosine < 0.90), while protein sequences, ticker symbols and
|
||||
// base64 are not sufficiently unlike English in position (unsorted
|
||||
// cosine ≥ 0.60) — so none of them are routed to OCR.
|
||||
let permuted_language = self.english_cosine() < 0.60 && self.english_shape_cosine() >= 0.90;
|
||||
|
||||
case_shifts || permuted_language
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct TextQualityReport {
|
||||
pub(crate) pages_needing_ocr: Vec<u32>,
|
||||
pub(crate) has_encoding_issues: bool,
|
||||
pub(crate) reasons_by_page: BTreeMap<u32, Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PageTextQualityEvidence {
|
||||
chars: usize,
|
||||
replacement_chars: usize,
|
||||
replacement_spans: usize,
|
||||
longest_replacement_run: usize,
|
||||
cipher_garble: CipherGarbleStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum TextSpanIssueKind {
|
||||
Replacement,
|
||||
Strong,
|
||||
}
|
||||
|
||||
pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
|
||||
let mut reasons_by_page = BTreeMap::new();
|
||||
let mut evidence_by_page = BTreeMap::<u32, PageTextQualityEvidence>::new();
|
||||
|
||||
for item in items {
|
||||
if !matches!(item.item_type, crate::types::ItemType::Text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let evidence = evidence_by_page.entry(item.page).or_default();
|
||||
evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
|
||||
evidence.cipher_garble.add_text(&item.text);
|
||||
|
||||
match text_span_decoding_issue_kind(&item.text) {
|
||||
Some(TextSpanIssueKind::Strong) => {
|
||||
add_ocr_reason(
|
||||
&mut reasons_by_page,
|
||||
item.page,
|
||||
OCR_REASON_SUSPECTED_GARBLED_TEXT,
|
||||
);
|
||||
}
|
||||
Some(TextSpanIssueKind::Replacement) => {
|
||||
let stats = replacement_text_stats(&item.text);
|
||||
evidence.replacement_chars += stats.0;
|
||||
evidence.replacement_spans += 1;
|
||||
evidence.longest_replacement_run = evidence.longest_replacement_run.max(stats.1);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
|
||||
for (page, evidence) in evidence_by_page {
|
||||
if reasons_by_page.contains_key(&page) {
|
||||
continue;
|
||||
}
|
||||
if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
|
||||
{
|
||||
add_ocr_reason(
|
||||
&mut reasons_by_page,
|
||||
page,
|
||||
OCR_REASON_SUSPECTED_GARBLED_TEXT,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let pages_needing_ocr: Vec<u32> = reasons_by_page.keys().copied().collect();
|
||||
TextQualityReport {
|
||||
has_encoding_issues: !pages_needing_ocr.is_empty(),
|
||||
pages_needing_ocr,
|
||||
reasons_by_page,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
|
||||
items.iter().any(|item| {
|
||||
matches!(item.item_type, crate::types::ItemType::Text)
|
||||
&& text_span_has_decoding_issue(&item.text)
|
||||
})
|
||||
}
|
||||
|
||||
fn text_span_has_decoding_issue(text: &str) -> bool {
|
||||
text_span_decoding_issue_kind(text).is_some()
|
||||
}
|
||||
|
||||
fn text_span_decoding_issue_kind(text: &str) -> Option<TextSpanIssueKind> {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if has_dollar_as_space_pattern(text)
|
||||
|| has_private_use_text_run(text)
|
||||
|| is_cid_garbage(text)
|
||||
|| has_cid_control_token(text)
|
||||
{
|
||||
return Some(TextSpanIssueKind::Strong);
|
||||
}
|
||||
|
||||
if has_replacement_text_run(text) {
|
||||
return Some(TextSpanIssueKind::Replacement);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn replacement_text_stats(text: &str) -> (usize, usize) {
|
||||
let mut replacement = 0usize;
|
||||
let mut current_run = 0usize;
|
||||
let mut longest_run = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch == '\u{FFFD}' {
|
||||
replacement += 1;
|
||||
current_run += 1;
|
||||
longest_run = longest_run.max(current_run);
|
||||
} else {
|
||||
current_run = 0;
|
||||
}
|
||||
}
|
||||
|
||||
(replacement, longest_run)
|
||||
}
|
||||
|
||||
fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
|
||||
if evidence.replacement_chars == 0 || evidence.chars == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If the entire page is only a short broken text layer, even a short
|
||||
// replacement run is enough evidence. On otherwise text-heavy pages,
|
||||
// require density so math formulas do not force full-page OCR.
|
||||
if evidence.chars <= 80 && evidence.longest_replacement_run >= 2 {
|
||||
return true;
|
||||
}
|
||||
|
||||
let replacement_density_bps = evidence.replacement_chars * 10_000 / evidence.chars;
|
||||
let enough_bad_text = evidence.replacement_chars >= 12 && replacement_density_bps >= 500;
|
||||
let repeated_bad_spans = evidence.replacement_spans >= 3 && replacement_density_bps >= 250;
|
||||
let long_bad_run = evidence.longest_replacement_run >= 8 && replacement_density_bps >= 250;
|
||||
|
||||
enough_bad_text || repeated_bad_spans || long_bad_run
|
||||
}
|
||||
|
||||
fn has_replacement_text_run(text: &str) -> bool {
|
||||
let (replacement, longest_run) = replacement_text_stats(text);
|
||||
longest_run >= 2 || replacement >= 3
|
||||
}
|
||||
|
||||
fn has_private_use_text_run(text: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut private_use = 0usize;
|
||||
let mut current_run = 0usize;
|
||||
let mut longest_run = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
current_run = 0;
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
if is_private_use_char(ch) {
|
||||
private_use += 1;
|
||||
current_run += 1;
|
||||
longest_run = longest_run.max(current_run);
|
||||
} else {
|
||||
current_run = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if private_use == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
|
||||
}
|
||||
|
||||
fn has_cid_control_token(text: &str) -> bool {
|
||||
text.split_whitespace().any(token_has_cid_control)
|
||||
}
|
||||
|
||||
fn token_has_cid_control(token: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
|
||||
for ch in token.chars() {
|
||||
total += 1;
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total >= 5 && c1_control >= 2 && c1_control * 20 >= total
|
||||
}
|
||||
|
||||
fn is_private_use_char(ch: char) -> bool {
|
||||
matches!(
|
||||
ch as u32,
|
||||
0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if extracted text is predominantly garbage (non-alphanumeric).
|
||||
///
|
||||
/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---."
|
||||
/// where most characters are punctuation/symbols. Real text in any language
|
||||
/// has >50% alphanumeric characters.
|
||||
pub(crate) fn is_garbage_text(markdown: &str) -> bool {
|
||||
let mut alphanum = 0usize;
|
||||
let mut non_alphanum = 0usize;
|
||||
|
||||
let chars: Vec<char> = markdown.chars().collect();
|
||||
let mut i = 0usize;
|
||||
while i < chars.len() {
|
||||
let ch = chars[i];
|
||||
let mut run_end = i + 1;
|
||||
while run_end < chars.len() && chars[run_end] == ch {
|
||||
run_end += 1;
|
||||
}
|
||||
|
||||
let is_decorative_leader = matches!(ch, '.' | '_' | '·') && run_end - i >= 3;
|
||||
if !is_decorative_leader {
|
||||
for &run_ch in &chars[i..run_end] {
|
||||
if run_ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
// Skip markdown syntax chars that we add (not from the PDF)
|
||||
if matches!(run_ch, '#' | '*' | '|' | '-' | '\n') {
|
||||
continue;
|
||||
}
|
||||
if run_ch.is_alphanumeric() {
|
||||
alphanum += 1;
|
||||
} else {
|
||||
non_alphanum += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
i = run_end;
|
||||
}
|
||||
|
||||
let total = alphanum + non_alphanum;
|
||||
total >= 50 && alphanum * 2 < total
|
||||
}
|
||||
|
||||
/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts.
|
||||
///
|
||||
/// When CID values don't correspond to Unicode codepoints, the raw bytes often
|
||||
/// produce characters in the C1 control range (U+0080–U+009F) or Private Use
|
||||
/// Area, mixed with random Latin Extended characters. Valid text in any
|
||||
/// language almost never contains C1 controls. We also fall back to the
|
||||
/// general `is_garbage_text` check for non-alphanumeric-heavy patterns.
|
||||
pub(crate) fn is_cid_garbage(text: &str) -> bool {
|
||||
if is_garbage_text(text) {
|
||||
return true;
|
||||
}
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
let mut high_latin = 0usize;
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
// C1 control characters (U+0080–U+009F) — almost never in real text
|
||||
if ch == '·' {
|
||||
continue;
|
||||
}
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
// High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text
|
||||
// but when combined with ASCII in CID passthrough, indicates mojibake
|
||||
// from CID values being misinterpreted as Latin-1 characters.
|
||||
if ('\u{00A0}'..='\u{00FF}').contains(&ch) {
|
||||
high_latin += 1;
|
||||
}
|
||||
}
|
||||
if total < 5 {
|
||||
return false;
|
||||
}
|
||||
// If ≥5% of non-whitespace chars are C1 controls, it's garbage
|
||||
if c1_control >= 2 && c1_control * 20 >= total {
|
||||
return true;
|
||||
}
|
||||
// If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
|
||||
// ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
|
||||
// where CID values 0x80-0xFF become accented Latin characters). Keep a
|
||||
// minimum length so short math tokens like "2×()×" do not route a clean
|
||||
// page to OCR.
|
||||
let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
|
||||
total >= 20 && high_latin * 5 >= total * 2 && ascii_letters * 3 < total
|
||||
}
|
||||
Reference in New Issue
Block a user