fix(fonts): suppress garbage CID text on OCR-flagged pages (#9)
When Identity-H or Type3 fonts lack a ToUnicode CMap and the CID-as-Unicode passthrough doesn't produce valid text, the raw CID byte values appear as mojibake (random Latin Extended characters mixed with C1 control codes). The detector already flags these pages in pages_needing_ocr, but the markdown pipeline still emitted the garbage. Now, for TextBased PDFs, we check each OCR-flagged page's extracted text for CID garbage (C1 control characters U+0080–U+009F at ≥5% density) and strip items from pages that fail the check. This is scoped to TextBased PDFs only — Mixed PDFs flag pages for OCR due to template images, not font encoding issues. Adds test fixture (shinagawa_identity_h.pdf) and integration test. Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
6c005ef4c2
commit
aa387503c3
BIN
Binary file not shown.
@@ -1037,3 +1037,27 @@ fn test_firecrawl_tagged_pdf_struct_tree() {
|
||||
// Fences come in open/close pairs
|
||||
assert_eq!(fence_count % 2, 0, "Code fences should be balanced");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_identity_h_no_tounicode_suppresses_garbage() {
|
||||
// shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no
|
||||
// ToUnicode CMap. The raw CID values look like random Latin characters.
|
||||
// We should suppress the garbage and flag the page for OCR.
|
||||
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
|
||||
let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
|
||||
|
||||
// Page 1 should be flagged for OCR
|
||||
assert!(
|
||||
result.pages_needing_ocr.contains(&1),
|
||||
"Page with Identity-H font without ToUnicode should be flagged for OCR"
|
||||
);
|
||||
|
||||
// Markdown should be empty (garbage suppressed)
|
||||
let md = result.markdown.unwrap_or_default();
|
||||
assert!(
|
||||
md.trim().is_empty(),
|
||||
"Garbage CID text should be suppressed, got {} chars: {:?}",
|
||||
md.len(),
|
||||
&md[..md.len().min(100)]
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user