feat(detect): flag Identity-H fonts without ToUnicode for OCR (#7)

* feat(detect): flag Identity-H fonts without ToUnicode for OCR Cyrillic (and other non-Latin) PDFs with Type0/Identity-H encoded fonts and no ToUnicode CMap produce garbage text from direct extraction. Two fixes: 1. Detector: new `page_has_identity_h_no_tounicode` check adds affected pages to `pages_needing_ocr` regardless of PDF classification. 2. Extraction: extend garbage-text safety net to TextBased PDFs — when extracted text is <50% alphanumeric, drop the markdown, set `has_encoding_issues`, and flag all pages for OCR. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor: replace integration tests with synthetic unit tests Remove PDF fixture dependencies from detector and lib tests. Use in-memory lopdf documents to test Identity-H/ToUnicode detection logic. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-19 15:40:48 -07:00
parent 57560f1692
commit 587c4bed95
3 changed files with 192 additions and 3 deletions
@@ -268,6 +268,9 @@ fn main() {
                        eprintln!("Pages: {}", result.page_count);
                        eprintln!("Processing time: {}ms", result.processing_time_ms);
                        print_layout_info(&result.layout);
+                        if !result.pages_needing_ocr.is_empty() {
+                            eprintln!("Pages needing OCR: {:?}", result.pages_needing_ocr);
+                        }

                        if let Some(markdown) = &result.markdown {
                            if let Some(output) = output_file {
@@ -197,12 +197,12 @@ pub(crate) fn detect_from_document(
            let analysis = analyze_page_content(doc, page_id);
            pages_actually_sampled += 1;
            log::debug!(
-                "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={}",
+                "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={}",
                page_num, analysis.text_operator_count, analysis.has_images,
                analysis.image_count, analysis.has_template_image,
                analysis.unique_text_chars, analysis.unique_alphanum_chars,
                analysis.path_op_count, analysis.has_vector_text,
-                analysis.total_image_area
+                analysis.total_image_area, analysis.has_identity_h_no_tounicode
            );
            let is_image_dominated = analysis.image_count > 10
                && analysis.image_count > analysis.text_operator_count * 3;
@@ -292,7 +292,7 @@ pub(crate) fn detect_from_document(
    };

    // Phase 2: Build per-page OCR list
-    let pages_needing_ocr = match pdf_type {
+    let mut pages_needing_ocr = match pdf_type {
        PdfType::TextBased => Vec::new(),
        PdfType::Scanned | PdfType::ImageBased => (1..=total_pages).collect(),
        PdfType::Mixed => {
@@ -319,6 +319,29 @@ pub(crate) fn detect_from_document(
        }
    };

+    // Phase 3: Flag pages with Identity-H/V fonts lacking ToUnicode for OCR.
+    // These fonts produce garbage text (raw CID values) for non-Latin scripts.
+    for (&page_num, analysis) in &analysis_cache {
+        if analysis.has_identity_h_no_tounicode && !pages_needing_ocr.contains(&page_num) {
+            pages_needing_ocr.push(page_num);
+        }
+    }
+    // Check uncached pages too (when not all pages were sampled)
+    if pages_needing_ocr.len() < total_pages as usize {
+        for page_num in 1..=total_pages {
+            if analysis_cache.contains_key(&page_num) || pages_needing_ocr.contains(&page_num) {
+                continue;
+            }
+            if let Some(&page_id) = pages.get(&page_num) {
+                if page_has_identity_h_no_tounicode(doc, page_id) {
+                    pages_needing_ocr.push(page_num);
+                }
+            }
+        }
+    }
+    pages_needing_ocr.sort();
+    pages_needing_ocr.dedup();
+
    // Try to get title from metadata
    let title = get_document_title(doc);

@@ -390,6 +413,9 @@ struct PageAnalysis {
    path_op_count: u32,
    /// Whether the page has vector-outlined text (massive path ops, minimal text ops)
    has_vector_text: bool,
+    /// Whether the page has Type0 fonts with Identity-H/V encoding but no ToUnicode CMap.
+    /// These fonts produce garbage text because CID values can't be mapped to Unicode.
+    has_identity_h_no_tounicode: bool,
 }

 /// Analyze a page's content stream for text operators and images
@@ -461,6 +487,10 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis {
        .filter(|b| b.is_ascii_alphanumeric())
        .count() as u32;

+    // Check for Identity-H/V fonts without ToUnicode — these produce garbage text
+    let has_identity_h_no_tounicode =
+        text_ops > 0 && page_has_identity_h_no_tounicode(doc, page_id);
+
    PageAnalysis {
        text_operator_count: text_ops,
        has_images,
@@ -471,9 +501,52 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis {
        unique_alphanum_chars,
        path_op_count: path_ops,
        has_vector_text,
+        has_identity_h_no_tounicode,
    }
 }

+/// Check if a page has Type0 fonts with Identity-H/V encoding and no ToUnicode CMap.
+/// These fonts encode text as raw CID values that can't be mapped to Unicode without
+/// a ToUnicode CMap, producing garbage output for non-Latin scripts (e.g. Cyrillic).
+fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool {
+    let fonts = match doc.get_page_fonts(page_id) {
+        Ok(f) => f,
+        Err(_) => return false,
+    };
+    for font_dict in fonts.values() {
+        let subtype = font_dict
+            .get(b"Subtype")
+            .ok()
+            .and_then(|o| o.as_name().ok());
+        if subtype != Some(b"Type0") {
+            continue;
+        }
+        let encoding = font_dict
+            .get(b"Encoding")
+            .ok()
+            .and_then(|o| o.as_name().ok());
+        let is_identity = matches!(encoding, Some(b"Identity-H") | Some(b"Identity-V"));
+        if !is_identity {
+            continue;
+        }
+        // Has ToUnicode? Then the font is decodable.
+        if font_dict.get(b"ToUnicode").is_ok() {
+            continue;
+        }
+        // Identity-H/V without ToUnicode — flag it
+        log::debug!(
+            "page has Identity-H/V font without ToUnicode: {:?}",
+            font_dict
+                .get(b"BaseFont")
+                .ok()
+                .and_then(|o| o.as_name().ok())
+                .map(|n| String::from_utf8_lossy(n).to_string())
+        );
+        return true;
+    }
+    false
+}
+
 fn scan_xobjects_in_resources(
    doc: &Document,
    resources: &lopdf::Dictionary,
@@ -1131,4 +1204,83 @@ mod tests {
        );
        assert!(result.ocr_recommended);
    }
+
+    #[test]
+    fn test_page_has_identity_h_no_tounicode_positive() {
+        // Build a minimal PDF with a Type0 Identity-H font and no ToUnicode.
+        use lopdf::dictionary;
+        let mut doc = Document::with_version("1.4");
+        let pages_id = doc.new_object_id();
+        let page_id = doc.new_object_id();
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"Type0".to_vec()),
+            "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()),
+            "Encoding" => Object::Name(b"Identity-H".to_vec()),
+        });
+        let resources = dictionary! {
+            "Font" => dictionary! {
+                "F1" => Object::Reference(font_id),
+            },
+        };
+        doc.objects.insert(
+            page_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Page",
+                "Parent" => Object::Reference(pages_id),
+                "Resources" => resources,
+            }),
+        );
+        doc.objects.insert(
+            pages_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Pages",
+                "Kids" => vec![Object::Reference(page_id)],
+                "Count" => Object::Integer(1),
+            }),
+        );
+        assert!(page_has_identity_h_no_tounicode(&doc, page_id));
+    }
+
+    #[test]
+    fn test_page_has_identity_h_with_tounicode_negative() {
+        // Type0 Identity-H font WITH ToUnicode — should NOT flag.
+        use lopdf::dictionary;
+        let mut doc = Document::with_version("1.4");
+        let pages_id = doc.new_object_id();
+        let page_id = doc.new_object_id();
+        let cmap_id = doc.add_object(Object::Stream(lopdf::Stream::new(
+            dictionary! {},
+            b"fake cmap".to_vec(),
+        )));
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"Type0".to_vec()),
+            "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()),
+            "Encoding" => Object::Name(b"Identity-H".to_vec()),
+            "ToUnicode" => Object::Reference(cmap_id),
+        });
+        let resources = dictionary! {
+            "Font" => dictionary! {
+                "F1" => Object::Reference(font_id),
+            },
+        };
+        doc.objects.insert(
+            page_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Page",
+                "Parent" => Object::Reference(pages_id),
+                "Resources" => resources,
+            }),
+        );
+        doc.objects.insert(
+            pages_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Pages",
+                "Kids" => vec![Object::Reference(page_id)],
+                "Count" => Object::Integer(1),
+            }),
+        );
+        assert!(!page_has_identity_h_no_tounicode(&doc, page_id));
+    }
 }
@@ -437,8 +437,23 @@ fn process_document(
            (pdf_type, markdown, confidence)
        };

+    // If a TextBased PDF produces garbage text, the fonts are undecodable
+    // (e.g. Identity-H without ToUnicode for non-Latin scripts like Cyrillic).
+    // Drop the useless markdown and flag all pages for OCR.
+    let (markdown, has_encoding_issues, force_ocr_all) = if pdf_type == PdfType::TextBased
+        && markdown.as_ref().is_some_and(|m| is_garbage_text(m))
+    {
+        log::debug!("TextBased PDF has garbage text — flagging all pages for OCR");
+        (None, true, true)
+    } else {
+        (markdown, has_encoding_issues, false)
+    };
+
    // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list
    let mut pages_needing_ocr = pages_needing_ocr;
+    if force_ocr_all {
+        pages_needing_ocr = (1..=page_count).collect();
+    }
    if !gid_pages.is_empty() {
        log::debug!("pages with gid-encoded fonts (need OCR): {:?}", gid_pages);
        for page in gid_pages {
@@ -827,4 +842,23 @@ mod tests {
        let text = "a$b c$d e$f";
        assert!(!detect_encoding_issues(text));
    }
+
+    #[test]
+    fn test_garbage_text_detection() {
+        // Simulates garbage output from Identity-H fonts without ToUnicode.
+        // Needs >= 50 non-whitespace chars and < 50% alphanumeric.
+        let garbage = ",&<X ~%5&8-!A ~*(!,-!U (/#!U X ~#/=U 9/%*(!U !(  X \
+                       (%U-(-/ V %&((8-#&&< *,(6--< %5&8-!( (,(/! #/<5U X \
+                       º&( >/5 /5&(#(8-!5 *,(6--( *,%@/-A W";
+        assert!(is_garbage_text(garbage));
+
+        // Normal text should not be garbage
+        let normal = "This is a normal paragraph with words and sentences that contains enough characters to pass the threshold.";
+        assert!(!is_garbage_text(normal));
+
+        // Cyrillic text should not be garbage
+        let cyrillic =
+            "Роботизированные технологии комплексы для производства металлургических предприятий";
+        assert!(!is_garbage_text(cyrillic));
+    }
 }