From 0a9c120a6bb5c921328763fb73e132e5abfad4d3 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 14 Apr 2026 21:42:19 -0700 Subject: [PATCH] fix: reduce false OCR recommendations for text PDFs with figures (#38) * fix: reduce false OCR recommendations for text PDFs with figure images Two fixes in the detector: 1. Fix Tf operator parsing: some PDFs concatenate Tf directly with the next operator (e.g. "25 Tf[<01>...") without whitespace. The scanner now accepts [, (, <, / as valid followers, fixing font_changes being reported as 0. 2. Distinguish text-with-figures from scanned-with-OCR: pages with multiple images (image_count > 1) and strong text signals (text_ops >= 50, alphanum >= 10) are recognized as text pages with figures, not scanned templates. Scanned PDFs have exactly 1 full-page image. This prevents academic papers, reports with charts, and similar PDFs from being incorrectly classified as Mixed/OCR-needed when their text is perfectly extractable. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: bump napi version to 0.7.2 Co-Authored-By: Claude Opus 4.6 (1M context) * fix: remove template image influence from page classification Template images (large background/figure images) no longer affect pages_needing_ocr. In the region-based pipeline, text regions are extracted independently from image regions, and per-region needs_ocr quality checks handle scanned-with-OCR garbage text. Also makes the invisible text retry (for OCR text layers) trigger on text quality rather than PDF type, so it works regardless of classification. Co-Authored-By: Claude Opus 4.6 (1M context) * Revert "fix: remove template image influence from page classification" This reverts commit 100cbe54532935bc6bfb4e79b302a39bbb430e6b. --------- Co-authored-by: Claude Opus 4.6 (1M context) --- napi/package.json | 2 +- src/detector.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/napi/package.json b/napi/package.json index 626cefd..53f480e 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-pdf-inspector", - "version": "0.7.1", + "version": "0.7.2", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/src/detector.rs b/src/detector.rs index 787925b..e3151a4 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -223,7 +223,15 @@ pub(crate) fn detect_from_document( if analysis.has_images { pages_with_images += 1; } - if analysis.has_template_image { + // Only count as a template-image page if it looks like a scan + // (single full-page image) rather than a text page with figures. + // Scanned-with-OCR PDFs have 1 large image per page + OCR text overlay; + // text PDFs with figures have multiple smaller images alongside real text. + if analysis.has_template_image + && (analysis.image_count <= 1 + || analysis.text_operator_count < 50 + || analysis.unique_alphanum_chars < 10) + { pages_with_template_images += 1; } if analysis.has_vector_text { @@ -350,7 +358,12 @@ pub(crate) fn detect_from_document( } else { continue; }; - if analysis.has_template_image + // Template images only need OCR when it looks like a scan + // (single full-page image) rather than figures alongside text. + let looks_like_scan = analysis.image_count <= 1 + || analysis.text_operator_count < 50 + || analysis.unique_alphanum_chars < 10; + if (analysis.has_template_image && looks_like_scan) || analysis.has_vector_text || (analysis.text_operator_count < config.min_text_ops_per_page && analysis.has_images) @@ -865,10 +878,17 @@ fn scan_content_for_text_operators( } } else if next == b'f' { // Tf = set font operator + // Some PDFs concatenate Tf with the next operator without + // whitespace (e.g. "25 Tf[<01>..." or "25 Tf(..."), + // so also accept '[', '(', '<', '/' as valid followers. if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() || content[i + 2] == b'\n' || content[i + 2] == b'\r' + || content[i + 2] == b'[' + || content[i + 2] == b'(' + || content[i + 2] == b'<' + || content[i + 2] == b'/' { font_changes += 1; } @@ -1613,6 +1633,39 @@ mod tests { assert_eq!(fonts, 2); } + #[test] + fn test_tf_without_trailing_whitespace() { + // Some PDFs concatenate Tf directly with the next operator's operand, + // e.g. "25 Tf[<01>..." or "25 Tf(..." + let mut uchars = HashSet::new(); + + // Tf followed by '[' (TJ array start) + let content = b"BT /F1 25 Tf[<01>1<02>-1] TJ ET"; + let (ops, _, _, fonts) = scan_content_for_text_operators(content, &mut uchars); + assert_eq!(fonts, 1, "Tf followed by '[' should be counted"); + assert_eq!(ops, 1); + + // Tf followed by '(' (literal string) + uchars.clear(); + let content2 = b"BT /F1 12 Tf(Hello) Tj ET"; + let (ops2, _, _, fonts2) = scan_content_for_text_operators(content2, &mut uchars); + assert_eq!(fonts2, 1, "Tf followed by '(' should be counted"); + assert_eq!(ops2, 1); + + // Tf followed by '<' (hex string) + uchars.clear(); + let content3 = b"BT /F1 12 Tf<0102> Tj ET"; + let (ops3, _, _, fonts3) = scan_content_for_text_operators(content3, &mut uchars); + assert_eq!(fonts3, 1, "Tf followed by '<' should be counted"); + assert_eq!(ops3, 1); + + // Tf followed by '/' (next font name) + uchars.clear(); + let content4 = b"BT /F1 12 Tf/F2 10 Tf (x) Tj ET"; + let (_, _, _, fonts4) = scan_content_for_text_operators(content4, &mut uchars); + assert_eq!(fonts4, 2, "Tf followed by '/' should be counted"); + } + #[test] fn test_newspaper_heuristic_thresholds() { // Newspaper page: high text ops, moderate font changes, low ratio