diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ff4a3c1..ca1e450 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -25,6 +25,7 @@ pub(crate) fn extract_page_text_items( page_id: ObjectId, page_num: u32, font_cmaps: &FontCMaps, + include_invisible: bool, ) -> Result { use lopdf::content::Content; @@ -252,8 +253,10 @@ pub(crate) fn extract_page_text_items( } continue; } - // Skip invisible (Tr=3) text but still advance text matrix - if text_rendering_mode == 3 { + // Skip invisible (Tr=3) text but still advance text matrix. + // For Mixed/template PDFs, include_invisible=true extracts + // the OCR text layer that sits behind scanned images. + if text_rendering_mode == 3 && !include_invisible { if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -311,7 +314,8 @@ pub(crate) fn extract_page_text_items( if in_text_block && !op.operands.is_empty() { if let Ok(array) = op.operands[0].as_array() { let font_info = font_widths.get(¤t_font); - let is_invisible = text_rendering_mode == 3 || suppress_glyph_extraction; + let is_invisible = (text_rendering_mode == 3 && !include_invisible) + || suppress_glyph_extraction; // Compute space threshold based on font metrics when available let space_threshold = if let Some(font_info) = font_info { @@ -471,7 +475,7 @@ pub(crate) fn extract_page_text_items( line_matrix[4] += (-tl) * line_matrix[2]; line_matrix[5] += (-tl) * line_matrix[3]; text_matrix = line_matrix; - if !(text_rendering_mode == 3 + if !((text_rendering_mode == 3 && !include_invisible) || suppress_glyph_extraction || op.operands.is_empty()) { diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 2b85eed..9be4977 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -149,6 +149,25 @@ pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, +) -> Result<(PageExtraction, PageThresholds), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, false) +} + +/// Extract with option to include invisible (Tr=3) text. +/// Used for Mixed/template PDFs where the OCR text layer is invisible. +pub(crate) fn extract_positioned_text_include_invisible( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, +) -> Result<(PageExtraction, PageThresholds), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, true) +} + +fn extract_positioned_text_impl( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, + include_invisible: bool, ) -> Result<(PageExtraction, PageThresholds), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); @@ -167,7 +186,7 @@ pub(crate) fn extract_positioned_text_from_doc( } } let (mut items, rects, lines) = - extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?; let threshold = crate::text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { page_thresholds.insert(*page_num, threshold); diff --git a/src/lib.rs b/src/lib.rs index d5acd94..db15348 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -342,7 +342,38 @@ fn process_document( // Step 2 — Extraction (reuses the already-loaded document) let extracted = { let font_cmaps = FontCMaps::from_doc(&doc); - extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, options.page_filter.as_ref()) + let result = extractor::extract_positioned_text_from_doc( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ); + + // For Mixed/template PDFs: if normal extraction produces garbage text + // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. + // This unlocks OCR text layers behind scanned images. + if pdf_type == PdfType::Mixed { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _)| e) { + let sample: String = items.iter().take(200).map(|i| i.text.as_str()).collect(); + if is_garbage_text(&sample) || sample.trim().is_empty() { + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } else { + result + } + } else { + // Normal extraction failed — try invisible as fallback + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } + } else { + result + } }; // For Mixed PDFs, extraction failure is non-fatal diff --git a/src/tounicode.rs b/src/tounicode.rs index eeb3e10..727e89f 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -815,6 +815,28 @@ fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option { } } } + // Fallback: Windows Unicode BMP (3,1) — maps Unicode codepoints to GIDs. + // For single-byte fonts, try each byte value as a Unicode codepoint. + // Common in OCR-generated PDFs where byte values correspond to Unicode + // codepoints but the declared encoding (WinAnsiEncoding) is wrong. + if !used_encoding_cmap { + for subtable in cmap_table.subtables { + if subtable.platform_id == ttf_parser::PlatformId::Windows + && subtable.encoding_id == 1 + { + for code in 0x20..=0xFF_u32 { + if let Some(gid) = subtable.glyph_index(code) { + if let Some(&ch) = gid_to_unicode.get(&gid.0) { + let ch = strip_pua_char(ch); + cmap.char_map.entry(code as u16).or_insert(ch.to_string()); + } + } + } + used_encoding_cmap = true; + break; + } + } + } } if !used_encoding_cmap { @@ -1898,13 +1920,6 @@ impl FontCMaps { if font_dict.get(b"ToUnicode").is_ok() { continue; } - // Skip fonts with explicit encoding — they can be decoded by the - // standard encoding path (lopdf) and don't need a fallback CMap. - if let Ok(enc) = font_dict.get(b"Encoding") { - if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() { - continue; - } - } let subtype = match font_dict .get(b"Subtype") .ok() @@ -1916,6 +1931,18 @@ impl FontCMaps { if subtype == b"Type0" { continue; } + // Skip non-TrueType fonts with explicit encoding — they can be + // decoded by the standard encoding path and don't need a fallback. + // TrueType fonts are NOT skipped: OCR-generated PDFs often declare + // WinAnsiEncoding but the embedded font's cmap has the real mapping. + if subtype != b"TrueType" { + if let Ok(enc) = font_dict.get(b"Encoding") { + if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() + { + continue; + } + } + } let font_descriptor = font_dict.get(b"FontDescriptor").ok().and_then(|o| match o { Object::Reference(r) => doc.get_dictionary(*r).ok(),