feat(detect): flag Type3-only pages without ToUnicode for OCR

Type3 fonts render each glyph as a custom drawing/bitmap. Without a
ToUnicode CMap the character codes can't be mapped to Unicode, so
extracted text is garbage. Pages using only Type3 fonts are now
excluded from pages_with_text and added to pages_needing_ocr.

Fixes Korean/CJK PDFs using Type3 fonts (e.g. D2Coding) being
classified as TextBased when no usable text can be extracted.
This commit is contained in:
Abimael Martell
2026-03-20 17:26:20 -07:00
parent 672be2e9e5
commit 2a8e628396
+55 -6
View File
@@ -197,12 +197,13 @@ pub(crate) fn detect_from_document(
let analysis = analyze_page_content(doc, page_id);
pages_actually_sampled += 1;
log::debug!(
"page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={}",
"page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={} type3_only={}",
page_num, analysis.text_operator_count, analysis.has_images,
analysis.image_count, analysis.has_template_image,
analysis.unique_text_chars, analysis.unique_alphanum_chars,
analysis.path_op_count, analysis.has_vector_text,
analysis.total_image_area, analysis.has_identity_h_no_tounicode
analysis.total_image_area, analysis.has_identity_h_no_tounicode,
analysis.has_only_type3_fonts
);
let is_image_dominated = analysis.image_count > 10
&& analysis.image_count > analysis.text_operator_count * 3;
@@ -215,6 +216,7 @@ pub(crate) fn detect_from_document(
&& !is_image_dominated
&& analysis.unique_text_chars >= 5
&& !analysis.has_vector_text
&& !analysis.has_only_type3_fonts
{
pages_with_text += 1;
}
@@ -319,10 +321,13 @@ pub(crate) fn detect_from_document(
}
};
// Phase 3: Flag pages with Identity-H/V fonts lacking ToUnicode for OCR.
// These fonts produce garbage text (raw CID values) for non-Latin scripts.
// Phase 3: Flag pages with undecodable fonts for OCR.
// - Identity-H/V without ToUnicode: raw CID values can't map to Unicode
// - Type3-only without ToUnicode: glyph bitmaps can't map to Unicode
for (&page_num, analysis) in &analysis_cache {
if analysis.has_identity_h_no_tounicode && !pages_needing_ocr.contains(&page_num) {
if (analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts)
&& !pages_needing_ocr.contains(&page_num)
{
pages_needing_ocr.push(page_num);
}
}
@@ -333,7 +338,9 @@ pub(crate) fn detect_from_document(
continue;
}
if let Some(&page_id) = pages.get(&page_num) {
if page_has_identity_h_no_tounicode(doc, page_id) {
if page_has_identity_h_no_tounicode(doc, page_id)
|| page_has_only_type3_fonts(doc, page_id)
{
pages_needing_ocr.push(page_num);
}
}
@@ -416,6 +423,10 @@ struct PageAnalysis {
/// Whether the page has Type0 fonts with Identity-H/V encoding but no ToUnicode CMap.
/// These fonts produce garbage text because CID values can't be mapped to Unicode.
has_identity_h_no_tounicode: bool,
/// Whether the page uses only Type3 fonts (no normal text fonts).
/// Type3 fonts render each glyph as a custom drawing/bitmap — without a
/// ToUnicode CMap, the character codes can't be mapped to Unicode.
has_only_type3_fonts: bool,
}
/// Analyze a page's content stream for text operators and images
@@ -491,6 +502,9 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis {
let has_identity_h_no_tounicode =
text_ops > 0 && page_has_identity_h_no_tounicode(doc, page_id);
// Check for Type3-only fonts — glyph bitmaps without Unicode mapping
let has_only_type3_fonts = text_ops > 0 && page_has_only_type3_fonts(doc, page_id);
PageAnalysis {
text_operator_count: text_ops,
has_images,
@@ -502,6 +516,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis {
path_op_count: path_ops,
has_vector_text,
has_identity_h_no_tounicode,
has_only_type3_fonts,
}
}
@@ -547,6 +562,40 @@ fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool {
false
}
/// Returns true if every font on the page is Type3 (no normal text fonts).
/// Type3 fonts render glyphs as custom drawings/bitmaps. Without a ToUnicode
/// CMap, character codes can't be mapped to Unicode — the page needs OCR.
fn page_has_only_type3_fonts(doc: &Document, page_id: ObjectId) -> bool {
let fonts = match doc.get_page_fonts(page_id) {
Ok(f) => f,
Err(_) => return false,
};
if fonts.is_empty() {
return false;
}
let mut has_type3 = false;
for font_dict in fonts.values() {
let subtype = font_dict
.get(b"Subtype")
.ok()
.and_then(|o| o.as_name().ok());
if subtype == Some(b"Type3") {
// Type3 with a ToUnicode CMap can still produce usable text
if font_dict.get(b"ToUnicode").is_ok() {
return false;
}
has_type3 = true;
} else {
// Has a non-Type3 font — page has real text fonts
return false;
}
}
if has_type3 {
log::debug!("page has only Type3 fonts without ToUnicode — text is undecodable");
}
has_type3
}
fn scan_xobjects_in_resources(
doc: &Document,
resources: &lopdf::Dictionary,