From 6c005ef4c22afa1d74ae0bb6ba6623d7fa0456a9 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 09:41:10 -0700 Subject: [PATCH] fix(fonts): CID-as-Unicode passthrough and subscript merging (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(fonts): CID-as-Unicode passthrough and subscript/superscript merging Add smart CID-as-Unicode passthrough for Identity-H fonts without ToUnicode maps. Uses /W array median CID heuristic to distinguish Unicode-CID PDFs (Chromium-generated) from GID-based subsets. Add merge_subscript_items() pass that merges small-font items (<75% of dominant font size, ≤4 chars, tightly adjacent) into parent items. Fixes chemical formulas (NH3, H2O, KClO3), footnote references, and subscript notation (vf, Hfg, m3/kg) that were previously orphaned as separate text items. Co-Authored-By: Claude Opus 4.6 * fix(subscripts): restrict merge to purely numeric text only Tighten subscript merging to only merge items containing ASCII digits (0-9). This avoids false positives with ordinal indicators (º), letter subscripts (sol, vf), and small bullet characters (▶) that caused table restructuring regressions. Numeric-only keeps the primary wins: chemical formulas (NH3, H2O), footnote references, and unit notation (m2, m3). Co-Authored-By: Claude Opus 4.6 * fix(subscripts): restrict merge to parent text ending with a letter Only merge numeric subscripts when the parent item's text ends with an alphabetic character. Prevents false merges like "33" + "1" in fractions (33 1/3%), table credit numbers after spaces, and footnote refs after punctuation (land.1 → land. 1). Chemical formulas (NH3, H2O, KClO3) still merge correctly since parent ends with a letter. Reduces pdf-eval regressions from 13 to 2 (both are correct reversions of over-aggressive footnote merging from the prior commit). Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/extractor/content_stream.rs | 1 + src/extractor/mod.rs | 203 ++++++++++++++++++++++++++++++ src/tounicode.rs | 155 +++++++++++++++++++---- tests/snapshots/thermo-freon12.md | 4 +- 4 files changed, 339 insertions(+), 24 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index c1952b7..e703b79 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -880,6 +880,7 @@ pub(crate) fn extract_page_text_items( } let items = super::merge_text_items(items); + let items = super::merge_subscript_items(items); Ok(((items, rects, lines), has_gid_fonts)) } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index e8a2a90..15554db 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -363,6 +363,92 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { merged } +/// Merge subscript/superscript items into their adjacent parent items. +/// +/// Subscripts (e.g. "2" in H₂O) are rendered as separate text items with a +/// much smaller font size and a slight Y offset. This pass finds such items +/// and absorbs them into the preceding normal-sized item so that downstream +/// table detection and line grouping see complete text (e.g. "H2O" not "H"+"2"+"O"). +pub(crate) fn merge_subscript_items(items: Vec) -> Vec { + if items.len() < 2 { + return items; + } + + // Group items by (page, approximate Y) with generous tolerance to capture + // both the parent line and the subscript/superscript offset. + let y_tolerance = 5.0; + let mut line_groups: Vec<(u32, f32, Vec)> = Vec::new(); + + for item in items { + let found = line_groups + .iter_mut() + .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance); + if let Some((_, _, group)) = found { + group.push(item); + } else { + let page = item.page; + let y = item.y; + line_groups.push((page, y, vec![item])); + } + } + + let mut result = Vec::new(); + + for (_, _, mut group) in line_groups { + // Sort by X position + group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + + // Find the dominant (most common) font size in this group + let max_fs = group.iter().map(|i| i.font_size).fold(0.0_f32, f32::max); + + if max_fs < 1.0 { + result.extend(group); + continue; + } + + let sub_threshold = max_fs * 0.75; + + // Walk through items and merge subscripts into their preceding parent + let mut merged: Vec = Vec::new(); + for item in group { + if item.font_size < sub_threshold + && item.font_size > 0.0 + && item.text.len() <= 4 + && item.text.chars().all(|c| c.is_ascii_digit()) + { + // This is a candidate numeric subscript/superscript (e.g. "2" in H₂O). + // Only merge purely numeric text to avoid false positives with small + // bullets, ordinal indicators, or letter-based labels. + if let Some(parent) = merged.last_mut() { + // Only merge into a parent that is normal-sized, not another subscript, + // and whose text ends with a letter. This prevents merging into numbers + // (e.g. "33" + "1" in "33 1/3%") or punctuation, while preserving + // chemical formulas (NH + "3") and footnote refs (word + "2"). + let ends_with_letter = parent + .text + .chars() + .last() + .is_some_and(|c| c.is_alphabetic()); + if parent.font_size >= sub_threshold && ends_with_letter { + let parent_right = parent.x + parent.width; + let gap = item.x - parent_right; + // Subscripts must be tightly adjacent (within ~1pt) + if gap < parent.font_size * 0.2 && gap > -parent.font_size * 0.3 { + parent.text.push_str(&item.text); + parent.width = (item.x + item.width) - parent.x; + continue; + } + } + } + } + merged.push(item); + } + result.extend(merged); + } + + result +} + /// Helper to get f32 from Object pub(crate) fn get_number(obj: &Object) -> Option { match obj { @@ -1018,4 +1104,121 @@ mod tests { ]; assert!(!is_newspaper_layout(&[col1, col2], &cols)); } + + fn make_item_fs(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: font_size, + font: "F1".into(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn test_merge_subscript_items_chemical_formula() { + // NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7 + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 4.7), + make_item_fs("Cl", 100.0, 499.0, 7.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "NH3"); + assert_eq!(merged[1].text, "Cl"); + } + + #[test] + fn test_merge_subscript_items_h2o() { + // H₂O: "H" then subscript "2" then "O" + let items = vec![ + make_item_fs("H", 250.0, 499.0, 5.0, 8.0), + make_item_fs("2", 255.0, 496.0, 2.3, 4.7), + make_item_fs("O", 257.5, 499.0, 6.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "H2"); + assert_eq!(merged[1].text, "O"); + } + + #[test] + fn test_merge_subscript_items_no_merge_far_gap() { + // Subscript-sized item that's far from the parent should NOT merge + let items = vec![ + make_item_fs("Text", 78.0, 499.0, 20.0, 8.0), + make_item_fs("▶", 120.0, 498.0, 3.0, 3.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "Text"); + assert_eq!(merged[1].text, "▶"); + } + + #[test] + fn test_merge_subscript_items_no_merge_long_text() { + // Long subscript-sized text should NOT merge (not a true subscript) + let items = vec![ + make_item_fs("Title", 78.0, 499.0, 30.0, 8.0), + make_item_fs("footnote", 108.0, 496.0, 20.0, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_same_font_size() { + // Same font size items should NOT be treated as subscripts + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_non_numeric() { + // Non-numeric subscript text (e.g. "sol", "º", "vf") should NOT merge + let items = vec![ + make_item_fs("∆", 200.0, 639.0, 5.5, 8.0), + make_item_fs("sol", 205.8, 636.9, 5.7, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "∆"); + assert_eq!(merged[1].text, "sol"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_digit() { + // "33" + "1" in "33 1/3%" — parent ends with digit, should NOT merge + let items = vec![ + make_item_fs("33", 78.0, 499.0, 10.0, 8.0), + make_item_fs("1", 88.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "33"); + assert_eq!(merged[1].text, "1"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_space() { + // "Health " + "1" — parent ends with space (table credit), should NOT merge + let items = vec![ + make_item_fs("Health ", 78.0, 499.0, 30.0, 8.0), + make_item_fs("1", 108.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } } diff --git a/src/tounicode.rs b/src/tounicode.rs index 2a757ab..6e13f4a 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -18,6 +18,9 @@ pub struct ToUnicodeCMap { pub ranges: Vec<(u16, u16, u32)>, /// Byte width of source codes (1 or 2), determined from codespace and CMap entries pub code_byte_length: u8, + /// When true, unmapped CIDs are interpreted as Unicode codepoints directly. + /// Used as a last resort for Identity-H fonts without ToUnicode/cmap/glyph names. + pub cid_passthrough: bool, } pub(crate) fn build_cmap_entry_from_stream( @@ -467,10 +470,24 @@ impl ToUnicodeCMap { match self.lookup(cid) { Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s), _ => { - // Do NOT blindly interpret CIDs as Unicode codepoints. - // CIDs are font-internal indices, not Unicode values. - // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. - unmapped_count += 1; + if self.cid_passthrough { + // Last-resort: treat CID as Unicode codepoint. + // Valid for Identity-H fonts where the PDF generator + // used Unicode values as CIDs but stripped the cmap. + if let Some(ch) = char::from_u32(cid as u32) { + if !ch.is_control() || ch == '\t' || ch == '\n' { + result.push(ch); + } else { + unmapped_count += 1; + } + } else { + unmapped_count += 1; + } + } else { + // CIDs are font-internal indices, not Unicode values. + // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. + unmapped_count += 1; + } } } } @@ -1628,6 +1645,68 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap base } +/// Check if a CIDFont's /W (widths) array contains CID values that look like +/// Unicode codepoints rather than low-value GIDs. +/// +/// Returns true if the median CID is >= 0x41 (letter 'A'), indicating +/// the PDF generator likely used Unicode codepoints as CIDs. +fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool { + let w_arr = match cid_font_dict.get(b"W").ok() { + Some(Object::Array(arr)) => arr, + _ => return false, + }; + + // The /W array format: [cid [w1 w2 ...]] or [cid_start cid_end w] + // We extract all CID values (the first element of each group). + let mut cids: Vec = Vec::new(); + let mut i = 0; + while i < w_arr.len() { + if let Ok(cid) = w_arr[i].as_i64() { + cids.push(cid as u16); + // Skip the width data + if i + 1 < w_arr.len() { + match &w_arr[i + 1] { + Object::Array(widths) => { + // [cid [w1 w2 ...]] — CIDs are cid, cid+1, ..., cid+len-1 + for j in 1..widths.len() { + cids.push((cid as u16).wrapping_add(j as u16)); + } + i += 2; + } + _ => { + // [cid_start cid_end w] — range of CIDs + if i + 2 < w_arr.len() { + if let Ok(cid_end) = w_arr[i + 1].as_i64() { + for c in (cid as u16)..=(cid_end as u16) { + cids.push(c); + } + } + i += 3; + } else { + i += 1; + } + } + } + } else { + i += 1; + } + } else { + i += 1; + } + } + + if cids.is_empty() { + return false; + } + + cids.sort_unstable(); + let median = cids[cids.len() / 2]; + // Unicode text CIDs are typically >= 0x20 (space) with letters at 0x41+. + // GID-based subsets typically start at low values (0-based). + // Use median >= 0x41 as a heuristic for Unicode CIDs. + median >= 0x41 +} + /// Build a ToUnicodeCMap from predefined CID→Unicode mapping based on CIDSystemInfo. /// /// Supports Adobe-Korea1 (Korean) character collection. Can be extended for @@ -1874,23 +1953,25 @@ impl FontCMaps { // Try parsing embedded TrueType/OpenType cmap if let Some(ff_ref) = font_file_ref { if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) { - if let Ok(data) = stream.decompressed_content() { - if let Some(cmap) = build_cmap_from_truetype(&data) { - debug!( - "TrueType CMap obj={:<6} (embedded font) char_map={}", - lookup_key, - cmap.char_map.len() - ); - by_obj_num.insert( - lookup_key, - CMapEntry { - primary: cmap, - remapped: None, - fallback: None, - }, - ); - resolved = true; - } + let data = match stream.decompressed_content() { + Ok(d) => d, + Err(_) => stream.content.clone(), + }; + if let Some(cmap) = build_cmap_from_truetype(&data) { + debug!( + "TrueType CMap obj={:<6} (embedded font) char_map={}", + lookup_key, + cmap.char_map.len() + ); + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + resolved = true; } } } @@ -1911,6 +1992,38 @@ impl FontCMaps { fallback: None, }, ); + resolved = true; + } + } + + // Last resort: CID-as-Unicode passthrough. + // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H encoding where + // CID values ARE Unicode codepoints, but strip the cmap table and omit + // ToUnicode. We detect this by checking the /W (widths) array: if CID values + // fall in typical Unicode letter/digit ranges (0x41+), CIDs are likely Unicode. + // If CIDs are low values (< 0x41), they're GIDs in a subset font. + if !resolved { + if cid_values_look_like_unicode(cid_font_dict) { + debug!( + "Identity-H font obj={}: W array CIDs look like Unicode — using passthrough", + lookup_key + ); + let mut cmap = ToUnicodeCMap::new(); + cmap.code_byte_length = 2; + cmap.cid_passthrough = true; + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + } else { + debug!( + "Identity-H font obj={}: no decoding possible (stripped cmap, GID-based CIDs)", + lookup_key + ); } } } diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 08a217a..789ed1a 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -48,7 +48,7 @@ l |Temp|Pressure|Volume||Density|||Enthalpy||Entropy||Temp| |---|---|---|---|---|---|---|---|---|---|---|---| -|°C|[kPa]|[m Liquid v|/kg] Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| +|°C|[kPa]|[m3/kg] Liquid v|Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| |-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99| |-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98| @@ -104,7 +104,5 @@ l |-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48| |-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47| -**3** - **f g f g f** fg **g f g**