fix(fonts): CID-as-Unicode passthrough and subscript merging (#8)

* fix(fonts): CID-as-Unicode passthrough and subscript/superscript merging Add smart CID-as-Unicode passthrough for Identity-H fonts without ToUnicode maps. Uses /W array median CID heuristic to distinguish Unicode-CID PDFs (Chromium-generated) from GID-based subsets. Add merge_subscript_items() pass that merges small-font items (<75% of dominant font size, ≤4 chars, tightly adjacent) into parent items. Fixes chemical formulas (NH3, H2O, KClO3), footnote references, and subscript notation (vf, Hfg, m3/kg) that were previously orphaned as separate text items. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(subscripts): restrict merge to purely numeric text only Tighten subscript merging to only merge items containing ASCII digits (0-9). This avoids false positives with ordinal indicators (º), letter subscripts (sol, vf), and small bullet characters (▶) that caused table restructuring regressions. Numeric-only keeps the primary wins: chemical formulas (NH3, H2O), footnote references, and unit notation (m2, m3). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(subscripts): restrict merge to parent text ending with a letter Only merge numeric subscripts when the parent item's text ends with an alphabetic character. Prevents false merges like "33" + "1" in fractions (33 1/3%), table credit numbers after spaces, and footnote refs after punctuation (land.1 → land. 1). Chemical formulas (NH3, H2O, KClO3) still merge correctly since parent ends with a letter. Reduces pdf-eval regressions from 13 to 2 (both are correct reversions of over-aggressive footnote merging from the prior commit). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-23 09:41:10 -07:00
parent 7f2995ad0c
commit 6c005ef4c2
4 changed files with 339 additions and 24 deletions
@@ -880,6 +880,7 @@ pub(crate) fn extract_page_text_items(
    }

    let items = super::merge_text_items(items);
+    let items = super::merge_subscript_items(items);
    Ok(((items, rects, lines), has_gid_fonts))
 }

@@ -363,6 +363,92 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
    merged
 }

+/// Merge subscript/superscript items into their adjacent parent items.
+///
+/// Subscripts (e.g. "2" in H₂O) are rendered as separate text items with a
+/// much smaller font size and a slight Y offset. This pass finds such items
+/// and absorbs them into the preceding normal-sized item so that downstream
+/// table detection and line grouping see complete text (e.g. "H2O" not "H"+"2"+"O").
+pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
+    if items.len() < 2 {
+        return items;
+    }
+
+    // Group items by (page, approximate Y) with generous tolerance to capture
+    // both the parent line and the subscript/superscript offset.
+    let y_tolerance = 5.0;
+    let mut line_groups: Vec<(u32, f32, Vec<TextItem>)> = Vec::new();
+
+    for item in items {
+        let found = line_groups
+            .iter_mut()
+            .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance);
+        if let Some((_, _, group)) = found {
+            group.push(item);
+        } else {
+            let page = item.page;
+            let y = item.y;
+            line_groups.push((page, y, vec![item]));
+        }
+    }
+
+    let mut result = Vec::new();
+
+    for (_, _, mut group) in line_groups {
+        // Sort by X position
+        group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal));
+
+        // Find the dominant (most common) font size in this group
+        let max_fs = group.iter().map(|i| i.font_size).fold(0.0_f32, f32::max);
+
+        if max_fs < 1.0 {
+            result.extend(group);
+            continue;
+        }
+
+        let sub_threshold = max_fs * 0.75;
+
+        // Walk through items and merge subscripts into their preceding parent
+        let mut merged: Vec<TextItem> = Vec::new();
+        for item in group {
+            if item.font_size < sub_threshold
+                && item.font_size > 0.0
+                && item.text.len() <= 4
+                && item.text.chars().all(|c| c.is_ascii_digit())
+            {
+                // This is a candidate numeric subscript/superscript (e.g. "2" in H₂O).
+                // Only merge purely numeric text to avoid false positives with small
+                // bullets, ordinal indicators, or letter-based labels.
+                if let Some(parent) = merged.last_mut() {
+                    // Only merge into a parent that is normal-sized, not another subscript,
+                    // and whose text ends with a letter. This prevents merging into numbers
+                    // (e.g. "33" + "1" in "33 1/3%") or punctuation, while preserving
+                    // chemical formulas (NH + "3") and footnote refs (word + "2").
+                    let ends_with_letter = parent
+                        .text
+                        .chars()
+                        .last()
+                        .is_some_and(|c| c.is_alphabetic());
+                    if parent.font_size >= sub_threshold && ends_with_letter {
+                        let parent_right = parent.x + parent.width;
+                        let gap = item.x - parent_right;
+                        // Subscripts must be tightly adjacent (within ~1pt)
+                        if gap < parent.font_size * 0.2 && gap > -parent.font_size * 0.3 {
+                            parent.text.push_str(&item.text);
+                            parent.width = (item.x + item.width) - parent.x;
+                            continue;
+                        }
+                    }
+                }
+            }
+            merged.push(item);
+        }
+        result.extend(merged);
+    }
+
+    result
+}
+
 /// Helper to get f32 from Object
 pub(crate) fn get_number(obj: &Object) -> Option<f32> {
    match obj {
@@ -1018,4 +1104,121 @@ mod tests {
        ];
        assert!(!is_newspaper_layout(&[col1, col2], &cols));
    }
+
+    fn make_item_fs(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem {
+        TextItem {
+            text: text.into(),
+            x,
+            y,
+            width,
+            height: font_size,
+            font: "F1".into(),
+            font_size,
+            page: 1,
+            is_bold: false,
+            is_italic: false,
+            item_type: ItemType::Text,
+            mcid: None,
+        }
+    }
+
+    #[test]
+    fn test_merge_subscript_items_chemical_formula() {
+        // NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7
+        let items = vec![
+            make_item_fs("NH", 78.0, 499.0, 12.0, 8.0),
+            make_item_fs("3", 90.0, 496.0, 2.3, 4.7),
+            make_item_fs("Cl", 100.0, 499.0, 7.0, 8.0),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "NH3");
+        assert_eq!(merged[1].text, "Cl");
+    }
+
+    #[test]
+    fn test_merge_subscript_items_h2o() {
+        // H₂O: "H" then subscript "2" then "O"
+        let items = vec![
+            make_item_fs("H", 250.0, 499.0, 5.0, 8.0),
+            make_item_fs("2", 255.0, 496.0, 2.3, 4.7),
+            make_item_fs("O", 257.5, 499.0, 6.0, 8.0),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "H2");
+        assert_eq!(merged[1].text, "O");
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_far_gap() {
+        // Subscript-sized item that's far from the parent should NOT merge
+        let items = vec![
+            make_item_fs("Text", 78.0, 499.0, 20.0, 8.0),
+            make_item_fs("▶", 120.0, 498.0, 3.0, 3.7),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "Text");
+        assert_eq!(merged[1].text, "▶");
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_long_text() {
+        // Long subscript-sized text should NOT merge (not a true subscript)
+        let items = vec![
+            make_item_fs("Title", 78.0, 499.0, 30.0, 8.0),
+            make_item_fs("footnote", 108.0, 496.0, 20.0, 4.7),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_same_font_size() {
+        // Same font size items should NOT be treated as subscripts
+        let items = vec![
+            make_item_fs("NH", 78.0, 499.0, 12.0, 8.0),
+            make_item_fs("3", 90.0, 496.0, 2.3, 8.0),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_non_numeric() {
+        // Non-numeric subscript text (e.g. "sol", "º", "vf") should NOT merge
+        let items = vec![
+            make_item_fs("∆", 200.0, 639.0, 5.5, 8.0),
+            make_item_fs("sol", 205.8, 636.9, 5.7, 4.7),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "∆");
+        assert_eq!(merged[1].text, "sol");
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_parent_ends_with_digit() {
+        // "33" + "1" in "33 1/3%" — parent ends with digit, should NOT merge
+        let items = vec![
+            make_item_fs("33", 78.0, 499.0, 10.0, 8.0),
+            make_item_fs("1", 88.0, 496.0, 2.3, 4.7),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "33");
+        assert_eq!(merged[1].text, "1");
+    }
+
+    #[test]
+    fn test_merge_subscript_items_no_merge_parent_ends_with_space() {
+        // "Health " + "1" — parent ends with space (table credit), should NOT merge
+        let items = vec![
+            make_item_fs("Health ", 78.0, 499.0, 30.0, 8.0),
+            make_item_fs("1", 108.0, 496.0, 2.3, 4.7),
+        ];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 2);
+    }
 }
@@ -18,6 +18,9 @@ pub struct ToUnicodeCMap {
    pub ranges: Vec<(u16, u16, u32)>,
    /// Byte width of source codes (1 or 2), determined from codespace and CMap entries
    pub code_byte_length: u8,
+    /// When true, unmapped CIDs are interpreted as Unicode codepoints directly.
+    /// Used as a last resort for Identity-H fonts without ToUnicode/cmap/glyph names.
+    pub cid_passthrough: bool,
 }

 pub(crate) fn build_cmap_entry_from_stream(
@@ -467,10 +470,24 @@ impl ToUnicodeCMap {
                    match self.lookup(cid) {
                        Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s),
                        _ => {
-                            // Do NOT blindly interpret CIDs as Unicode codepoints.
-                            // CIDs are font-internal indices, not Unicode values.
-                            // Unmapped 2-byte CIDs are skipped to avoid CJK garbage.
-                            unmapped_count += 1;
+                            if self.cid_passthrough {
+                                // Last-resort: treat CID as Unicode codepoint.
+                                // Valid for Identity-H fonts where the PDF generator
+                                // used Unicode values as CIDs but stripped the cmap.
+                                if let Some(ch) = char::from_u32(cid as u32) {
+                                    if !ch.is_control() || ch == '\t' || ch == '\n' {
+                                        result.push(ch);
+                                    } else {
+                                        unmapped_count += 1;
+                                    }
+                                } else {
+                                    unmapped_count += 1;
+                                }
+                            } else {
+                                // CIDs are font-internal indices, not Unicode values.
+                                // Unmapped 2-byte CIDs are skipped to avoid CJK garbage.
+                                unmapped_count += 1;
+                            }
                        }
                    }
                }
@@ -1628,6 +1645,68 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap
    base
 }

+/// Check if a CIDFont's /W (widths) array contains CID values that look like
+/// Unicode codepoints rather than low-value GIDs.
+///
+/// Returns true if the median CID is >= 0x41 (letter 'A'), indicating
+/// the PDF generator likely used Unicode codepoints as CIDs.
+fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool {
+    let w_arr = match cid_font_dict.get(b"W").ok() {
+        Some(Object::Array(arr)) => arr,
+        _ => return false,
+    };
+
+    // The /W array format: [cid [w1 w2 ...]] or [cid_start cid_end w]
+    // We extract all CID values (the first element of each group).
+    let mut cids: Vec<u16> = Vec::new();
+    let mut i = 0;
+    while i < w_arr.len() {
+        if let Ok(cid) = w_arr[i].as_i64() {
+            cids.push(cid as u16);
+            // Skip the width data
+            if i + 1 < w_arr.len() {
+                match &w_arr[i + 1] {
+                    Object::Array(widths) => {
+                        // [cid [w1 w2 ...]] — CIDs are cid, cid+1, ..., cid+len-1
+                        for j in 1..widths.len() {
+                            cids.push((cid as u16).wrapping_add(j as u16));
+                        }
+                        i += 2;
+                    }
+                    _ => {
+                        // [cid_start cid_end w] — range of CIDs
+                        if i + 2 < w_arr.len() {
+                            if let Ok(cid_end) = w_arr[i + 1].as_i64() {
+                                for c in (cid as u16)..=(cid_end as u16) {
+                                    cids.push(c);
+                                }
+                            }
+                            i += 3;
+                        } else {
+                            i += 1;
+                        }
+                    }
+                }
+            } else {
+                i += 1;
+            }
+        } else {
+            i += 1;
+        }
+    }
+
+    if cids.is_empty() {
+        return false;
+    }
+
+    cids.sort_unstable();
+    let median = cids[cids.len() / 2];
+    // Unicode text CIDs are typically >= 0x20 (space) with letters at 0x41+.
+    // GID-based subsets typically start at low values (0-based).
+    // Use median >= 0x41 as a heuristic for Unicode CIDs.
+    median >= 0x41
+}
+
 /// Build a ToUnicodeCMap from predefined CID→Unicode mapping based on CIDSystemInfo.
 ///
 /// Supports Adobe-Korea1 (Korean) character collection. Can be extended for
@@ -1874,23 +1953,25 @@ impl FontCMaps {
            // Try parsing embedded TrueType/OpenType cmap
            if let Some(ff_ref) = font_file_ref {
                if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) {
-                    if let Ok(data) = stream.decompressed_content() {
-                        if let Some(cmap) = build_cmap_from_truetype(&data) {
-                            debug!(
-                                "TrueType CMap obj={:<6} (embedded font) char_map={}",
-                                lookup_key,
-                                cmap.char_map.len()
-                            );
-                            by_obj_num.insert(
-                                lookup_key,
-                                CMapEntry {
-                                    primary: cmap,
-                                    remapped: None,
-                                    fallback: None,
-                                },
-                            );
-                            resolved = true;
-                        }
+                    let data = match stream.decompressed_content() {
+                        Ok(d) => d,
+                        Err(_) => stream.content.clone(),
+                    };
+                    if let Some(cmap) = build_cmap_from_truetype(&data) {
+                        debug!(
+                            "TrueType CMap obj={:<6} (embedded font) char_map={}",
+                            lookup_key,
+                            cmap.char_map.len()
+                        );
+                        by_obj_num.insert(
+                            lookup_key,
+                            CMapEntry {
+                                primary: cmap,
+                                remapped: None,
+                                fallback: None,
+                            },
+                        );
+                        resolved = true;
                    }
                }
            }
@@ -1911,6 +1992,38 @@ impl FontCMaps {
                            fallback: None,
                        },
                    );
+                    resolved = true;
+                }
+            }
+
+            // Last resort: CID-as-Unicode passthrough.
+            // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H encoding where
+            // CID values ARE Unicode codepoints, but strip the cmap table and omit
+            // ToUnicode. We detect this by checking the /W (widths) array: if CID values
+            // fall in typical Unicode letter/digit ranges (0x41+), CIDs are likely Unicode.
+            // If CIDs are low values (< 0x41), they're GIDs in a subset font.
+            if !resolved {
+                if cid_values_look_like_unicode(cid_font_dict) {
+                    debug!(
+                        "Identity-H font obj={}: W array CIDs look like Unicode — using passthrough",
+                        lookup_key
+                    );
+                    let mut cmap = ToUnicodeCMap::new();
+                    cmap.code_byte_length = 2;
+                    cmap.cid_passthrough = true;
+                    by_obj_num.insert(
+                        lookup_key,
+                        CMapEntry {
+                            primary: cmap,
+                            remapped: None,
+                            fallback: None,
+                        },
+                    );
+                } else {
+                    debug!(
+                        "Identity-H font obj={}: no decoding possible (stripped cmap, GID-based CIDs)",
+                        lookup_key
+                    );
                }
            }
        }
@@ -48,7 +48,7 @@ l

 |Temp|Pressure|Volume||Density|||Enthalpy||Entropy||Temp|
 |---|---|---|---|---|---|---|---|---|---|---|---|
-|°C|[kPa]|[m Liquid v|/kg] Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C|
+|°C|[kPa]|[m3/kg] Liquid v|Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C|
 |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100|
 |-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99|
 |-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98|
@@ -104,7 +104,5 @@ l
 |-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48|
 |-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47|

-**3**
-
 **f g f g f** fg **g f g**