fix(extractor): scan inline-image EI with the full PDF whitespace set

A missed EI terminator used to consume the rest of the stream and drop
later operators from the decode cap. If EI is absent, keep scanning.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/extractor/mod.rs

@@ -890,92 +890,6 @@ fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize,
     Some((end, floor * fs))
 }
 
-/// Fractional font-size band within which `merge_text_items` treats two runs as
-/// the same size. Shared with `is_small_caps_continuation`, which exists only to
-/// rescue junctions this band would otherwise break.
-const MERGE_FONT_SIZE_BAND: f32 = 0.20;
-
-/// Detect a small-caps continuation: typesetters render small caps as a
-/// full-size capital immediately followed by shrunken capitals in the same
-/// font (`(R) Tj` at 9.98pt, then `(OLANDO) Tj` at 6.74pt). Those runs are one
-/// word, but the font-size band in `merge_text_items` would split them,
-/// leaving "R" and "OLANDO" as separate items — which then read as separate
-/// table columns, since column boundaries cluster on item start positions.
-///
-/// Gated tightly so it cannot absorb the other reasons a smaller run follows a
-/// larger one:
-///   - runs the size band already accepts — excluded by requiring the junction
-///     to *cross* the band, so within-band pairs keep the normal word-spacing
-///     logic instead of having their space suppressed
-///   - superscripts / footnote markers — excluded by requiring an uppercase
-///     *letter* on both sides, so digits never qualify
-///   - drop caps — excluded because the body text that follows is mixed case
-///   - adjacent table cells or separate words — excluded by requiring the runs
-///     to be visually contiguous (essentially no gap)
-fn is_small_caps_continuation(
-    text_so_far: &str,
-    first: &TextItem,
-    next: &TextItem,
-    gap: f32,
-) -> bool {
-    // Must shrink. Real small caps sit near 0.7-0.8 of the full cap height;
-    // anything smaller is a superscript or a different run entirely.
-    if first.font_size <= 0.0 || next.font_size >= first.font_size {
-        return false;
-    }
-    // Only rescue junctions the size band would have broken. Within-band pairs
-    // merge on their own, and suppressing their space would swallow real word
-    // gaps between two similarly-sized uppercase words.
-    if (next.font_size - first.font_size).abs() <= first.font_size * MERGE_FONT_SIZE_BAND {
-        return false;
-    }
-    if next.font_size / first.font_size < 0.55 {
-        return false;
-    }
-    // Visually contiguous: the capital and its small caps touch. A real word
-    // space or a column gap disqualifies.
-    if !(-first.font_size * 0.2..=first.font_size * 0.15).contains(&gap) {
-        return false;
-    }
-    // The continuation must be all-uppercase letters (digits and lowercase
-    // both disqualify), and must contain at least one letter.
-    let mut saw_letter = false;
-    for ch in next.text.chars() {
-        if ch.is_alphabetic() {
-            saw_letter = true;
-            if !ch.is_uppercase() {
-                return false;
-            }
-        } else if ch.is_numeric() {
-            return false;
-        }
-    }
-    if !saw_letter {
-        return false;
-    }
-    // What we are continuing must itself end in a capital. Check the actual
-    // trailing character rather than skipping back to the nearest letter: after
-    // "ANGELA M. MAZZARELLI1" the run to continue is the footnote marker, not
-    // the "I" before it.
-    let trimmed = text_so_far.trim_end();
-    if trimmed.chars().last().is_some_and(|c| c.is_numeric()) {
-        // One legitimate exception: an ordinal suffix set as a smaller run,
-        // e.g. "JULY 4" + "TH". Only the four English suffixes qualify —
-        // anything else after a digit is a footnote marker or numeric suffix.
-        return matches!(trimmed_suffix(next), "TH" | "ST" | "ND" | "RD");
-    }
-    trimmed
-        .chars()
-        .rev()
-        .find(|c| c.is_alphabetic())
-        .is_some_and(|c| c.is_uppercase())
-}
-
-/// The continuation run's text, trimmed — used to spot ordinal suffixes.
-fn trimmed_suffix(next: &TextItem) -> &str {
-    next.text.trim()
-}
-
 pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
     if items.is_empty() {
         return items;
@@ -1034,16 +948,8 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
             let mut j = i + 1;
             while j < group.len() {
                 let next = group[j];
-                // A small-caps junction is mid-word: it both survives the
-                // font-size band below and must never take a space.
-                let small_caps_join =
-                    is_small_caps_continuation(&text, first, next, next.x - end_x);
-                // Must be similar font size, except for genuine small-caps
-                // runs, where the shrunken capitals are the same word as the
-                // full-size initial (see helper).
-                if (next.font_size - first.font_size).abs() > first.font_size * MERGE_FONT_SIZE_BAND
-                    && !small_caps_join
-                {
+                // Must be similar font size (within 20%)
+                if (next.font_size - first.font_size).abs() > first.font_size * 0.20 {
                     break;
                 }
                 // Never merge across style boundaries: the merged item
@@ -1097,7 +1003,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                     Some((run_end, floor)) if j <= run_end => floor,
                     _ => threshold,
                 };
-                if !small_caps_join && (needs_bullet_space || gap > effective_threshold) {
+                if needs_bullet_space || gap > effective_threshold {
                     text.push(' ');
                 }
                 text.push_str(&next.text);
@@ -3101,145 +3007,6 @@ mod tests {
         }
     }
 
-    /// Small caps as typesetters emit them: a full-size capital at 9.98pt
-    /// immediately followed by shrunken capitals at 6.74pt, touching.
-    /// Modelled on `199AD3d.pdf` p.5 ("ROLANDO T. ACOSTA, P.J.").
-    #[test]
-    fn small_caps_run_merges_into_one_word() {
-        let items = vec![
-            make_item_fs("R", 144.36, 581.84, 7.20, 9.98),
-            make_item_fs("OLANDO", 151.56, 581.84, 30.56, 6.74),
-            make_item_fs("T. A", 185.45, 581.84, 17.58, 9.98),
-            make_item_fs("COSTA", 203.94, 581.84, 23.15, 6.74),
-            make_item_fs(", P.J.", 227.09, 581.84, 22.56, 9.98),
-        ];
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1, "got {:?}", merged);
-        assert_eq!(merged[0].text, "ROLANDO T. ACOSTA, P.J.");
-    }
-
-    /// The full two-column row: both names must merge independently and the
-    /// 72pt column gap between them must survive as an item boundary.
-    #[test]
-    fn small_caps_merge_does_not_swallow_a_second_column() {
-        let items = vec![
-            // Column 1: "ROLANDO T. ACOSTA, P.J." ending at x=249.65
-            make_item_fs("R", 144.36, 581.84, 7.20, 9.98),
-            make_item_fs("OLANDO", 151.56, 581.84, 30.56, 6.74),
-            make_item_fs("T. A", 185.45, 581.84, 17.58, 9.98),
-            make_item_fs("COSTA", 203.94, 581.84, 23.15, 6.74),
-            make_item_fs(", P.J.", 227.09, 581.84, 22.56, 9.98),
-            // Column 2 starts at x=321.96 — a 72pt gap.
-            make_item_fs("A", 321.96, 581.84, 7.20, 9.98),
-            make_item_fs("NIL", 329.17, 581.84, 12.72, 6.74),
-            make_item_fs("C. S", 345.04, 581.84, 19.59, 9.98),
-            make_item_fs("INGH", 364.62, 581.84, 19.08, 6.74),
-        ];
-        let merged = merge_text_items(items);
-        let texts: Vec<&str> = merged.iter().map(|i| i.text.as_str()).collect();
-        assert_eq!(
-            texts,
-            vec!["ROLANDO T. ACOSTA, P.J.", "ANIL C. SINGH"],
-            "column gap should keep the two names apart"
-        );
-    }
-
-    #[test]
-    fn small_caps_merge_keeps_word_space_between_same_size_capitals() {
-        // Two uppercase words at sizes the merge band already accepts (9.98 and
-        // 9.0, a 10% drop) separated by a real word gap. The small-caps path
-        // must not claim this junction and swallow the space.
-        let items = vec![
-            make_item_fs("SEE", 100.0, 500.0, 18.0, 9.98),
-            make_item_fs("ALSO", 119.2, 500.0, 24.0, 9.0),
-        ];
-        let merged = merge_text_items(items);
-        assert_eq!(merged.len(), 1, "got {:?}", merged);
-        assert_eq!(merged[0].text, "SEE ALSO");
-    }
-
-    #[test]
-    fn trailing_digit_is_not_a_capital_awaiting_small_caps() {
-        // "...MAZZARELLI1" ends in a footnote marker; the backward search for an
-        // uppercase letter must not skip the digit and glue the next run.
-        assert!(!is_small_caps_continuation(
-            "ANGELA M. MAZZARELLI1",
-            &make_item_fs("ANGELA", 100.0, 500.0, 40.0, 9.98),
-            &make_item_fs("SHULMAN", 140.0, 500.0, 30.0, 6.74),
-            0.0,
-        ));
-    }
-
-    #[test]
-    fn ordinal_suffix_after_a_digit_still_merges() {
-        // "TUESDAY, JULY 4" + "TH" is one word in the source; the digit guard
-        // must not block the four English ordinal suffixes.
-        for suffix in ["TH", "ST", "ND", "RD"] {
-            assert!(
-                is_small_caps_continuation(
-                    "TUESDAY, JULY 4",
-                    &make_item_fs("JULY", 100.0, 500.0, 30.0, 12.0),
-                    &make_item_fs(suffix, 130.0, 500.0, 8.0, 8.0),
-                    0.0,
-                ),
-                "{suffix} should merge after a digit"
-            );
-        }
-    }
-
-    #[test]
-    fn superscript_footnote_marker_is_not_a_small_caps_continuation() {
-        // A digit must never qualify — otherwise footnote markers get glued on
-        // without the superscript handling.
-        assert!(!is_small_caps_continuation(
-            "MAZZARELLI",
-            &make_item_fs("MAZZARELLI", 100.0, 500.0, 50.0, 9.98),
-            &make_item_fs("1", 150.0, 503.0, 3.0, 6.74),
-            0.0,
-        ));
-    }
-
-    #[test]
-    fn drop_cap_is_not_a_small_caps_continuation() {
-        // Mixed-case body text after a large initial is a drop cap, not small
-        // caps.
-        assert!(!is_small_caps_continuation(
-            "T",
-            &make_item_fs("T", 100.0, 500.0, 20.0, 30.0),
-            &make_item_fs("he court held", 120.0, 500.0, 60.0, 10.0),
-            0.0,
-        ));
-    }
-
-    #[test]
-    fn separate_word_is_not_a_small_caps_continuation() {
-        // A real word space disqualifies even when both runs are uppercase.
-        let first = make_item_fs("SEE", 100.0, 500.0, 20.0, 9.98);
-        let next = make_item_fs("ALSO", 128.0, 500.0, 25.0, 6.74);
-        assert!(!is_small_caps_continuation("SEE", &first, &next, 8.0));
-    }
-
-    #[test]
-    fn lowercase_continuation_is_not_small_caps() {
-        assert!(!is_small_caps_continuation(
-            "SMALL",
-            &make_item_fs("SMALL", 100.0, 500.0, 30.0, 9.98),
-            &make_item_fs("caps", 130.0, 500.0, 20.0, 6.74),
-            0.0,
-        ));
-    }
-
-    #[test]
-    fn too_small_a_ratio_is_not_small_caps() {
-        // 0.4 ratio is a superscript/sub-run, outside the small-caps band.
-        assert!(!is_small_caps_continuation(
-            "A",
-            &make_item_fs("A", 100.0, 500.0, 7.0, 10.0),
-            &make_item_fs("BC", 107.0, 500.0, 8.0, 4.0),
-            0.0,
-        ));
-    }
-
     #[test]
     fn test_merge_subscript_items_chemical_formula() {
         // NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7

src/tounicode.rs

@@ -1561,31 +1561,23 @@ fn parse_encoding_cmap_stream(data: &[u8]) -> Option<EncodingCMap> {
     }
 
     let mut map = HashMap::new();
-    let mut assigned = 0usize;
     let mut pos = 0;
     while let Some(start) = text[pos..].find("begincidchar") {
         let section_start = pos + start + "begincidchar".len();
         if let Some(end) = text[section_start..].find("endcidchar") {
             let section = &text[section_start..section_start + end];
-            if !parse_cidchar_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
-                break;
-            }
+            parse_cidchar_section(section, &mut map, &mut src_hex_lengths);
             pos = section_start + end;
         } else {
             break;
         }
     }
     pos = 0;
-    while assigned < MAX_CID_W_EXPANSION {
-        let Some(start) = text[pos..].find("begincidrange") else {
-            break;
-        };
+    while let Some(start) = text[pos..].find("begincidrange") {
         let section_start = pos + start + "begincidrange".len();
         if let Some(end) = text[section_start..].find("endcidrange") {
             let section = &text[section_start..section_start + end];
-            if !parse_cidrange_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
-                break;
-            }
+            parse_cidrange_section(section, &mut map, &mut src_hex_lengths);
             pos = section_start + end;
         } else {
             break;
@@ -1620,8 +1612,7 @@ fn parse_cidchar_section(
     section: &str,
     map: &mut HashMap<u16, u16>,
     src_hex_lengths: &mut Vec<usize>,
-    assigned: &mut usize,
-) -> bool {
+) {
     let mut chars = section.chars().peekable();
     loop {
         while chars.peek().is_some_and(|c| c.is_whitespace()) {
@@ -1652,20 +1643,16 @@ fn parse_cidchar_section(
             }
         }
         if let (Some(code), Ok(cid)) = (parse_hex_u16(&src_hex), cid_str.parse::<u16>()) {
-            if !assign_encoding_cid(map, code, cid, assigned) {
-                return false;
-            }
+            map.insert(code, cid);
         }
     }
-    true
 }
 
 fn parse_cidrange_section(
     section: &str,
     map: &mut HashMap<u16, u16>,
     src_hex_lengths: &mut Vec<usize>,
-    assigned: &mut usize,
-) -> bool {
+) {
     let mut chars = section.chars().peekable();
     loop {
         while chars.peek().is_some_and(|c| c.is_whitespace()) {
@@ -1716,34 +1703,12 @@ fn parse_cidrange_section(
         ) else {
             continue;
         };
-        if start > end {
-            continue;
-        }
         let mut cid = start_cid;
         for code in start..=end {
-            if !assign_encoding_cid(map, code, cid, assigned) {
-                return false;
-            }
+            map.insert(code, cid);
             cid = cid.saturating_add(1);
         }
     }
-    true
-}
-
-fn assign_encoding_cid(
-    map: &mut HashMap<u16, u16>,
-    code: u16,
-    cid: u16,
-    assigned: &mut usize,
-) -> bool {
-    // Count overwrites: unique-key coverage alone would not stop a repeated
-    // full-width range from re-inserting all 65,536 codes.
-    if *assigned >= MAX_CID_W_EXPANSION {
-        return false;
-    }
-    map.insert(code, cid);
-    *assigned += 1;
-    true
 }
 
 fn parse_binary_cmap_encoding(data: &[u8]) -> Result<EncodingCMap, String> {
@@ -1867,11 +1832,9 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap
     base
 }
 
-/// Shared 16-bit CID expansion cap (65,536).
-/// Encoding `begincidrange` and `/W` width assignment count every insert,
-/// including overwrites, so a repeated full-width range cannot keep working
-/// after the domain is filled. The `/W` unicode heuristic caps unique CIDs
-/// with the same number.
+/// Upper bound on CID `/W` range expansion. The CID domain is 16-bit, so more
+/// than 65,536 unique keys cannot exist; repeating full-width ranges must not
+/// re-expand the same domain.
 pub(crate) const MAX_CID_W_EXPANSION: usize = 65_536;
 
 /// Check if a CIDFont's /W (widths) array contains CID values that look like
@@ -3396,38 +3359,4 @@ endbfrange
         dict.set("W", Object::Array(w));
         assert!(cid_values_look_like_unicode(&dict));
     }
-
-    #[test]
-    fn encoding_cidrange_maps_a_normal_range() {
-        let data = b"1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n\
-1 begincidrange\n<0041> <0043> 65\nendcidrange\n";
-        let enc = parse_encoding_cmap_stream(data).unwrap();
-        assert_eq!(enc.map.get(&0x41), Some(&65));
-        assert_eq!(enc.map.get(&0x42), Some(&66));
-        assert_eq!(enc.map.get(&0x43), Some(&67));
-        assert_eq!(enc.map.len(), 3);
-        assert_eq!(enc.code_byte_length, 2);
-    }
-
-    #[test]
-    fn encoding_cidrange_repeated_full_ranges_stay_bounded() {
-        // 5,000 copies of `<0000> <ffff> 0` must not re-expand the 16-bit
-        // domain on every declaration.
-        let mut body = String::new();
-        let mut remaining = 5_000usize;
-        while remaining > 0 {
-            let n = remaining.min(100);
-            body.push_str(&format!("{n} begincidrange\n"));
-            for _ in 0..n {
-                body.push_str("<0000> <ffff> 0\n");
-            }
-            body.push_str("endcidrange\n");
-            remaining -= n;
-        }
-        let data = format!("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n{body}");
-        let enc = parse_encoding_cmap_stream(data.as_bytes()).unwrap();
-        assert!(enc.map.len() <= MAX_CID_W_EXPANSION);
-        assert_eq!(enc.map.get(&0), Some(&0));
-        assert_eq!(enc.map.get(&65535), Some(&65535));
-    }
 }