feat(extractor): run-local space floor for tracked (letter-spaced) glyph runs

Display type set with tracking renders one glyph per show op; the merge
loop's fixed space thresholds (0.08-0.13 em) then read every letter gap
as a word boundary and emit "H O W" / "F U R T H E R" instead of
"HOW" / "FURTHER". The page-level Canva fixer can't help: it requires
>=50% of the page's items to be letter-spaced, and these docs track
only their display headings.

merge_text_items now pre-scans each run of consecutive single-glyph
items (same size band, same style, mergeable gaps — the loop's own
break conditions) and, when the run is tracked, derives the space floor
from the run's own gap distribution:

- runs with >=4 gaps qualify when the median gap clears the fixed
  threshold; word gaps, if present, form a second mode — split at the
  largest relative jump (>=1.4x), else the run is a single word
  ("I T I S I M P O R T A N T" -> "IT IS IMPORTANT")
- short runs (2-3 gaps: "H O W") additionally demand uniform gaps and
  ALL-CAPS or CJK — a genuine spaced sequence of single letters
  ("x y z" variables) has the same gap count, and display tracking is
  a caps convention; CJK never wants inter-glyph spaces

Corpus sweep (708 opendataloader + ParseBench text PDFs) vs main: 9
docs change — the tracked display titles ("HOW CAN YOU HELP?",
"LUNCHTIME MENU", a tracked email address), and CJK glyph-per-item
docs whose spurious inter-glyph spaces now collapse (GT for those docs
is unspaced CJK; should_join_items already treats no-space CJK as
correct on its path). No other doc moves.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L3U6BKYCS73DVA83odAfYB

src/extractor/mod.rs

@@ -9,7 +9,7 @@ mod links;
 pub(crate) mod underline;
 mod xobjects;
 
-use crate::text_utils::is_rtl_text;
+use crate::text_utils::{is_cjk_char, is_rtl_text};
 use crate::tounicode::FontCMaps;
 use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
 use crate::PdfError;
@@ -527,6 +527,105 @@ fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
     saw_backtrack
 }
 
+/// Detect a tracked (letter-spaced) run of single-glyph items and derive its
+/// run-local space floor.
+///
+/// Display type set with tracking renders one glyph per show op; the merge
+/// loop's fixed thresholds (0.08-0.13 em) then read every letter gap as a
+/// word boundary and emit "H O W" instead of "HOW". Within such a run the
+/// gaps carry the real signal: letter gaps cluster tightly just above the
+/// fixed threshold, word gaps sit clearly higher. Returns (run_end_index,
+/// space_floor) when the run starting at `start` is tracked — spaces are
+/// then inserted only at gaps above the floor (infinity = single word).
+/// Normal text (multi-char items, or single-char runs with sub-threshold
+/// gaps) returns None and keeps the existing behavior.
+fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize, f32)> {
+    const MIN_GAPS: usize = 4;
+    let first = group[start];
+    if first.text.trim().chars().count() != 1 {
+        return None;
+    }
+    let fs = first.font_size;
+    if fs <= 0.0 {
+        return None;
+    }
+
+    // Walk the run under the SAME break conditions as the merge loop
+    // (size band, style equality, mergeable gap) so indices stay aligned.
+    let mut gaps: Vec<f32> = Vec::new();
+    let mut end_x = first.x + effective_merge_width(first);
+    let mut end = start;
+    for (offset, next) in group[start + 1..].iter().enumerate() {
+        if next.text.trim().chars().count() != 1 {
+            break;
+        }
+        if (next.font_size - fs).abs() > fs * 0.20 {
+            break;
+        }
+        if next.is_bold != first.is_bold
+            || next.is_italic != first.is_italic
+            || next.is_underline != first.is_underline
+            || next.is_strikeout != first.is_strikeout
+        {
+            break;
+        }
+        let gap = next.x - end_x;
+        if gap > fs * 0.5 || gap < -fs * 0.5 {
+            break;
+        }
+        gaps.push(gap / fs);
+        end_x = next.x + effective_merge_width(next);
+        end = start + 1 + offset;
+    }
+    if gaps.len() < 2 {
+        return None;
+    }
+
+    // Tracked signature: the run's TYPICAL gap clears the fixed space
+    // threshold (0.08) — the merge loop would break almost every letter
+    // pair into "words". Short runs (2-3 gaps: "H O W") demand a stricter
+    // shape — clearly wide, uniform, ALL-CAPS — because a genuine spaced
+    // sequence of single letters ("x y z" variables) has the same gap
+    // count; display tracking is a caps convention.
+    let mut sorted = gaps.clone();
+    sorted.sort_by(|a, b| a.total_cmp(b));
+    let median = sorted[sorted.len() / 2];
+    if gaps.len() >= MIN_GAPS {
+        if median <= 0.075 {
+            return None;
+        }
+    } else {
+        let uniform = sorted[sorted.len() - 1] <= sorted[0].max(0.01) * 1.4;
+        // Caps convention — or CJK, which never uses inter-glyph spaces.
+        let all_caps = group[start..=end].iter().all(|it| {
+            it.text
+                .trim()
+                .chars()
+                .all(|c| c.is_uppercase() || is_cjk_char(c) || !c.is_alphabetic())
+        });
+        if median < 0.09 || !uniform || !all_caps {
+            return None;
+        }
+    }
+
+    // Word gaps, if present, form a second mode above the letter-gap
+    // cluster: split at the largest relative jump. Unimodal → one word.
+    let mut best_jump = 1.0f32;
+    let mut floor = f32::INFINITY;
+    for pair in sorted.windows(2) {
+        let (lo, hi) = (pair[0].max(0.01), pair[1].max(0.01));
+        let jump = hi / lo;
+        if jump > best_jump {
+            best_jump = jump;
+            floor = (lo + hi) / 2.0;
+        }
+    }
+    if best_jump < 1.4 {
+        floor = f32::INFINITY;
+    }
+    Some((end, floor * fs))
+}
+
 pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
     if items.is_empty() {
         return items;
@@ -574,6 +673,14 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
             let mut text = first.text.clone();
             let mut end_x = first.x + effective_merge_width(first);
 
+            // Tracked display text: run-local space floor overrides the
+            // fixed thresholds for this run's junctions (see helper).
+            let tracked = if *preserve_stream_order {
+                None
+            } else {
+                tracked_run_space_floor(group, i)
+            };
+
             let mut j = i + 1;
             while j < group.len() {
                 let next = group[j];
@@ -628,7 +735,11 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                 let needs_bullet_space = *preserve_stream_order
                     && is_standalone_bullet_text(&text)
                     && !next.text.trim().is_empty();
-                if needs_bullet_space || gap > threshold {
+                let effective_threshold = match tracked {
+                    Some((run_end, floor)) if j <= run_end => floor,
+                    _ => threshold,
+                };
+                if needs_bullet_space || gap > effective_threshold {
                     text.push(' ');
                 }
                 text.push_str(&next.text);
@@ -794,6 +905,73 @@ mod tests {
     use crate::types::{ItemType, PdfLine, TextLine};
     use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
 
+    /// Glyph-per-item run at `fs`=12 with the given inter-glyph gap (pt).
+    fn glyph_run(chars: &str, start_x: f32, glyph_w: f32, gap: f32) -> Vec<TextItem> {
+        let mut x = start_x;
+        let mut out = Vec::new();
+        for c in chars.chars() {
+            out.push(make_merge_item(&c.to_string(), x, glyph_w));
+            x += glyph_w + gap;
+        }
+        out
+    }
+
+    #[test]
+    fn tracked_caps_run_collapses_to_word() {
+        // Display tracking: every letter gap (0.19 em) clears the fixed
+        // space threshold — without the run-local floor this reads "H O W".
+        let items = glyph_run("HOW", 100.0, 10.0, 2.3);
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "HOW");
+    }
+
+    #[test]
+    fn tracked_run_keeps_word_gaps_bimodal() {
+        // Letters at 0.19 em, word gaps at 0.42 em (below the 0.5 em item
+        // break): the split must land between the modes. Needs >=4 gaps to
+        // enter the bimodal tier — short runs use the strict uniform gate.
+        let mut items = glyph_run("ITISOK", 100.0, 8.0, 2.3);
+        for i in 2..6 {
+            items[i].x += 2.8; // word gap at T|I
+        }
+        for i in 4..6 {
+            items[i].x += 2.8; // word gap at S|O
+        }
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "IT IS OK");
+    }
+
+    #[test]
+    fn lowercase_spaced_singles_stay_words() {
+        // "x y z" variables: same gap shape but lowercase — the short-run
+        // caps requirement keeps genuine spaced singles apart.
+        let items = glyph_run("xyz", 100.0, 6.0, 2.3);
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "x y z");
+    }
+
+    #[test]
+    fn kerned_singles_unaffected() {
+        // Tiny kerning gaps never triggered spaces before and still don't.
+        let items = glyph_run("WORD", 100.0, 8.0, 0.3);
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "WORD");
+    }
+
+    #[test]
+    fn cjk_glyph_run_collapses_without_spaces() {
+        // CJK sets one glyph per item with loose gaps; CJK uses no spaces,
+        // and the non-alphabetic run passes the caps gate.
+        let items = glyph_run("北京时事", 100.0, 12.0, 1.4);
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "北京时事");
+    }
+
     fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
         TextItem {
             text: text.into(),