From 325a7bfa628ddcc7d910a16ce28fb4f0b7923aab Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:57:59 -0700 Subject: [PATCH] feat(extractor): run-local space floor for tracked (letter-spaced) glyph runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Display type set with tracking renders one glyph per show op; the merge loop's fixed space thresholds (0.08-0.13 em) then read every letter gap as a word boundary and emit "H O W" / "F U R T H E R" instead of "HOW" / "FURTHER". The page-level Canva fixer can't help: it requires >=50% of the page's items to be letter-spaced, and these docs track only their display headings. merge_text_items now pre-scans each run of consecutive single-glyph items (same size band, same style, mergeable gaps — the loop's own break conditions) and, when the run is tracked, derives the space floor from the run's own gap distribution: - runs with >=4 gaps qualify when the median gap clears the fixed threshold; word gaps, if present, form a second mode — split at the largest relative jump (>=1.4x), else the run is a single word ("I T I S I M P O R T A N T" -> "IT IS IMPORTANT") - short runs (2-3 gaps: "H O W") additionally demand uniform gaps and ALL-CAPS or CJK — a genuine spaced sequence of single letters ("x y z" variables) has the same gap count, and display tracking is a caps convention; CJK never wants inter-glyph spaces Corpus sweep (708 opendataloader + ParseBench text PDFs) vs main: 9 docs change — the tracked display titles ("HOW CAN YOU HELP?", "LUNCHTIME MENU", a tracked email address), and CJK glyph-per-item docs whose spurious inter-glyph spaces now collapse (GT for those docs is unspaced CJK; should_join_items already treats no-space CJK as correct on its path). No other doc moves. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01L3U6BKYCS73DVA83odAfYB --- src/extractor/mod.rs | 182 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 180 insertions(+), 2 deletions(-) diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 4f2b9b0..9f75b05 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -9,7 +9,7 @@ mod links; pub(crate) mod underline; mod xobjects; -use crate::text_utils::is_rtl_text; +use crate::text_utils::{is_cjk_char, is_rtl_text}; use crate::tounicode::FontCMaps; use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem}; use crate::PdfError; @@ -527,6 +527,105 @@ fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool { saw_backtrack } +/// Detect a tracked (letter-spaced) run of single-glyph items and derive its +/// run-local space floor. +/// +/// Display type set with tracking renders one glyph per show op; the merge +/// loop's fixed thresholds (0.08-0.13 em) then read every letter gap as a +/// word boundary and emit "H O W" instead of "HOW". Within such a run the +/// gaps carry the real signal: letter gaps cluster tightly just above the +/// fixed threshold, word gaps sit clearly higher. Returns (run_end_index, +/// space_floor) when the run starting at `start` is tracked — spaces are +/// then inserted only at gaps above the floor (infinity = single word). +/// Normal text (multi-char items, or single-char runs with sub-threshold +/// gaps) returns None and keeps the existing behavior. +fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize, f32)> { + const MIN_GAPS: usize = 4; + let first = group[start]; + if first.text.trim().chars().count() != 1 { + return None; + } + let fs = first.font_size; + if fs <= 0.0 { + return None; + } + + // Walk the run under the SAME break conditions as the merge loop + // (size band, style equality, mergeable gap) so indices stay aligned. + let mut gaps: Vec = Vec::new(); + let mut end_x = first.x + effective_merge_width(first); + let mut end = start; + for (offset, next) in group[start + 1..].iter().enumerate() { + if next.text.trim().chars().count() != 1 { + break; + } + if (next.font_size - fs).abs() > fs * 0.20 { + break; + } + if next.is_bold != first.is_bold + || next.is_italic != first.is_italic + || next.is_underline != first.is_underline + || next.is_strikeout != first.is_strikeout + { + break; + } + let gap = next.x - end_x; + if gap > fs * 0.5 || gap < -fs * 0.5 { + break; + } + gaps.push(gap / fs); + end_x = next.x + effective_merge_width(next); + end = start + 1 + offset; + } + if gaps.len() < 2 { + return None; + } + + // Tracked signature: the run's TYPICAL gap clears the fixed space + // threshold (0.08) — the merge loop would break almost every letter + // pair into "words". Short runs (2-3 gaps: "H O W") demand a stricter + // shape — clearly wide, uniform, ALL-CAPS — because a genuine spaced + // sequence of single letters ("x y z" variables) has the same gap + // count; display tracking is a caps convention. + let mut sorted = gaps.clone(); + sorted.sort_by(|a, b| a.total_cmp(b)); + let median = sorted[sorted.len() / 2]; + if gaps.len() >= MIN_GAPS { + if median <= 0.075 { + return None; + } + } else { + let uniform = sorted[sorted.len() - 1] <= sorted[0].max(0.01) * 1.4; + // Caps convention — or CJK, which never uses inter-glyph spaces. + let all_caps = group[start..=end].iter().all(|it| { + it.text + .trim() + .chars() + .all(|c| c.is_uppercase() || is_cjk_char(c) || !c.is_alphabetic()) + }); + if median < 0.09 || !uniform || !all_caps { + return None; + } + } + + // Word gaps, if present, form a second mode above the letter-gap + // cluster: split at the largest relative jump. Unimodal → one word. + let mut best_jump = 1.0f32; + let mut floor = f32::INFINITY; + for pair in sorted.windows(2) { + let (lo, hi) = (pair[0].max(0.01), pair[1].max(0.01)); + let jump = hi / lo; + if jump > best_jump { + best_jump = jump; + floor = (lo + hi) / 2.0; + } + } + if best_jump < 1.4 { + floor = f32::INFINITY; + } + Some((end, floor * fs)) +} + pub(crate) fn merge_text_items(items: Vec) -> Vec { if items.is_empty() { return items; @@ -574,6 +673,14 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { let mut text = first.text.clone(); let mut end_x = first.x + effective_merge_width(first); + // Tracked display text: run-local space floor overrides the + // fixed thresholds for this run's junctions (see helper). + let tracked = if *preserve_stream_order { + None + } else { + tracked_run_space_floor(group, i) + }; + let mut j = i + 1; while j < group.len() { let next = group[j]; @@ -628,7 +735,11 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { let needs_bullet_space = *preserve_stream_order && is_standalone_bullet_text(&text) && !next.text.trim().is_empty(); - if needs_bullet_space || gap > threshold { + let effective_threshold = match tracked { + Some((run_end, floor)) if j <= run_end => floor, + _ => threshold, + }; + if needs_bullet_space || gap > effective_threshold { text.push(' '); } text.push_str(&next.text); @@ -794,6 +905,73 @@ mod tests { use crate::types::{ItemType, PdfLine, TextLine}; use layout::{detect_columns, is_newspaper_layout, ColumnRegion}; + /// Glyph-per-item run at `fs`=12 with the given inter-glyph gap (pt). + fn glyph_run(chars: &str, start_x: f32, glyph_w: f32, gap: f32) -> Vec { + let mut x = start_x; + let mut out = Vec::new(); + for c in chars.chars() { + out.push(make_merge_item(&c.to_string(), x, glyph_w)); + x += glyph_w + gap; + } + out + } + + #[test] + fn tracked_caps_run_collapses_to_word() { + // Display tracking: every letter gap (0.19 em) clears the fixed + // space threshold — without the run-local floor this reads "H O W". + let items = glyph_run("HOW", 100.0, 10.0, 2.3); + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "HOW"); + } + + #[test] + fn tracked_run_keeps_word_gaps_bimodal() { + // Letters at 0.19 em, word gaps at 0.42 em (below the 0.5 em item + // break): the split must land between the modes. Needs >=4 gaps to + // enter the bimodal tier — short runs use the strict uniform gate. + let mut items = glyph_run("ITISOK", 100.0, 8.0, 2.3); + for i in 2..6 { + items[i].x += 2.8; // word gap at T|I + } + for i in 4..6 { + items[i].x += 2.8; // word gap at S|O + } + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "IT IS OK"); + } + + #[test] + fn lowercase_spaced_singles_stay_words() { + // "x y z" variables: same gap shape but lowercase — the short-run + // caps requirement keeps genuine spaced singles apart. + let items = glyph_run("xyz", 100.0, 6.0, 2.3); + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "x y z"); + } + + #[test] + fn kerned_singles_unaffected() { + // Tiny kerning gaps never triggered spaces before and still don't. + let items = glyph_run("WORD", 100.0, 8.0, 0.3); + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "WORD"); + } + + #[test] + fn cjk_glyph_run_collapses_without_spaces() { + // CJK sets one glyph per item with loose gaps; CJK uses no spaces, + // and the non-alphabetic run passes the caps gate. + let items = glyph_run("北京时事", 100.0, 12.0, 1.4); + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "北京时事"); + } + fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem { TextItem { text: text.into(),