Normalize typographic spaces (U+2000–U+200A) to ASCII space

EM SPACE from PDF ActualText entries was lost by text.trim(), breaking
spacing after bullets and numbered list markers. Normalizing to ASCII
space lets should_join_items detect word boundaries naturally. NBSP
(U+00A0) is excluded as it's handled by coordinate-based spacing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-24 12:08:24 -08:00
co-authored by Claude Opus 4.6
parent 1afc1fdaa2
commit e72152ce63
+19
View File
@@ -133,6 +133,11 @@ pub(crate) fn expand_ligatures(text: &str) -> String {
'\u{FEFF}' => {} // BOM / zero-width no-break space
'\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ
'\u{2060}' => {} // word joiner
// Normalize typographic spaces to ASCII space so downstream
// spacing logic (should_join_items) can detect word boundaries.
// Excludes NBSP (U+00A0) which is common in PDFs and handled
// correctly by existing coordinate-based spacing.
'\u{2000}'..='\u{200A}' => result.push(' '), // en/em/thin/hair spaces etc.
_ => result.push(ch),
}
}
@@ -406,4 +411,18 @@ mod tests {
fn ligatures_still_expand() {
assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl");
}
#[test]
fn normalize_typographic_spaces() {
// EM SPACE, EN SPACE, THIN SPACE → ASCII space
assert_eq!(expand_ligatures("\u{2003}text"), "• text");
assert_eq!(expand_ligatures("a\u{2002}b"), "a b");
assert_eq!(expand_ligatures("x\u{2009}y"), "x y");
}
#[test]
fn nbsp_preserved() {
// NBSP (U+00A0) should NOT be normalized
assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b");
}
}