Normalize typographic spaces (U+2000–U+200A) to ASCII space
EM SPACE from PDF ActualText entries was lost by text.trim(), breaking spacing after bullets and numbered list markers. Normalizing to ASCII space lets should_join_items detect word boundaries naturally. NBSP (U+00A0) is excluded as it's handled by coordinate-based spacing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
1afc1fdaa2
commit
e72152ce63
@@ -133,6 +133,11 @@ pub(crate) fn expand_ligatures(text: &str) -> String {
|
||||
'\u{FEFF}' => {} // BOM / zero-width no-break space
|
||||
'\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ
|
||||
'\u{2060}' => {} // word joiner
|
||||
// Normalize typographic spaces to ASCII space so downstream
|
||||
// spacing logic (should_join_items) can detect word boundaries.
|
||||
// Excludes NBSP (U+00A0) which is common in PDFs and handled
|
||||
// correctly by existing coordinate-based spacing.
|
||||
'\u{2000}'..='\u{200A}' => result.push(' '), // en/em/thin/hair spaces etc.
|
||||
_ => result.push(ch),
|
||||
}
|
||||
}
|
||||
@@ -406,4 +411,18 @@ mod tests {
|
||||
fn ligatures_still_expand() {
|
||||
assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_typographic_spaces() {
|
||||
// EM SPACE, EN SPACE, THIN SPACE → ASCII space
|
||||
assert_eq!(expand_ligatures("•\u{2003}text"), "• text");
|
||||
assert_eq!(expand_ligatures("a\u{2002}b"), "a b");
|
||||
assert_eq!(expand_ligatures("x\u{2009}y"), "x y");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nbsp_preserved() {
|
||||
// NBSP (U+00A0) should NOT be normalized
|
||||
assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user