Normalize typographic spaces (U+2000–U+200A) to ASCII space
EM SPACE from PDF ActualText entries was lost by text.trim(), breaking spacing after bullets and numbered list markers. Normalizing to ASCII space lets should_join_items detect word boundaries naturally. NBSP (U+00A0) is excluded as it's handled by coordinate-based spacing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
1afc1fdaa2
commit
e72152ce63
@@ -133,6 +133,11 @@ pub(crate) fn expand_ligatures(text: &str) -> String {
|
|||||||
'\u{FEFF}' => {} // BOM / zero-width no-break space
|
'\u{FEFF}' => {} // BOM / zero-width no-break space
|
||||||
'\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ
|
'\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ
|
||||||
'\u{2060}' => {} // word joiner
|
'\u{2060}' => {} // word joiner
|
||||||
|
// Normalize typographic spaces to ASCII space so downstream
|
||||||
|
// spacing logic (should_join_items) can detect word boundaries.
|
||||||
|
// Excludes NBSP (U+00A0) which is common in PDFs and handled
|
||||||
|
// correctly by existing coordinate-based spacing.
|
||||||
|
'\u{2000}'..='\u{200A}' => result.push(' '), // en/em/thin/hair spaces etc.
|
||||||
_ => result.push(ch),
|
_ => result.push(ch),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -406,4 +411,18 @@ mod tests {
|
|||||||
fn ligatures_still_expand() {
|
fn ligatures_still_expand() {
|
||||||
assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl");
|
assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalize_typographic_spaces() {
|
||||||
|
// EM SPACE, EN SPACE, THIN SPACE → ASCII space
|
||||||
|
assert_eq!(expand_ligatures("•\u{2003}text"), "• text");
|
||||||
|
assert_eq!(expand_ligatures("a\u{2002}b"), "a b");
|
||||||
|
assert_eq!(expand_ligatures("x\u{2009}y"), "x y");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nbsp_preserved() {
|
||||||
|
// NBSP (U+00A0) should NOT be normalized
|
||||||
|
assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user