From e72152ce63dfa05a01db47799a1055620ed96523 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 24 Feb 2026 12:08:24 -0800 Subject: [PATCH] =?UTF-8?q?Normalize=20typographic=20spaces=20(U+2000?= =?UTF-8?q?=E2=80=93U+200A)=20to=20ASCII=20space?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EM SPACE from PDF ActualText entries was lost by text.trim(), breaking spacing after bullets and numbered list markers. Normalizing to ASCII space lets should_join_items detect word boundaries naturally. NBSP (U+00A0) is excluded as it's handled by coordinate-based spacing. Co-Authored-By: Claude Opus 4.6 --- src/text_utils.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/text_utils.rs b/src/text_utils.rs index 6800d44..a6929bb 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -133,6 +133,11 @@ pub(crate) fn expand_ligatures(text: &str) -> String { '\u{FEFF}' => {} // BOM / zero-width no-break space '\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ '\u{2060}' => {} // word joiner + // Normalize typographic spaces to ASCII space so downstream + // spacing logic (should_join_items) can detect word boundaries. + // Excludes NBSP (U+00A0) which is common in PDFs and handled + // correctly by existing coordinate-based spacing. + '\u{2000}'..='\u{200A}' => result.push(' '), // en/em/thin/hair spaces etc. _ => result.push(ch), } } @@ -406,4 +411,18 @@ mod tests { fn ligatures_still_expand() { assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl"); } + + #[test] + fn normalize_typographic_spaces() { + // EM SPACE, EN SPACE, THIN SPACE → ASCII space + assert_eq!(expand_ligatures("•\u{2003}text"), "• text"); + assert_eq!(expand_ligatures("a\u{2002}b"), "a b"); + assert_eq!(expand_ligatures("x\u{2009}y"), "x y"); + } + + #[test] + fn nbsp_preserved() { + // NBSP (U+00A0) should NOT be normalized + assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b"); + } }