diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ee9b469..dd99cc2 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -594,7 +594,7 @@ pub(crate) fn extract_page_text_items( .map(|s| s.as_str()) .unwrap_or(¤t_font); items.push(TextItem { - text: at, + text: expand_ligatures(&at), x, y, width, diff --git a/src/text_utils.rs b/src/text_utils.rs index 7bb77ff..6800d44 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -127,6 +127,12 @@ pub(crate) fn expand_ligatures(text: &str) -> String { '\u{FB03}' => result.push_str("ffi"), '\u{FB04}' => result.push_str("ffl"), '\u{FB05}' | '\u{FB06}' => result.push_str("st"), + // Strip invisible Unicode characters that pollute markdown output + '\u{00AD}' => {} // soft hyphen + '\u{200B}' => {} // zero-width space + '\u{FEFF}' => {} // BOM / zero-width no-break space + '\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ + '\u{2060}' => {} // word joiner _ => result.push(ch), } } @@ -366,3 +372,38 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strip_soft_hyphen() { + assert_eq!(expand_ligatures("con\u{00AD}tent"), "content"); + } + + #[test] + fn strip_zero_width_space() { + assert_eq!(expand_ligatures("hello\u{200B}world"), "helloworld"); + } + + #[test] + fn strip_bom() { + assert_eq!(expand_ligatures("\u{FEFF}text"), "text"); + } + + #[test] + fn strip_zwnj_zwj_word_joiner() { + assert_eq!(expand_ligatures("a\u{200C}b\u{200D}c\u{2060}d"), "abcd"); + } + + #[test] + fn ligature_plus_invisible_chars() { + assert_eq!(expand_ligatures("\u{FB01}rst\u{00AD}ly"), "firstly"); + } + + #[test] + fn ligatures_still_expand() { + assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl"); + } +}