Strip invisible Unicode chars and expand ligatures on ActualText path

Soft hyphens, zero-width spaces, BOM, ZWNJ/ZWJ, and word joiners now get
stripped in expand_ligatures(). Also call expand_ligatures() on the ActualText
code path which was the only TextItem creation site missing it.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/extractor/content_stream.rs

@@ -594,7 +594,7 @@ pub(crate) fn extract_page_text_items(
                                 .map(|s| s.as_str())
                                 .unwrap_or(&current_font);
                             items.push(TextItem {
-                                text: at,
+                                text: expand_ligatures(&at),
                                 x,
                                 y,
                                 width,

src/text_utils.rs

@@ -127,6 +127,12 @@ pub(crate) fn expand_ligatures(text: &str) -> String {
             '\u{FB03}' => result.push_str("ffi"),
             '\u{FB04}' => result.push_str("ffl"),
             '\u{FB05}' | '\u{FB06}' => result.push_str("st"),
+            // Strip invisible Unicode characters that pollute markdown output
+            '\u{00AD}' => {}              // soft hyphen
+            '\u{200B}' => {}              // zero-width space
+            '\u{FEFF}' => {}              // BOM / zero-width no-break space
+            '\u{200C}' | '\u{200D}' => {} // ZWNJ / ZWJ
+            '\u{2060}' => {}              // word joiner
             _ => result.push(ch),
         }
     }
@@ -366,3 +372,38 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn strip_soft_hyphen() {
+        assert_eq!(expand_ligatures("con\u{00AD}tent"), "content");
+    }
+
+    #[test]
+    fn strip_zero_width_space() {
+        assert_eq!(expand_ligatures("hello\u{200B}world"), "helloworld");
+    }
+
+    #[test]
+    fn strip_bom() {
+        assert_eq!(expand_ligatures("\u{FEFF}text"), "text");
+    }
+
+    #[test]
+    fn strip_zwnj_zwj_word_joiner() {
+        assert_eq!(expand_ligatures("a\u{200C}b\u{200D}c\u{2060}d"), "abcd");
+    }
+
+    #[test]
+    fn ligature_plus_invisible_chars() {
+        assert_eq!(expand_ligatures("\u{FB01}rst\u{00AD}ly"), "firstly");
+    }
+
+    #[test]
+    fn ligatures_still_expand() {
+        assert_eq!(expand_ligatures("\u{FB00}\u{FB01}\u{FB02}"), "fffifl");
+    }
+}