diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index a83308b..e6f3b78 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -1010,6 +1010,13 @@ fn score_text(text: &str) -> i32 { digits += 1; } else if ch.is_control() || ch == '\u{FFFD}' { other += 3; + } else if ('\u{4E00}'..='\u{9FFF}').contains(&ch) + || ('\u{3040}'..='\u{309F}').contains(&ch) + || ('\u{30A0}'..='\u{30FF}').contains(&ch) + || ('\u{3400}'..='\u{4DBF}').contains(&ch) + || ('\u{F900}'..='\u{FAFF}').contains(&ch) + { + letters += 1; // CJK ideographs / kana count as valid text } else { other += 1; } @@ -1025,3 +1032,45 @@ fn score_text(text: &str) -> i32 { } score } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn score_text_cjk() { + // Correct Japanese text should score well + let japanese = "2026年9月期 1Q 業績報告"; + // Garbled output (random CJK from wrong remap) + let garbled = "\u{FFFD}\u{FFFD}\u{FFFD}"; + + let s_jp = score_text(japanese); + let s_garbled = score_text(garbled); + assert!( + s_jp > s_garbled, + "Japanese text ({s_jp}) should score higher than garbled ({s_garbled})" + ); + } + + #[test] + fn score_text_cjk_vs_ascii_garbage() { + // Real CJK text + let cjk = "株式会社の業績についてご報告いたします"; + // Ascii garbage of similar length + let garbage = "}{|~`^@#$%&*()!<>[];:',./"; + + let s_cjk = score_text(cjk); + let s_garbage = score_text(garbage); + assert!( + s_cjk > s_garbage, + "CJK text ({s_cjk}) should score higher than garbage ({s_garbage})" + ); + } + + #[test] + fn score_text_english_still_works() { + let good = "the quick brown fox and the lazy dog"; + let bad = "###!!!@@@$$$"; + assert!(score_text(good) > score_text(bad)); + } +} diff --git a/src/tounicode.rs b/src/tounicode.rs index 40af597..eeb3e10 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -44,6 +44,26 @@ pub(crate) fn build_cmap_entry_from_stream( } } + // When a sequential remap was applied and a TrueType fallback has more + // entries than the primary ToUnicode CMap, prefer the TrueType cmap. + // Subset fonts number GIDs by document encounter order, so the sorted + // sequential remap scrambles characters. The TrueType cmap table maps + // the real GID→Unicode and is authoritative. + if remapped.is_some() { + if let Some(ref fb) = fallback { + let fb_entries = fb.char_map.len() + fb.ranges.len(); + if fb_entries > primary_entries { + debug!( + "ToUnicode CMap obj={}: TrueType fallback ({} entries) > primary ({}); promoting over sequential remap", + obj_num, fb_entries, primary_entries + ); + let old_remap = remapped.take().unwrap(); + remapped = fallback.take(); + fallback = Some(old_remap); + } + } + } + return Some(CMapEntry { primary, remapped, @@ -2385,4 +2405,111 @@ endbfchar "Unmapped 2-byte CIDs should not produce CJK" ); } + + #[test] + fn fallback_promotion_when_larger_than_primary() { + // Simulate: primary has 5 char_map entries, remapped exists (sequential), + // fallback has 20 entries (TrueType cmap). The fallback should be + // promoted to `remapped` and the old remap demoted to `fallback`. + let mut primary = ToUnicodeCMap::new(); + for i in 0..5u16 { + primary + .char_map + .insert(100 + i, char::from(b'A' + i as u8).to_string()); + } + primary.code_byte_length = 2; + + let mut sequential_remap = ToUnicodeCMap::new(); + for i in 0..5u16 { + sequential_remap + .char_map + .insert(i, char::from(b'A' + i as u8).to_string()); + } + sequential_remap.code_byte_length = 2; + + let mut truetype_fb = ToUnicodeCMap::new(); + for i in 0..20u16 { + truetype_fb + .char_map + .insert(i, format!("U+{:04X}", 0x4E00 + i)); + } + truetype_fb.code_byte_length = 2; + + let primary_entries = primary.char_map.len() + primary.ranges.len(); + let mut remapped: Option = Some(sequential_remap); + let mut fallback: Option = Some(truetype_fb); + + // Apply the same promotion logic as build_cmap_entry_from_stream + if remapped.is_some() { + if let Some(ref fb) = fallback { + let fb_entries = fb.char_map.len() + fb.ranges.len(); + if fb_entries > primary_entries { + let old_remap = remapped.take().unwrap(); + remapped = fallback.take(); + fallback = Some(old_remap); + } + } + } + + // The TrueType fallback (20 entries) should now be in `remapped` + let r = remapped.unwrap(); + assert_eq!( + r.char_map.len(), + 20, + "TrueType cmap should be promoted to remapped" + ); + + // The old sequential remap (5 entries) should now be in `fallback` + let f = fallback.unwrap(); + assert_eq!( + f.char_map.len(), + 5, + "Sequential remap should be demoted to fallback" + ); + } + + #[test] + fn no_fallback_promotion_when_smaller() { + // When fallback has fewer entries than primary, no swap should occur. + let mut primary = ToUnicodeCMap::new(); + for i in 0..50u16 { + primary + .char_map + .insert(100 + i, format!("U+{:04X}", 0x0041 + i)); + } + primary.code_byte_length = 2; + + let mut sequential_remap = ToUnicodeCMap::new(); + for i in 0..50u16 { + sequential_remap + .char_map + .insert(i, format!("U+{:04X}", 0x0041 + i)); + } + sequential_remap.code_byte_length = 2; + + let mut small_fb = ToUnicodeCMap::new(); + for i in 0..10u16 { + small_fb.char_map.insert(i, format!("U+{:04X}", 0x4E00 + i)); + } + small_fb.code_byte_length = 2; + + let primary_entries = primary.char_map.len() + primary.ranges.len(); + let mut remapped: Option = Some(sequential_remap); + let mut fallback: Option = Some(small_fb); + + if remapped.is_some() { + if let Some(ref fb) = fallback { + let fb_entries = fb.char_map.len() + fb.ranges.len(); + if fb_entries > primary_entries { + let old_remap = remapped.take().unwrap(); + remapped = fallback.take(); + fallback = Some(old_remap); + } + } + } + + // No swap: remapped should still have 50 entries + assert_eq!(remapped.unwrap().char_map.len(), 50); + assert_eq!(fallback.unwrap().char_map.len(), 10); + } }