fix(fonts): prefer TrueType cmap over sequential remap for CJK subset fonts
Subset fonts number GIDs by document encounter order, not sorted order. The sequential remap scrambled character mappings, losing 2,532+ CJK characters. Now promotes TrueType cmap fallback when it has more entries than the primary ToUnicode CMap. Also adds CJK scoring to score_text() as defense-in-depth. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
cc92e554d0
commit
f1b32b320b
@@ -1010,6 +1010,13 @@ fn score_text(text: &str) -> i32 {
|
||||
digits += 1;
|
||||
} else if ch.is_control() || ch == '\u{FFFD}' {
|
||||
other += 3;
|
||||
} else if ('\u{4E00}'..='\u{9FFF}').contains(&ch)
|
||||
|| ('\u{3040}'..='\u{309F}').contains(&ch)
|
||||
|| ('\u{30A0}'..='\u{30FF}').contains(&ch)
|
||||
|| ('\u{3400}'..='\u{4DBF}').contains(&ch)
|
||||
|| ('\u{F900}'..='\u{FAFF}').contains(&ch)
|
||||
{
|
||||
letters += 1; // CJK ideographs / kana count as valid text
|
||||
} else {
|
||||
other += 1;
|
||||
}
|
||||
@@ -1025,3 +1032,45 @@ fn score_text(text: &str) -> i32 {
|
||||
}
|
||||
score
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn score_text_cjk() {
|
||||
// Correct Japanese text should score well
|
||||
let japanese = "2026年9月期 1Q 業績報告";
|
||||
// Garbled output (random CJK from wrong remap)
|
||||
let garbled = "\u{FFFD}\u{FFFD}\u{FFFD}";
|
||||
|
||||
let s_jp = score_text(japanese);
|
||||
let s_garbled = score_text(garbled);
|
||||
assert!(
|
||||
s_jp > s_garbled,
|
||||
"Japanese text ({s_jp}) should score higher than garbled ({s_garbled})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_text_cjk_vs_ascii_garbage() {
|
||||
// Real CJK text
|
||||
let cjk = "株式会社の業績についてご報告いたします";
|
||||
// Ascii garbage of similar length
|
||||
let garbage = "}{|~`^@#$%&*()!<>[];:',./";
|
||||
|
||||
let s_cjk = score_text(cjk);
|
||||
let s_garbage = score_text(garbage);
|
||||
assert!(
|
||||
s_cjk > s_garbage,
|
||||
"CJK text ({s_cjk}) should score higher than garbage ({s_garbage})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_text_english_still_works() {
|
||||
let good = "the quick brown fox and the lazy dog";
|
||||
let bad = "###!!!@@@$$$";
|
||||
assert!(score_text(good) > score_text(bad));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,26 @@ pub(crate) fn build_cmap_entry_from_stream(
|
||||
}
|
||||
}
|
||||
|
||||
// When a sequential remap was applied and a TrueType fallback has more
|
||||
// entries than the primary ToUnicode CMap, prefer the TrueType cmap.
|
||||
// Subset fonts number GIDs by document encounter order, so the sorted
|
||||
// sequential remap scrambles characters. The TrueType cmap table maps
|
||||
// the real GID→Unicode and is authoritative.
|
||||
if remapped.is_some() {
|
||||
if let Some(ref fb) = fallback {
|
||||
let fb_entries = fb.char_map.len() + fb.ranges.len();
|
||||
if fb_entries > primary_entries {
|
||||
debug!(
|
||||
"ToUnicode CMap obj={}: TrueType fallback ({} entries) > primary ({}); promoting over sequential remap",
|
||||
obj_num, fb_entries, primary_entries
|
||||
);
|
||||
let old_remap = remapped.take().unwrap();
|
||||
remapped = fallback.take();
|
||||
fallback = Some(old_remap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Some(CMapEntry {
|
||||
primary,
|
||||
remapped,
|
||||
@@ -2385,4 +2405,111 @@ endbfchar
|
||||
"Unmapped 2-byte CIDs should not produce CJK"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_promotion_when_larger_than_primary() {
|
||||
// Simulate: primary has 5 char_map entries, remapped exists (sequential),
|
||||
// fallback has 20 entries (TrueType cmap). The fallback should be
|
||||
// promoted to `remapped` and the old remap demoted to `fallback`.
|
||||
let mut primary = ToUnicodeCMap::new();
|
||||
for i in 0..5u16 {
|
||||
primary
|
||||
.char_map
|
||||
.insert(100 + i, char::from(b'A' + i as u8).to_string());
|
||||
}
|
||||
primary.code_byte_length = 2;
|
||||
|
||||
let mut sequential_remap = ToUnicodeCMap::new();
|
||||
for i in 0..5u16 {
|
||||
sequential_remap
|
||||
.char_map
|
||||
.insert(i, char::from(b'A' + i as u8).to_string());
|
||||
}
|
||||
sequential_remap.code_byte_length = 2;
|
||||
|
||||
let mut truetype_fb = ToUnicodeCMap::new();
|
||||
for i in 0..20u16 {
|
||||
truetype_fb
|
||||
.char_map
|
||||
.insert(i, format!("U+{:04X}", 0x4E00 + i));
|
||||
}
|
||||
truetype_fb.code_byte_length = 2;
|
||||
|
||||
let primary_entries = primary.char_map.len() + primary.ranges.len();
|
||||
let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
|
||||
let mut fallback: Option<ToUnicodeCMap> = Some(truetype_fb);
|
||||
|
||||
// Apply the same promotion logic as build_cmap_entry_from_stream
|
||||
if remapped.is_some() {
|
||||
if let Some(ref fb) = fallback {
|
||||
let fb_entries = fb.char_map.len() + fb.ranges.len();
|
||||
if fb_entries > primary_entries {
|
||||
let old_remap = remapped.take().unwrap();
|
||||
remapped = fallback.take();
|
||||
fallback = Some(old_remap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The TrueType fallback (20 entries) should now be in `remapped`
|
||||
let r = remapped.unwrap();
|
||||
assert_eq!(
|
||||
r.char_map.len(),
|
||||
20,
|
||||
"TrueType cmap should be promoted to remapped"
|
||||
);
|
||||
|
||||
// The old sequential remap (5 entries) should now be in `fallback`
|
||||
let f = fallback.unwrap();
|
||||
assert_eq!(
|
||||
f.char_map.len(),
|
||||
5,
|
||||
"Sequential remap should be demoted to fallback"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fallback_promotion_when_smaller() {
|
||||
// When fallback has fewer entries than primary, no swap should occur.
|
||||
let mut primary = ToUnicodeCMap::new();
|
||||
for i in 0..50u16 {
|
||||
primary
|
||||
.char_map
|
||||
.insert(100 + i, format!("U+{:04X}", 0x0041 + i));
|
||||
}
|
||||
primary.code_byte_length = 2;
|
||||
|
||||
let mut sequential_remap = ToUnicodeCMap::new();
|
||||
for i in 0..50u16 {
|
||||
sequential_remap
|
||||
.char_map
|
||||
.insert(i, format!("U+{:04X}", 0x0041 + i));
|
||||
}
|
||||
sequential_remap.code_byte_length = 2;
|
||||
|
||||
let mut small_fb = ToUnicodeCMap::new();
|
||||
for i in 0..10u16 {
|
||||
small_fb.char_map.insert(i, format!("U+{:04X}", 0x4E00 + i));
|
||||
}
|
||||
small_fb.code_byte_length = 2;
|
||||
|
||||
let primary_entries = primary.char_map.len() + primary.ranges.len();
|
||||
let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
|
||||
let mut fallback: Option<ToUnicodeCMap> = Some(small_fb);
|
||||
|
||||
if remapped.is_some() {
|
||||
if let Some(ref fb) = fallback {
|
||||
let fb_entries = fb.char_map.len() + fb.ranges.len();
|
||||
if fb_entries > primary_entries {
|
||||
let old_remap = remapped.take().unwrap();
|
||||
remapped = fallback.take();
|
||||
fallback = Some(old_remap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No swap: remapped should still have 50 entries
|
||||
assert_eq!(remapped.unwrap().char_map.len(), 50);
|
||||
assert_eq!(fallback.unwrap().char_map.len(), 10);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user