fix(fonts): prefer TrueType cmap over sequential remap for CJK subset fonts

Subset fonts number GIDs by document encounter order, not sorted order.
The sequential remap scrambled character mappings, losing 2,532+ CJK
characters. Now promotes TrueType cmap fallback when it has more entries
than the primary ToUnicode CMap. Also adds CJK scoring to score_text()
as defense-in-depth.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-11 17:37:42 -07:00
co-authored by Claude Opus 4.6
parent cc92e554d0
commit f1b32b320b
2 changed files with 176 additions and 0 deletions
+49
View File
@@ -1010,6 +1010,13 @@ fn score_text(text: &str) -> i32 {
digits += 1;
} else if ch.is_control() || ch == '\u{FFFD}' {
other += 3;
} else if ('\u{4E00}'..='\u{9FFF}').contains(&ch)
|| ('\u{3040}'..='\u{309F}').contains(&ch)
|| ('\u{30A0}'..='\u{30FF}').contains(&ch)
|| ('\u{3400}'..='\u{4DBF}').contains(&ch)
|| ('\u{F900}'..='\u{FAFF}').contains(&ch)
{
letters += 1; // CJK ideographs / kana count as valid text
} else {
other += 1;
}
@@ -1025,3 +1032,45 @@ fn score_text(text: &str) -> i32 {
}
score
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn score_text_cjk() {
// Correct Japanese text should score well
let japanese = "2026年9月期 1Q 業績報告";
// Garbled output (random CJK from wrong remap)
let garbled = "\u{FFFD}\u{FFFD}\u{FFFD}";
let s_jp = score_text(japanese);
let s_garbled = score_text(garbled);
assert!(
s_jp > s_garbled,
"Japanese text ({s_jp}) should score higher than garbled ({s_garbled})"
);
}
#[test]
fn score_text_cjk_vs_ascii_garbage() {
// Real CJK text
let cjk = "株式会社の業績についてご報告いたします";
// Ascii garbage of similar length
let garbage = "}{|~`^@#$%&*()!<>[];:',./";
let s_cjk = score_text(cjk);
let s_garbage = score_text(garbage);
assert!(
s_cjk > s_garbage,
"CJK text ({s_cjk}) should score higher than garbage ({s_garbage})"
);
}
#[test]
fn score_text_english_still_works() {
let good = "the quick brown fox and the lazy dog";
let bad = "###!!!@@@$$$";
assert!(score_text(good) > score_text(bad));
}
}
+127
View File
@@ -44,6 +44,26 @@ pub(crate) fn build_cmap_entry_from_stream(
}
}
// When a sequential remap was applied and a TrueType fallback has more
// entries than the primary ToUnicode CMap, prefer the TrueType cmap.
// Subset fonts number GIDs by document encounter order, so the sorted
// sequential remap scrambles characters. The TrueType cmap table maps
// the real GID→Unicode and is authoritative.
if remapped.is_some() {
if let Some(ref fb) = fallback {
let fb_entries = fb.char_map.len() + fb.ranges.len();
if fb_entries > primary_entries {
debug!(
"ToUnicode CMap obj={}: TrueType fallback ({} entries) > primary ({}); promoting over sequential remap",
obj_num, fb_entries, primary_entries
);
let old_remap = remapped.take().unwrap();
remapped = fallback.take();
fallback = Some(old_remap);
}
}
}
return Some(CMapEntry {
primary,
remapped,
@@ -2385,4 +2405,111 @@ endbfchar
"Unmapped 2-byte CIDs should not produce CJK"
);
}
#[test]
fn fallback_promotion_when_larger_than_primary() {
// Simulate: primary has 5 char_map entries, remapped exists (sequential),
// fallback has 20 entries (TrueType cmap). The fallback should be
// promoted to `remapped` and the old remap demoted to `fallback`.
let mut primary = ToUnicodeCMap::new();
for i in 0..5u16 {
primary
.char_map
.insert(100 + i, char::from(b'A' + i as u8).to_string());
}
primary.code_byte_length = 2;
let mut sequential_remap = ToUnicodeCMap::new();
for i in 0..5u16 {
sequential_remap
.char_map
.insert(i, char::from(b'A' + i as u8).to_string());
}
sequential_remap.code_byte_length = 2;
let mut truetype_fb = ToUnicodeCMap::new();
for i in 0..20u16 {
truetype_fb
.char_map
.insert(i, format!("U+{:04X}", 0x4E00 + i));
}
truetype_fb.code_byte_length = 2;
let primary_entries = primary.char_map.len() + primary.ranges.len();
let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
let mut fallback: Option<ToUnicodeCMap> = Some(truetype_fb);
// Apply the same promotion logic as build_cmap_entry_from_stream
if remapped.is_some() {
if let Some(ref fb) = fallback {
let fb_entries = fb.char_map.len() + fb.ranges.len();
if fb_entries > primary_entries {
let old_remap = remapped.take().unwrap();
remapped = fallback.take();
fallback = Some(old_remap);
}
}
}
// The TrueType fallback (20 entries) should now be in `remapped`
let r = remapped.unwrap();
assert_eq!(
r.char_map.len(),
20,
"TrueType cmap should be promoted to remapped"
);
// The old sequential remap (5 entries) should now be in `fallback`
let f = fallback.unwrap();
assert_eq!(
f.char_map.len(),
5,
"Sequential remap should be demoted to fallback"
);
}
#[test]
fn no_fallback_promotion_when_smaller() {
// When fallback has fewer entries than primary, no swap should occur.
let mut primary = ToUnicodeCMap::new();
for i in 0..50u16 {
primary
.char_map
.insert(100 + i, format!("U+{:04X}", 0x0041 + i));
}
primary.code_byte_length = 2;
let mut sequential_remap = ToUnicodeCMap::new();
for i in 0..50u16 {
sequential_remap
.char_map
.insert(i, format!("U+{:04X}", 0x0041 + i));
}
sequential_remap.code_byte_length = 2;
let mut small_fb = ToUnicodeCMap::new();
for i in 0..10u16 {
small_fb.char_map.insert(i, format!("U+{:04X}", 0x4E00 + i));
}
small_fb.code_byte_length = 2;
let primary_entries = primary.char_map.len() + primary.ranges.len();
let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
let mut fallback: Option<ToUnicodeCMap> = Some(small_fb);
if remapped.is_some() {
if let Some(ref fb) = fallback {
let fb_entries = fb.char_map.len() + fb.ranges.len();
if fb_entries > primary_entries {
let old_remap = remapped.take().unwrap();
remapped = fallback.take();
fallback = Some(old_remap);
}
}
}
// No swap: remapped should still have 50 entries
assert_eq!(remapped.unwrap().char_map.len(), 50);
assert_eq!(fallback.unwrap().char_map.len(), 10);
}
}