mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix PUA glyph decoding, Symbol font cmap mapping, and UTF-8 detection

Browse Source 
- Strip PUA F000 offset for "uniF0XX" glyph names (e.g. uniF072 → 'r')
- Use Mac Roman/Symbol cmap subtables for proper code→GID→Unicode mapping
  instead of assuming GID equals character code in subsetted fonts
- Skip fallback CMap building for fonts with explicit encoding
- Move UTF-8 detection before lopdf single-byte encoding decoder

Fixes systematic letter substitutions (CITY→CITQ), missing French letters,
and UTF-8 mojibake (José→José). Eval: missing_text -32%, encoding_issue 7→4.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-21 13:54:12 -08:00
co-authored by Claude Opus 4.6
parent 97910e86bf
commit a4a2e1f267
3 changed files with 94 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/fonts.rs
+9
View File
@@ -851,6 +851,15 @@ pub(crate) fn extract_text_from_operand(
}
}
// Check for UTF-8 encoded strings before single-byte encoding decoding.
// Some PDFs incorrectly embed UTF-8 bytes in single-byte encoded fonts
// (e.g. "José" as UTF-8 [C3 A9] instead of WinAnsi [E9]).
if bytes.iter().any(|&b| b > 0x7F) {
if let Ok(text) = std::str::from_utf8(bytes) {
return Some(text.to_string());
}
}
// Try to decode using cached font encoding from lopdf
if let Some(encoding) = encoding_cache.get(current_font) {
if let Ok(text) = Document::decode_text(encoding, bytes) {
src/glyph_names.rs
+6
View File
@@ -4569,6 +4569,12 @@ pub fn glyph_to_char(name: &str) -> Option<char> {
// Try to parse uniXXXX format
if name.starts_with("uni") && name.len() >= 7 {
if let Ok(code) = u32::from_str_radix(&name[3..7], 16) {
// Strip PUA F000 offset: uniF0XX → U+00XX (Windows Symbol encoding convention)
let code = if (0xF000..=0xF0FF).contains(&code) {
code - 0xF000
} else {
code
};
return char::from_u32(code);
}
}
src/tounicode.rs
+79 -17
View File
@@ -753,27 +753,72 @@ fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option<ToUnicodeCMap> {
let gid_to_unicode = build_gid_to_unicode(&face)?;
let mut cmap = ToUnicodeCMap::new();
for (gid, ch) in gid_to_unicode {
if gid <= 0xFF {
cmap.char_map.insert(gid, ch.to_string());
}
}
// Fill missing single-byte codes from glyph names (helps with ligatures like "t_i").
for gid in 0..face.number_of_glyphs() {
let gid = ttf_parser::GlyphId(gid);
let gid_val = gid.0;
if gid_val > 0xFF || cmap.char_map.contains_key(&gid_val) {
continue;
}
if let Some(name) = face.glyph_name(gid) {
if gid_val == 0x1B {
debug!("simple cmap glyph gid=0x1B name={:?}", name);
// Use the font's encoding cmap subtable for proper code→GID→Unicode mapping.
// In subsetted TrueType fonts, GID ≠ character code, so we need the cmap table
// to translate byte codes (as used in the PDF content stream) to GIDs.
let mut used_encoding_cmap = false;
if let Some(cmap_table) = face.tables().cmap {
// Prefer Mac Roman (1,0): maps byte codes 0–255 directly to GIDs.
for subtable in cmap_table.subtables {
if subtable.platform_id == ttf_parser::PlatformId::Macintosh
&& subtable.encoding_id == 0
{
for code in 0x20..=0xFF_u32 {
if let Some(gid) = subtable.glyph_index(code) {
if let Some(&ch) = gid_to_unicode.get(&gid.0) {
let ch = strip_pua_char(ch);
cmap.char_map.entry(code as u16).or_insert(ch.to_string());
}
}
}
used_encoding_cmap = true;
break;
}
if let Some(s) = glyph_name_to_string(name) {
cmap.char_map.insert(gid_val, s);
}
// Fallback: Windows Symbol (3,0) — maps F000+byte to GIDs.
if !used_encoding_cmap {
for subtable in cmap_table.subtables {
if subtable.platform_id == ttf_parser::PlatformId::Windows
&& subtable.encoding_id == 0
{
for code in 0x20..=0xFF_u32 {
if let Some(gid) = subtable.glyph_index(code + 0xF000) {
if let Some(&ch) = gid_to_unicode.get(&gid.0) {
let ch = strip_pua_char(ch);
cmap.char_map.entry(code as u16).or_insert(ch.to_string());
}
}
}
used_encoding_cmap = true;
break;
}
}
}
}
if !used_encoding_cmap {
// No encoding cmap found — fall back to treating GID as code.
for (&gid, &ch) in &gid_to_unicode {
if gid <= 0xFF {
cmap.char_map.insert(gid, ch.to_string());
}
}
// Fill missing single-byte codes from glyph names (helps with ligatures like "t_i").
for gid_idx in 0..face.number_of_glyphs() {
let gid = ttf_parser::GlyphId(gid_idx);
let gid_val = gid.0;
if gid_val > 0xFF || cmap.char_map.contains_key(&gid_val) {
continue;
}
if let Some(name) = face.glyph_name(gid) {
if let Some(s) = glyph_name_to_string(name) {
cmap.char_map.insert(gid_val, s);
}
}
}
}
if cmap.char_map.is_empty() {
return None;
}
@@ -785,6 +830,16 @@ fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option<ToUnicodeCMap> {
Some(cmap)
}
/// Strip Private Use Area F000 offset (Windows Symbol encoding convention).
fn strip_pua_char(ch: char) -> char {
let cp = ch as u32;
if (0xF000..=0xF0FF).contains(&cp) {
char::from_u32(cp - 0xF000).unwrap_or(ch)
} else {
ch
}
}
fn glyph_name_to_string(name: &str) -> Option<String> {
let base = name.split('.').next().unwrap_or(name);
if let Some(ch) = glyph_to_char(base) {
@@ -1823,6 +1878,13 @@ impl FontCMaps {
if font_dict.get(b"ToUnicode").is_ok() {
continue;
}
// Skip fonts with explicit encoding — they can be decoded by the
// standard encoding path (lopdf) and don't need a fallback CMap.
if let Ok(enc) = font_dict.get(b"Encoding") {
if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() {
continue;
}
}
let subtype = match font_dict
.get(b"Subtype")
.ok()