fix(text): Fix ligature corruption and expand ligature characters
Fix Differences encoding path dropping standard characters when only ligature bytes matched the sparse encoding map. Now combines Differences entries with Latin-1 fallback for printable bytes instead of using filter_map which silently dropped unmapped bytes. Add expand_ligatures() to replace Unicode ligature characters (U+FB00-FB04) with their ASCII components (ff, fi, fl, ffi, ffl) at all TextItem creation sites. Add underscore-variant glyph names (f_f, f_f_i, f_f_l). Eliminates 669 ligature corruptions across the eval suite. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
56cfd895b5
commit
906051b832
@@ -0,0 +1,182 @@
|
||||
use pdf_inspector::extract_text_with_positions;
|
||||
use pdf_inspector::tounicode::FontCMaps;
|
||||
|
||||
fn main() {
|
||||
let path = std::env::args()
|
||||
.nth(1)
|
||||
.expect("Usage: debug_ligatures <pdf>");
|
||||
|
||||
// Load PDF and extract CMaps
|
||||
let pdf_bytes = std::fs::read(&path).unwrap();
|
||||
let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes);
|
||||
|
||||
println!("=== Font CMaps ===");
|
||||
if font_cmaps.by_name.is_empty() && font_cmaps.by_obj_num.is_empty() {
|
||||
println!(" (none found)");
|
||||
}
|
||||
for (name, cmap) in &font_cmaps.by_name {
|
||||
println!(
|
||||
" font={:30} code_byte_length={} char_map_entries={} ranges={}",
|
||||
name,
|
||||
cmap.code_byte_length,
|
||||
cmap.char_map.len(),
|
||||
cmap.ranges.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Load with lopdf to inspect font Differences arrays
|
||||
let doc = lopdf::Document::load_mem(&pdf_bytes).unwrap();
|
||||
let pages = doc.get_pages();
|
||||
|
||||
println!("\n=== Font Encoding Differences ===");
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
println!("--- Page {} ---", page_num);
|
||||
let fonts = match doc.get_page_fonts(page_id) {
|
||||
Ok(f) => f,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for (font_name_bytes, font_dict) in &fonts {
|
||||
let font_name = String::from_utf8_lossy(font_name_bytes).to_string();
|
||||
|
||||
// Check for Encoding
|
||||
if let Ok(encoding_obj) = font_dict.get(b"Encoding") {
|
||||
let enc_dict = match encoding_obj {
|
||||
lopdf::Object::Dictionary(d) => Some(d.clone()),
|
||||
lopdf::Object::Reference(r) => doc.get_dictionary(*r).ok().cloned(),
|
||||
lopdf::Object::Name(name) => {
|
||||
println!(
|
||||
" font={}: Encoding={}",
|
||||
font_name,
|
||||
String::from_utf8_lossy(name)
|
||||
);
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
if let Some(enc_dict) = enc_dict {
|
||||
// Check BaseEncoding
|
||||
if let Ok(base) = enc_dict.get(b"BaseEncoding") {
|
||||
if let lopdf::Object::Name(name) = base {
|
||||
println!(
|
||||
" font={}: BaseEncoding={}",
|
||||
font_name,
|
||||
String::from_utf8_lossy(name)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Dump Differences
|
||||
if let Ok(diff_obj) = enc_dict.get(b"Differences") {
|
||||
let diff_array = match diff_obj {
|
||||
lopdf::Object::Array(arr) => Some(arr.clone()),
|
||||
lopdf::Object::Reference(r) => {
|
||||
if let Ok(lopdf::Object::Array(arr)) = doc.get_object(*r) {
|
||||
Some(arr.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
if let Some(diff_array) = diff_array {
|
||||
let mut current_code: u8 = 0;
|
||||
let mut entries = Vec::new();
|
||||
let mut total_glyphs = 0;
|
||||
|
||||
for item in &diff_array {
|
||||
match item {
|
||||
lopdf::Object::Integer(n) => {
|
||||
current_code = *n as u8;
|
||||
}
|
||||
lopdf::Object::Name(name) => {
|
||||
let glyph = String::from_utf8_lossy(name).to_string();
|
||||
entries.push((current_code, glyph));
|
||||
current_code = current_code.wrapping_add(1);
|
||||
total_glyphs += 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
println!(
|
||||
" font={}: Differences has {} glyph entries",
|
||||
font_name, total_glyphs
|
||||
);
|
||||
|
||||
// Show ligature entries specifically
|
||||
for (code, glyph) in &entries {
|
||||
if glyph == "fi"
|
||||
|| glyph == "fl"
|
||||
|| glyph == "ffi"
|
||||
|| glyph == "ffl"
|
||||
{
|
||||
println!(
|
||||
" code=0x{:02X} ({:3}) glyph={:?} (LIGATURE)",
|
||||
code, code, glyph
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Check coverage: does it have standard ASCII letters?
|
||||
let has_a = entries.iter().any(|(_, g)| g == "a");
|
||||
let has_space = entries.iter().any(|(_, g)| g == "space");
|
||||
let has_period = entries.iter().any(|(_, g)| g == "period");
|
||||
println!(
|
||||
" has 'a': {}, has 'space': {}, has 'period': {}",
|
||||
has_a, has_space, has_period
|
||||
);
|
||||
|
||||
// Show first 10 and last 5 entries
|
||||
println!(" First 10 entries:");
|
||||
for (code, glyph) in entries.iter().take(10) {
|
||||
println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph);
|
||||
}
|
||||
if entries.len() > 15 {
|
||||
println!(" ...");
|
||||
println!(" Last 5 entries:");
|
||||
for (code, glyph) in entries
|
||||
.iter()
|
||||
.rev()
|
||||
.take(5)
|
||||
.collect::<Vec<_>>()
|
||||
.iter()
|
||||
.rev()
|
||||
{
|
||||
println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!(" font={}: no Encoding", font_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now extract text and look for ligatures
|
||||
let items = extract_text_with_positions(&path).unwrap();
|
||||
|
||||
println!("\n=== Items containing fi or fl (first 10) ===");
|
||||
let mut count = 0;
|
||||
for item in items.iter() {
|
||||
if item.text.contains('\u{FB01}') || item.text.contains('\u{FB02}') {
|
||||
println!(
|
||||
" page={} font={} text={:?}",
|
||||
item.page, item.font, item.text
|
||||
);
|
||||
count += 1;
|
||||
if count >= 10 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_lig = items
|
||||
.iter()
|
||||
.filter(|i| i.text.contains('\u{FB01}') || i.text.contains('\u{FB02}'))
|
||||
.count();
|
||||
println!(" Total items with ligatures: {}", total_lig);
|
||||
}
|
||||
+46
-12
@@ -982,7 +982,7 @@ fn extract_page_text_items(
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text,
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
@@ -1081,7 +1081,7 @@ fn extract_page_text_items(
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text: combined_text,
|
||||
text: expand_ligatures(&combined_text),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
@@ -1129,7 +1129,7 @@ fn extract_page_text_items(
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text,
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
y,
|
||||
width: 0.0,
|
||||
@@ -1403,7 +1403,7 @@ fn extract_form_xobject_text(
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text,
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
@@ -1497,7 +1497,7 @@ fn extract_form_xobject_text(
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text: combined_text,
|
||||
text: expand_ligatures(&combined_text),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
@@ -1786,14 +1786,30 @@ fn extract_text_from_operand(
|
||||
}
|
||||
}
|
||||
|
||||
// Try our custom encoding map from Differences arrays
|
||||
// Try our custom encoding map from Differences arrays.
|
||||
// The Differences array overrides specific codes in a base encoding (typically
|
||||
// WinAnsiEncoding). We must combine Differences entries with the base encoding
|
||||
// rather than using filter_map which silently drops unmapped bytes.
|
||||
if let Some(encoding_map) = font_encodings.get(current_font) {
|
||||
let decoded: String = bytes
|
||||
.iter()
|
||||
.filter_map(|&b| encoding_map.get(&b).copied())
|
||||
.collect();
|
||||
if !decoded.is_empty() {
|
||||
return Some(decoded);
|
||||
let has_diff_match = bytes.iter().any(|b| encoding_map.contains_key(b));
|
||||
if has_diff_match {
|
||||
let decoded: String = bytes
|
||||
.iter()
|
||||
.filter_map(|&b| {
|
||||
if let Some(&ch) = encoding_map.get(&b) {
|
||||
Some(ch)
|
||||
} else if b >= 0x20 {
|
||||
// Base encoding fallback for printable bytes.
|
||||
// For codes 0x20-0x7E this matches all standard PDF encodings.
|
||||
Some(b as char)
|
||||
} else {
|
||||
None // Skip unmapped control characters
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
if !decoded.is_empty() {
|
||||
return Some(decoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1822,6 +1838,24 @@ fn extract_text_from_operand(
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand Unicode ligature characters to their component characters.
|
||||
/// This makes extracted text more searchable and semantically correct.
|
||||
fn expand_ligatures(text: &str) -> String {
|
||||
if !text.contains('\u{FB00}')
|
||||
&& !text.contains('\u{FB01}')
|
||||
&& !text.contains('\u{FB02}')
|
||||
&& !text.contains('\u{FB03}')
|
||||
&& !text.contains('\u{FB04}')
|
||||
{
|
||||
return text.to_string();
|
||||
}
|
||||
text.replace('\u{FB00}', "ff")
|
||||
.replace('\u{FB01}', "fi")
|
||||
.replace('\u{FB02}', "fl")
|
||||
.replace('\u{FB03}', "ffi")
|
||||
.replace('\u{FB04}', "ffl")
|
||||
}
|
||||
|
||||
/// Estimate the width of a text item, falling back to a character-count heuristic when width is 0.
|
||||
fn effective_width(item: &TextItem) -> f32 {
|
||||
if item.width > 0.0 {
|
||||
|
||||
@@ -219,6 +219,12 @@ pub static GLYPH_TO_UNICODE: LazyLock<HashMap<&'static str, char>> = LazyLock::n
|
||||
m.insert("ff", '\u{FB00}'); // ff
|
||||
m.insert("ffi", '\u{FB03}'); // ffi
|
||||
m.insert("ffl", '\u{FB04}'); // ffl
|
||||
// Alternative naming with underscores (used by some PDF producers)
|
||||
m.insert("f_i", '\u{FB01}'); // fi
|
||||
m.insert("f_l", '\u{FB02}'); // fl
|
||||
m.insert("f_f", '\u{FB00}'); // ff
|
||||
m.insert("f_f_i", '\u{FB03}'); // ffi
|
||||
m.insert("f_f_l", '\u{FB04}'); // ffl
|
||||
|
||||
// Quotes and dashes
|
||||
m.insert("endash", '–');
|
||||
|
||||
Reference in New Issue
Block a user