From 3d6772fc2073eb118a9ad114ce9e648a9d785299 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 9 Feb 2026 12:45:08 -0800 Subject: [PATCH] improve spacing detection, and layout detection --- src/bin/debug_spacing.rs | 32 ++++++++ src/bin/detect_pdf.rs | 37 +++++---- src/detector.rs | 96 +++++++++++++++++++++--- src/extractor.rs | 158 ++++++++++++++++++++++++++++++--------- src/markdown.rs | 71 ++++++++++++++++-- src/tables.rs | 23 ++++-- src/tounicode.rs | 50 +++++++++++-- 7 files changed, 385 insertions(+), 82 deletions(-) create mode 100644 src/bin/debug_spacing.rs diff --git a/src/bin/debug_spacing.rs b/src/bin/debug_spacing.rs new file mode 100644 index 0000000..4d95d1e --- /dev/null +++ b/src/bin/debug_spacing.rs @@ -0,0 +1,32 @@ +use pdf_inspector::extract_text_with_positions; +use std::env; + +fn main() { + let path = env::args().nth(1).expect("Need PDF path"); + let items = extract_text_with_positions(&path).expect("Failed"); + + // Look at consecutive items on same Y line + let mut prev_item: Option<&pdf_inspector::TextItem> = None; + for item in items.iter() { + if let Some(prev) = prev_item { + // Same line (similar Y) + if (item.y - prev.y).abs() < 5.0 && item.x > prev.x { + let gap = item.x - prev.x - prev.width; + let char_width = if prev.width > 0.0 && !prev.text.is_empty() { + prev.width / prev.text.len() as f32 + } else { + prev.font_size * 0.5 // Approximate + }; + + println!( + "Gap: {:.1} (charW: {:.1}) | '{}' -> '{}'", + gap, + char_width, + prev.text.chars().take(20).collect::(), + item.text.chars().take(20).collect::() + ); + } + } + prev_item = Some(item); + } +} diff --git a/src/bin/detect_pdf.rs b/src/bin/detect_pdf.rs index f53b596..59792c4 100644 --- a/src/bin/detect_pdf.rs +++ b/src/bin/detect_pdf.rs @@ -25,7 +25,7 @@ fn main() { if json_output { println!( - r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"detection_time_ms":{}}}"#, + r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"detection_time_ms":{}}}"#, match result.pdf_type { PdfType::TextBased => "text_based", PdfType::Scanned => "scanned", @@ -41,6 +41,7 @@ fn main() { .as_ref() .map(|t| format!("\"{}\"", t.replace('"', "\\\""))) .unwrap_or_else(|| "null".to_string()), + result.ocr_recommended, elapsed.as_millis() ); } else { @@ -62,6 +63,10 @@ fn main() { println!("Page count: {}", result.page_count); println!("Pages sampled: {}", result.pages_sampled); println!("Pages with text: {}", result.pages_with_text); + println!( + "OCR recommended: {}", + if result.ocr_recommended { "YES" } else { "NO" } + ); if let Some(title) = &result.title { println!("Title: {}", title); } @@ -70,21 +75,23 @@ fn main() { println!(); // Recommendations - match result.pdf_type { - PdfType::TextBased => { - println!("Recommendation: Use direct text extraction (fast)"); - } - PdfType::Scanned => { - println!("Recommendation: Use OCR (MinerU or similar)"); - } - PdfType::ImageBased => { - println!("Recommendation: Use OCR for best results"); - } - PdfType::Mixed => { - println!( - "Recommendation: Try text extraction first, use OCR for image pages" - ); + if result.ocr_recommended { + match result.pdf_type { + PdfType::Mixed => { + println!("Recommendation: Use OCR - images provide essential context (template PDF)"); + } + PdfType::Scanned => { + println!("Recommendation: Use OCR (MinerU or similar)"); + } + PdfType::ImageBased => { + println!("Recommendation: Use OCR for best results"); + } + _ => { + println!("Recommendation: Use OCR for complete extraction"); + } } + } else { + println!("Recommendation: Use direct text extraction (fast)"); } } } diff --git a/src/detector.rs b/src/detector.rs index 5237e22..afffb69 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -36,6 +36,9 @@ pub struct PdfTypeResult { pub confidence: f32, /// Title from metadata (if available) pub title: Option, + /// Whether OCR is recommended for better extraction + /// True when images provide essential context (e.g., template-based PDFs) + pub ocr_recommended: bool, } /// Configuration for PDF type detection @@ -140,6 +143,7 @@ fn detect_from_document( let mut pages_with_text = 0u32; let mut pages_with_images = 0u32; + let mut pages_with_template_images = 0u32; let mut total_text_ops = 0u32; for page_num in &sample_indices { @@ -151,6 +155,9 @@ fn detect_from_document( if analysis.has_images { pages_with_images += 1; } + if analysis.has_template_image { + pages_with_template_images += 1; + } total_text_ops += analysis.text_operator_count; } } @@ -162,20 +169,44 @@ fn detect_from_document( 0.0 }; + // Check if this is a template-based PDF (images provide essential context) + // Template PDFs have text AND large background images on most pages + let has_template_images = pages_with_template_images > 0; + let template_ratio = if pages_sampled > 0 { + pages_with_template_images as f32 / pages_sampled as f32 + } else { + 0.0 + }; + + // OCR is recommended when: + // 1. Template images are present (text alone is insufficient), OR + // 2. PDF is scanned/image-based + let ocr_recommended: bool; + // Classification logic - let (pdf_type, confidence) = if text_ratio >= config.text_page_ratio_threshold { + let (pdf_type, confidence) = if has_template_images && pages_with_text > 0 { + // Template-based PDF: has text but images provide essential context + // Classify as Mixed with lower confidence + ocr_recommended = true; + (PdfType::Mixed, 0.5 + (0.3 * (1.0 - template_ratio))) + } else if text_ratio >= config.text_page_ratio_threshold { + ocr_recommended = false; (PdfType::TextBased, text_ratio) } else if pages_with_text == 0 && pages_with_images > 0 { + ocr_recommended = true; if total_text_ops == 0 { (PdfType::Scanned, 0.95) } else { (PdfType::ImageBased, 0.8) } } else if pages_with_text > 0 && pages_with_images > 0 { + ocr_recommended = true; (PdfType::Mixed, 0.7) } else if total_text_ops == 0 { + ocr_recommended = true; (PdfType::Scanned, 0.9) } else { + ocr_recommended = false; (PdfType::TextBased, text_ratio.max(0.5)) }; @@ -189,6 +220,7 @@ fn detect_from_document( pages_with_text, confidence, title, + ocr_recommended, }) } @@ -196,6 +228,11 @@ fn detect_from_document( struct PageAnalysis { text_operator_count: u32, has_images: bool, + /// Whether page has a large background/template image (>50% coverage) + has_template_image: bool, + /// Total image area in pixels (reserved for future use) + #[allow(dead_code)] + total_image_area: u64, } /// Analyze a page's content stream for text operators and images @@ -221,14 +258,18 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { } } - // Also check for XObject images in page resources - if !has_images { - has_images = page_has_images(doc, page_id); + // Check for XObject images and calculate coverage + let (found_images, total_image_area, has_template_image) = analyze_page_images(doc, page_id); + + if found_images { + has_images = true; } PageAnalysis { text_operator_count: text_ops, has_images, + has_template_image, + total_image_area, } } @@ -278,10 +319,22 @@ fn scan_content_for_text_operators(content: &[u8]) -> (u32, bool) { (text_ops, has_images) } -/// Check if page has image XObjects in resources -fn page_has_images(doc: &Document, page_id: ObjectId) -> bool { +/// Analyze page images: returns (has_images, total_area, has_template_image) +/// +/// A template image is one that covers >50% of a standard page area. +/// Standard page: 612x792 points (US Letter) = ~485,000 sq points +/// At 2x resolution that's ~1.9M pixels, so we use 250K pixels as threshold +/// (accounting for varying DPI and page sizes) +fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { + // Threshold: image covering roughly half a page at 150+ DPI + // 612 * 792 / 2 * (150/72)^2 ≈ 1M pixels, but we'll be conservative + const TEMPLATE_IMAGE_THRESHOLD: u64 = 500_000; // 500K pixels + + let mut has_images = false; + let mut total_area: u64 = 0; + let mut has_template_image = false; + if let Ok(page_dict) = doc.get_dictionary(page_id) { - // Get Resources let resources = match page_dict.get(b"Resources") { Ok(Object::Reference(id)) => doc.get_dictionary(*id).ok(), Ok(Object::Dictionary(dict)) => Some(dict), @@ -289,7 +342,6 @@ fn page_has_images(doc: &Document, page_id: ObjectId) -> bool { }; if let Some(resources) = resources { - // Check XObject dictionary if let Ok(xobject) = resources.get(b"XObject") { let xobject_dict = match xobject { Object::Reference(id) => doc.get_dictionary(*id).ok(), @@ -306,7 +358,31 @@ fn page_has_images(doc: &Document, page_id: ObjectId) -> bool { if let Ok(subtype) = stream.dict.get(b"Subtype") { if let Ok(name) = subtype.as_name() { if name == b"Image" { - return true; + has_images = true; + + // Get image dimensions + let width = stream + .dict + .get(b"Width") + .ok() + .and_then(|w| w.as_i64().ok()) + .unwrap_or(0) + as u64; + let height = stream + .dict + .get(b"Height") + .ok() + .and_then(|h| h.as_i64().ok()) + .unwrap_or(0) + as u64; + + let area = width * height; + total_area += area; + + // Check if this is a large template image + if area >= TEMPLATE_IMAGE_THRESHOLD { + has_template_image = true; + } } } } @@ -319,7 +395,7 @@ fn page_has_images(doc: &Document, page_id: ObjectId) -> bool { } } - false + (has_images, total_area, has_template_image) } /// Get document title from Info dictionary diff --git a/src/extractor.rs b/src/extractor.rs index a0daff4..0722fd7 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -63,19 +63,26 @@ impl TextLine { // Previous item was subscript/superscript (returning to normal size) let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; - // Detect word fragments that should be joined without space - // This happens when a word is broken across text elements - // e.g., "ve" + "ntos" should become "ventos" not "ve ntos" - let is_word_fragment = is_word_continuation(&result, text); + // Use position-based spacing detection + // This is more reliable than character-case heuristics for determining + // whether text fragments should be joined (e.g., "CONST" + "ANCIA" → "CONSTANCIA") + let should_join = should_join_items(prev_item, item); + + // Check if space already exists to avoid double spaces + let prev_ends_with_space = result.ends_with(' '); + let curr_starts_with_space = text.starts_with(' '); + let space_already_exists = prev_ends_with_space || curr_starts_with_space; if prev_ends_with_hyphen || curr_is_hyphen || curr_starts_with_hyphen || is_sub_super || was_sub_super - || is_word_fragment + || should_join + || space_already_exists { - // No space for hyphenated words, subscript/superscript, or word fragments + // No space for hyphenated words, subscript/superscript, closely positioned items, + // or when space already exists result.push_str(text); } else { result.push(' '); @@ -87,38 +94,81 @@ impl TextLine { } } -/// Check if the current text is a continuation of a word from the previous text -/// Returns true if the items should be joined without a space -fn is_word_continuation(prev_text: &str, curr_text: &str) -> bool { - // Get the last character of previous text (excluding trailing spaces) - let prev_trimmed = prev_text.trim_end(); - let last_char = match prev_trimmed.chars().last() { - Some(c) => c, - None => return false, - }; +/// Determine if two adjacent text items should be joined without a space +/// based on their physical positions on the page and character case. +/// Uses a hybrid approach: position-based with case-aware thresholds. +fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { + // If either text explicitly has leading/trailing spaces, respect them + if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') { + return false; + } - // Get the first character of current text (excluding leading spaces) - let curr_trimmed = curr_text.trim_start(); - let first_char = match curr_trimmed.chars().next() { - Some(c) => c, - None => return false, - }; + // Get the last character of previous and first character of current + let prev_last = prev_item.text.trim_end().chars().last(); + let curr_first = curr_item.text.trim_start().chars().next(); - // If previous ends with a letter and current starts with a lowercase letter, - // this is likely a word fragment that should be joined - // e.g., "ve" + "ntos" -> "ventos" - if last_char.is_alphabetic() && first_char.is_lowercase() { - // Additional check: previous should not end with a space - // and current should not start with a space in the original - let prev_ends_with_space = prev_text.ends_with(' '); - let curr_starts_with_space = curr_text.starts_with(' '); - - if !prev_ends_with_space && !curr_starts_with_space { + // Always join if current starts with punctuation that typically follows without space + // e.g., "www" + ".com" → "www.com", not "www .com" + if let Some(c) = curr_first { + if matches!(c, '.' | ',' | ';' | '!' | '?' | ')' | ']' | '}' | '\'') { return true; } } - false + // After colons, add space if followed by alphanumeric (typical label:value pattern) + // e.g., "Clave:" + "T9N2I6" → "Clave: T9N2I6" + if let (Some(p), Some(c)) = (prev_last, curr_first) { + if p == ':' && c.is_alphanumeric() { + return false; + } + } + + // Estimate the average character width from font size + // Use a conservative estimate (0.45) since fonts vary + let char_width = prev_item.font_size * 0.45; + + // Estimate the width of the previous text + let prev_text_len = prev_item.text.chars().count() as f32; + let estimated_prev_width = if prev_item.width > 0.0 { + prev_item.width // Use actual width if available + } else { + prev_text_len * char_width + }; + + // Calculate expected end position of previous item + let prev_end_x = prev_item.x + estimated_prev_width; + + // Calculate gap between items + let gap = curr_item.x - prev_end_x; + + // Use different thresholds based on character case + // Same-case sequences (ALL CAPS or all lowercase) are more likely to be + // word fragments that got split. Mixed case suggests word boundaries. + match (prev_last, curr_first) { + (Some(p), Some(c)) if p.is_alphabetic() && c.is_alphabetic() => { + let same_case = + (p.is_uppercase() && c.is_uppercase()) || (p.is_lowercase() && c.is_lowercase()); + if same_case { + // Same case: use generous threshold (likely same word fragment) + // e.g., "CONST" + "ANCIA" → "CONSTANCIA" + gap < char_width * 0.8 + } else if p.is_lowercase() && c.is_uppercase() { + // Lowercase to uppercase transition (e.g., "presente" → "CONSTANCIA") + // This is typically a word boundary. In Spanish/English, words don't + // transition from lowercase to uppercase mid-word. + // Always add a space for this case, regardless of position. + false + } else { + // Uppercase to lowercase (e.g., "REGISTRO" → "para") + // Use stricter threshold (likely word boundary) + gap < char_width * 0.3 + } + } + _ => { + // Non-alphabetic: use moderate threshold + gap < char_width * 0.5 + } + } } /// Extract text from PDF file as plain string @@ -207,15 +257,23 @@ fn extract_page_text_items( // Get fonts for encoding let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); - // Build a map of font resource names to their base font names (for CMap lookup) + // Build maps of font resource names to their base font names and ToUnicode object refs let mut font_base_names: std::collections::HashMap = std::collections::HashMap::new(); + let mut font_tounicode_refs: std::collections::HashMap = + std::collections::HashMap::new(); for (font_name, font_dict) in &fonts { let resource_name = String::from_utf8_lossy(font_name).to_string(); if let Ok(base_font) = font_dict.get(b"BaseFont") { if let Ok(name) = base_font.as_name() { let base_name = String::from_utf8_lossy(name).to_string(); - font_base_names.insert(resource_name, base_name); + font_base_names.insert(resource_name.clone(), base_name); + } + } + // Track ToUnicode object reference + if let Ok(tounicode) = font_dict.get(b"ToUnicode") { + if let Ok(obj_ref) = tounicode.as_reference() { + font_tounicode_refs.insert(resource_name, obj_ref.0 as u32); } } } @@ -322,6 +380,7 @@ fn extract_page_text_items( ¤t_font, font_cmaps, &font_base_names, + &font_tounicode_refs, ) { if !text.trim().is_empty() { let rendered_size = @@ -356,6 +415,7 @@ fn extract_page_text_items( ¤t_font, font_cmaps, &font_base_names, + &font_tounicode_refs, ) { combined_text.push_str(&text); } @@ -392,6 +452,7 @@ fn extract_page_text_items( ¤t_font, font_cmaps, &font_base_names, + &font_tounicode_refs, ) { if !text.trim().is_empty() { let rendered_size = @@ -450,13 +511,36 @@ fn extract_text_from_operand( current_font: &str, font_cmaps: &FontCMaps, font_base_names: &std::collections::HashMap, + font_tounicode_refs: &std::collections::HashMap, ) -> Option { if let Object::String(bytes, _) = obj { - // First, check if this font has a ToUnicode CMap we can use - // This is especially important for Identity-H encoded fonts (Type0/CIDFont) + // First, try to look up CMap by ToUnicode object reference (most reliable) + // This handles cases where multiple fonts have the same BaseFont but different ToUnicode + if let Some(&obj_num) = font_tounicode_refs.get(current_font) { + if let Some(cmap) = font_cmaps.get_by_obj(obj_num) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Fall back to base name lookup with object number + if let (Some(base_name), Some(&obj_num)) = ( + font_base_names.get(current_font), + font_tounicode_refs.get(current_font), + ) { + if let Some(cmap) = font_cmaps.get_with_obj(base_name, obj_num) { + let decoded = cmap.decode_cids(bytes); + if !decoded.is_empty() { + return Some(decoded); + } + } + } + + // Try base name only (legacy fallback) if let Some(base_name) = font_base_names.get(current_font) { if let Some(cmap) = font_cmaps.get(base_name) { - // Use the ToUnicode CMap to decode CID bytes let decoded = cmap.decode_cids(bytes); if !decoded.is_empty() { return Some(decoded); diff --git a/src/markdown.rs b/src/markdown.rs index 7cfd4e7..85a390a 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -866,10 +866,9 @@ fn remove_page_numbers(text: &str) -> String { for (i, line) in lines.iter().enumerate() { let trimmed = line.trim(); - // Check if this line is just a number (1-4 digits) - if trimmed.len() <= 4 && !trimmed.is_empty() && trimmed.chars().all(|c| c.is_ascii_digit()) - { - // Check context to determine if this is a page number + // Check for page number patterns + if is_page_number_line(trimmed) { + // Check context to determine if this is isolated let prev_is_break = i > 0 && lines[i - 1].trim() == "---"; let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---"; let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty(); @@ -880,7 +879,6 @@ fn remove_page_numbers(text: &str) -> String { && (next_is_break || next_is_empty || i + 1 == lines.len()); // Also remove numbers that appear right before a page break - // (common pattern: content ends, page number, then ---) let before_break = i + 1 < lines.len() && (lines[i + 1].trim() == "---" || (i + 2 < lines.len() @@ -898,6 +896,69 @@ fn remove_page_numbers(text: &str) -> String { result.join("\n") } +/// Check if a line looks like a page number +fn is_page_number_line(trimmed: &str) -> bool { + // Empty lines are not page numbers + if trimmed.is_empty() { + return false; + } + + // Pattern 1: Just a number (1-4 digits) + if trimmed.len() <= 4 && trimmed.chars().all(|c| c.is_ascii_digit()) { + return true; + } + + // Pattern 2: "Page X of Y" or "Page X" or "Page of" (placeholder) + let lower = trimmed.to_lowercase(); + if let Some(rest) = lower.strip_prefix("page") { + let rest = rest.trim(); + // "Page of" (empty page numbers) + if rest == "of" || rest.starts_with("of ") { + return true; + } + // "Page X" or "Page X of Y" + if rest + .chars() + .next() + .map(|c| c.is_ascii_digit()) + .unwrap_or(false) + { + return true; + } + // Just "Page" followed by whitespace and maybe "of" + if rest.is_empty() + || rest + .split_whitespace() + .all(|w| w == "of" || w.chars().all(|c| c.is_ascii_digit())) + { + return true; + } + } + + // Pattern 3: "X of Y" where X and Y are numbers + if let Some(of_idx) = trimmed.find(" of ") { + let before = trimmed[..of_idx].trim(); + let after = trimmed[of_idx + 4..].trim(); + if before.chars().all(|c| c.is_ascii_digit()) + && after.chars().all(|c| c.is_ascii_digit()) + && !before.is_empty() + && !after.is_empty() + { + return true; + } + } + + // Pattern 4: "- X -" centered page number + if trimmed.starts_with('-') && trimmed.ends_with('-') { + let inner = trimmed[1..trimmed.len() - 1].trim(); + if inner.chars().all(|c| c.is_ascii_digit()) && !inner.is_empty() { + return true; + } + } + + false +} + /// Convert URLs to markdown links fn format_urls(text: &str) -> String { use once_cell::sync::Lazy; diff --git a/src/tables.rs b/src/tables.rs index de3e5e7..d86cdd8 100644 --- a/src/tables.rs +++ b/src/tables.rs @@ -251,7 +251,12 @@ fn is_key_value_layout(cells: &[Vec]) -> bool { // Check if first column looks like a label (ends with : or is all caps) let first = row.first().map(|s| s.trim()).unwrap_or(""); - if first.ends_with(':') || (first.len() > 3 && first.chars().all(|c| c.is_uppercase() || c.is_whitespace() || c == '(' || c == ')')) { + if first.ends_with(':') + || (first.len() > 3 + && first + .chars() + .all(|c| c.is_uppercase() || c.is_whitespace() || c == '(' || c == ')')) + { label_like_first_col += 1; } } @@ -339,7 +344,8 @@ fn looks_like_number(s: &str) -> bool { } // Handle common number formats: 9.0, 10, 8.6, etc. - s.chars().all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+') + s.chars() + .all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+') && s.chars().any(|c| c.is_ascii_digit()) } @@ -654,11 +660,7 @@ fn find_first_table_row( // Build string cells for analysis let cells: Vec> = cell_items .iter() - .map(|row| { - row.iter() - .map(|col| join_cell_items(col)) - .collect() - }) + .map(|row| row.iter().map(|col| join_cell_items(col)).collect()) .collect(); if cells.is_empty() { @@ -733,7 +735,12 @@ fn find_first_table_row( }); // If next row is dense or has data (and no form patterns), this row starts the table - if (next_fill_ratio >= 0.4 || next_row.iter().filter(|c| looks_like_number(c.trim())).count() >= 2) + if (next_fill_ratio >= 0.4 + || next_row + .iter() + .filter(|c| looks_like_number(c.trim())) + .count() + >= 2) && !next_has_form { first_table_row = row_idx; diff --git a/src/tounicode.rs b/src/tounicode.rs index cee7b00..af9289d 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -106,7 +106,8 @@ impl ToUnicodeCMap { chars.next(); // consume > // Parse and store mapping - if let (Some(src), Some(dst)) = (parse_hex_u16(&src_hex), hex_to_unicode_string(&dst_hex)) + if let (Some(src), Some(dst)) = + (parse_hex_u16(&src_hex), hex_to_unicode_string(&dst_hex)) { self.char_map.insert(src, dst); } @@ -345,7 +346,9 @@ pub fn extract_tounicode_cmaps(pdf_bytes: &[u8]) -> HashMap // Skip whitespace let mut p = ref_start; - while p < pdf_bytes.len() && (pdf_bytes[p] == b' ' || pdf_bytes[p] == b'\n' || pdf_bytes[p] == b'\r') { + while p < pdf_bytes.len() + && (pdf_bytes[p] == b' ' || pdf_bytes[p] == b'\n' || pdf_bytes[p] == b'\r') + { p += 1; } @@ -376,6 +379,8 @@ pub fn extract_tounicode_cmaps(pdf_bytes: &[u8]) -> HashMap pub struct FontCMaps { /// Map of font name (e.g., "FNotoSans0") to ToUnicodeCMap pub by_name: HashMap, + /// Map of ToUnicode object number to CMap (for direct lookup) + pub by_obj_num: HashMap, } impl FontCMaps { @@ -400,8 +405,8 @@ impl FontCMaps { // Search backwards and forwards for << and >> let dict_start = find_dict_start(&pdf_bytes[..font_start]); - let dict_end = find_pattern(&pdf_bytes[font_start..], b">>") - .map(|e| font_start + e + 2); + let dict_end = + find_pattern(&pdf_bytes[font_start..], b">>").map(|e| font_start + e + 2); if let (Some(start), Some(end)) = (dict_start, dict_end) { let dict_region = &pdf_bytes[start..end]; @@ -413,6 +418,11 @@ impl FontCMaps { let ref_part = &dict_region[tounicode_idx + 10..]; if let Some(obj_num) = extract_obj_reference(ref_part) { if let Some(cmap) = cmaps_by_obj.get(&obj_num) { + // Use combined key to handle multiple fonts with same BaseFont + let unique_key = format!("{}_{}", font_name, obj_num); + by_name.insert(unique_key, cmap.clone()); + // Also keep base font name for backwards compatibility + // (last one wins, but that's better than nothing) by_name.insert(font_name, cmap.clone()); } } @@ -426,7 +436,13 @@ impl FontCMaps { } } - FontCMaps { by_name } + // Copy the by_obj map + let by_obj_num = cmaps_by_obj; + + FontCMaps { + by_name, + by_obj_num, + } } /// Get a CMap for a font name @@ -446,6 +462,22 @@ impl FontCMaps { None } + + /// Get a CMap by ToUnicode object number + pub fn get_by_obj(&self, obj_num: u32) -> Option<&ToUnicodeCMap> { + self.by_obj_num.get(&obj_num) + } + + /// Get a CMap for a base font name with specific ToUnicode object number + pub fn get_with_obj(&self, font_name: &str, obj_num: u32) -> Option<&ToUnicodeCMap> { + // Try the unique key first + let unique_key = format!("{}_{}", font_name, obj_num); + if let Some(cmap) = self.by_name.get(&unique_key) { + return Some(cmap); + } + // Fall back to direct object lookup + self.by_obj_num.get(&obj_num) + } } /// Find the start of a dictionary (<<) searching backwards from a position @@ -464,7 +496,7 @@ fn extract_font_name(dict: &[u8]) -> Option { // Look for /BaseFont /Name if let Some(idx) = find_pattern(dict, b"/BaseFont") { let after = &dict[idx + 9..]; // "/BaseFont" is 9 chars - // Skip whitespace + // Skip whitespace let mut p = 0; while p < after.len() && (after[p] == b' ' || after[p] == b'\n' || after[p] == b'\r') { p += 1; @@ -473,7 +505,11 @@ fn extract_font_name(dict: &[u8]) -> Option { if p < after.len() && after[p] == b'/' { p += 1; let mut name = String::new(); - while p < after.len() && !after[p].is_ascii_whitespace() && after[p] != b'/' && after[p] != b'>' { + while p < after.len() + && !after[p].is_ascii_whitespace() + && after[p] != b'/' + && after[p] != b'>' + { name.push(after[p] as char); p += 1; }