diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index d0b2f9e..2191c87 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -186,7 +186,7 @@ fn main() { .map(|p| p.to_string()) .collect(); println!( - r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}]}}"#, + r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{}}}"#, pdf_type_str, result.page_count, result.processing_time_ms, @@ -194,6 +194,7 @@ fn main() { result.layout.is_complex, table_pages.join(","), col_pages.join(","), + result.has_encoding_issues, ); } else { eprintln!("Type: {}", pdf_type_str); @@ -231,7 +232,7 @@ fn main() { .map(|p| p.to_string()) .collect(); println!( - r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"markdown":"{}"}}"#, + r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{},"markdown":"{}"}}"#, match result.pdf_type { PdfType::TextBased => "text_based", PdfType::Scanned => "scanned", @@ -246,6 +247,7 @@ fn main() { result.layout.is_complex, table_pages.join(","), col_pages.join(","), + result.has_encoding_issues, md_escaped ); } else if raw_output { diff --git a/src/lib.rs b/src/lib.rs index e650f99..07e4b09 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,9 @@ pub struct PdfProcessResult { pub confidence: f32, /// Layout complexity analysis (tables, multi-column detection). pub layout: LayoutComplexity, + /// True when broken font encodings are detected (garbled text, replacement + /// characters). Clients should fall back to OCR for affected pages. + pub has_encoding_issues: bool, } /// Process a PDF file with smart detection and extraction @@ -78,6 +81,7 @@ pub fn process_pdf>(path: P) -> Result>(path: P) -> Result { @@ -103,12 +108,13 @@ pub fn process_pdf>(path: P) -> Result { // Try to extract what we can with position-aware reading order let extracted = extractor::extract_text_with_positions_and_rects(&path, None).ok(); - let (markdown, layout) = match extracted { + let (markdown, layout, has_encoding_issues) = match extracted { Some((items, rects)) => { let layout = compute_layout_complexity(&items, &rects); let md = to_markdown_from_items_with_rects( @@ -116,9 +122,10 @@ pub fn process_pdf>(path: P) -> Result (None, LayoutComplexity::default()), + None => (None, LayoutComplexity::default(), false), }; PdfProcessResult { @@ -131,6 +138,7 @@ pub fn process_pdf>(path: P) -> Result>( title, confidence, layout: LayoutComplexity::default(), + has_encoding_issues: false, }); } @@ -198,6 +207,9 @@ pub fn process_pdf_with_config_pages>( &rects, )) }; + let has_encoding_issues = markdown + .as_ref() + .is_some_and(|md| detect_encoding_issues(md)); PdfProcessResult { pdf_type, @@ -209,6 +221,7 @@ pub fn process_pdf_with_config_pages>( title, confidence, layout, + has_encoding_issues, } } PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { @@ -221,11 +234,12 @@ pub fn process_pdf_with_config_pages>( title, confidence, layout: LayoutComplexity::default(), + has_encoding_issues: false, }, PdfType::Mixed => { let extracted = extractor::extract_text_with_positions_and_rects(&path, page_filter).ok(); - let (markdown, layout) = match extracted { + let (markdown, layout, has_encoding_issues) = match extracted { Some((items, rects)) => { let layout = compute_layout_complexity(&items, &rects); let md = if markdown_options.process_mode == ProcessMode::Analyze { @@ -237,9 +251,10 @@ pub fn process_pdf_with_config_pages>( &rects, )) }; - (md, layout) + let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m)); + (md, layout, enc) } - None => (None, LayoutComplexity::default()), + None => (None, LayoutComplexity::default(), false), }; PdfProcessResult { @@ -252,6 +267,7 @@ pub fn process_pdf_with_config_pages>( title, confidence, layout, + has_encoding_issues, } } }; @@ -297,6 +313,7 @@ pub fn process_pdf_mem_with_config( title, confidence, layout: LayoutComplexity::default(), + has_encoding_issues: false, }); } @@ -315,6 +332,9 @@ pub fn process_pdf_mem_with_config( &rects, )) }; + let has_encoding_issues = markdown + .as_ref() + .is_some_and(|md| detect_encoding_issues(md)); PdfProcessResult { pdf_type, @@ -326,6 +346,7 @@ pub fn process_pdf_mem_with_config( title, confidence, layout, + has_encoding_issues, } } PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { @@ -338,10 +359,11 @@ pub fn process_pdf_mem_with_config( title, confidence, layout: LayoutComplexity::default(), + has_encoding_issues: false, }, PdfType::Mixed => { let extracted = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); - let (markdown, layout) = match extracted { + let (markdown, layout, has_encoding_issues) = match extracted { Some((items, rects)) => { let layout = compute_layout_complexity(&items, &rects); let md = if markdown_options.process_mode == ProcessMode::Analyze { @@ -353,9 +375,10 @@ pub fn process_pdf_mem_with_config( &rects, )) }; - (md, layout) + let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m)); + (md, layout, enc) } - None => (None, LayoutComplexity::default()), + None => (None, LayoutComplexity::default(), false), }; PdfProcessResult { @@ -368,6 +391,7 @@ pub fn process_pdf_mem_with_config( title, confidence, layout, + has_encoding_issues, } } }; @@ -375,6 +399,40 @@ pub fn process_pdf_mem_with_config( Ok(result) } +/// Detect broken font encodings in extracted markdown text. +/// +/// Two heuristics: +/// 1. **U+FFFD**: Any replacement character indicates decode failures. +/// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a +/// word separator due to broken ToUnicode CMaps. Triggers when >50% of `$` +/// characters appear between letters AND the count exceeds 10. +fn detect_encoding_issues(markdown: &str) -> bool { + // Heuristic 1: U+FFFD replacement characters + if markdown.contains('\u{FFFD}') { + return true; + } + + // Heuristic 2: dollar-as-space pattern + let total_dollars = markdown.matches('$').count(); + if total_dollars > 10 { + let bytes = markdown.as_bytes(); + let mut letter_dollar_letter = 0usize; + for i in 1..bytes.len().saturating_sub(1) { + if bytes[i] == b'$' + && bytes[i - 1].is_ascii_alphabetic() + && bytes[i + 1].is_ascii_alphabetic() + { + letter_dollar_letter += 1; + } + } + if letter_dollar_letter * 2 > total_dollars { + return true; + } + } + + false +} + /// Analyse extracted items and rects for layout complexity. fn compute_layout_complexity( items: &[types::TextItem], @@ -573,3 +631,43 @@ pub(crate) fn validate_pdf_file>(path: P) -> Result<(), PdfError> let n = file.read(&mut buf)?; validate_pdf_bytes(&buf[..n]) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_detect_encoding_issues_fffd() { + assert!(detect_encoding_issues( + "Some text with \u{FFFD} replacement" + )); + } + + #[test] + fn test_detect_encoding_issues_dollar_as_space() { + // Simulates broken CMap: "$Workshop$on$Chest$Wall$Deformities$and$..." + let garbled = "Last$advanced$Book$Programm$3th$Workshop$on$Chest$Wall$Deformities$and$More"; + assert!(detect_encoding_issues(garbled)); + } + + #[test] + fn test_detect_encoding_issues_financial_text() { + // Legitimate dollar signs in financial text should NOT trigger + let financial = "Revenue was $100M in Q1, up from $90M. Costs: $50M, $30M, $20M, $15M, $12M, $8M, $5M, $3M, $2M, $1M, $500K."; + assert!(!detect_encoding_issues(financial)); + } + + #[test] + fn test_detect_encoding_issues_clean_text() { + assert!(!detect_encoding_issues( + "Normal markdown text with no issues." + )); + } + + #[test] + fn test_detect_encoding_issues_few_dollars() { + // Under threshold of 10 total dollars — should not trigger + let text = "a$b c$d e$f"; + assert!(!detect_encoding_issues(text)); + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index a05b74d..03e31d8 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -950,6 +950,7 @@ fn test_pages_needing_ocr_field_accessible() { title: None, confidence: 1.0, layout: pdf_inspector::LayoutComplexity::default(), + has_encoding_issues: false, }; assert_eq!(process_result.pages_needing_ocr, vec![1, 3]); }