diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 0829440..7696563 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -20,6 +20,60 @@ use super::fonts::{ use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; use super::{get_number, multiply_matrices}; +/// Strip PDF comments (% to end of line) from content stream bytes. +/// +/// Some PDF generators (e.g. PD4ML) embed comments in content streams that +/// confuse lopdf's `Content::decode` parser. Comments inside string literals +/// (parentheses) are NOT stripped — only top-level comments. +fn strip_pdf_comments(data: &[u8]) -> Vec { + // Quick check: if no '%' present, return as-is (common case) + if !data.contains(&b'%') { + return data.to_vec(); + } + + let mut result = Vec::with_capacity(data.len()); + let mut i = 0; + let mut in_string = 0i32; // parenthesis nesting depth + let mut in_hex_string = false; + + while i < data.len() { + let b = data[i]; + match b { + b'(' if !in_hex_string => { + in_string += 1; + result.push(b); + } + b')' if !in_hex_string && in_string > 0 => { + in_string -= 1; + result.push(b); + } + b'<' if in_string == 0 && !in_hex_string => { + in_hex_string = true; + result.push(b); + } + b'>' if in_hex_string => { + in_hex_string = false; + result.push(b); + } + b'%' if in_string == 0 && !in_hex_string => { + // Skip until end of line + while i < data.len() && data[i] != b'\n' && data[i] != b'\r' { + i += 1; + } + // Replace comment with a space to preserve token separation + result.push(b' '); + continue; // Don't increment i again + } + _ => { + result.push(b); + } + } + i += 1; + } + + result +} + /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates /// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( @@ -113,6 +167,11 @@ pub(crate) fn extract_page_text_items( .get_page_content(page_id) .map_err(|e| PdfError::Parse(e.to_string()))?; + // Strip PDF comments (% to end of line) from the content stream. + // Some PDF generators (e.g. PD4ML) embed comments that confuse lopdf's + // Content::decode parser, causing it to skip operators like ET and Q. + let content_data = strip_pdf_comments(&content_data); + let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; const MAX_OPERATIONS: usize = 1_000_000; @@ -1174,4 +1233,36 @@ mod tests { assert!(rects.is_empty()); assert!(lines.is_empty()); } + + #[test] + fn test_strip_pdf_comments() { + // Basic comment stripping + let input = b"BT\n% comment\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"BT\n \nTj\nET\n"); + + // No comments = unchanged + let input = b"BT\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, input.to_vec()); + + // Don't strip inside string literals + let input = b"(text with % not a comment)\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"(text with % not a comment)\n \n"); + + // Don't strip inside hex strings + let input = b"<0033% not a comment>\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"<0033% not a comment>\n \n"); + + // PD4ML style: comment between Tj and ET + let input = b"<0033> Tj\n\t% Mission Statement\n\tET\n"; + let output = strip_pdf_comments(input); + let output_str = String::from_utf8_lossy(&output); + assert!( + output_str.contains("ET"), + "ET should be preserved after comment stripping" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 2f4611b..64eb001 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -520,6 +520,27 @@ fn process_document( pages_needing_ocr.sort_unstable(); } + // Detect sparse extraction: when a TEXT-BASED PDF produces very few + // characters per page, the text is likely embedded in images/forms + // that need OCR. Flag all pages for OCR in this case. + // Only check when markdown was actually generated (not in Analyze mode). + if pdf_type == PdfType::TextBased + && page_count > 0 + && pages_needing_ocr.is_empty() + && markdown.is_some() + { + let md_len = markdown.as_ref().map_or(0, |m| m.len()); + let chars_per_page = md_len as f32 / page_count as f32; + if chars_per_page < 50.0 && md_len < 500 { + log::debug!( + "sparse extraction: {:.0} chars/page — recommending OCR for all {} pages", + chars_per_page, + page_count + ); + pages_needing_ocr = (1..=page_count).collect(); + } + } + let markdown = if all_gid { log::debug!( "all {} pages have gid-encoded fonts — suppressing markdown output", @@ -622,6 +643,7 @@ fn is_cid_garbage(text: &str) -> bool { } let mut total = 0usize; let mut c1_control = 0usize; + let mut high_latin = 0usize; for ch in text.chars() { if ch.is_whitespace() { continue; @@ -631,9 +653,25 @@ fn is_cid_garbage(text: &str) -> bool { if ('\u{0080}'..='\u{009F}').contains(&ch) { c1_control += 1; } + // High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text + // but when combined with ASCII in CID passthrough, indicates mojibake + // from CID values being misinterpreted as Latin-1 characters. + if ('\u{00A0}'..='\u{00FF}').contains(&ch) { + high_latin += 1; + } + } + if total < 5 { + return false; } // If ≥5% of non-whitespace chars are C1 controls, it's garbage - total >= 20 && c1_control * 20 >= total + if c1_control * 20 >= total { + return true; + } + // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few + // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs + // where CID values 0x80-0xFF become accented Latin characters). + let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count(); + high_latin * 5 >= total * 2 && ascii_letters * 3 < total } /// Analyse extracted items and rects for layout complexity.