diff --git a/.gitignore b/.gitignore index 95bd90f..1429ec2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,10 @@ scripts/ # Test output test_output/ + +# Debug/diagnostic binaries +src/bin/debug_*.rs +src/bin/dump_*.rs +src/bin/profile_*.rs +src/bin/detection_report.rs +REPORT.md diff --git a/Cargo.toml b/Cargo.toml index a2936be..5f63730 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,3 +50,15 @@ path = "src/bin/debug_ygaps.rs" [[bin]] name = "debug_pages" path = "src/bin/debug_pages.rs" + +[[bin]] +name = "debug_fonts" +path = "src/bin/debug_fonts.rs" + +[[bin]] +name = "dump_ops" +path = "src/bin/dump_ops.rs" + +[[bin]] +name = "debug_spaces" +path = "src/bin/debug_spaces.rs" diff --git a/src/bin/debug_ligatures.rs b/src/bin/debug_ligatures.rs deleted file mode 100644 index a19ce6c..0000000 --- a/src/bin/debug_ligatures.rs +++ /dev/null @@ -1,180 +0,0 @@ -use pdf_inspector::extract_text_with_positions; -use pdf_inspector::tounicode::FontCMaps; - -fn main() { - let path = std::env::args() - .nth(1) - .expect("Usage: debug_ligatures "); - - // Load PDF and extract CMaps - let pdf_bytes = std::fs::read(&path).unwrap(); - let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes); - - println!("=== Font CMaps ==="); - if font_cmaps.by_name.is_empty() && font_cmaps.by_obj_num.is_empty() { - println!(" (none found)"); - } - for (name, cmap) in &font_cmaps.by_name { - println!( - " font={:30} code_byte_length={} char_map_entries={} ranges={}", - name, - cmap.code_byte_length, - cmap.char_map.len(), - cmap.ranges.len() - ); - } - - // Load with lopdf to inspect font Differences arrays - let doc = lopdf::Document::load_mem(&pdf_bytes).unwrap(); - let pages = doc.get_pages(); - - println!("\n=== Font Encoding Differences ==="); - for (page_num, &page_id) in pages.iter() { - println!("--- Page {} ---", page_num); - let fonts = match doc.get_page_fonts(page_id) { - Ok(f) => f, - Err(_) => continue, - }; - for (font_name_bytes, font_dict) in &fonts { - let font_name = String::from_utf8_lossy(font_name_bytes).to_string(); - - // Check for Encoding - if let Ok(encoding_obj) = font_dict.get(b"Encoding") { - let enc_dict = match encoding_obj { - lopdf::Object::Dictionary(d) => Some(d.clone()), - lopdf::Object::Reference(r) => doc.get_dictionary(*r).ok().cloned(), - lopdf::Object::Name(name) => { - println!( - " font={}: Encoding={}", - font_name, - String::from_utf8_lossy(name) - ); - None - } - _ => None, - }; - - if let Some(enc_dict) = enc_dict { - // Check BaseEncoding - if let Ok(lopdf::Object::Name(name)) = enc_dict.get(b"BaseEncoding") { - println!( - " font={}: BaseEncoding={}", - font_name, - String::from_utf8_lossy(name) - ); - } - - // Dump Differences - if let Ok(diff_obj) = enc_dict.get(b"Differences") { - let diff_array = match diff_obj { - lopdf::Object::Array(arr) => Some(arr.clone()), - lopdf::Object::Reference(r) => { - if let Ok(lopdf::Object::Array(arr)) = doc.get_object(*r) { - Some(arr.clone()) - } else { - None - } - } - _ => None, - }; - - if let Some(diff_array) = diff_array { - let mut current_code: u8 = 0; - let mut entries = Vec::new(); - let mut total_glyphs = 0; - - for item in &diff_array { - match item { - lopdf::Object::Integer(n) => { - current_code = *n as u8; - } - lopdf::Object::Name(name) => { - let glyph = String::from_utf8_lossy(name).to_string(); - entries.push((current_code, glyph)); - current_code = current_code.wrapping_add(1); - total_glyphs += 1; - } - _ => {} - } - } - - println!( - " font={}: Differences has {} glyph entries", - font_name, total_glyphs - ); - - // Show ligature entries specifically - for (code, glyph) in &entries { - if glyph == "fi" - || glyph == "fl" - || glyph == "ffi" - || glyph == "ffl" - { - println!( - " code=0x{:02X} ({:3}) glyph={:?} (LIGATURE)", - code, code, glyph - ); - } - } - - // Check coverage: does it have standard ASCII letters? - let has_a = entries.iter().any(|(_, g)| g == "a"); - let has_space = entries.iter().any(|(_, g)| g == "space"); - let has_period = entries.iter().any(|(_, g)| g == "period"); - println!( - " has 'a': {}, has 'space': {}, has 'period': {}", - has_a, has_space, has_period - ); - - // Show first 10 and last 5 entries - println!(" First 10 entries:"); - for (code, glyph) in entries.iter().take(10) { - println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph); - } - if entries.len() > 15 { - println!(" ..."); - println!(" Last 5 entries:"); - for (code, glyph) in entries - .iter() - .rev() - .take(5) - .collect::>() - .iter() - .rev() - { - println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph); - } - } - } - } - } - } else { - println!(" font={}: no Encoding", font_name); - } - } - } - - // Now extract text and look for ligatures - let items = extract_text_with_positions(&path).unwrap(); - - println!("\n=== Items containing fi or fl (first 10) ==="); - let mut count = 0; - for item in items.iter() { - if item.text.contains('\u{FB01}') || item.text.contains('\u{FB02}') { - println!( - " page={} font={} text={:?}", - item.page, item.font, item.text - ); - count += 1; - if count >= 10 { - break; - } - } - } - - let total_lig = items - .iter() - .filter(|i| i.text.contains('\u{FB01}') || i.text.contains('\u{FB02}')) - .count(); - println!(" Total items with ligatures: {}", total_lig); -} diff --git a/src/bin/debug_order.rs b/src/bin/debug_order.rs deleted file mode 100644 index 406d85b..0000000 --- a/src/bin/debug_order.rs +++ /dev/null @@ -1,46 +0,0 @@ -use pdf_inspector::extract_text_with_positions; - -fn main() { - let items = extract_text_with_positions("samples/tables/doc.pdf").unwrap(); - - // Find items containing "Description" or section numbers - println!("Items containing section markers:"); - for item in items.iter().filter(|i| i.page == 1) { - if item.text.contains("Description") - || item.text.starts_with("3 ") - || item.text.starts_with("3 ") - { - println!(" x={:6.1} y={:6.1} \"{}\"", item.x, item.y, item.text); - } - } - - // Look at the Y range for right column - let right_col: Vec<_> = items - .iter() - .filter(|i| i.page == 1 && i.x > 300.0 && i.x < 400.0) - .collect(); - - let y_min = right_col.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); - let y_max = right_col - .iter() - .map(|i| i.y) - .fold(f32::NEG_INFINITY, f32::max); - println!( - "\nRight column (x=300-400) Y range: {:.1} to {:.1}", - y_min, y_max - ); - - // Show items near Y=675 (where "3 Description" appears) - println!("\nItems near Y=675 (±10):"); - for item in items - .iter() - .filter(|i| i.page == 1 && (i.y - 675.0).abs() < 10.0) - { - println!( - " x={:6.1} y={:6.1} \"{}\"", - item.x, - item.y, - &item.text[..item.text.len().min(50)] - ); - } -} diff --git a/src/bin/debug_pages.rs b/src/bin/debug_pages.rs deleted file mode 100644 index f385378..0000000 --- a/src/bin/debug_pages.rs +++ /dev/null @@ -1,31 +0,0 @@ -use pdf_inspector::extract_text_with_positions; -use std::env; - -fn main() { - let args: Vec = env::args().collect(); - if args.len() < 2 { - eprintln!("Usage: debug_pages [max_page | min-max]"); - std::process::exit(1); - } - - let range = args.get(2).map(|s| s.as_str()).unwrap_or("1-3"); - let (min_page, max_page) = if let Some((a, b)) = range.split_once('-') { - (a.parse().unwrap_or(1), b.parse().unwrap_or(3)) - } else { - (1, range.parse().unwrap_or(3)) - }; - - let items = extract_text_with_positions(&args[1]).expect("Failed to extract"); - - for page in min_page..=max_page { - let page_items: Vec<_> = items.iter().filter(|i| i.page == page).collect(); - println!("=== PAGE {} ({} items) ===", page, page_items.len()); - for item in &page_items { - println!( - " x={:7.1} y={:7.1} w={:7.1} fs={:5.1} text={:?}", - item.x, item.y, item.width, item.font_size, item.text - ); - } - println!(); - } -} diff --git a/src/bin/debug_ygaps.rs b/src/bin/debug_ygaps.rs deleted file mode 100644 index d4c65a3..0000000 --- a/src/bin/debug_ygaps.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Debug tool: Print Y positions and gaps between consecutive lines -//! -//! Usage: debug_ygaps [page_number] -//! -//! Shows text lines grouped by page with Y coordinates, gaps from previous line, -//! font sizes, and whether each gap would be treated as a paragraph break. - -use pdf_inspector::extract_text_with_positions; -use pdf_inspector::extractor::{group_into_lines, TextLine}; -use std::env; -use std::process; - -fn main() { - let args: Vec = env::args().collect(); - - if args.len() < 2 { - eprintln!("Usage: {} [page_number]", args[0]); - eprintln!(); - eprintln!("Prints Y positions and gaps between consecutive text lines."); - eprintln!("If page_number is given, only that page is shown."); - process::exit(1); - } - - let pdf_path = &args[1]; - let filter_page: Option = args.get(2).and_then(|s| s.parse().ok()); - - let items = match extract_text_with_positions(pdf_path) { - Ok(items) => items, - Err(e) => { - eprintln!("Error extracting text: {}", e); - process::exit(1); - } - }; - - if items.is_empty() { - eprintln!("No text items found in PDF."); - process::exit(0); - } - - // Compute base font size (most common font size >= 9pt) - let base_size = { - let mut size_counts: std::collections::HashMap = - std::collections::HashMap::new(); - for item in &items { - if item.font_size >= 9.0 { - let key = (item.font_size * 10.0) as i32; - *size_counts.entry(key).or_insert(0) += 1; - } - } - size_counts - .into_iter() - .max_by_key(|&(_, count)| count) - .map(|(size_key, _)| size_key as f32 / 10.0) - .unwrap_or(10.0) - }; - - eprintln!("Base font size: {:.1}pt", base_size); - eprintln!( - "Paragraph break threshold: y_gap > {:.1} (base * 1.8)", - base_size * 1.8 - ); - eprintln!(); - - // Group into lines - let lines = group_into_lines(items); - - // Get unique pages - let mut pages: Vec = lines.iter().map(|l| l.page).collect(); - pages.sort(); - pages.dedup(); - - for page in pages { - if let Some(fp) = filter_page { - if page != fp { - continue; - } - } - - let page_lines: Vec<&TextLine> = lines.iter().filter(|l| l.page == page).collect(); - - println!("===== PAGE {} ({} lines) =====", page, page_lines.len()); - println!( - "{:>8} {:>8} {:>8} {:>6} {:>5} Text (first 80 chars)", - "Y", "Gap", "GapRatio", "Font", "Bold" - ); - println!("{}", "-".repeat(120)); - - let mut prev_y: Option = None; - - for line in &page_lines { - let font_size = line.items.first().map(|i| i.font_size).unwrap_or(0.0); - let is_bold = line.items.first().map(|i| i.is_bold).unwrap_or(false); - let text = line.text(); - let display_text: String = text.chars().take(80).collect(); - - let (gap_str, ratio_str, marker) = if let Some(py) = prev_y { - let gap = py - line.y; - let ratio = gap / base_size; - let is_para = gap > base_size * 1.8; - let marker = if is_para { " <>" } else { "" }; - ( - format!("{:8.1}", gap), - format!("{:8.2}", ratio), - marker.to_string(), - ) - } else { - ( - " ---".to_string(), - " ---".to_string(), - String::new(), - ) - }; - - println!( - "{:8.1} {} {} {:6.1} {:>5} {}{}", - line.y, - gap_str, - ratio_str, - font_size, - if is_bold { "B" } else { "" }, - display_text, - marker - ); - - prev_y = Some(line.y); - } - - println!(); - - // Summary statistics for this page - let mut gaps: Vec = Vec::new(); - let mut prev_y: Option = None; - for line in &page_lines { - if let Some(py) = prev_y { - let gap = py - line.y; - if gap > 0.0 && gap < 200.0 { - gaps.push(gap); - } - } - prev_y = Some(line.y); - } - - if !gaps.is_empty() { - gaps.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let min = gaps.first().unwrap(); - let max = gaps.last().unwrap(); - let median = gaps[gaps.len() / 2]; - let mean: f32 = gaps.iter().sum::() / gaps.len() as f32; - - println!(" Gap statistics for page {}:", page); - println!(" Count: {}", gaps.len()); - println!(" Min: {:6.1} (ratio: {:.2})", min, min / base_size); - println!(" Max: {:6.1} (ratio: {:.2})", max, max / base_size); - println!( - " Median: {:6.1} (ratio: {:.2})", - median, - median / base_size - ); - println!(" Mean: {:6.1} (ratio: {:.2})", mean, mean / base_size); - - // Histogram of gap ratios - println!(); - println!(" Gap ratio histogram (gap / base_size):"); - let buckets: Vec = vec![ - 0.0, - 0.5, - 1.0, - 1.2, - 1.5, - 1.8, - 2.0, - 2.5, - 3.0, - 5.0, - 10.0, - f32::INFINITY, - ]; - for i in 0..buckets.len() - 1 { - let count = gaps - .iter() - .filter(|&&g| { - let r = g / base_size; - r >= buckets[i] && r < buckets[i + 1] - }) - .count(); - if count > 0 { - let label = if buckets[i + 1] == f32::INFINITY { - format!("{:4.1}+ ", buckets[i]) - } else { - format!("{:4.1}-{:<4.1}", buckets[i], buckets[i + 1]) - }; - let bar: String = "#".repeat(count.min(60)); - println!(" {} | {:3} {}", label, count, bar); - } - } - println!(); - } - } -} diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index 20caa34..7e66977 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -1,10 +1,47 @@ //! CLI tool for PDF to Markdown conversion -use pdf_inspector::{process_pdf, PdfType}; +use pdf_inspector::{process_pdf_with_config_pages, DetectionConfig, MarkdownOptions, PdfType}; +use std::collections::HashSet; use std::env; use std::fs; use std::process; +/// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers. +fn parse_page_spec(spec: &str) -> Result, String> { + let mut pages = HashSet::new(); + for part in spec.split(',') { + let part = part.trim(); + if let Some((start, end)) = part.split_once('-') { + let start: u32 = start + .trim() + .parse() + .map_err(|_| format!("invalid page number: {}", start.trim()))?; + let end: u32 = end + .trim() + .parse() + .map_err(|_| format!("invalid page number: {}", end.trim()))?; + if start == 0 || end == 0 { + return Err("page numbers are 1-indexed".to_string()); + } + if start > end { + return Err(format!("invalid range: {}-{}", start, end)); + } + for p in start..=end { + pages.insert(p); + } + } else { + let p: u32 = part + .parse() + .map_err(|_| format!("invalid page number: {}", part))?; + if p == 0 { + return Err("page numbers are 1-indexed".to_string()); + } + pages.insert(p); + } + } + Ok(pages) +} + fn main() { let args: Vec = env::args().collect(); @@ -17,20 +54,51 @@ fn main() { eprintln!("Returns early if PDF is scanned (OCR needed)."); eprintln!(); eprintln!("Options:"); - eprintln!(" --json Output result as JSON"); - eprintln!(" --raw Output only markdown (no headers)"); + eprintln!(" --json Output result as JSON"); + eprintln!(" --raw Output only markdown (no headers)"); + eprintln!(" --pages Insert page break markers ()"); + eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)"); process::exit(1); } let pdf_path = &args[1]; let json_output = args.iter().any(|a| a == "--json"); let raw_output = args.iter().any(|a| a == "--raw"); + let page_numbers = args.iter().any(|a| a == "--pages"); + + // Parse --select-pages value + let page_filter = args + .iter() + .position(|a| a == "--select-pages") + .map(|i| { + args.get(i + 1) + .unwrap_or_else(|| { + eprintln!("Error: --select-pages requires a value (e.g. 1,3,5-10)"); + process::exit(1); + }) + .as_str() + }) + .map(|spec| { + parse_page_spec(spec).unwrap_or_else(|e| { + eprintln!("Error: invalid --select-pages value: {}", e); + process::exit(1); + }) + }); + let output_file = args .get(2) .filter(|a| !a.starts_with("--")) .map(|s| s.as_str()); - match process_pdf(pdf_path) { + let mut md_options = MarkdownOptions::default(); + md_options.include_page_numbers = page_numbers; + + match process_pdf_with_config_pages( + pdf_path, + DetectionConfig::default(), + md_options, + page_filter.as_ref(), + ) { Ok(result) => { if json_output { let md_escaped = result diff --git a/src/extractor.rs b/src/extractor.rs index cb1b60b..0cafd94 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -6,7 +6,7 @@ use crate::glyph_names::glyph_to_char; use crate::tounicode::FontCMaps; use crate::PdfError; use lopdf::{Document, Encoding, Object, ObjectId}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; /// Font encoding map: maps byte codes to Unicode characters @@ -747,10 +747,10 @@ fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { } // Single-character fragment joined to a multi-character item: use a - // more generous threshold to rejoin split words like "b" + "illion" - // or "C" + "ultural". + // moderately generous threshold to rejoin split words like "b" + "illion" + // or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words. if (prev_chars == 1) != (curr_chars == 1) { - return gap < font_size * 0.25; + return gap < font_size * 0.20; } // Both single-char: per-glyph positioning. For numeric characters @@ -874,34 +874,70 @@ fn extract_text_from_doc(doc: &Document) -> Result { /// Extract text with position information from PDF file pub fn extract_text_with_positions>(path: P) -> Result, PdfError> { + extract_text_with_positions_pages(path, None) +} + +/// Extract text with positions from a file, limited to specific pages. +/// +/// `page_filter` is an optional set of 1-indexed page numbers to process. +/// When `None`, all pages are processed. +pub fn extract_text_with_positions_pages>( + path: P, + page_filter: Option<&HashSet>, +) -> Result, PdfError> { // Read the raw PDF bytes for ToUnicode extraction let pdf_bytes = std::fs::read(path.as_ref())?; crate::validate_pdf_bytes(&pdf_bytes)?; let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes); let doc = Document::load_mem(&pdf_bytes)?; - extract_positioned_text_from_doc(&doc, &font_cmaps) + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) } /// Extract text with positions from memory buffer pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result, PdfError> { + extract_text_with_positions_mem_pages(buffer, None) +} + +/// Extract text with positions from memory buffer, limited to specific pages. +pub fn extract_text_with_positions_mem_pages( + buffer: &[u8], + page_filter: Option<&HashSet>, +) -> Result, PdfError> { crate::validate_pdf_bytes(buffer)?; // Extract ToUnicode CMaps from raw PDF bytes let font_cmaps = FontCMaps::from_pdf_bytes(buffer); let doc = Document::load_mem(buffer)?; - extract_positioned_text_from_doc(&doc, &font_cmaps) + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) } /// Extract positioned text from loaded document fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, ) -> Result, PdfError> { + // If raw byte scanning found no CMaps, populate from the document model. + // This handles PDFs with compressed object streams where raw scanning fails. + let mut font_cmaps_owned; + let font_cmaps = if font_cmaps.by_obj_num.is_empty() { + font_cmaps_owned = font_cmaps.clone(); + populate_cmaps_from_doc(doc, &mut font_cmaps_owned); + &font_cmaps_owned + } else { + font_cmaps + }; + let pages = doc.get_pages(); let mut all_items = Vec::new(); for (page_num, &page_id) in pages.iter() { + if let Some(filter) = page_filter { + if !filter.contains(page_num) { + continue; + } + } let items = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; all_items.extend(items); @@ -913,6 +949,54 @@ fn extract_positioned_text_from_doc( Ok(all_items) } +/// Populate FontCMaps from the lopdf document model for ToUnicode streams +/// that weren't found by raw byte scanning (e.g. in compressed object streams). +fn populate_cmaps_from_doc(doc: &Document, font_cmaps: &mut FontCMaps) { + use crate::tounicode::ToUnicodeCMap; + + for (_page_num, &page_id) in doc.get_pages().iter() { + let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); + for (font_name, font_dict) in &fonts { + if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") { + if let Ok(obj_ref) = tounicode_ref.as_reference() { + let obj_num = obj_ref.0; + if font_cmaps.by_obj_num.contains_key(&obj_num) { + continue; + } + // Try to get the stream content via lopdf + if let Ok(stream) = doc.get_object(obj_ref) { + if let Ok(stream) = stream.as_stream() { + if let Ok(data) = stream.decompressed_content() { + if let Some(cmap) = ToUnicodeCMap::parse(&data) { + let resource_name = + String::from_utf8_lossy(font_name).to_string(); + let base_name = font_dict + .get(b"BaseFont") + .ok() + .and_then(|o| o.as_name().ok()) + .map(|n| String::from_utf8_lossy(n).to_string()); + + // Store by object number + font_cmaps.by_obj_num.insert(obj_num, cmap.clone()); + // Store by resource name + font_cmaps + .by_name + .insert(resource_name.clone(), cmap.clone()); + if let Some(base) = base_name { + let unique_key = format!("{}_{}", base, obj_num); + font_cmaps.by_name.insert(unique_key, cmap.clone()); + font_cmaps.by_name.insert(base, cmap); + } + } + } + } + } + } + } + } + } +} + /// Multiply two 2D transformation matrices /// Matrix format: [a, b, c, d, e, f] representing: /// | a b 0 | @@ -1146,28 +1230,28 @@ fn extract_page_text_items( &font_encodings, &encoding_cache, ) { - if !text.trim().is_empty() { - let rendered_size = - effective_font_size(current_font_size, &text_matrix); - let combined = multiply_matrices(&text_matrix, &ctm); - let (x, y) = (combined[4], combined[5]); - let width = if let Some(font_info) = font_widths.get(¤t_font) { - if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { - let w_ts = compute_string_width_ts( - raw_bytes, - font_info, - current_font_size, - ); - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - (w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])) - .abs() - } else { - 0.0 - } + let rendered_size = effective_font_size(current_font_size, &text_matrix); + let combined = multiply_matrices(&text_matrix, &ctm); + let (x, y) = (combined[4], combined[5]); + let width = if let Some(font_info) = font_widths.get(¤t_font) { + if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { + let w_ts = compute_string_width_ts( + raw_bytes, + font_info, + current_font_size, + ); + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + (w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])).abs() } else { 0.0 - }; + } + } else { + 0.0 + }; + // Only create text item for non-whitespace; whitespace + // still advances the text matrix above so gap detection works + if !text.trim().is_empty() { let base_font = font_base_names .get(¤t_font) .map(|s| s.as_str()) @@ -1659,30 +1743,31 @@ fn extract_form_xobject_text( &font_encodings, &encoding_cache, ) { - if !text.trim().is_empty() { - let rendered_size = - effective_font_size(current_font_size, &text_matrix); - let combined = multiply_matrices(&text_matrix, parent_ctm); - let (x, y) = (combined[4], combined[5]); - let width = if let Some(font_info) = font_widths.get(¤t_font) { - if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { - let w_ts = compute_string_width_ts( - raw_bytes, - font_info, - current_font_size, - ); - text_matrix[4] += w_ts * text_matrix[0]; - text_matrix[5] += w_ts * text_matrix[1]; - (w_ts - * (text_matrix[0] * parent_ctm[0] - + text_matrix[1] * parent_ctm[2])) - .abs() - } else { - 0.0 - } + let rendered_size = effective_font_size(current_font_size, &text_matrix); + let combined = multiply_matrices(&text_matrix, parent_ctm); + let (x, y) = (combined[4], combined[5]); + let width = if let Some(font_info) = font_widths.get(¤t_font) { + if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { + let w_ts = compute_string_width_ts( + raw_bytes, + font_info, + current_font_size, + ); + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + (w_ts + * (text_matrix[0] * parent_ctm[0] + + text_matrix[1] * parent_ctm[2])) + .abs() } else { 0.0 - }; + } + } else { + 0.0 + }; + // Only create text item for non-whitespace; whitespace + // still advances the text matrix above so gap detection works + if !text.trim().is_empty() { let base_font = font_base_names .get(¤t_font) .map(|s| s.as_str()) diff --git a/src/lib.rs b/src/lib.rs index cb06a20..93be6e6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,9 @@ pub use detector::{ detect_pdf_type, detect_pdf_type_mem, detect_pdf_type_mem_with_config, detect_pdf_type_with_config, DetectionConfig, PdfType, PdfTypeResult, ScanStrategy, }; -pub use extractor::{extract_text, extract_text_with_positions, TextItem}; +pub use extractor::{ + extract_text, extract_text_with_positions, extract_text_with_positions_pages, TextItem, +}; pub use markdown::{to_markdown, to_markdown_from_items, MarkdownOptions}; use std::path::Path; @@ -112,10 +114,24 @@ pub fn process_pdf>(path: P) -> Result>( path: P, config: DetectionConfig, + markdown_options: MarkdownOptions, +) -> Result { + process_pdf_with_config_pages(path, config, markdown_options, None) +} + +/// Process a PDF file with custom configuration and optional page filter. +/// +/// `page_filter` limits extraction to the given 1-indexed page numbers. +/// When `None`, all pages are processed. +pub fn process_pdf_with_config_pages>( + path: P, + config: DetectionConfig, + markdown_options: MarkdownOptions, + page_filter: Option<&std::collections::HashSet>, ) -> Result { let start = std::time::Instant::now(); @@ -130,8 +146,8 @@ pub fn process_pdf_with_config>( let result = match pdf_type { PdfType::TextBased => { - let items = extract_text_with_positions(&path)?; - let markdown = to_markdown_from_items(items, MarkdownOptions::default()); + let items = extract_text_with_positions_pages(&path, page_filter)?; + let markdown = to_markdown_from_items(items, markdown_options); PdfProcessResult { pdf_type, @@ -155,8 +171,8 @@ pub fn process_pdf_with_config>( confidence, }, PdfType::Mixed => { - let items = extract_text_with_positions(&path).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default())); + let items = extract_text_with_positions_pages(&path, page_filter).ok(); + let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone())); PdfProcessResult { pdf_type, @@ -235,10 +251,11 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { Ok(result) } -/// Process PDF from memory buffer with custom detection configuration +/// Process PDF from memory buffer with custom detection and markdown configuration pub fn process_pdf_mem_with_config( buffer: &[u8], config: DetectionConfig, + markdown_options: MarkdownOptions, ) -> Result { let start = std::time::Instant::now(); @@ -254,7 +271,7 @@ pub fn process_pdf_mem_with_config( let result = match pdf_type { PdfType::TextBased => { let items = extractor::extract_text_with_positions_mem(buffer)?; - let markdown = to_markdown_from_items(items, MarkdownOptions::default()); + let markdown = to_markdown_from_items(items, markdown_options); PdfProcessResult { pdf_type, @@ -279,7 +296,7 @@ pub fn process_pdf_mem_with_config( }, PdfType::Mixed => { let items = extractor::extract_text_with_positions_mem(buffer).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default())); + let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone())); PdfProcessResult { pdf_type, diff --git a/src/markdown.rs b/src/markdown.rs index 6414fd1..455d0e2 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -36,6 +36,8 @@ pub struct MarkdownOptions { pub include_images: bool, /// Include extracted hyperlinks pub include_links: bool, + /// Insert page break markers () between pages + pub include_page_numbers: bool, } impl Default for MarkdownOptions { @@ -52,6 +54,7 @@ impl Default for MarkdownOptions { detect_italic: true, include_images: true, include_links: true, + include_page_numbers: false, } } } @@ -518,6 +521,10 @@ fn to_markdown_from_lines_with_tables_and_images( current_page = line.page; prev_y = f32::MAX; + + if options.include_page_numbers { + output.push_str(&format!("\n\n", current_page)); + } } // Check if we should insert a table before this line @@ -757,6 +764,10 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> prev_y = f32::MAX; in_list = false; last_list_x = None; + + if options.include_page_numbers { + output.push_str(&format!("\n\n", current_page)); + } } // Paragraph break (large Y gap relative to document's typical line spacing) diff --git a/src/tounicode.rs b/src/tounicode.rs index da470e1..1a8056c 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -535,7 +535,7 @@ pub fn extract_tounicode_cmaps(pdf_bytes: &[u8]) -> HashMap } /// Collection of ToUnicode CMaps indexed by font name -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct FontCMaps { /// Map of font name (e.g., "FNotoSans0") to ToUnicodeCMap pub by_name: HashMap, @@ -605,26 +605,9 @@ impl FontCMaps { } } - /// Get a CMap for a font name + /// Get a CMap for a font name (exact match only) pub fn get(&self, font_name: &str) -> Option<&ToUnicodeCMap> { - // Try exact match first - if let Some(cmap) = self.by_name.get(font_name) { - return Some(cmap); - } - - // Try without leading 'F' if present (resource names sometimes differ) - // but only if the stripped name is long enough to avoid false matches - // (e.g., "F1" → "1" would match too many things) - let stripped = font_name.strip_prefix('F').unwrap_or(font_name); - if stripped.len() >= 4 { - for (name, cmap) in &self.by_name { - if name.contains(stripped) || stripped.contains(name.as_str()) { - return Some(cmap); - } - } - } - - None + self.by_name.get(font_name) } /// Get a CMap by ToUnicode object number diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e053c38..9710284 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -265,6 +265,7 @@ fn test_markdown_options_custom() { detect_italic: false, include_images: false, include_links: false, + include_page_numbers: false, }; assert!(!opts.detect_headers); assert!(opts.detect_lists);