diff --git a/Cargo.toml b/Cargo.toml index 4e0c4c4..7f1d5d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,3 +45,7 @@ path = "src/bin/detect_pdf.rs" [[bin]] name = "debug_ygaps" path = "src/bin/debug_ygaps.rs" + +[[bin]] +name = "debug_pages" +path = "src/bin/debug_pages.rs" diff --git a/src/bin/debug_pages.rs b/src/bin/debug_pages.rs new file mode 100644 index 0000000..f385378 --- /dev/null +++ b/src/bin/debug_pages.rs @@ -0,0 +1,31 @@ +use pdf_inspector::extract_text_with_positions; +use std::env; + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: debug_pages [max_page | min-max]"); + std::process::exit(1); + } + + let range = args.get(2).map(|s| s.as_str()).unwrap_or("1-3"); + let (min_page, max_page) = if let Some((a, b)) = range.split_once('-') { + (a.parse().unwrap_or(1), b.parse().unwrap_or(3)) + } else { + (1, range.parse().unwrap_or(3)) + }; + + let items = extract_text_with_positions(&args[1]).expect("Failed to extract"); + + for page in min_page..=max_page { + let page_items: Vec<_> = items.iter().filter(|i| i.page == page).collect(); + println!("=== PAGE {} ({} items) ===", page, page_items.len()); + for item in &page_items { + println!( + " x={:7.1} y={:7.1} w={:7.1} fs={:5.1} text={:?}", + item.x, item.y, item.width, item.font_size, item.text + ); + } + println!(); + } +} diff --git a/src/markdown.rs b/src/markdown.rs index 3d486cb..9972b46 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -380,6 +380,46 @@ fn count_table_columns(table_md: &str) -> usize { 0 } +/// Flush any remaining tables and images for a given page +fn flush_page_tables_and_images( + page: u32, + page_tables: &std::collections::HashMap>, + page_images: &std::collections::HashMap>, + inserted_tables: &mut HashSet<(u32, usize)>, + inserted_images: &mut HashSet<(u32, usize)>, + output: &mut String, + in_paragraph: &mut bool, +) { + if let Some(tables) = page_tables.get(&page) { + for (idx, (_, table_md)) in tables.iter().enumerate() { + if !inserted_tables.contains(&(page, idx)) { + if *in_paragraph { + output.push_str("\n\n"); + *in_paragraph = false; + } + output.push('\n'); + output.push_str(table_md); + output.push('\n'); + inserted_tables.insert((page, idx)); + } + } + } + if let Some(images) = page_images.get(&page) { + for (idx, (_, image_md)) in images.iter().enumerate() { + if !inserted_images.contains(&(page, idx)) { + if *in_paragraph { + output.push_str("\n\n"); + *in_paragraph = false; + } + output.push('\n'); + output.push_str(image_md); + output.push('\n'); + inserted_images.insert((page, idx)); + } + } + } +} + /// Convert text lines to markdown, inserting tables and images at appropriate Y positions fn to_markdown_from_lines_with_tables_and_images( lines: Vec, @@ -421,45 +461,61 @@ fn to_markdown_from_lines_with_tables_and_images( let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); + // Collect all pages that have tables or images (including image-only pages) + let mut all_content_pages: Vec = page_tables + .keys() + .chain(page_images.keys()) + .copied() + .collect(); + all_content_pages.sort(); + all_content_pages.dedup(); + for line in lines { // Page break if line.page != current_page { - // Before leaving the current page, insert any remaining tables and images + // Flush current page's remaining tables and images if current_page > 0 { - if let Some(tables) = page_tables.get(¤t_page) { - for (idx, (_, table_md)) in tables.iter().enumerate() { - if !inserted_tables.contains(&(current_page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(table_md); - output.push('\n'); - inserted_tables.insert((current_page, idx)); - } - } - } - if let Some(images) = page_images.get(¤t_page) { - for (idx, (_, image_md)) in images.iter().enumerate() { - if !inserted_images.contains(&(current_page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(image_md); - output.push('\n'); - inserted_images.insert((current_page, idx)); - } - } - } + flush_page_tables_and_images( + current_page, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); if in_paragraph { output.push_str("\n\n"); in_paragraph = false; } output.push_str("\n\n"); } + + // Flush any intermediate pages (image-only or table-only) between + // current_page and line.page that have no text lines + for &p in &all_content_pages { + if p <= current_page { + continue; + } + if p >= line.page { + break; + } + flush_page_tables_and_images( + p, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str("\n\n"); + } + current_page = line.page; prev_y = f32::MAX; } @@ -621,45 +677,30 @@ fn to_markdown_from_lines_with_tables_and_images( in_paragraph = true; } - // Insert any remaining tables and images for all pages - // (handles the case where all text was consumed by tables, leaving no lines to iterate) - let mut remaining_pages: Vec = page_tables - .keys() - .chain(page_images.keys()) - .copied() - .collect(); - remaining_pages.sort(); - remaining_pages.dedup(); - - for page in remaining_pages { - if let Some(tables) = page_tables.get(&page) { - for (idx, (_, table_md)) in tables.iter().enumerate() { - if !inserted_tables.contains(&(page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(table_md); - output.push('\n'); - inserted_tables.insert((page, idx)); - } - } - } - if let Some(images) = page_images.get(&page) { - for (idx, (_, image_md)) in images.iter().enumerate() { - if !inserted_images.contains(&(page, idx)) { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - output.push('\n'); - output.push_str(image_md); - output.push('\n'); - inserted_images.insert((page, idx)); - } - } + // Flush current page and any remaining pages with tables/images + // (handles table-only pages after the last text line, and trailing image-only pages) + flush_page_tables_and_images( + current_page, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); + for &p in &all_content_pages { + if p <= current_page { + continue; } + flush_page_tables_and_images( + p, + &page_tables, + &page_images, + &mut inserted_tables, + &mut inserted_images, + &mut output, + &mut in_paragraph, + ); } // Close final paragraph @@ -1310,6 +1351,9 @@ fn is_monospace_font(font_name: &str) -> bool { /// Clean up markdown output with post-processing fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String { + // Collapse dot leaders (e.g. TOC entries: "Introduction...............................1") + text = collapse_dot_leaders(&text); + // Fix hyphenation first (before other processing) if options.fix_hyphenation { text = fix_hyphenation(&text); @@ -1337,6 +1381,15 @@ fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String { text } +/// Collapse dot leaders (runs of 4+ dots) into " ... " +/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" +fn collapse_dot_leaders(text: &str) -> String { + use once_cell::sync::Lazy; + static DOT_LEADER_RE: Lazy = Lazy::new(|| Regex::new(r"\.{4,}").unwrap()); + + DOT_LEADER_RE.replace_all(text, " ... ").to_string() +} + /// Fix words broken across lines with spaces before the continuation /// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte" fn fix_hyphenation(text: &str) -> String {