From c2c528ff239fa32b3f8cb1b07180d5d41b89dee6 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Sat, 7 Feb 2026 14:12:19 -0800 Subject: [PATCH] url formatting, page no detection, additional cleanup --- Cargo.toml | 4 ++ src/bin/pdf2md.rs | 90 ++++++++++++++---------- src/markdown.rs | 140 +++++++++++++++++++++++++++++++++++-- tests/integration_tests.rs | 6 ++ 4 files changed, 200 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 70a6a55..7282a13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,10 @@ rayon = "1.10" # Logging log = "0.4" +# Text processing +regex = "1.10" +once_cell = "1.19" + [dev-dependencies] tempfile = "3.3" diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index e1c32ed..075a364 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -4,7 +4,6 @@ use pdf_inspector::{process_pdf, PdfType}; use std::env; use std::fs; use std::process; -use std::time::Instant; fn main() { let args: Vec = env::args().collect(); @@ -12,22 +11,27 @@ fn main() { if args.len() < 2 { eprintln!("Usage: {} [output_file]", args[0]); eprintln!(" {} --json", args[0]); + eprintln!(" {} --raw", args[0]); eprintln!(); eprintln!("Converts PDF to Markdown with smart type detection."); eprintln!("Returns early if PDF is scanned (OCR needed)."); + eprintln!(); + eprintln!("Options:"); + eprintln!(" --json Output result as JSON"); + eprintln!(" --raw Output only markdown (no headers)"); process::exit(1); } let pdf_path = &args[1]; - let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false); - let output_file = if !json_output { args.get(2) } else { None }; - - let start = Instant::now(); + let json_output = args.iter().any(|a| a == "--json"); + let raw_output = args.iter().any(|a| a == "--raw"); + let output_file = args + .get(2) + .filter(|a| !a.starts_with("--")) + .map(|s| s.as_str()); match process_pdf(pdf_path) { Ok(result) => { - let _elapsed = start.elapsed(); - if json_output { let md_escaped = result .markdown @@ -53,34 +57,48 @@ fn main() { result.markdown.as_ref().map(|m| m.len()).unwrap_or(0), md_escaped ); + } else if raw_output { + // Raw output - just the markdown, no headers + match result.pdf_type { + PdfType::TextBased | PdfType::Mixed => { + if let Some(markdown) = &result.markdown { + print!("{}", markdown); + } + } + PdfType::Scanned | PdfType::ImageBased => { + eprintln!("Error: PDF requires OCR (type: {:?})", result.pdf_type); + process::exit(2); + } + } } else { - println!("PDF to Markdown Conversion"); - println!("=========================="); - println!("File: {}", pdf_path); - println!(); + // Verbose output with headers + eprintln!("PDF to Markdown Conversion"); + eprintln!("=========================="); + eprintln!("File: {}", pdf_path); + eprintln!(); match result.pdf_type { PdfType::TextBased => { - println!("Type: TEXT-BASED (direct extraction)"); - println!("Pages: {}", result.page_count); - println!("Processing time: {}ms", result.processing_time_ms); + eprintln!("Type: TEXT-BASED (direct extraction)"); + eprintln!("Pages: {}", result.page_count); + eprintln!("Processing time: {}ms", result.processing_time_ms); if let Some(markdown) = &result.markdown { if let Some(output) = output_file { fs::write(output, markdown).expect("Failed to write output file"); - println!(); - println!("Markdown written to: {}", output); - println!("Length: {} characters", markdown.len()); + eprintln!(); + eprintln!("Markdown written to: {}", output); + eprintln!("Length: {} characters", markdown.len()); } else { - println!(); - println!("--- Markdown Output ---"); - println!(); + eprintln!(); + eprintln!("--- Markdown Output ---"); + eprintln!(); println!("{}", markdown); } } } PdfType::Scanned | PdfType::ImageBased => { - println!( + eprintln!( "Type: {} (OCR required)", if result.pdf_type == PdfType::Scanned { "SCANNED" @@ -88,30 +106,30 @@ fn main() { "IMAGE-BASED" } ); - println!("Pages: {}", result.page_count); - println!("Processing time: {}ms", result.processing_time_ms); - println!(); - println!("This PDF requires OCR for text extraction."); - println!("Consider using MinerU or similar OCR tool."); + eprintln!("Pages: {}", result.page_count); + eprintln!("Processing time: {}ms", result.processing_time_ms); + eprintln!(); + eprintln!("This PDF requires OCR for text extraction."); + eprintln!("Consider using MinerU or similar OCR tool."); process::exit(2); } PdfType::Mixed => { - println!("Type: MIXED (partial text extraction)"); - println!("Pages: {}", result.page_count); - println!("Processing time: {}ms", result.processing_time_ms); + eprintln!("Type: MIXED (partial text extraction)"); + eprintln!("Pages: {}", result.page_count); + eprintln!("Processing time: {}ms", result.processing_time_ms); if let Some(markdown) = &result.markdown { - println!(); - println!("Note: Some pages may contain images that require OCR."); - println!(); + eprintln!(); + eprintln!("Note: Some pages may contain images that require OCR."); + eprintln!(); if let Some(output) = output_file { fs::write(output, markdown).expect("Failed to write output file"); - println!("Markdown written to: {}", output); - println!("Length: {} characters", markdown.len()); + eprintln!("Markdown written to: {}", output); + eprintln!("Length: {} characters", markdown.len()); } else { - println!("--- Markdown Output ---"); - println!(); + eprintln!("--- Markdown Output ---"); + eprintln!(); println!("{}", markdown); } } diff --git a/src/markdown.rs b/src/markdown.rs index cee7a01..984870d 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -9,6 +9,8 @@ use crate::extractor::{group_into_lines, TextItem, TextLine}; use std::collections::HashMap; +use regex::Regex; + /// Options for markdown conversion #[derive(Debug, Clone)] pub struct MarkdownOptions { @@ -20,6 +22,12 @@ pub struct MarkdownOptions { pub detect_code: bool, /// Base font size for comparison pub base_font_size: Option, + /// Remove standalone page numbers + pub remove_page_numbers: bool, + /// Convert URLs to markdown links + pub format_urls: bool, + /// Fix hyphenation (broken words across lines) + pub fix_hyphenation: bool, } impl Default for MarkdownOptions { @@ -29,6 +37,9 @@ impl Default for MarkdownOptions { detect_lists: true, detect_code: true, base_font_size: None, + remove_page_numbers: true, + format_urls: true, + fix_hyphenation: true, } } } @@ -211,8 +222,8 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> output.push('\n'); } - // Clean up excessive newlines - clean_markdown(output) + // Clean up and post-process + clean_markdown(output, &options) } /// Merge drop caps with the appropriate line @@ -474,8 +485,23 @@ fn is_monospace_font(font_name: &str) -> bool { patterns.iter().any(|p| lower.contains(p)) } -/// Clean up markdown output -fn clean_markdown(mut text: String) -> String { +/// Clean up markdown output with post-processing +fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String { + // Fix hyphenation first (before other processing) + if options.fix_hyphenation { + text = fix_hyphenation(&text); + } + + // Remove standalone page numbers + if options.remove_page_numbers { + text = remove_page_numbers(&text); + } + + // Format URLs as markdown links + if options.format_urls { + text = format_urls(&text); + } + // Remove excessive newlines (more than 2 in a row) while text.contains("\n\n\n") { text = text.replace("\n\n\n", "\n\n"); @@ -488,6 +514,112 @@ fn clean_markdown(mut text: String) -> String { text } +/// Fix words broken across lines with spaces before the continuation +/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte" +fn fix_hyphenation(text: &str) -> String { + use once_cell::sync::Lazy; + + // Fix "word - word" patterns that should be "word-word" (compound words) + // But be careful not to break list items (which start with "- ") + static SPACED_HYPHEN_RE: Lazy = Lazy::new(|| { + Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap() + }); + + let result = SPACED_HYPHEN_RE + .replace_all(text, |caps: ®ex::Captures| { + format!("{}-{}", &caps[1], &caps[2]) + }) + .to_string(); + + result +} + +/// Remove standalone page numbers (lines that are just 1-4 digit numbers) +fn remove_page_numbers(text: &str) -> String { + let mut result = Vec::new(); + let lines: Vec<&str> = text.lines().collect(); + + for (i, line) in lines.iter().enumerate() { + let trimmed = line.trim(); + + // Check if this line is just a number (1-4 digits) + if trimmed.len() <= 4 && !trimmed.is_empty() && trimmed.chars().all(|c| c.is_ascii_digit()) + { + // Check context to determine if this is a page number + let prev_is_break = i > 0 && lines[i - 1].trim() == "---"; + let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---"; + let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty(); + let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty(); + + // Check if it's on its own line (surrounded by empty lines or page breaks) + let is_isolated = (prev_is_break || prev_is_empty || i == 0) + && (next_is_break || next_is_empty || i + 1 == lines.len()); + + // Also remove numbers that appear right before a page break + // (common pattern: content ends, page number, then ---) + let before_break = i + 1 < lines.len() + && (lines[i + 1].trim() == "---" + || (i + 2 < lines.len() + && lines[i + 1].trim().is_empty() + && lines[i + 2].trim() == "---")); + + if is_isolated || before_break { + continue; + } + } + + result.push(*line); + } + + result.join("\n") +} + +/// Convert URLs to markdown links +fn format_urls(text: &str) -> String { + use once_cell::sync::Lazy; + + // Match URLs - we'll check context manually to avoid formatting already-linked URLs + static URL_RE: Lazy = + Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap()); + + let mut result = String::with_capacity(text.len()); + let mut last_end = 0; + + for mat in URL_RE.find_iter(text) { + let start = mat.start(); + let url = mat.as_str(); + + // Check if this URL is already in a markdown link by looking at preceding chars + let before = if start >= 2 { + &text[start - 2..start] + } else { + "" + }; + let already_linked = before.ends_with("](") || before.ends_with("]("); + + // Also check if it's inside square brackets (link text) + let prefix = &text[..start]; + let open_brackets = prefix.matches('[').count(); + let close_brackets = prefix.matches(']').count(); + let inside_link_text = open_brackets > close_brackets; + + if already_linked || inside_link_text { + // Already formatted, keep as-is + result.push_str(&text[last_end..mat.end()]); + } else { + // Add text before this URL + result.push_str(&text[last_end..start]); + // Format as markdown link + result.push_str(&format!("[{}]({})", url, url)); + } + last_end = mat.end(); + } + + // Add remaining text + result.push_str(&text[last_end..]); + result +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index f6db682..da1c123 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -250,11 +250,17 @@ fn test_markdown_options_custom() { detect_lists: true, detect_code: false, base_font_size: Some(14.0), + remove_page_numbers: false, + format_urls: false, + fix_hyphenation: false, }; assert!(!opts.detect_headers); assert!(opts.detect_lists); assert!(!opts.detect_code); assert_eq!(opts.base_font_size, Some(14.0)); + assert!(!opts.remove_page_numbers); + assert!(!opts.format_urls); + assert!(!opts.fix_hyphenation); } // ============================================================================