url formatting, page no detection, additional cleanup

Cargo.toml

@@ -20,6 +20,10 @@ rayon = "1.10"
 # Logging
 log = "0.4"
 
+# Text processing
+regex = "1.10"
+once_cell = "1.19"
+
 [dev-dependencies]
 tempfile = "3.3"
 

src/bin/pdf2md.rs

@@ -4,7 +4,6 @@ use pdf_inspector::{process_pdf, PdfType};
 use std::env;
 use std::fs;
 use std::process;
-use std::time::Instant;
 
 fn main() {
     let args: Vec<String> = env::args().collect();
@@ -12,22 +11,27 @@ fn main() {
     if args.len() < 2 {
         eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
         eprintln!("       {} <pdf_file> --json", args[0]);
+        eprintln!("       {} <pdf_file> --raw", args[0]);
         eprintln!();
         eprintln!("Converts PDF to Markdown with smart type detection.");
         eprintln!("Returns early if PDF is scanned (OCR needed).");
+        eprintln!();
+        eprintln!("Options:");
+        eprintln!("  --json    Output result as JSON");
+        eprintln!("  --raw     Output only markdown (no headers)");
         process::exit(1);
     }
 
     let pdf_path = &args[1];
-    let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false);
+    let json_output = args.iter().any(|a| a == "--json");
-    let output_file = if !json_output { args.get(2) } else { None };
+    let raw_output = args.iter().any(|a| a == "--raw");
-
+    let output_file = args
-    let start = Instant::now();
+        .get(2)
+        .filter(|a| !a.starts_with("--"))
+        .map(|s| s.as_str());
 
     match process_pdf(pdf_path) {
         Ok(result) => {
-            let _elapsed = start.elapsed();
-
             if json_output {
                 let md_escaped = result
                     .markdown
@@ -53,34 +57,48 @@ fn main() {
                     result.markdown.as_ref().map(|m| m.len()).unwrap_or(0),
                     md_escaped
                 );
+            } else if raw_output {
+                // Raw output - just the markdown, no headers
+                match result.pdf_type {
+                    PdfType::TextBased | PdfType::Mixed => {
+                        if let Some(markdown) = &result.markdown {
+                            print!("{}", markdown);
+                        }
+                    }
+                    PdfType::Scanned | PdfType::ImageBased => {
+                        eprintln!("Error: PDF requires OCR (type: {:?})", result.pdf_type);
+                        process::exit(2);
+                    }
+                }
             } else {
-                println!("PDF to Markdown Conversion");
+                // Verbose output with headers
-                println!("==========================");
+                eprintln!("PDF to Markdown Conversion");
-                println!("File: {}", pdf_path);
+                eprintln!("==========================");
-                println!();
+                eprintln!("File: {}", pdf_path);
+                eprintln!();
 
                 match result.pdf_type {
                     PdfType::TextBased => {
-                        println!("Type: TEXT-BASED (direct extraction)");
+                        eprintln!("Type: TEXT-BASED (direct extraction)");
-                        println!("Pages: {}", result.page_count);
+                        eprintln!("Pages: {}", result.page_count);
-                        println!("Processing time: {}ms", result.processing_time_ms);
+                        eprintln!("Processing time: {}ms", result.processing_time_ms);
 
                         if let Some(markdown) = &result.markdown {
                             if let Some(output) = output_file {
                                 fs::write(output, markdown).expect("Failed to write output file");
-                                println!();
+                                eprintln!();
-                                println!("Markdown written to: {}", output);
+                                eprintln!("Markdown written to: {}", output);
-                                println!("Length: {} characters", markdown.len());
+                                eprintln!("Length: {} characters", markdown.len());
                             } else {
-                                println!();
+                                eprintln!();
-                                println!("--- Markdown Output ---");
+                                eprintln!("--- Markdown Output ---");
-                                println!();
+                                eprintln!();
                                 println!("{}", markdown);
                             }
                         }
                     }
                     PdfType::Scanned | PdfType::ImageBased => {
-                        println!(
+                        eprintln!(
                             "Type: {} (OCR required)",
                             if result.pdf_type == PdfType::Scanned {
                                 "SCANNED"
@@ -88,30 +106,30 @@ fn main() {
                                 "IMAGE-BASED"
                             }
                         );
-                        println!("Pages: {}", result.page_count);
+                        eprintln!("Pages: {}", result.page_count);
-                        println!("Processing time: {}ms", result.processing_time_ms);
+                        eprintln!("Processing time: {}ms", result.processing_time_ms);
-                        println!();
+                        eprintln!();
-                        println!("This PDF requires OCR for text extraction.");
+                        eprintln!("This PDF requires OCR for text extraction.");
-                        println!("Consider using MinerU or similar OCR tool.");
+                        eprintln!("Consider using MinerU or similar OCR tool.");
                         process::exit(2);
                     }
                     PdfType::Mixed => {
-                        println!("Type: MIXED (partial text extraction)");
+                        eprintln!("Type: MIXED (partial text extraction)");
-                        println!("Pages: {}", result.page_count);
+                        eprintln!("Pages: {}", result.page_count);
-                        println!("Processing time: {}ms", result.processing_time_ms);
+                        eprintln!("Processing time: {}ms", result.processing_time_ms);
 
                         if let Some(markdown) = &result.markdown {
-                            println!();
+                            eprintln!();
-                            println!("Note: Some pages may contain images that require OCR.");
+                            eprintln!("Note: Some pages may contain images that require OCR.");
-                            println!();
+                            eprintln!();
 
                             if let Some(output) = output_file {
                                 fs::write(output, markdown).expect("Failed to write output file");
-                                println!("Markdown written to: {}", output);
+                                eprintln!("Markdown written to: {}", output);
-                                println!("Length: {} characters", markdown.len());
+                                eprintln!("Length: {} characters", markdown.len());
                             } else {
-                                println!("--- Markdown Output ---");
+                                eprintln!("--- Markdown Output ---");
-                                println!();
+                                eprintln!();
                                 println!("{}", markdown);
                             }
                         }

src/markdown.rs

@@ -9,6 +9,8 @@
 use crate::extractor::{group_into_lines, TextItem, TextLine};
 use std::collections::HashMap;
 
+use regex::Regex;
+
 /// Options for markdown conversion
 #[derive(Debug, Clone)]
 pub struct MarkdownOptions {
@@ -20,6 +22,12 @@ pub struct MarkdownOptions {
     pub detect_code: bool,
     /// Base font size for comparison
     pub base_font_size: Option<f32>,
+    /// Remove standalone page numbers
+    pub remove_page_numbers: bool,
+    /// Convert URLs to markdown links
+    pub format_urls: bool,
+    /// Fix hyphenation (broken words across lines)
+    pub fix_hyphenation: bool,
 }
 
 impl Default for MarkdownOptions {
@@ -29,6 +37,9 @@ impl Default for MarkdownOptions {
             detect_lists: true,
             detect_code: true,
             base_font_size: None,
+            remove_page_numbers: true,
+            format_urls: true,
+            fix_hyphenation: true,
         }
     }
 }
@@ -211,8 +222,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
         output.push('\n');
     }
 
-    // Clean up excessive newlines
+    // Clean up and post-process
-    clean_markdown(output)
+    clean_markdown(output, &options)
 }
 
 /// Merge drop caps with the appropriate line
@@ -474,8 +485,23 @@ fn is_monospace_font(font_name: &str) -> bool {
     patterns.iter().any(|p| lower.contains(p))
 }
 
-/// Clean up markdown output
+/// Clean up markdown output with post-processing
-fn clean_markdown(mut text: String) -> String {
+fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String {
+    // Fix hyphenation first (before other processing)
+    if options.fix_hyphenation {
+        text = fix_hyphenation(&text);
+    }
+
+    // Remove standalone page numbers
+    if options.remove_page_numbers {
+        text = remove_page_numbers(&text);
+    }
+
+    // Format URLs as markdown links
+    if options.format_urls {
+        text = format_urls(&text);
+    }
+
     // Remove excessive newlines (more than 2 in a row)
     while text.contains("\n\n\n") {
         text = text.replace("\n\n\n", "\n\n");
@@ -488,6 +514,112 @@ fn clean_markdown(mut text: String) -> String {
     text
 }
 
+/// Fix words broken across lines with spaces before the continuation
+/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte"
+fn fix_hyphenation(text: &str) -> String {
+    use once_cell::sync::Lazy;
+
+    // Fix "word - word" patterns that should be "word-word" (compound words)
+    // But be careful not to break list items (which start with "- ")
+    static SPACED_HYPHEN_RE: Lazy<Regex> = Lazy::new(|| {
+        Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap()
+    });
+
+    let result = SPACED_HYPHEN_RE
+        .replace_all(text, |caps: &regex::Captures| {
+            format!("{}-{}", &caps[1], &caps[2])
+        })
+        .to_string();
+
+    result
+}
+
+/// Remove standalone page numbers (lines that are just 1-4 digit numbers)
+fn remove_page_numbers(text: &str) -> String {
+    let mut result = Vec::new();
+    let lines: Vec<&str> = text.lines().collect();
+
+    for (i, line) in lines.iter().enumerate() {
+        let trimmed = line.trim();
+
+        // Check if this line is just a number (1-4 digits)
+        if trimmed.len() <= 4 && !trimmed.is_empty() && trimmed.chars().all(|c| c.is_ascii_digit())
+        {
+            // Check context to determine if this is a page number
+            let prev_is_break = i > 0 && lines[i - 1].trim() == "---";
+            let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---";
+            let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty();
+            let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty();
+
+            // Check if it's on its own line (surrounded by empty lines or page breaks)
+            let is_isolated = (prev_is_break || prev_is_empty || i == 0)
+                && (next_is_break || next_is_empty || i + 1 == lines.len());
+
+            // Also remove numbers that appear right before a page break
+            // (common pattern: content ends, page number, then ---)
+            let before_break = i + 1 < lines.len()
+                && (lines[i + 1].trim() == "---"
+                    || (i + 2 < lines.len()
+                        && lines[i + 1].trim().is_empty()
+                        && lines[i + 2].trim() == "---"));
+
+            if is_isolated || before_break {
+                continue;
+            }
+        }
+
+        result.push(*line);
+    }
+
+    result.join("\n")
+}
+
+/// Convert URLs to markdown links
+fn format_urls(text: &str) -> String {
+    use once_cell::sync::Lazy;
+
+    // Match URLs - we'll check context manually to avoid formatting already-linked URLs
+    static URL_RE: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap());
+
+    let mut result = String::with_capacity(text.len());
+    let mut last_end = 0;
+
+    for mat in URL_RE.find_iter(text) {
+        let start = mat.start();
+        let url = mat.as_str();
+
+        // Check if this URL is already in a markdown link by looking at preceding chars
+        let before = if start >= 2 {
+            &text[start - 2..start]
+        } else {
+            ""
+        };
+        let already_linked = before.ends_with("](") || before.ends_with("](");
+
+        // Also check if it's inside square brackets (link text)
+        let prefix = &text[..start];
+        let open_brackets = prefix.matches('[').count();
+        let close_brackets = prefix.matches(']').count();
+        let inside_link_text = open_brackets > close_brackets;
+
+        if already_linked || inside_link_text {
+            // Already formatted, keep as-is
+            result.push_str(&text[last_end..mat.end()]);
+        } else {
+            // Add text before this URL
+            result.push_str(&text[last_end..start]);
+            // Format as markdown link
+            result.push_str(&format!("[{}]({})", url, url));
+        }
+        last_end = mat.end();
+    }
+
+    // Add remaining text
+    result.push_str(&text[last_end..]);
+    result
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

tests/integration_tests.rs

@@ -250,11 +250,17 @@ fn test_markdown_options_custom() {
         detect_lists: true,
         detect_code: false,
         base_font_size: Some(14.0),
+        remove_page_numbers: false,
+        format_urls: false,
+        fix_hyphenation: false,
     };
     assert!(!opts.detect_headers);
     assert!(opts.detect_lists);
     assert!(!opts.detect_code);
     assert_eq!(opts.base_font_size, Some(14.0));
+    assert!(!opts.remove_page_numbers);
+    assert!(!opts.format_urls);
+    assert!(!opts.fix_hyphenation);
 }
 
 // ============================================================================