url formatting, page no detection, additional cleanup
This commit is contained in:
Abimael Martell
@@ -20,6 +20,10 @@ rayon = "1.10"
|
||||
# Logging
|
||||
log = "0.4"
|
||||
|
||||
# Text processing
|
||||
regex = "1.10"
|
||||
once_cell = "1.19"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.3"
|
||||
|
||||
|
||||
+54
-36
@@ -4,7 +4,6 @@ use pdf_inspector::{process_pdf, PdfType};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::process;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
@@ -12,22 +11,27 @@ fn main() {
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
|
||||
eprintln!(" {} <pdf_file> --json", args[0]);
|
||||
eprintln!(" {} <pdf_file> --raw", args[0]);
|
||||
eprintln!();
|
||||
eprintln!("Converts PDF to Markdown with smart type detection.");
|
||||
eprintln!("Returns early if PDF is scanned (OCR needed).");
|
||||
eprintln!();
|
||||
eprintln!("Options:");
|
||||
eprintln!(" --json Output result as JSON");
|
||||
eprintln!(" --raw Output only markdown (no headers)");
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let pdf_path = &args[1];
|
||||
let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false);
|
||||
let output_file = if !json_output { args.get(2) } else { None };
|
||||
|
||||
let start = Instant::now();
|
||||
let json_output = args.iter().any(|a| a == "--json");
|
||||
let raw_output = args.iter().any(|a| a == "--raw");
|
||||
let output_file = args
|
||||
.get(2)
|
||||
.filter(|a| !a.starts_with("--"))
|
||||
.map(|s| s.as_str());
|
||||
|
||||
match process_pdf(pdf_path) {
|
||||
Ok(result) => {
|
||||
let _elapsed = start.elapsed();
|
||||
|
||||
if json_output {
|
||||
let md_escaped = result
|
||||
.markdown
|
||||
@@ -53,34 +57,48 @@ fn main() {
|
||||
result.markdown.as_ref().map(|m| m.len()).unwrap_or(0),
|
||||
md_escaped
|
||||
);
|
||||
} else if raw_output {
|
||||
// Raw output - just the markdown, no headers
|
||||
match result.pdf_type {
|
||||
PdfType::TextBased | PdfType::Mixed => {
|
||||
if let Some(markdown) = &result.markdown {
|
||||
print!("{}", markdown);
|
||||
}
|
||||
}
|
||||
PdfType::Scanned | PdfType::ImageBased => {
|
||||
eprintln!("Error: PDF requires OCR (type: {:?})", result.pdf_type);
|
||||
process::exit(2);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!("PDF to Markdown Conversion");
|
||||
println!("==========================");
|
||||
println!("File: {}", pdf_path);
|
||||
println!();
|
||||
// Verbose output with headers
|
||||
eprintln!("PDF to Markdown Conversion");
|
||||
eprintln!("==========================");
|
||||
eprintln!("File: {}", pdf_path);
|
||||
eprintln!();
|
||||
|
||||
match result.pdf_type {
|
||||
PdfType::TextBased => {
|
||||
println!("Type: TEXT-BASED (direct extraction)");
|
||||
println!("Pages: {}", result.page_count);
|
||||
println!("Processing time: {}ms", result.processing_time_ms);
|
||||
eprintln!("Type: TEXT-BASED (direct extraction)");
|
||||
eprintln!("Pages: {}", result.page_count);
|
||||
eprintln!("Processing time: {}ms", result.processing_time_ms);
|
||||
|
||||
if let Some(markdown) = &result.markdown {
|
||||
if let Some(output) = output_file {
|
||||
fs::write(output, markdown).expect("Failed to write output file");
|
||||
println!();
|
||||
println!("Markdown written to: {}", output);
|
||||
println!("Length: {} characters", markdown.len());
|
||||
eprintln!();
|
||||
eprintln!("Markdown written to: {}", output);
|
||||
eprintln!("Length: {} characters", markdown.len());
|
||||
} else {
|
||||
println!();
|
||||
println!("--- Markdown Output ---");
|
||||
println!();
|
||||
eprintln!();
|
||||
eprintln!("--- Markdown Output ---");
|
||||
eprintln!();
|
||||
println!("{}", markdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
PdfType::Scanned | PdfType::ImageBased => {
|
||||
println!(
|
||||
eprintln!(
|
||||
"Type: {} (OCR required)",
|
||||
if result.pdf_type == PdfType::Scanned {
|
||||
"SCANNED"
|
||||
@@ -88,30 +106,30 @@ fn main() {
|
||||
"IMAGE-BASED"
|
||||
}
|
||||
);
|
||||
println!("Pages: {}", result.page_count);
|
||||
println!("Processing time: {}ms", result.processing_time_ms);
|
||||
println!();
|
||||
println!("This PDF requires OCR for text extraction.");
|
||||
println!("Consider using MinerU or similar OCR tool.");
|
||||
eprintln!("Pages: {}", result.page_count);
|
||||
eprintln!("Processing time: {}ms", result.processing_time_ms);
|
||||
eprintln!();
|
||||
eprintln!("This PDF requires OCR for text extraction.");
|
||||
eprintln!("Consider using MinerU or similar OCR tool.");
|
||||
process::exit(2);
|
||||
}
|
||||
PdfType::Mixed => {
|
||||
println!("Type: MIXED (partial text extraction)");
|
||||
println!("Pages: {}", result.page_count);
|
||||
println!("Processing time: {}ms", result.processing_time_ms);
|
||||
eprintln!("Type: MIXED (partial text extraction)");
|
||||
eprintln!("Pages: {}", result.page_count);
|
||||
eprintln!("Processing time: {}ms", result.processing_time_ms);
|
||||
|
||||
if let Some(markdown) = &result.markdown {
|
||||
println!();
|
||||
println!("Note: Some pages may contain images that require OCR.");
|
||||
println!();
|
||||
eprintln!();
|
||||
eprintln!("Note: Some pages may contain images that require OCR.");
|
||||
eprintln!();
|
||||
|
||||
if let Some(output) = output_file {
|
||||
fs::write(output, markdown).expect("Failed to write output file");
|
||||
println!("Markdown written to: {}", output);
|
||||
println!("Length: {} characters", markdown.len());
|
||||
eprintln!("Markdown written to: {}", output);
|
||||
eprintln!("Length: {} characters", markdown.len());
|
||||
} else {
|
||||
println!("--- Markdown Output ---");
|
||||
println!();
|
||||
eprintln!("--- Markdown Output ---");
|
||||
eprintln!();
|
||||
println!("{}", markdown);
|
||||
}
|
||||
}
|
||||
|
||||
+136
-4
@@ -9,6 +9,8 @@
|
||||
use crate::extractor::{group_into_lines, TextItem, TextLine};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// Options for markdown conversion
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MarkdownOptions {
|
||||
@@ -20,6 +22,12 @@ pub struct MarkdownOptions {
|
||||
pub detect_code: bool,
|
||||
/// Base font size for comparison
|
||||
pub base_font_size: Option<f32>,
|
||||
/// Remove standalone page numbers
|
||||
pub remove_page_numbers: bool,
|
||||
/// Convert URLs to markdown links
|
||||
pub format_urls: bool,
|
||||
/// Fix hyphenation (broken words across lines)
|
||||
pub fix_hyphenation: bool,
|
||||
}
|
||||
|
||||
impl Default for MarkdownOptions {
|
||||
@@ -29,6 +37,9 @@ impl Default for MarkdownOptions {
|
||||
detect_lists: true,
|
||||
detect_code: true,
|
||||
base_font_size: None,
|
||||
remove_page_numbers: true,
|
||||
format_urls: true,
|
||||
fix_hyphenation: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,8 +222,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
output.push('\n');
|
||||
}
|
||||
|
||||
// Clean up excessive newlines
|
||||
clean_markdown(output)
|
||||
// Clean up and post-process
|
||||
clean_markdown(output, &options)
|
||||
}
|
||||
|
||||
/// Merge drop caps with the appropriate line
|
||||
@@ -474,8 +485,23 @@ fn is_monospace_font(font_name: &str) -> bool {
|
||||
patterns.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
/// Clean up markdown output
|
||||
fn clean_markdown(mut text: String) -> String {
|
||||
/// Clean up markdown output with post-processing
|
||||
fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String {
|
||||
// Fix hyphenation first (before other processing)
|
||||
if options.fix_hyphenation {
|
||||
text = fix_hyphenation(&text);
|
||||
}
|
||||
|
||||
// Remove standalone page numbers
|
||||
if options.remove_page_numbers {
|
||||
text = remove_page_numbers(&text);
|
||||
}
|
||||
|
||||
// Format URLs as markdown links
|
||||
if options.format_urls {
|
||||
text = format_urls(&text);
|
||||
}
|
||||
|
||||
// Remove excessive newlines (more than 2 in a row)
|
||||
while text.contains("\n\n\n") {
|
||||
text = text.replace("\n\n\n", "\n\n");
|
||||
@@ -488,6 +514,112 @@ fn clean_markdown(mut text: String) -> String {
|
||||
text
|
||||
}
|
||||
|
||||
/// Fix words broken across lines with spaces before the continuation
|
||||
/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte"
|
||||
fn fix_hyphenation(text: &str) -> String {
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
// Fix "word - word" patterns that should be "word-word" (compound words)
|
||||
// But be careful not to break list items (which start with "- ")
|
||||
static SPACED_HYPHEN_RE: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap()
|
||||
});
|
||||
|
||||
let result = SPACED_HYPHEN_RE
|
||||
.replace_all(text, |caps: ®ex::Captures| {
|
||||
format!("{}-{}", &caps[1], &caps[2])
|
||||
})
|
||||
.to_string();
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Remove standalone page numbers (lines that are just 1-4 digit numbers)
|
||||
fn remove_page_numbers(text: &str) -> String {
|
||||
let mut result = Vec::new();
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
|
||||
for (i, line) in lines.iter().enumerate() {
|
||||
let trimmed = line.trim();
|
||||
|
||||
// Check if this line is just a number (1-4 digits)
|
||||
if trimmed.len() <= 4 && !trimmed.is_empty() && trimmed.chars().all(|c| c.is_ascii_digit())
|
||||
{
|
||||
// Check context to determine if this is a page number
|
||||
let prev_is_break = i > 0 && lines[i - 1].trim() == "---";
|
||||
let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---";
|
||||
let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty();
|
||||
let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty();
|
||||
|
||||
// Check if it's on its own line (surrounded by empty lines or page breaks)
|
||||
let is_isolated = (prev_is_break || prev_is_empty || i == 0)
|
||||
&& (next_is_break || next_is_empty || i + 1 == lines.len());
|
||||
|
||||
// Also remove numbers that appear right before a page break
|
||||
// (common pattern: content ends, page number, then ---)
|
||||
let before_break = i + 1 < lines.len()
|
||||
&& (lines[i + 1].trim() == "---"
|
||||
|| (i + 2 < lines.len()
|
||||
&& lines[i + 1].trim().is_empty()
|
||||
&& lines[i + 2].trim() == "---"));
|
||||
|
||||
if is_isolated || before_break {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
result.push(*line);
|
||||
}
|
||||
|
||||
result.join("\n")
|
||||
}
|
||||
|
||||
/// Convert URLs to markdown links
|
||||
fn format_urls(text: &str) -> String {
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
// Match URLs - we'll check context manually to avoid formatting already-linked URLs
|
||||
static URL_RE: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap());
|
||||
|
||||
let mut result = String::with_capacity(text.len());
|
||||
let mut last_end = 0;
|
||||
|
||||
for mat in URL_RE.find_iter(text) {
|
||||
let start = mat.start();
|
||||
let url = mat.as_str();
|
||||
|
||||
// Check if this URL is already in a markdown link by looking at preceding chars
|
||||
let before = if start >= 2 {
|
||||
&text[start - 2..start]
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let already_linked = before.ends_with("](") || before.ends_with("](");
|
||||
|
||||
// Also check if it's inside square brackets (link text)
|
||||
let prefix = &text[..start];
|
||||
let open_brackets = prefix.matches('[').count();
|
||||
let close_brackets = prefix.matches(']').count();
|
||||
let inside_link_text = open_brackets > close_brackets;
|
||||
|
||||
if already_linked || inside_link_text {
|
||||
// Already formatted, keep as-is
|
||||
result.push_str(&text[last_end..mat.end()]);
|
||||
} else {
|
||||
// Add text before this URL
|
||||
result.push_str(&text[last_end..start]);
|
||||
// Format as markdown link
|
||||
result.push_str(&format!("[{}]({})", url, url));
|
||||
}
|
||||
last_end = mat.end();
|
||||
}
|
||||
|
||||
// Add remaining text
|
||||
result.push_str(&text[last_end..]);
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -250,11 +250,17 @@ fn test_markdown_options_custom() {
|
||||
detect_lists: true,
|
||||
detect_code: false,
|
||||
base_font_size: Some(14.0),
|
||||
remove_page_numbers: false,
|
||||
format_urls: false,
|
||||
fix_hyphenation: false,
|
||||
};
|
||||
assert!(!opts.detect_headers);
|
||||
assert!(opts.detect_lists);
|
||||
assert!(!opts.detect_code);
|
||||
assert_eq!(opts.base_font_size, Some(14.0));
|
||||
assert!(!opts.remove_page_numbers);
|
||||
assert!(!opts.format_urls);
|
||||
assert!(!opts.fix_hyphenation);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
||||
Reference in New Issue
Block a user