From 135ce518c1ffd0ebab7abc4e847c464f8d2aa891 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 6 Feb 2026 11:49:23 -0800 Subject: [PATCH] Initial commit: Rust PDF-to-Markdown library - Smart PDF type detection (text vs scanned) without full document load - Text extraction using lopdf directly - Markdown conversion with header/list/code detection - CLI tools: detect-pdf, pdf2md Co-Authored-By: Claude Opus 4.5 --- .gitignore | 25 +++ Cargo.toml | 34 ++++ src/bin/detect_pdf.rs | 94 ++++++++++ src/bin/pdf2md.rs | 127 ++++++++++++++ src/detector.rs | 376 +++++++++++++++++++++++++++++++++++++++ src/extractor.rs | 371 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 150 ++++++++++++++++ src/markdown.rs | 397 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1574 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/bin/detect_pdf.rs create mode 100644 src/bin/pdf2md.rs create mode 100644 src/detector.rs create mode 100644 src/extractor.rs create mode 100644 src/lib.rs create mode 100644 src/markdown.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..635ee68 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Rust build artifacts +/target/ +debug/ +*.pdb + +# Cargo lock (optional for libraries) +Cargo.lock + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Test files +*.pdf +!tests/fixtures/*.pdf + +# Build cache +**/*.rs.bk diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..75f7b77 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "pdf-to-markdown" +version = "0.1.0" +edition = "2021" +authors = ["Firecrawl Team"] +description = "Fast PDF to Markdown conversion with smart detection" +license = "MIT" + +[dependencies] +# PDF parsing - use local firecrawl lopdf +lopdf = { path = "../../lopdf", features = ["rayon"] } + +# Error handling +thiserror = "2.0" + +# Parallel processing +rayon = "1.10" + +# Logging +log = "0.4" + +[dev-dependencies] +tempfile = "3.3" + +[features] +default = [] + +[[bin]] +name = "pdf2md" +path = "src/bin/pdf2md.rs" + +[[bin]] +name = "detect-pdf" +path = "src/bin/detect_pdf.rs" diff --git a/src/bin/detect_pdf.rs b/src/bin/detect_pdf.rs new file mode 100644 index 0000000..92808fd --- /dev/null +++ b/src/bin/detect_pdf.rs @@ -0,0 +1,94 @@ +//! CLI tool for detecting PDF type (text-based vs scanned) + +use pdf_to_markdown::{detect_pdf_type, PdfType}; +use std::env; +use std::process; +use std::time::Instant; + +fn main() { + let args: Vec = env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: {} ", args[0]); + eprintln!(" {} --json", args[0]); + process::exit(1); + } + + let pdf_path = &args[1]; + let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false); + + let start = Instant::now(); + + match detect_pdf_type(pdf_path) { + Ok(result) => { + let elapsed = start.elapsed(); + + if json_output { + println!( + r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"detection_time_ms":{}}}"#, + match result.pdf_type { + PdfType::TextBased => "text_based", + PdfType::Scanned => "scanned", + PdfType::ImageBased => "image_based", + PdfType::Mixed => "mixed", + }, + result.page_count, + result.pages_sampled, + result.pages_with_text, + result.confidence, + result.title.as_ref().map(|t| format!("\"{}\"", t.replace('"', "\\\""))).unwrap_or_else(|| "null".to_string()), + elapsed.as_millis() + ); + } else { + println!("PDF Type Detection Results"); + println!("=========================="); + println!("File: {}", pdf_path); + println!(); + println!( + "Type: {}", + match result.pdf_type { + PdfType::TextBased => "TEXT-BASED (extractable text)", + PdfType::Scanned => "SCANNED (OCR needed)", + PdfType::ImageBased => "IMAGE-BASED (mostly images, OCR may help)", + PdfType::Mixed => "MIXED (some text, some images)", + } + ); + println!("Confidence: {:.0}%", result.confidence * 100.0); + println!(); + println!("Page count: {}", result.page_count); + println!("Pages sampled: {}", result.pages_sampled); + println!("Pages with text: {}", result.pages_with_text); + if let Some(title) = &result.title { + println!("Title: {}", title); + } + println!(); + println!("Detection time: {}ms", elapsed.as_millis()); + println!(); + + // Recommendations + match result.pdf_type { + PdfType::TextBased => { + println!("Recommendation: Use direct text extraction (fast)"); + } + PdfType::Scanned => { + println!("Recommendation: Use OCR (MinerU or similar)"); + } + PdfType::ImageBased => { + println!("Recommendation: Use OCR for best results"); + } + PdfType::Mixed => { + println!("Recommendation: Try text extraction first, use OCR for image pages"); + } + } + } + } + Err(e) => { + if json_output { + println!(r#"{{"error":"{}"}}"#, e); + } else { + eprintln!("Error: {}", e); + } + process::exit(1); + } + } +} diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs new file mode 100644 index 0000000..33c3cf0 --- /dev/null +++ b/src/bin/pdf2md.rs @@ -0,0 +1,127 @@ +//! CLI tool for PDF to Markdown conversion + +use pdf_to_markdown::{process_pdf, PdfType}; +use std::env; +use std::fs; +use std::process; +use std::time::Instant; + +fn main() { + let args: Vec = env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: {} [output_file]", args[0]); + eprintln!(" {} --json", args[0]); + eprintln!(); + eprintln!("Converts PDF to Markdown with smart type detection."); + eprintln!("Returns early if PDF is scanned (OCR needed)."); + process::exit(1); + } + + let pdf_path = &args[1]; + let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false); + let output_file = if !json_output { args.get(2) } else { None }; + + let start = Instant::now(); + + match process_pdf(pdf_path) { + Ok(result) => { + let _elapsed = start.elapsed(); + + if json_output { + let md_escaped = result + .markdown + .as_ref() + .map(|m| m.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n")) + .unwrap_or_default(); + + println!( + r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"markdown":"{}"}}"#, + match result.pdf_type { + PdfType::TextBased => "text_based", + PdfType::Scanned => "scanned", + PdfType::ImageBased => "image_based", + PdfType::Mixed => "mixed", + }, + result.page_count, + result.text.is_some(), + result.processing_time_ms, + result.markdown.as_ref().map(|m| m.len()).unwrap_or(0), + md_escaped + ); + } else { + println!("PDF to Markdown Conversion"); + println!("=========================="); + println!("File: {}", pdf_path); + println!(); + + match result.pdf_type { + PdfType::TextBased => { + println!("Type: TEXT-BASED (direct extraction)"); + println!("Pages: {}", result.page_count); + println!("Processing time: {}ms", result.processing_time_ms); + + if let Some(markdown) = &result.markdown { + if let Some(output) = output_file { + fs::write(output, markdown).expect("Failed to write output file"); + println!(); + println!("Markdown written to: {}", output); + println!("Length: {} characters", markdown.len()); + } else { + println!(); + println!("--- Markdown Output ---"); + println!(); + println!("{}", markdown); + } + } + } + PdfType::Scanned | PdfType::ImageBased => { + println!( + "Type: {} (OCR required)", + if result.pdf_type == PdfType::Scanned { + "SCANNED" + } else { + "IMAGE-BASED" + } + ); + println!("Pages: {}", result.page_count); + println!("Processing time: {}ms", result.processing_time_ms); + println!(); + println!("This PDF requires OCR for text extraction."); + println!("Consider using MinerU or similar OCR tool."); + process::exit(2); + } + PdfType::Mixed => { + println!("Type: MIXED (partial text extraction)"); + println!("Pages: {}", result.page_count); + println!("Processing time: {}ms", result.processing_time_ms); + + if let Some(markdown) = &result.markdown { + println!(); + println!("Note: Some pages may contain images that require OCR."); + println!(); + + if let Some(output) = output_file { + fs::write(output, markdown).expect("Failed to write output file"); + println!("Markdown written to: {}", output); + println!("Length: {} characters", markdown.len()); + } else { + println!("--- Markdown Output ---"); + println!(); + println!("{}", markdown); + } + } + } + } + } + } + Err(e) => { + if json_output { + println!(r#"{{"error":"{}"}}"#, e); + } else { + eprintln!("Error: {}", e); + } + process::exit(1); + } + } +} diff --git a/src/detector.rs b/src/detector.rs new file mode 100644 index 0000000..9e56366 --- /dev/null +++ b/src/detector.rs @@ -0,0 +1,376 @@ +//! Smart PDF type detection without full document load +//! +//! This module detects whether a PDF is text-based, scanned, or image-based +//! by sampling content streams for text operators (Tj/TJ) without loading +//! all objects. + +use crate::PdfError; +use lopdf::{Document, Object, ObjectId}; +use std::path::Path; + +/// PDF type classification +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PdfType { + /// PDF has extractable text (Tj/TJ operators found) + TextBased, + /// PDF appears to be scanned (images only, no text operators) + Scanned, + /// PDF contains mostly images with minimal/no text + ImageBased, + /// PDF has mix of text and image-heavy pages + Mixed, +} + +/// Result of PDF type detection +#[derive(Debug)] +pub struct PdfTypeResult { + /// Detected PDF type + pub pdf_type: PdfType, + /// Number of pages in the document + pub page_count: u32, + /// Number of pages sampled for detection + pub pages_sampled: u32, + /// Number of pages with text operators found + pub pages_with_text: u32, + /// Confidence score (0.0 - 1.0) + pub confidence: f32, + /// Title from metadata (if available) + pub title: Option, +} + +/// Configuration for PDF type detection +#[derive(Debug, Clone)] +pub struct DetectionConfig { + /// Maximum number of pages to sample (default: 5) + pub max_pages_to_sample: u32, + /// Minimum text operator count per page to consider as text-based + pub min_text_ops_per_page: u32, + /// Threshold ratio of text pages to total pages for classification + pub text_page_ratio_threshold: f32, +} + +impl Default for DetectionConfig { + fn default() -> Self { + Self { + max_pages_to_sample: 5, + min_text_ops_per_page: 3, + text_page_ratio_threshold: 0.6, + } + } +} + +/// Detect PDF type from file path +pub fn detect_pdf_type>(path: P) -> Result { + detect_pdf_type_with_config(path, DetectionConfig::default()) +} + +/// Detect PDF type from file path with custom configuration +pub fn detect_pdf_type_with_config>( + path: P, + config: DetectionConfig, +) -> Result { + // First, load metadata only (fast operation) + let metadata = Document::load_metadata(&path)?; + + // Then load the full document for content inspection + // We use filtered loading to skip heavy objects we don't need + let doc = Document::load(&path)?; + + detect_from_document(&doc, metadata.page_count, &config) +} + +/// Detect PDF type from memory buffer +pub fn detect_pdf_type_mem(buffer: &[u8]) -> Result { + detect_pdf_type_mem_with_config(buffer, DetectionConfig::default()) +} + +/// Detect PDF type from memory buffer with custom configuration +pub fn detect_pdf_type_mem_with_config( + buffer: &[u8], + config: DetectionConfig, +) -> Result { + // Load metadata first (fast) + let metadata = Document::load_metadata_mem(buffer)?; + + // Load document for inspection + let doc = Document::load_mem(buffer)?; + + detect_from_document(&doc, metadata.page_count, &config) +} + +/// Internal detection logic on a loaded document +fn detect_from_document( + doc: &Document, + page_count: u32, + config: &DetectionConfig, +) -> Result { + let pages = doc.get_pages(); + let total_pages = pages.len() as u32; + + // Sample pages for text operator detection + let pages_to_sample = std::cmp::min(config.max_pages_to_sample, total_pages); + + // Sample strategy: first page, last page, and evenly distributed pages + let sample_indices: Vec = if pages_to_sample >= total_pages { + (1..=total_pages).collect() + } else { + let mut indices = Vec::with_capacity(pages_to_sample as usize); + indices.push(1); // Always sample first page + + if pages_to_sample > 1 { + indices.push(total_pages); // Always sample last page + } + + // Add evenly distributed pages in between + let remaining = pages_to_sample.saturating_sub(2); + if remaining > 0 && total_pages > 2 { + let step = (total_pages - 2) / (remaining + 1); + for i in 1..=remaining { + let idx = 1 + (step * i); + if idx > 1 && idx < total_pages && !indices.contains(&idx) { + indices.push(idx); + } + } + } + + indices.sort(); + indices.dedup(); + indices + }; + + let mut pages_with_text = 0u32; + let mut pages_with_images = 0u32; + let mut total_text_ops = 0u32; + + for page_num in &sample_indices { + if let Some(&page_id) = pages.get(page_num) { + let analysis = analyze_page_content(doc, page_id); + if analysis.text_operator_count >= config.min_text_ops_per_page { + pages_with_text += 1; + } + if analysis.has_images { + pages_with_images += 1; + } + total_text_ops += analysis.text_operator_count; + } + } + + let pages_sampled = sample_indices.len() as u32; + let text_ratio = if pages_sampled > 0 { + pages_with_text as f32 / pages_sampled as f32 + } else { + 0.0 + }; + + // Classification logic + let (pdf_type, confidence) = if text_ratio >= config.text_page_ratio_threshold { + (PdfType::TextBased, text_ratio) + } else if pages_with_text == 0 && pages_with_images > 0 { + if total_text_ops == 0 { + (PdfType::Scanned, 0.95) + } else { + (PdfType::ImageBased, 0.8) + } + } else if pages_with_text > 0 && pages_with_images > 0 { + (PdfType::Mixed, 0.7) + } else if total_text_ops == 0 { + (PdfType::Scanned, 0.9) + } else { + (PdfType::TextBased, text_ratio.max(0.5)) + }; + + // Try to get title from metadata + let title = get_document_title(doc); + + Ok(PdfTypeResult { + pdf_type, + page_count, + pages_sampled, + pages_with_text, + confidence, + title, + }) +} + +/// Page content analysis result +struct PageAnalysis { + text_operator_count: u32, + has_images: bool, +} + +/// Analyze a page's content stream for text operators and images +fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { + let mut text_ops = 0u32; + let mut has_images = false; + + // Get content streams for this page + let content_streams = doc.get_page_contents(page_id); + + for content_id in content_streams { + if let Ok(Object::Stream(stream)) = doc.get_object(content_id) { + // Try to decompress and scan content + let content = match stream.decompressed_content() { + Ok(data) => data, + Err(_) => stream.content.clone(), + }; + + // Scan for text operators (Tj, TJ) + let (ops, imgs) = scan_content_for_text_operators(&content); + text_ops += ops; + has_images = has_images || imgs; + } + } + + // Also check for XObject images in page resources + if !has_images { + has_images = page_has_images(doc, page_id); + } + + PageAnalysis { + text_operator_count: text_ops, + has_images, + } +} + +/// Fast scan of content stream bytes for text operators +/// +/// This is a fast heuristic scan that looks for: +/// - "Tj" - show text string +/// - "TJ" - show text with individual glyph positioning +/// - "'" - move to next line and show text +/// - "\"" - set word/char spacing, move to next line, show text +fn scan_content_for_text_operators(content: &[u8]) -> (u32, bool) { + let mut text_ops = 0u32; + let mut has_images = false; + + // Simple state machine to find operators + let mut i = 0; + while i < content.len() { + let b = content[i]; + + // Look for 'T' followed by 'j' or 'J' + if b == b'T' && i + 1 < content.len() { + let next = content[i + 1]; + if next == b'j' || next == b'J' { + // Verify it's an operator (followed by whitespace or newline) + if i + 2 >= content.len() + || content[i + 2].is_ascii_whitespace() + || content[i + 2] == b'\n' + || content[i + 2] == b'\r' + { + text_ops += 1; + } + } + } + + // Look for BT (Begin Text) as additional confirmation + if b == b'B' && i + 1 < content.len() && content[i + 1] == b'T' { + if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() { + // BT found - text block marker + } + } + + // Look for 'Do' operator (XObject/image placement) + if b == b'D' && i + 1 < content.len() && content[i + 1] == b'o' { + if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() { + has_images = true; + } + } + + i += 1; + } + + (text_ops, has_images) +} + +/// Check if page has image XObjects in resources +fn page_has_images(doc: &Document, page_id: ObjectId) -> bool { + if let Ok(page_dict) = doc.get_dictionary(page_id) { + // Get Resources + let resources = match page_dict.get(b"Resources") { + Ok(Object::Reference(id)) => doc.get_dictionary(*id).ok(), + Ok(Object::Dictionary(dict)) => Some(dict), + _ => None, + }; + + if let Some(resources) = resources { + // Check XObject dictionary + if let Ok(xobject) = resources.get(b"XObject") { + let xobject_dict = match xobject { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(dict) => Some(dict), + _ => None, + }; + + if let Some(xobject_dict) = xobject_dict { + for (_, value) in xobject_dict.iter() { + if let Ok(xobj_ref) = value.as_reference() { + if let Ok(xobj) = doc.get_object(xobj_ref) { + if let Ok(stream) = xobj.as_stream() { + // Check if it's an Image subtype + if let Ok(subtype) = stream.dict.get(b"Subtype") { + if let Ok(name) = subtype.as_name() { + if name == b"Image" { + return true; + } + } + } + } + } + } + } + } + } + } + } + + false +} + +/// Get document title from Info dictionary +fn get_document_title(doc: &Document) -> Option { + let info_ref = doc.trailer.get(b"Info").ok()?.as_reference().ok()?; + let info = doc.get_dictionary(info_ref).ok()?; + let title_obj = info.get(b"Title").ok()?; + + match title_obj { + Object::String(bytes, _) => { + // Handle UTF-16BE encoding (BOM: 0xFE 0xFF) + if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { + let utf16: Vec = bytes[2..] + .chunks_exact(2) + .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) + .collect(); + Some(String::from_utf16_lossy(&utf16)) + } else { + Some(String::from_utf8_lossy(bytes).to_string()) + } + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_scan_content_operators() { + // Sample PDF content stream with text operators + let content = b"BT /F1 12 Tf 100 700 Td (Hello World) Tj ET"; + let (ops, imgs) = scan_content_for_text_operators(content); + assert_eq!(ops, 1); + assert!(!imgs); + + // Content with TJ array + let content2 = b"BT /F1 12 Tf 100 700 Td [(H) 10 (ello)] TJ ET"; + let (ops2, _) = scan_content_for_text_operators(content2); + assert_eq!(ops2, 1); + + // Content with Do (image) + let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q"; + let (ops3, imgs3) = scan_content_for_text_operators(content3); + assert_eq!(ops3, 0); + assert!(imgs3); + } +} diff --git a/src/extractor.rs b/src/extractor.rs new file mode 100644 index 0000000..b24be8b --- /dev/null +++ b/src/extractor.rs @@ -0,0 +1,371 @@ +//! Text extraction from PDF using lopdf +//! +//! This module extracts text with position information for structure detection. + +use crate::PdfError; +use lopdf::{Document, Object, ObjectId}; +use std::path::Path; + +/// A text item with position information +#[derive(Debug, Clone)] +pub struct TextItem { + /// The text content + pub text: String, + /// X position on page + pub x: f32, + /// Y position on page (PDF coordinates, origin at bottom-left) + pub y: f32, + /// Width of text + pub width: f32, + /// Height (approximated from font size) + pub height: f32, + /// Font name + pub font: String, + /// Font size + pub font_size: f32, + /// Page number (1-indexed) + pub page: u32, +} + +/// A line of text (grouped text items) +#[derive(Debug, Clone)] +pub struct TextLine { + pub items: Vec, + pub y: f32, + pub page: u32, +} + +impl TextLine { + pub fn text(&self) -> String { + self.items.iter().map(|i| i.text.as_str()).collect::>().join(" ") + } +} + +/// Extract text from PDF file as plain string +pub fn extract_text>(path: P) -> Result { + let doc = Document::load(path)?; + extract_text_from_doc(&doc) +} + +/// Extract text from PDF memory buffer +pub fn extract_text_mem(buffer: &[u8]) -> Result { + let doc = Document::load_mem(buffer)?; + extract_text_from_doc(&doc) +} + +/// Extract text from loaded document +fn extract_text_from_doc(doc: &Document) -> Result { + let pages = doc.get_pages(); + let page_nums: Vec = pages.keys().cloned().collect(); + + doc.extract_text(&page_nums) + .map_err(|e| PdfError::Parse(e.to_string())) +} + +/// Extract text with position information from PDF file +pub fn extract_text_with_positions>(path: P) -> Result, PdfError> { + let doc = Document::load(path)?; + extract_positioned_text_from_doc(&doc) +} + +/// Extract text with positions from memory buffer +pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result, PdfError> { + let doc = Document::load_mem(buffer)?; + extract_positioned_text_from_doc(&doc) +} + +/// Extract positioned text from loaded document +fn extract_positioned_text_from_doc(doc: &Document) -> Result, PdfError> { + let pages = doc.get_pages(); + let mut all_items = Vec::new(); + + for (page_num, &page_id) in pages.iter() { + let items = extract_page_text_items(doc, page_id, *page_num)?; + all_items.extend(items); + } + + Ok(all_items) +} + +/// Extract text items from a single page +fn extract_page_text_items( + doc: &Document, + page_id: ObjectId, + page_num: u32, +) -> Result, PdfError> { + use lopdf::content::Content; + + let mut items = Vec::new(); + + // Get fonts for encoding + let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); + + // Get content + let content_data = doc.get_page_content(page_id) + .map_err(|e| PdfError::Parse(e.to_string()))?; + + let content = Content::decode(&content_data) + .map_err(|e| PdfError::Parse(e.to_string()))?; + + // Text state tracking + let mut current_font = String::new(); + let mut current_font_size: f32 = 12.0; + let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut in_text_block = false; + + for op in &content.operations { + match op.operator.as_str() { + "BT" => { + // Begin text block + in_text_block = true; + text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + line_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + } + "ET" => { + // End text block + in_text_block = false; + } + "Tf" => { + // Set font and size + if op.operands.len() >= 2 { + if let Ok(name) = op.operands[0].as_name() { + current_font = String::from_utf8_lossy(name).to_string(); + } + if let Ok(size) = op.operands[1].as_f32() { + current_font_size = size; + } else if let Ok(size) = op.operands[1].as_i64() { + current_font_size = size as f32; + } + } + } + "Td" | "TD" => { + // Move text position + if op.operands.len() >= 2 { + let tx = get_number(&op.operands[0]).unwrap_or(0.0); + let ty = get_number(&op.operands[1]).unwrap_or(0.0); + line_matrix[4] += tx; + line_matrix[5] += ty; + text_matrix = line_matrix; + } + } + "Tm" => { + // Set text matrix + if op.operands.len() >= 6 { + for (i, operand) in op.operands.iter().take(6).enumerate() { + text_matrix[i] = get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 }); + } + line_matrix = text_matrix; + } + } + "T*" => { + // Move to start of next line + line_matrix[5] -= current_font_size * 1.2; // Approximate line height + text_matrix = line_matrix; + } + "Tj" => { + // Show text string + if in_text_block && !op.operands.is_empty() { + if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font) { + if !text.trim().is_empty() { + items.push(TextItem { + text, + x: text_matrix[4], + y: text_matrix[5], + width: 0.0, // Would need glyph widths + height: current_font_size, + font: current_font.clone(), + font_size: current_font_size, + page: page_num, + }); + } + } + } + } + "TJ" => { + // Show text with positioning + if in_text_block && !op.operands.is_empty() { + if let Ok(array) = op.operands[0].as_array() { + let mut combined_text = String::new(); + for item in array { + if let Some(text) = extract_text_from_operand(item, doc, &fonts, ¤t_font) { + combined_text.push_str(&text); + } + } + if !combined_text.trim().is_empty() { + items.push(TextItem { + text: combined_text, + x: text_matrix[4], + y: text_matrix[5], + width: 0.0, + height: current_font_size, + font: current_font.clone(), + font_size: current_font_size, + page: page_num, + }); + } + } + } + } + "'" => { + // Move to next line and show text + line_matrix[5] -= current_font_size * 1.2; + text_matrix = line_matrix; + if !op.operands.is_empty() { + if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font) { + if !text.trim().is_empty() { + items.push(TextItem { + text, + x: text_matrix[4], + y: text_matrix[5], + width: 0.0, + height: current_font_size, + font: current_font.clone(), + font_size: current_font_size, + page: page_num, + }); + } + } + } + } + _ => {} + } + } + + Ok(items) +} + +/// Helper to get f32 from Object +fn get_number(obj: &Object) -> Option { + match obj { + Object::Integer(i) => Some(*i as f32), + Object::Real(r) => Some(*r as f32), + _ => None, + } +} + +/// Extract text from a text operand, handling encoding +fn extract_text_from_operand( + obj: &Object, + doc: &Document, + fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, + current_font: &str, +) -> Option { + if let Object::String(bytes, _) = obj { + // Try to decode using font encoding + if let Some(font_dict) = fonts.get(current_font.as_bytes()) { + if let Ok(encoding) = font_dict.get_font_encoding(doc) { + if let Ok(text) = Document::decode_text(&encoding, bytes) { + return Some(text); + } + } + } + + // Fallback: try UTF-16BE then Latin-1 + if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF { + let utf16: Vec = bytes[2..] + .chunks_exact(2) + .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) + .collect(); + return Some(String::from_utf16_lossy(&utf16)); + } + + // Latin-1 fallback + Some(bytes.iter().map(|&b| b as char).collect()) + } else { + None + } +} + +/// Group text items into lines based on Y position +pub fn group_into_lines(items: Vec) -> Vec { + if items.is_empty() { + return Vec::new(); + } + + // Sort by page, then by Y (descending for PDF coords), then by X + let mut sorted = items; + sorted.sort_by(|a, b| { + a.page.cmp(&b.page) + .then(b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)) + .then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)) + }); + + let mut lines = Vec::new(); + let mut current_line: Option = None; + let y_tolerance = 3.0; // Tolerance for same-line grouping + + for item in sorted { + match &mut current_line { + Some(line) if line.page == item.page && (line.y - item.y).abs() < y_tolerance => { + // Same line + line.items.push(item); + } + _ => { + // New line + if let Some(line) = current_line.take() { + lines.push(line); + } + let y = item.y; + let page = item.page; + current_line = Some(TextLine { + items: vec![item], + y, + page, + }); + } + } + } + + if let Some(line) = current_line { + lines.push(line); + } + + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_group_into_lines() { + let items = vec![ + TextItem { + text: "Hello".into(), + x: 100.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + }, + TextItem { + text: "World".into(), + x: 160.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + }, + TextItem { + text: "Next line".into(), + x: 100.0, + y: 680.0, + width: 80.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + }, + ]; + + let lines = group_into_lines(items); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].text(), "Hello World"); + assert_eq!(lines[1].text(), "Next line"); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..65e3b96 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,150 @@ +//! Smart PDF detection and text extraction using lopdf +//! +//! This module provides: +//! - Fast detection of scanned vs text-based PDFs without full document load +//! - Direct text extraction from text-based PDFs +//! - Markdown conversion with structure detection + +pub mod detector; +pub mod extractor; +pub mod markdown; + +pub use detector::{detect_pdf_type, PdfType, PdfTypeResult}; +pub use extractor::{extract_text, extract_text_with_positions, TextItem}; +pub use markdown::{to_markdown, MarkdownOptions}; + +use std::path::Path; + +/// High-level PDF processing result +#[derive(Debug)] +pub struct PdfProcessResult { + /// The detected PDF type + pub pdf_type: PdfType, + /// Extracted text (if text-based PDF) + pub text: Option, + /// Markdown output (if text-based PDF) + pub markdown: Option, + /// Page count + pub page_count: u32, + /// Processing time in milliseconds + pub processing_time_ms: u64, +} + +/// Process a PDF file with smart detection and extraction +/// +/// This function will: +/// 1. Quickly detect if the PDF is text-based or scanned +/// 2. If text-based, extract text and convert to markdown +/// 3. If scanned, return early indicating OCR is needed +pub fn process_pdf>(path: P) -> Result { + let start = std::time::Instant::now(); + + // Step 1: Smart detection (fast, no full load) + let detection = detect_pdf_type(&path)?; + + let result = match detection.pdf_type { + PdfType::TextBased => { + // Step 2: Full extraction for text-based PDFs + let text = extract_text(&path)?; + let markdown = to_markdown(&text, MarkdownOptions::default()); + + PdfProcessResult { + pdf_type: PdfType::TextBased, + text: Some(text), + markdown: Some(markdown), + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + PdfType::Scanned | PdfType::ImageBased => { + // Return early - OCR needed + PdfProcessResult { + pdf_type: detection.pdf_type, + text: None, + markdown: None, + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + PdfType::Mixed => { + // Try to extract what we can + let text = extract_text(&path).ok(); + let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default())); + + PdfProcessResult { + pdf_type: PdfType::Mixed, + text, + markdown, + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + }; + + Ok(result) +} + +/// Process PDF from memory buffer +pub fn process_pdf_mem(buffer: &[u8]) -> Result { + let start = std::time::Instant::now(); + + // Step 1: Smart detection (fast, no full load) + let detection = detector::detect_pdf_type_mem(buffer)?; + + let result = match detection.pdf_type { + PdfType::TextBased => { + // Step 2: Full extraction for text-based PDFs + let text = extractor::extract_text_mem(buffer)?; + let markdown = to_markdown(&text, MarkdownOptions::default()); + + PdfProcessResult { + pdf_type: PdfType::TextBased, + text: Some(text), + markdown: Some(markdown), + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + PdfType::Scanned | PdfType::ImageBased => { + PdfProcessResult { + pdf_type: detection.pdf_type, + text: None, + markdown: None, + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + PdfType::Mixed => { + let text = extractor::extract_text_mem(buffer).ok(); + let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default())); + + PdfProcessResult { + pdf_type: PdfType::Mixed, + text, + markdown, + page_count: detection.page_count, + processing_time_ms: start.elapsed().as_millis() as u64, + } + } + }; + + Ok(result) +} + +#[derive(Debug, thiserror::Error)] +pub enum PdfError { + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("PDF parsing error: {0}")] + Parse(String), + #[error("PDF is encrypted")] + Encrypted, + #[error("Invalid PDF structure")] + InvalidStructure, +} + +impl From for PdfError { + fn from(e: lopdf::Error) -> Self { + PdfError::Parse(e.to_string()) + } +} diff --git a/src/markdown.rs b/src/markdown.rs new file mode 100644 index 0000000..052a5d4 --- /dev/null +++ b/src/markdown.rs @@ -0,0 +1,397 @@ +//! Markdown conversion with structure detection +//! +//! This module converts extracted text to markdown, detecting: +//! - Headers (by font size) +//! - Lists (bullet points, numbered lists) +//! - Code blocks (monospace fonts, indentation) +//! - Paragraphs + +use crate::extractor::{TextItem, TextLine, group_into_lines}; +use std::collections::HashMap; + +/// Options for markdown conversion +#[derive(Debug, Clone)] +pub struct MarkdownOptions { + /// Detect headers by font size + pub detect_headers: bool, + /// Detect list items + pub detect_lists: bool, + /// Detect code blocks + pub detect_code: bool, + /// Base font size for comparison + pub base_font_size: Option, +} + +impl Default for MarkdownOptions { + fn default() -> Self { + Self { + detect_headers: true, + detect_lists: true, + detect_code: true, + base_font_size: None, + } + } +} + +/// Convert plain text to markdown (basic conversion) +pub fn to_markdown(text: &str, options: MarkdownOptions) -> String { + let mut output = String::new(); + let mut in_list = false; + let mut in_code_block = false; + + for line in text.lines() { + let trimmed = line.trim(); + + if trimmed.is_empty() { + if in_list { + in_list = false; + } + if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } + output.push('\n'); + continue; + } + + // Detect list items + if options.detect_lists && is_list_item(trimmed) { + let formatted = format_list_item(trimmed); + output.push_str(&formatted); + output.push('\n'); + in_list = true; + continue; + } + + // Detect code blocks (indented lines) + if options.detect_code && is_code_like(trimmed) { + if !in_code_block { + output.push_str("```\n"); + in_code_block = true; + } + output.push_str(trimmed); + output.push('\n'); + continue; + } else if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } + + // Regular paragraph text + output.push_str(trimmed); + output.push('\n'); + } + + if in_code_block { + output.push_str("```\n"); + } + + output +} + +/// Convert positioned text items to markdown with structure detection +pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> String { + let lines = group_into_lines(items); + to_markdown_from_lines(lines, options) +} + +/// Convert text lines to markdown +pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> String { + if lines.is_empty() { + return String::new(); + } + + // Calculate font statistics + let font_stats = calculate_font_stats(&lines); + let base_size = options.base_font_size.unwrap_or(font_stats.most_common_size); + + let mut output = String::new(); + let mut current_page = 0u32; + let mut prev_y = f32::MAX; + let mut in_list = false; + + for line in lines { + // Page break + if line.page != current_page { + if current_page > 0 { + output.push_str("\n---\n\n"); + } + current_page = line.page; + prev_y = f32::MAX; + } + + // Paragraph break (large Y gap) + let y_gap = prev_y - line.y; + if y_gap > base_size * 2.0 && !output.ends_with("\n\n") { + if in_list { + in_list = false; + } + output.push('\n'); + } + prev_y = line.y; + + let text = line.text(); + let trimmed = text.trim(); + + if trimmed.is_empty() { + continue; + } + + // Detect headers by font size + if options.detect_headers { + let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + if let Some(header_level) = detect_header_level(line_font_size, base_size) { + let prefix = "#".repeat(header_level); + output.push_str(&format!("{} {}\n\n", prefix, trimmed)); + in_list = false; + continue; + } + } + + // Detect list items + if options.detect_lists && is_list_item(trimmed) { + let formatted = format_list_item(trimmed); + output.push_str(&formatted); + output.push('\n'); + in_list = true; + continue; + } else if in_list { + // Check if continuing list or ending + if !trimmed.starts_with(char::is_whitespace) { + in_list = false; + } + } + + // Detect code blocks by font + if options.detect_code { + let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); + if is_mono { + output.push_str(&format!("```\n{}\n```\n", trimmed)); + continue; + } + } + + // Regular text + output.push_str(trimmed); + output.push('\n'); + } + + // Clean up excessive newlines + clean_markdown(output) +} + +/// Font statistics for a document +struct FontStats { + most_common_size: f32, +} + +fn calculate_font_stats(lines: &[TextLine]) -> FontStats { + let mut size_counts: HashMap = HashMap::new(); + + for line in lines { + for item in &line.items { + let size_key = (item.font_size * 10.0) as i32; // Round to 0.1 + *size_counts.entry(size_key).or_insert(0) += 1; + } + } + + let most_common_size = size_counts + .iter() + .max_by_key(|(_, count)| *count) + .map(|(size, _)| *size as f32 / 10.0) + .unwrap_or(12.0); + + FontStats { + most_common_size, + } +} + +/// Detect header level from font size +fn detect_header_level(font_size: f32, base_size: f32) -> Option { + let ratio = font_size / base_size; + + if ratio >= 2.0 { + Some(1) // H1 + } else if ratio >= 1.5 { + Some(2) // H2 + } else if ratio >= 1.25 { + Some(3) // H3 + } else if ratio >= 1.1 { + Some(4) // H4 + } else { + None // Regular text + } +} + +/// Check if text looks like a list item +fn is_list_item(text: &str) -> bool { + let trimmed = text.trim_start(); + + // Bullet patterns + if trimmed.starts_with("• ") + || trimmed.starts_with("- ") + || trimmed.starts_with("* ") + || trimmed.starts_with("○ ") + || trimmed.starts_with("● ") + || trimmed.starts_with("◦ ") + { + return true; + } + + // Numbered list patterns: "1.", "1)", "(1)", "a.", "a)" + let first_chars: String = trimmed.chars().take(5).collect(); + if first_chars.contains(|c: char| c.is_ascii_digit()) { + // Check for "1.", "1)", "10." + if let Some(idx) = first_chars.find(|c: char| c == '.' || c == ')') { + let prefix = &first_chars[..idx]; + if prefix.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + // Letter list: "a.", "a)", "(a)" + if trimmed.len() >= 2 { + let first = trimmed.chars().next().unwrap(); + let second = trimmed.chars().nth(1).unwrap(); + if first.is_ascii_alphabetic() && (second == '.' || second == ')') { + return true; + } + if first == '(' && trimmed.chars().nth(2) == Some(')') { + return true; + } + } + + false +} + +/// Format list item to markdown +fn format_list_item(text: &str) -> String { + let trimmed = text.trim_start(); + + // Convert various bullet styles to markdown + if trimmed.starts_with("• ") + || trimmed.starts_with("○ ") + || trimmed.starts_with("● ") + || trimmed.starts_with("◦ ") + { + return format!("- {}", &trimmed[2..].trim_start()); + } + + if trimmed.starts_with("- ") || trimmed.starts_with("* ") { + return trimmed.to_string(); + } + + // Keep numbered lists as-is (markdown supports them) + trimmed.to_string() +} + +/// Check if text looks like code +fn is_code_like(text: &str) -> bool { + let trimmed = text.trim(); + + // Code patterns + let code_patterns = [ + // Language keywords + "import ", "export ", "from ", "const ", "let ", "var ", "function ", + "class ", "def ", "pub fn ", "fn ", "async fn ", "impl ", + // Syntax patterns + "=> ", "-> ", ":: ", ":= ", + // Common code endings + ]; + + for pattern in &code_patterns { + if trimmed.starts_with(pattern) { + return true; + } + } + + // Check for code-like syntax + let special_chars: usize = trimmed.chars() + .filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>')) + .count(); + + if special_chars >= 3 && trimmed.len() < 200 { + return true; + } + + // Ends with semicolon or braces + if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') { + return true; + } + + false +} + +/// Check if font name indicates monospace +fn is_monospace_font(font_name: &str) -> bool { + let lower = font_name.to_lowercase(); + let patterns = [ + "courier", "consolas", "monaco", "menlo", "mono", "fixed", + "terminal", "typewriter", "source code", "fira code", + "jetbrains", "inconsolata", "dejavu sans mono", "liberation mono", + ]; + + patterns.iter().any(|p| lower.contains(p)) +} + +/// Clean up markdown output +fn clean_markdown(mut text: String) -> String { + // Remove excessive newlines (more than 2 in a row) + while text.contains("\n\n\n") { + text = text.replace("\n\n\n", "\n\n"); + } + + // Ensure ends with single newline + text = text.trim_end().to_string(); + text.push('\n'); + + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_list_item() { + assert!(is_list_item("• Item one")); + assert!(is_list_item("- Item two")); + assert!(is_list_item("* Item three")); + assert!(is_list_item("1. First")); + assert!(is_list_item("2) Second")); + assert!(is_list_item("a. Letter item")); + assert!(!is_list_item("Regular text")); + } + + #[test] + fn test_format_list_item() { + assert_eq!(format_list_item("• Item"), "- Item"); + assert_eq!(format_list_item("- Item"), "- Item"); + assert_eq!(format_list_item("1. First"), "1. First"); + } + + #[test] + fn test_is_code_like() { + assert!(is_code_like("const x = 5;")); + assert!(is_code_like("function foo() {")); + assert!(is_code_like("import React from 'react'")); + assert!(!is_code_like("This is regular text.")); + } + + #[test] + fn test_detect_header_level() { + assert_eq!(detect_header_level(24.0, 12.0), Some(1)); + assert_eq!(detect_header_level(18.0, 12.0), Some(2)); + assert_eq!(detect_header_level(15.0, 12.0), Some(3)); + assert_eq!(detect_header_level(12.0, 12.0), None); + } + + #[test] + fn test_to_markdown() { + let text = "• First item\n• Second item\n\nRegular paragraph."; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("- First item")); + assert!(md.contains("- Second item")); + } +}