mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: Rust PDF-to-Markdown library

Browse Source 
- Smart PDF type detection (text vs scanned) without full document load
- Text extraction using lopdf directly
- Markdown conversion with header/list/code detection
- CLI tools: detect-pdf, pdf2md

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-06 11:51:41 -08:00
co-authored by Claude Opus 4.5
commit 135ce518c1
8 changed files with 1574 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+25
View File
@@ -0,0 +1,25 @@
# Rust build artifacts
/target/
debug/
*.pdb
# Cargo lock (optional for libraries)
Cargo.lock
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Test files
*.pdf
!tests/fixtures/*.pdf
# Build cache
**/*.rs.bk
Cargo.toml
+34
View File
@@ -0,0 +1,34 @@
[package]
name = "pdf-to-markdown"
version = "0.1.0"
edition = "2021"
authors = ["Firecrawl Team"]
description = "Fast PDF to Markdown conversion with smart detection"
license = "MIT"
[dependencies]
# PDF parsing - use local firecrawl lopdf
lopdf = { path = "../../lopdf", features = ["rayon"] }
# Error handling
thiserror = "2.0"
# Parallel processing
rayon = "1.10"
# Logging
log = "0.4"
[dev-dependencies]
tempfile = "3.3"
[features]
default = []
[[bin]]
name = "pdf2md"
path = "src/bin/pdf2md.rs"
[[bin]]
name = "detect-pdf"
path = "src/bin/detect_pdf.rs"
src/bin/detect_pdf.rs
+94
View File
@@ -0,0 +1,94 @@
//! CLI tool for detecting PDF type (text-based vs scanned)
use pdf_to_markdown::{detect_pdf_type, PdfType};
use std::env;
use std::process;
use std::time::Instant;
fn main() {
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: {} <pdf_file>", args[0]);
eprintln!(" {} <pdf_file> --json", args[0]);
process::exit(1);
}
let pdf_path = &args[1];
let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false);
let start = Instant::now();
match detect_pdf_type(pdf_path) {
Ok(result) => {
let elapsed = start.elapsed();
if json_output {
println!(
r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"detection_time_ms":{}}}"#,
match result.pdf_type {
PdfType::TextBased => "text_based",
PdfType::Scanned => "scanned",
PdfType::ImageBased => "image_based",
PdfType::Mixed => "mixed",
},
result.page_count,
result.pages_sampled,
result.pages_with_text,
result.confidence,
result.title.as_ref().map(|t| format!("\"{}\"", t.replace('"', "\\\""))).unwrap_or_else(|| "null".to_string()),
elapsed.as_millis()
);
} else {
println!("PDF Type Detection Results");
println!("==========================");
println!("File: {}", pdf_path);
println!();
println!(
"Type: {}",
match result.pdf_type {
PdfType::TextBased => "TEXT-BASED (extractable text)",
PdfType::Scanned => "SCANNED (OCR needed)",
PdfType::ImageBased => "IMAGE-BASED (mostly images, OCR may help)",
PdfType::Mixed => "MIXED (some text, some images)",
}
);
println!("Confidence: {:.0}%", result.confidence * 100.0);
println!();
println!("Page count: {}", result.page_count);
println!("Pages sampled: {}", result.pages_sampled);
println!("Pages with text: {}", result.pages_with_text);
if let Some(title) = &result.title {
println!("Title: {}", title);
}
println!();
println!("Detection time: {}ms", elapsed.as_millis());
println!();
// Recommendations
match result.pdf_type {
PdfType::TextBased => {
println!("Recommendation: Use direct text extraction (fast)");
}
PdfType::Scanned => {
println!("Recommendation: Use OCR (MinerU or similar)");
}
PdfType::ImageBased => {
println!("Recommendation: Use OCR for best results");
}
PdfType::Mixed => {
println!("Recommendation: Try text extraction first, use OCR for image pages");
}
}
}
}
Err(e) => {
if json_output {
println!(r#"{{"error":"{}"}}"#, e);
} else {
eprintln!("Error: {}", e);
}
process::exit(1);
}
}
}
src/bin/pdf2md.rs
+127
View File
@@ -0,0 +1,127 @@
//! CLI tool for PDF to Markdown conversion
use pdf_to_markdown::{process_pdf, PdfType};
use std::env;
use std::fs;
use std::process;
use std::time::Instant;
fn main() {
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
eprintln!(" {} <pdf_file> --json", args[0]);
eprintln!();
eprintln!("Converts PDF to Markdown with smart type detection.");
eprintln!("Returns early if PDF is scanned (OCR needed).");
process::exit(1);
}
let pdf_path = &args[1];
let json_output = args.get(2).map(|a| a == "--json").unwrap_or(false);
let output_file = if !json_output { args.get(2) } else { None };
let start = Instant::now();
match process_pdf(pdf_path) {
Ok(result) => {
let _elapsed = start.elapsed();
if json_output {
let md_escaped = result
.markdown
.as_ref()
.map(|m| m.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n"))
.unwrap_or_default();
println!(
r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"markdown":"{}"}}"#,
match result.pdf_type {
PdfType::TextBased => "text_based",
PdfType::Scanned => "scanned",
PdfType::ImageBased => "image_based",
PdfType::Mixed => "mixed",
},
result.page_count,
result.text.is_some(),
result.processing_time_ms,
result.markdown.as_ref().map(|m| m.len()).unwrap_or(0),
md_escaped
);
} else {
println!("PDF to Markdown Conversion");
println!("==========================");
println!("File: {}", pdf_path);
println!();
match result.pdf_type {
PdfType::TextBased => {
println!("Type: TEXT-BASED (direct extraction)");
println!("Pages: {}", result.page_count);
println!("Processing time: {}ms", result.processing_time_ms);
if let Some(markdown) = &result.markdown {
if let Some(output) = output_file {
fs::write(output, markdown).expect("Failed to write output file");
println!();
println!("Markdown written to: {}", output);
println!("Length: {} characters", markdown.len());
} else {
println!();
println!("--- Markdown Output ---");
println!();
println!("{}", markdown);
}
}
}
PdfType::Scanned | PdfType::ImageBased => {
println!(
"Type: {} (OCR required)",
if result.pdf_type == PdfType::Scanned {
"SCANNED"
} else {
"IMAGE-BASED"
}
);
println!("Pages: {}", result.page_count);
println!("Processing time: {}ms", result.processing_time_ms);
println!();
println!("This PDF requires OCR for text extraction.");
println!("Consider using MinerU or similar OCR tool.");
process::exit(2);
}
PdfType::Mixed => {
println!("Type: MIXED (partial text extraction)");
println!("Pages: {}", result.page_count);
println!("Processing time: {}ms", result.processing_time_ms);
if let Some(markdown) = &result.markdown {
println!();
println!("Note: Some pages may contain images that require OCR.");
println!();
if let Some(output) = output_file {
fs::write(output, markdown).expect("Failed to write output file");
println!("Markdown written to: {}", output);
println!("Length: {} characters", markdown.len());
} else {
println!("--- Markdown Output ---");
println!();
println!("{}", markdown);
}
}
}
}
}
}
Err(e) => {
if json_output {
println!(r#"{{"error":"{}"}}"#, e);
} else {
eprintln!("Error: {}", e);
}
process::exit(1);
}
}
}
src/detector.rs
+376
View File
@@ -0,0 +1,376 @@
//! Smart PDF type detection without full document load
//!
//! This module detects whether a PDF is text-based, scanned, or image-based
//! by sampling content streams for text operators (Tj/TJ) without loading
//! all objects.
use crate::PdfError;
use lopdf::{Document, Object, ObjectId};
use std::path::Path;
/// PDF type classification
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PdfType {
/// PDF has extractable text (Tj/TJ operators found)
TextBased,
/// PDF appears to be scanned (images only, no text operators)
Scanned,
/// PDF contains mostly images with minimal/no text
ImageBased,
/// PDF has mix of text and image-heavy pages
Mixed,
}
/// Result of PDF type detection
#[derive(Debug)]
pub struct PdfTypeResult {
/// Detected PDF type
pub pdf_type: PdfType,
/// Number of pages in the document
pub page_count: u32,
/// Number of pages sampled for detection
pub pages_sampled: u32,
/// Number of pages with text operators found
pub pages_with_text: u32,
/// Confidence score (0.0 - 1.0)
pub confidence: f32,
/// Title from metadata (if available)
pub title: Option<String>,
}
/// Configuration for PDF type detection
#[derive(Debug, Clone)]
pub struct DetectionConfig {
/// Maximum number of pages to sample (default: 5)
pub max_pages_to_sample: u32,
/// Minimum text operator count per page to consider as text-based
pub min_text_ops_per_page: u32,
/// Threshold ratio of text pages to total pages for classification
pub text_page_ratio_threshold: f32,
}
impl Default for DetectionConfig {
fn default() -> Self {
Self {
max_pages_to_sample: 5,
min_text_ops_per_page: 3,
text_page_ratio_threshold: 0.6,
}
}
}
/// Detect PDF type from file path
pub fn detect_pdf_type<P: AsRef<Path>>(path: P) -> Result<PdfTypeResult, PdfError> {
detect_pdf_type_with_config(path, DetectionConfig::default())
}
/// Detect PDF type from file path with custom configuration
pub fn detect_pdf_type_with_config<P: AsRef<Path>>(
path: P,
config: DetectionConfig,
) -> Result<PdfTypeResult, PdfError> {
// First, load metadata only (fast operation)
let metadata = Document::load_metadata(&path)?;
// Then load the full document for content inspection
// We use filtered loading to skip heavy objects we don't need
let doc = Document::load(&path)?;
detect_from_document(&doc, metadata.page_count, &config)
}
/// Detect PDF type from memory buffer
pub fn detect_pdf_type_mem(buffer: &[u8]) -> Result<PdfTypeResult, PdfError> {
detect_pdf_type_mem_with_config(buffer, DetectionConfig::default())
}
/// Detect PDF type from memory buffer with custom configuration
pub fn detect_pdf_type_mem_with_config(
buffer: &[u8],
config: DetectionConfig,
) -> Result<PdfTypeResult, PdfError> {
// Load metadata first (fast)
let metadata = Document::load_metadata_mem(buffer)?;
// Load document for inspection
let doc = Document::load_mem(buffer)?;
detect_from_document(&doc, metadata.page_count, &config)
}
/// Internal detection logic on a loaded document
fn detect_from_document(
doc: &Document,
page_count: u32,
config: &DetectionConfig,
) -> Result<PdfTypeResult, PdfError> {
let pages = doc.get_pages();
let total_pages = pages.len() as u32;
// Sample pages for text operator detection
let pages_to_sample = std::cmp::min(config.max_pages_to_sample, total_pages);
// Sample strategy: first page, last page, and evenly distributed pages
let sample_indices: Vec<u32> = if pages_to_sample >= total_pages {
(1..=total_pages).collect()
} else {
let mut indices = Vec::with_capacity(pages_to_sample as usize);
indices.push(1); // Always sample first page
if pages_to_sample > 1 {
indices.push(total_pages); // Always sample last page
}
// Add evenly distributed pages in between
let remaining = pages_to_sample.saturating_sub(2);
if remaining > 0 && total_pages > 2 {
let step = (total_pages - 2) / (remaining + 1);
for i in 1..=remaining {
let idx = 1 + (step * i);
if idx > 1 && idx < total_pages && !indices.contains(&idx) {
indices.push(idx);
}
}
}
indices.sort();
indices.dedup();
indices
};
let mut pages_with_text = 0u32;
let mut pages_with_images = 0u32;
let mut total_text_ops = 0u32;
for page_num in &sample_indices {
if let Some(&page_id) = pages.get(page_num) {
let analysis = analyze_page_content(doc, page_id);
if analysis.text_operator_count >= config.min_text_ops_per_page {
pages_with_text += 1;
}
if analysis.has_images {
pages_with_images += 1;
}
total_text_ops += analysis.text_operator_count;
}
}
let pages_sampled = sample_indices.len() as u32;
let text_ratio = if pages_sampled > 0 {
pages_with_text as f32 / pages_sampled as f32
} else {
0.0
};
// Classification logic
let (pdf_type, confidence) = if text_ratio >= config.text_page_ratio_threshold {
(PdfType::TextBased, text_ratio)
} else if pages_with_text == 0 && pages_with_images > 0 {
if total_text_ops == 0 {
(PdfType::Scanned, 0.95)
} else {
(PdfType::ImageBased, 0.8)
}
} else if pages_with_text > 0 && pages_with_images > 0 {
(PdfType::Mixed, 0.7)
} else if total_text_ops == 0 {
(PdfType::Scanned, 0.9)
} else {
(PdfType::TextBased, text_ratio.max(0.5))
};
// Try to get title from metadata
let title = get_document_title(doc);
Ok(PdfTypeResult {
pdf_type,
page_count,
pages_sampled,
pages_with_text,
confidence,
title,
})
}
/// Page content analysis result
struct PageAnalysis {
text_operator_count: u32,
has_images: bool,
}
/// Analyze a page's content stream for text operators and images
fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis {
let mut text_ops = 0u32;
let mut has_images = false;
// Get content streams for this page
let content_streams = doc.get_page_contents(page_id);
for content_id in content_streams {
if let Ok(Object::Stream(stream)) = doc.get_object(content_id) {
// Try to decompress and scan content
let content = match stream.decompressed_content() {
Ok(data) => data,
Err(_) => stream.content.clone(),
};
// Scan for text operators (Tj, TJ)
let (ops, imgs) = scan_content_for_text_operators(&content);
text_ops += ops;
has_images = has_images || imgs;
}
}
// Also check for XObject images in page resources
if !has_images {
has_images = page_has_images(doc, page_id);
}
PageAnalysis {
text_operator_count: text_ops,
has_images,
}
}
/// Fast scan of content stream bytes for text operators
///
/// This is a fast heuristic scan that looks for:
/// - "Tj" - show text string
/// - "TJ" - show text with individual glyph positioning
/// - "'" - move to next line and show text
/// - "\"" - set word/char spacing, move to next line, show text
fn scan_content_for_text_operators(content: &[u8]) -> (u32, bool) {
let mut text_ops = 0u32;
let mut has_images = false;
// Simple state machine to find operators
let mut i = 0;
while i < content.len() {
let b = content[i];
// Look for 'T' followed by 'j' or 'J'
if b == b'T' && i + 1 < content.len() {
let next = content[i + 1];
if next == b'j' || next == b'J' {
// Verify it's an operator (followed by whitespace or newline)
if i + 2 >= content.len()
|| content[i + 2].is_ascii_whitespace()
|| content[i + 2] == b'\n'
|| content[i + 2] == b'\r'
{
text_ops += 1;
}
}
}
// Look for BT (Begin Text) as additional confirmation
if b == b'B' && i + 1 < content.len() && content[i + 1] == b'T' {
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
// BT found - text block marker
}
}
// Look for 'Do' operator (XObject/image placement)
if b == b'D' && i + 1 < content.len() && content[i + 1] == b'o' {
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
has_images = true;
}
}
i += 1;
}
(text_ops, has_images)
}
/// Check if page has image XObjects in resources
fn page_has_images(doc: &Document, page_id: ObjectId) -> bool {
if let Ok(page_dict) = doc.get_dictionary(page_id) {
// Get Resources
let resources = match page_dict.get(b"Resources") {
Ok(Object::Reference(id)) => doc.get_dictionary(*id).ok(),
Ok(Object::Dictionary(dict)) => Some(dict),
_ => None,
};
if let Some(resources) = resources {
// Check XObject dictionary
if let Ok(xobject) = resources.get(b"XObject") {
let xobject_dict = match xobject {
Object::Reference(id) => doc.get_dictionary(*id).ok(),
Object::Dictionary(dict) => Some(dict),
_ => None,
};
if let Some(xobject_dict) = xobject_dict {
for (_, value) in xobject_dict.iter() {
if let Ok(xobj_ref) = value.as_reference() {
if let Ok(xobj) = doc.get_object(xobj_ref) {
if let Ok(stream) = xobj.as_stream() {
// Check if it's an Image subtype
if let Ok(subtype) = stream.dict.get(b"Subtype") {
if let Ok(name) = subtype.as_name() {
if name == b"Image" {
return true;
}
}
}
}
}
}
}
}
}
}
}
false
}
/// Get document title from Info dictionary
fn get_document_title(doc: &Document) -> Option<String> {
let info_ref = doc.trailer.get(b"Info").ok()?.as_reference().ok()?;
let info = doc.get_dictionary(info_ref).ok()?;
let title_obj = info.get(b"Title").ok()?;
match title_obj {
Object::String(bytes, _) => {
// Handle UTF-16BE encoding (BOM: 0xFE 0xFF)
if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF {
let utf16: Vec<u16> = bytes[2..]
.chunks_exact(2)
.map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
.collect();
Some(String::from_utf16_lossy(&utf16))
} else {
Some(String::from_utf8_lossy(bytes).to_string())
}
}
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_scan_content_operators() {
// Sample PDF content stream with text operators
let content = b"BT /F1 12 Tf 100 700 Td (Hello World) Tj ET";
let (ops, imgs) = scan_content_for_text_operators(content);
assert_eq!(ops, 1);
assert!(!imgs);
// Content with TJ array
let content2 = b"BT /F1 12 Tf 100 700 Td [(H) 10 (ello)] TJ ET";
let (ops2, _) = scan_content_for_text_operators(content2);
assert_eq!(ops2, 1);
// Content with Do (image)
let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q";
let (ops3, imgs3) = scan_content_for_text_operators(content3);
assert_eq!(ops3, 0);
assert!(imgs3);
}
}
src/extractor.rs
+371
View File
@@ -0,0 +1,371 @@
//! Text extraction from PDF using lopdf
//!
//! This module extracts text with position information for structure detection.
use crate::PdfError;
use lopdf::{Document, Object, ObjectId};
use std::path::Path;
/// A text item with position information
#[derive(Debug, Clone)]
pub struct TextItem {
/// The text content
pub text: String,
/// X position on page
pub x: f32,
/// Y position on page (PDF coordinates, origin at bottom-left)
pub y: f32,
/// Width of text
pub width: f32,
/// Height (approximated from font size)
pub height: f32,
/// Font name
pub font: String,
/// Font size
pub font_size: f32,
/// Page number (1-indexed)
pub page: u32,
}
/// A line of text (grouped text items)
#[derive(Debug, Clone)]
pub struct TextLine {
pub items: Vec<TextItem>,
pub y: f32,
pub page: u32,
}
impl TextLine {
pub fn text(&self) -> String {
self.items.iter().map(|i| i.text.as_str()).collect::<Vec<_>>().join(" ")
}
}
/// Extract text from PDF file as plain string
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
let doc = Document::load(path)?;
extract_text_from_doc(&doc)
}
/// Extract text from PDF memory buffer
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
let doc = Document::load_mem(buffer)?;
extract_text_from_doc(&doc)
}
/// Extract text from loaded document
fn extract_text_from_doc(doc: &Document) -> Result<String, PdfError> {
let pages = doc.get_pages();
let page_nums: Vec<u32> = pages.keys().cloned().collect();
doc.extract_text(&page_nums)
.map_err(|e| PdfError::Parse(e.to_string()))
}
/// Extract text with position information from PDF file
pub fn extract_text_with_positions<P: AsRef<Path>>(path: P) -> Result<Vec<TextItem>, PdfError> {
let doc = Document::load(path)?;
extract_positioned_text_from_doc(&doc)
}
/// Extract text with positions from memory buffer
pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result<Vec<TextItem>, PdfError> {
let doc = Document::load_mem(buffer)?;
extract_positioned_text_from_doc(&doc)
}
/// Extract positioned text from loaded document
fn extract_positioned_text_from_doc(doc: &Document) -> Result<Vec<TextItem>, PdfError> {
let pages = doc.get_pages();
let mut all_items = Vec::new();
for (page_num, &page_id) in pages.iter() {
let items = extract_page_text_items(doc, page_id, *page_num)?;
all_items.extend(items);
}
Ok(all_items)
}
/// Extract text items from a single page
fn extract_page_text_items(
doc: &Document,
page_id: ObjectId,
page_num: u32,
) -> Result<Vec<TextItem>, PdfError> {
use lopdf::content::Content;
let mut items = Vec::new();
// Get fonts for encoding
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
// Get content
let content_data = doc.get_page_content(page_id)
.map_err(|e| PdfError::Parse(e.to_string()))?;
let content = Content::decode(&content_data)
.map_err(|e| PdfError::Parse(e.to_string()))?;
// Text state tracking
let mut current_font = String::new();
let mut current_font_size: f32 = 12.0;
let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut in_text_block = false;
for op in &content.operations {
match op.operator.as_str() {
"BT" => {
// Begin text block
in_text_block = true;
text_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0];
line_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0];
}
"ET" => {
// End text block
in_text_block = false;
}
"Tf" => {
// Set font and size
if op.operands.len() >= 2 {
if let Ok(name) = op.operands[0].as_name() {
current_font = String::from_utf8_lossy(name).to_string();
}
if let Ok(size) = op.operands[1].as_f32() {
current_font_size = size;
} else if let Ok(size) = op.operands[1].as_i64() {
current_font_size = size as f32;
}
}
}
"Td" | "TD" => {
// Move text position
if op.operands.len() >= 2 {
let tx = get_number(&op.operands[0]).unwrap_or(0.0);
let ty = get_number(&op.operands[1]).unwrap_or(0.0);
line_matrix[4] += tx;
line_matrix[5] += ty;
text_matrix = line_matrix;
}
}
"Tm" => {
// Set text matrix
if op.operands.len() >= 6 {
for (i, operand) in op.operands.iter().take(6).enumerate() {
text_matrix[i] = get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 });
}
line_matrix = text_matrix;
}
}
"T*" => {
// Move to start of next line
line_matrix[5] -= current_font_size * 1.2; // Approximate line height
text_matrix = line_matrix;
}
"Tj" => {
// Show text string
if in_text_block && !op.operands.is_empty() {
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font) {
if !text.trim().is_empty() {
items.push(TextItem {
text,
x: text_matrix[4],
y: text_matrix[5],
width: 0.0, // Would need glyph widths
height: current_font_size,
font: current_font.clone(),
font_size: current_font_size,
page: page_num,
});
}
}
}
}
"TJ" => {
// Show text with positioning
if in_text_block && !op.operands.is_empty() {
if let Ok(array) = op.operands[0].as_array() {
let mut combined_text = String::new();
for item in array {
if let Some(text) = extract_text_from_operand(item, doc, &fonts, &current_font) {
combined_text.push_str(&text);
}
}
if !combined_text.trim().is_empty() {
items.push(TextItem {
text: combined_text,
x: text_matrix[4],
y: text_matrix[5],
width: 0.0,
height: current_font_size,
font: current_font.clone(),
font_size: current_font_size,
page: page_num,
});
}
}
}
}
"'" => {
// Move to next line and show text
line_matrix[5] -= current_font_size * 1.2;
text_matrix = line_matrix;
if !op.operands.is_empty() {
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font) {
if !text.trim().is_empty() {
items.push(TextItem {
text,
x: text_matrix[4],
y: text_matrix[5],
width: 0.0,
height: current_font_size,
font: current_font.clone(),
font_size: current_font_size,
page: page_num,
});
}
}
}
}
_ => {}
}
}
Ok(items)
}
/// Helper to get f32 from Object
fn get_number(obj: &Object) -> Option<f32> {
match obj {
Object::Integer(i) => Some(*i as f32),
Object::Real(r) => Some(*r as f32),
_ => None,
}
}
/// Extract text from a text operand, handling encoding
fn extract_text_from_operand(
obj: &Object,
doc: &Document,
fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
current_font: &str,
) -> Option<String> {
if let Object::String(bytes, _) = obj {
// Try to decode using font encoding
if let Some(font_dict) = fonts.get(current_font.as_bytes()) {
if let Ok(encoding) = font_dict.get_font_encoding(doc) {
if let Ok(text) = Document::decode_text(&encoding, bytes) {
return Some(text);
}
}
}
// Fallback: try UTF-16BE then Latin-1
if bytes.len() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF {
let utf16: Vec<u16> = bytes[2..]
.chunks_exact(2)
.map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
.collect();
return Some(String::from_utf16_lossy(&utf16));
}
// Latin-1 fallback
Some(bytes.iter().map(|&b| b as char).collect())
} else {
None
}
}
/// Group text items into lines based on Y position
pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
// Sort by page, then by Y (descending for PDF coords), then by X
let mut sorted = items;
sorted.sort_by(|a, b| {
a.page.cmp(&b.page)
.then(b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal))
.then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal))
});
let mut lines = Vec::new();
let mut current_line: Option<TextLine> = None;
let y_tolerance = 3.0; // Tolerance for same-line grouping
for item in sorted {
match &mut current_line {
Some(line) if line.page == item.page && (line.y - item.y).abs() < y_tolerance => {
// Same line
line.items.push(item);
}
_ => {
// New line
if let Some(line) = current_line.take() {
lines.push(line);
}
let y = item.y;
let page = item.page;
current_line = Some(TextLine {
items: vec![item],
y,
page,
});
}
}
}
if let Some(line) = current_line {
lines.push(line);
}
lines
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_group_into_lines() {
let items = vec![
TextItem {
text: "Hello".into(),
x: 100.0,
y: 700.0,
width: 50.0,
height: 12.0,
font: "F1".into(),
font_size: 12.0,
page: 1,
},
TextItem {
text: "World".into(),
x: 160.0,
y: 700.0,
width: 50.0,
height: 12.0,
font: "F1".into(),
font_size: 12.0,
page: 1,
},
TextItem {
text: "Next line".into(),
x: 100.0,
y: 680.0,
width: 80.0,
height: 12.0,
font: "F1".into(),
font_size: 12.0,
page: 1,
},
];
let lines = group_into_lines(items);
assert_eq!(lines.len(), 2);
assert_eq!(lines[0].text(), "Hello World");
assert_eq!(lines[1].text(), "Next line");
}
}
src/lib.rs
+150
View File
@@ -0,0 +1,150 @@
//! Smart PDF detection and text extraction using lopdf
//!
//! This module provides:
//! - Fast detection of scanned vs text-based PDFs without full document load
//! - Direct text extraction from text-based PDFs
//! - Markdown conversion with structure detection
pub mod detector;
pub mod extractor;
pub mod markdown;
pub use detector::{detect_pdf_type, PdfType, PdfTypeResult};
pub use extractor::{extract_text, extract_text_with_positions, TextItem};
pub use markdown::{to_markdown, MarkdownOptions};
use std::path::Path;
/// High-level PDF processing result
#[derive(Debug)]
pub struct PdfProcessResult {
/// The detected PDF type
pub pdf_type: PdfType,
/// Extracted text (if text-based PDF)
pub text: Option<String>,
/// Markdown output (if text-based PDF)
pub markdown: Option<String>,
/// Page count
pub page_count: u32,
/// Processing time in milliseconds
pub processing_time_ms: u64,
}
/// Process a PDF file with smart detection and extraction
///
/// This function will:
/// 1. Quickly detect if the PDF is text-based or scanned
/// 2. If text-based, extract text and convert to markdown
/// 3. If scanned, return early indicating OCR is needed
pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
// Step 1: Smart detection (fast, no full load)
let detection = detect_pdf_type(&path)?;
let result = match detection.pdf_type {
PdfType::TextBased => {
// Step 2: Full extraction for text-based PDFs
let text = extract_text(&path)?;
let markdown = to_markdown(&text, MarkdownOptions::default());
PdfProcessResult {
pdf_type: PdfType::TextBased,
text: Some(text),
markdown: Some(markdown),
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Scanned | PdfType::ImageBased => {
// Return early - OCR needed
PdfProcessResult {
pdf_type: detection.pdf_type,
text: None,
markdown: None,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Mixed => {
// Try to extract what we can
let text = extract_text(&path).ok();
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
PdfProcessResult {
pdf_type: PdfType::Mixed,
text,
markdown,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
};
Ok(result)
}
/// Process PDF from memory buffer
pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
// Step 1: Smart detection (fast, no full load)
let detection = detector::detect_pdf_type_mem(buffer)?;
let result = match detection.pdf_type {
PdfType::TextBased => {
// Step 2: Full extraction for text-based PDFs
let text = extractor::extract_text_mem(buffer)?;
let markdown = to_markdown(&text, MarkdownOptions::default());
PdfProcessResult {
pdf_type: PdfType::TextBased,
text: Some(text),
markdown: Some(markdown),
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Scanned | PdfType::ImageBased => {
PdfProcessResult {
pdf_type: detection.pdf_type,
text: None,
markdown: None,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Mixed => {
let text = extractor::extract_text_mem(buffer).ok();
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
PdfProcessResult {
pdf_type: PdfType::Mixed,
text,
markdown,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
};
Ok(result)
}
#[derive(Debug, thiserror::Error)]
pub enum PdfError {
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("PDF parsing error: {0}")]
Parse(String),
#[error("PDF is encrypted")]
Encrypted,
#[error("Invalid PDF structure")]
InvalidStructure,
}
impl From<lopdf::Error> for PdfError {
fn from(e: lopdf::Error) -> Self {
PdfError::Parse(e.to_string())
}
}
src/markdown.rs
+397
View File
@@ -0,0 +1,397 @@
//! Markdown conversion with structure detection
//!
//! This module converts extracted text to markdown, detecting:
//! - Headers (by font size)
//! - Lists (bullet points, numbered lists)
//! - Code blocks (monospace fonts, indentation)
//! - Paragraphs
use crate::extractor::{TextItem, TextLine, group_into_lines};
use std::collections::HashMap;
/// Options for markdown conversion
#[derive(Debug, Clone)]
pub struct MarkdownOptions {
/// Detect headers by font size
pub detect_headers: bool,
/// Detect list items
pub detect_lists: bool,
/// Detect code blocks
pub detect_code: bool,
/// Base font size for comparison
pub base_font_size: Option<f32>,
}
impl Default for MarkdownOptions {
fn default() -> Self {
Self {
detect_headers: true,
detect_lists: true,
detect_code: true,
base_font_size: None,
}
}
}
/// Convert plain text to markdown (basic conversion)
pub fn to_markdown(text: &str, options: MarkdownOptions) -> String {
let mut output = String::new();
let mut in_list = false;
let mut in_code_block = false;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
if in_list {
in_list = false;
}
if in_code_block {
output.push_str("```\n");
in_code_block = false;
}
output.push('\n');
continue;
}
// Detect list items
if options.detect_lists && is_list_item(trimmed) {
let formatted = format_list_item(trimmed);
output.push_str(&formatted);
output.push('\n');
in_list = true;
continue;
}
// Detect code blocks (indented lines)
if options.detect_code && is_code_like(trimmed) {
if !in_code_block {
output.push_str("```\n");
in_code_block = true;
}
output.push_str(trimmed);
output.push('\n');
continue;
} else if in_code_block {
output.push_str("```\n");
in_code_block = false;
}
// Regular paragraph text
output.push_str(trimmed);
output.push('\n');
}
if in_code_block {
output.push_str("```\n");
}
output
}
/// Convert positioned text items to markdown with structure detection
pub fn to_markdown_from_items(items: Vec<TextItem>, options: MarkdownOptions) -> String {
let lines = group_into_lines(items);
to_markdown_from_lines(lines, options)
}
/// Convert text lines to markdown
pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) -> String {
if lines.is_empty() {
return String::new();
}
// Calculate font statistics
let font_stats = calculate_font_stats(&lines);
let base_size = options.base_font_size.unwrap_or(font_stats.most_common_size);
let mut output = String::new();
let mut current_page = 0u32;
let mut prev_y = f32::MAX;
let mut in_list = false;
for line in lines {
// Page break
if line.page != current_page {
if current_page > 0 {
output.push_str("\n---\n\n");
}
current_page = line.page;
prev_y = f32::MAX;
}
// Paragraph break (large Y gap)
let y_gap = prev_y - line.y;
if y_gap > base_size * 2.0 && !output.ends_with("\n\n") {
if in_list {
in_list = false;
}
output.push('\n');
}
prev_y = line.y;
let text = line.text();
let trimmed = text.trim();
if trimmed.is_empty() {
continue;
}
// Detect headers by font size
if options.detect_headers {
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
if let Some(header_level) = detect_header_level(line_font_size, base_size) {
let prefix = "#".repeat(header_level);
output.push_str(&format!("{} {}\n\n", prefix, trimmed));
in_list = false;
continue;
}
}
// Detect list items
if options.detect_lists && is_list_item(trimmed) {
let formatted = format_list_item(trimmed);
output.push_str(&formatted);
output.push('\n');
in_list = true;
continue;
} else if in_list {
// Check if continuing list or ending
if !trimmed.starts_with(char::is_whitespace) {
in_list = false;
}
}
// Detect code blocks by font
if options.detect_code {
let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font));
if is_mono {
output.push_str(&format!("```\n{}\n```\n", trimmed));
continue;
}
}
// Regular text
output.push_str(trimmed);
output.push('\n');
}
// Clean up excessive newlines
clean_markdown(output)
}
/// Font statistics for a document
struct FontStats {
most_common_size: f32,
}
fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
let mut size_counts: HashMap<i32, usize> = HashMap::new();
for line in lines {
for item in &line.items {
let size_key = (item.font_size * 10.0) as i32; // Round to 0.1
*size_counts.entry(size_key).or_insert(0) += 1;
}
}
let most_common_size = size_counts
.iter()
.max_by_key(|(_, count)| *count)
.map(|(size, _)| *size as f32 / 10.0)
.unwrap_or(12.0);
FontStats {
most_common_size,
}
}
/// Detect header level from font size
fn detect_header_level(font_size: f32, base_size: f32) -> Option<usize> {
let ratio = font_size / base_size;
if ratio >= 2.0 {
Some(1) // H1
} else if ratio >= 1.5 {
Some(2) // H2
} else if ratio >= 1.25 {
Some(3) // H3
} else if ratio >= 1.1 {
Some(4) // H4
} else {
None // Regular text
}
}
/// Check if text looks like a list item
fn is_list_item(text: &str) -> bool {
let trimmed = text.trim_start();
// Bullet patterns
if trimmed.starts_with("• ")
|| trimmed.starts_with("- ")
|| trimmed.starts_with("* ")
|| trimmed.starts_with("○ ")
|| trimmed.starts_with("● ")
|| trimmed.starts_with("◦ ")
{
return true;
}
// Numbered list patterns: "1.", "1)", "(1)", "a.", "a)"
let first_chars: String = trimmed.chars().take(5).collect();
if first_chars.contains(|c: char| c.is_ascii_digit()) {
// Check for "1.", "1)", "10."
if let Some(idx) = first_chars.find(|c: char| c == '.' || c == ')') {
let prefix = &first_chars[..idx];
if prefix.chars().all(|c| c.is_ascii_digit()) {
return true;
}
}
}
// Letter list: "a.", "a)", "(a)"
if trimmed.len() >= 2 {
let first = trimmed.chars().next().unwrap();
let second = trimmed.chars().nth(1).unwrap();
if first.is_ascii_alphabetic() && (second == '.' || second == ')') {
return true;
}
if first == '(' && trimmed.chars().nth(2) == Some(')') {
return true;
}
}
false
}
/// Format list item to markdown
fn format_list_item(text: &str) -> String {
let trimmed = text.trim_start();
// Convert various bullet styles to markdown
if trimmed.starts_with("• ")
|| trimmed.starts_with("○ ")
|| trimmed.starts_with("● ")
|| trimmed.starts_with("◦ ")
{
return format!("- {}", &trimmed[2..].trim_start());
}
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
return trimmed.to_string();
}
// Keep numbered lists as-is (markdown supports them)
trimmed.to_string()
}
/// Check if text looks like code
fn is_code_like(text: &str) -> bool {
let trimmed = text.trim();
// Code patterns
let code_patterns = [
// Language keywords
"import ", "export ", "from ", "const ", "let ", "var ", "function ",
"class ", "def ", "pub fn ", "fn ", "async fn ", "impl ",
// Syntax patterns
"=> ", "-> ", ":: ", ":= ",
// Common code endings
];
for pattern in &code_patterns {
if trimmed.starts_with(pattern) {
return true;
}
}
// Check for code-like syntax
let special_chars: usize = trimmed.chars()
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
.count();
if special_chars >= 3 && trimmed.len() < 200 {
return true;
}
// Ends with semicolon or braces
if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') {
return true;
}
false
}
/// Check if font name indicates monospace
fn is_monospace_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
let patterns = [
"courier", "consolas", "monaco", "menlo", "mono", "fixed",
"terminal", "typewriter", "source code", "fira code",
"jetbrains", "inconsolata", "dejavu sans mono", "liberation mono",
];
patterns.iter().any(|p| lower.contains(p))
}
/// Clean up markdown output
fn clean_markdown(mut text: String) -> String {
// Remove excessive newlines (more than 2 in a row)
while text.contains("\n\n\n") {
text = text.replace("\n\n\n", "\n\n");
}
// Ensure ends with single newline
text = text.trim_end().to_string();
text.push('\n');
text
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_is_list_item() {
assert!(is_list_item("• Item one"));
assert!(is_list_item("- Item two"));
assert!(is_list_item("* Item three"));
assert!(is_list_item("1. First"));
assert!(is_list_item("2) Second"));
assert!(is_list_item("a. Letter item"));
assert!(!is_list_item("Regular text"));
}
#[test]
fn test_format_list_item() {
assert_eq!(format_list_item("• Item"), "- Item");
assert_eq!(format_list_item("- Item"), "- Item");
assert_eq!(format_list_item("1. First"), "1. First");
}
#[test]
fn test_is_code_like() {
assert!(is_code_like("const x = 5;"));
assert!(is_code_like("function foo() {"));
assert!(is_code_like("import React from 'react'"));
assert!(!is_code_like("This is regular text."));
}
#[test]
fn test_detect_header_level() {
assert_eq!(detect_header_level(24.0, 12.0), Some(1));
assert_eq!(detect_header_level(18.0, 12.0), Some(2));
assert_eq!(detect_header_level(15.0, 12.0), Some(3));
assert_eq!(detect_header_level(12.0, 12.0), None);
}
#[test]
fn test_to_markdown() {
let text = "• First item\n• Second item\n\nRegular paragraph.";
let md = to_markdown(text, MarkdownOptions::default());
assert!(md.contains("- First item"));
assert!(md.contains("- Second item"));
}
}