Abimael MartellandClaude Opus 4.6
a00ce46ab0
Eliminates double-parsing of PDFs by using the lopdf document model exclusively for ToUnicode CMap extraction. The old raw byte scanner parsed PDFs separately from lopdf and only handled FlateDecode, while lopdf handles FlateDecode + LZW + ASCII85. The new approach walks page fonts and Form XObject fonts via the document API, yielding ~7x speedup on text-heavy PDFs and ~1.2x overall. - Add FontCMaps::from_doc() with recursive Form XObject font walking - Remove ~270 lines of raw byte scanning code (from_pdf_bytes, extract_stream_from_raw_pdf, etc.) - Remove flate2 dependency (lopdf handles decompression internally) - Remove dead CMap lookup fallback branches (by_name, get_with_obj, base_font_name) - Remove unused font_base_names parameter from extract_text_from_operand - Skip U+FFFD replacement characters in CMap decode (PDF notdef glyph markers) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
48 lines
811 B
TOML
48 lines
811 B
TOML
[package]
|
|
name = "pdf-inspector"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
autobins = false
|
|
authors = ["Firecrawl Team"]
|
|
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
|
license = "MIT"
|
|
repository = "https://github.com/firecrawl/pdf-inspector"
|
|
|
|
[dependencies]
|
|
# PDF parsing
|
|
lopdf = { git = "https://github.com/J-F-Liu/lopdf", features = ["rayon"] }
|
|
|
|
# Error handling
|
|
thiserror = "2.0"
|
|
|
|
# Parallel processing
|
|
rayon = "1.10"
|
|
|
|
# Logging
|
|
log = "0.4"
|
|
env_logger = "0.11"
|
|
|
|
# Text processing
|
|
regex = "1.10"
|
|
once_cell = "1.19"
|
|
|
|
[dev-dependencies]
|
|
tempfile = "3.3"
|
|
|
|
[features]
|
|
default = []
|
|
|
|
[[bin]]
|
|
name = "pdf2md"
|
|
path = "src/bin/pdf2md.rs"
|
|
|
|
[[bin]]
|
|
name = "detect-pdf"
|
|
path = "src/bin/detect_pdf.rs"
|
|
|
|
[[bin]]
|
|
name = "dump_ops"
|
|
path = "src/bin/dump_ops.rs"
|
|
|
|
|