Compare commits

...
Author SHA1 Message Date
Abimael Martell 5cafddfa94 docs: add crates.io install instructions 2026-06-05 10:55:55 -07:00
Abimael Martell 4b6a05b505 chore: use crates.io lopdf 2026-06-05 10:46:44 -07:00
3 changed files with 32 additions and 31 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
# PDF parsing
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
lopdf = { version = "0.41.0", features = ["rayon"] }
# Error handling
thiserror = "2.0"
+24 -9
View File
@@ -1,5 +1,7 @@
# pdf-inspector
[![Crates.io](https://img.shields.io/crates/v/pdf-inspector.svg)](https://crates.io/crates/pdf-inspector)
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
@@ -71,9 +73,17 @@ console.log(result.markdown); // Markdown string or null
### Rust
Install from [crates.io](https://crates.io/crates/pdf-inspector):
```bash
cargo add pdf-inspector
```
Or add it manually:
```toml
[dependencies]
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
pdf-inspector = "0.1"
```
```rust
@@ -91,29 +101,34 @@ if let Some(markdown) = &result.markdown {
### CLI
```bash
# Install the CLI tools
cargo install pdf-inspector
# Convert PDF to Markdown
cargo run --bin pdf2md -- document.pdf
pdf2md document.pdf
# JSON output (for piping)
cargo run --bin pdf2md -- document.pdf --json
pdf2md document.pdf --json
# Raw markdown only (no headers)
cargo run --bin pdf2md -- document.pdf --raw
pdf2md document.pdf --raw
# Insert page break markers (<!-- Page N -->)
cargo run --bin pdf2md -- document.pdf --pages
pdf2md document.pdf --pages
# Process only specific pages
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
pdf2md document.pdf --select-pages 1,3,5-10
# Detection only (no extraction)
cargo run --bin detect-pdf -- document.pdf
cargo run --bin detect-pdf -- document.pdf --json
detect-pdf document.pdf
detect-pdf document.pdf --json
# Detection + layout analysis (tables, columns)
cargo run --bin detect-pdf -- document.pdf --analyze --json
detect-pdf document.pdf --analyze --json
```
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
## Architecture
```
+7 -21
View File
@@ -858,6 +858,13 @@ pub(crate) fn extract_text_from_operand(
// unmapped. Don't fall through to text-interpretation fallbacks
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
// 2-byte CIDs (Identity-H) are by far the common case; for
// an odd byte count we still emit at least one marker so
// detection downstream fires.
let cid_count = (bytes.len() / 2).max(1);
return Some("\u{FFFD}".repeat(cid_count));
}
// Try our custom encoding map from Differences arrays.
// The Differences array overrides specific codes in a base encoding (typically
@@ -966,27 +973,6 @@ pub(crate) fn extract_text_from_operand(
return Some(symbol_text);
}
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
// encodings — for these, an unmapped byte is a valid character
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
// production scrape_id 019de78c-... samples).
//
// For a CID font (has_cmap is set OR a /ToUnicode reference
// exists) with any non-ASCII bytes, emit a single U+FFFD per
// CID instead. This both replaces the mojibake with a proper
// "decode failed" marker AND keeps `detect_encoding_issues`
// tripping so the page is flagged for OCR — the existing
// garbage-detection path that the high-Latin-1 mojibake used
// to satisfy by accident.
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
// 2-byte CIDs (Identity-H) are by far the common case; for
// an odd byte count we still emit at least one marker so
// detection downstream fires.
let cid_count = (bytes.len() / 2).max(1);
return Some("\u{FFFD}".repeat(cid_count));
}
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
// use single-byte encodings where Latin-1 fallback is the