Compare commits

...
Author SHA1 Message Date
tomsideguide 13409859c9 fix(extractor): require a usable ToUnicode mapping to clear the gid flag
A mapping to U+FFFD (or an empty string) is rejected by extraction as
an invalid CMap result, so it must not count as decodable when deciding
whether gid-named Differences codes are resolvable.
2026-07-31 17:21:54 +01:00
tomsideguide 6796ad3efd fix(extractor): don't flag gid Differences names covered by ToUnicode
Pages were marked as having unresolvable gid-encoded fonts whenever any
font's /Differences array used gidNNNN glyph names, and when every page
carried such a font the whole document's markdown was suppressed.
LibreOffice exports do exactly this: subset fonts get /gidNNNN names in
Differences alongside a complete ToUnicode CMap that decodes them, so
ordinary text documents lost their entire markdown output even though
extraction decoded every glyph.

Track the character codes behind the gid names and only flag the font
when its ToUnicode CMap addresses none of them. Partially mapped codes
stay unflagged: an emoji ZWJ sequence maps whole on its first code, and
the remaining component-glyph codes are subset leftovers, not damage.
Fonts without ToUnicode, or whose CMap ignores the gid codes, are
flagged as before, and the downstream garbage/encoding checks still
catch partial breakage.
2026-07-31 16:50:39 +01:00
Abimael Martell 3c6eb8bf6b feat(site): add local WebAssembly demo (#181)
* feat(site): add local WebAssembly demo

* feat(site): parse PDFs on selection
2026-07-17 15:00:47 -07:00
Abimael Martell 5b287341a0 feat(wasm): add browser bindings (#180)
* feat(wasm): add browser bindings

* fix(wasm): address review feedback

* fix(wasm): preserve numeric plain text

* chore(wasm): prepare 0.1.2 release
2026-07-17 14:09:43 -07:00
21 changed files with 2815 additions and 54 deletions
+37
View File
@@ -42,6 +42,9 @@ jobs:
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check WASM formatting
run: cargo fmt --manifest-path wasm/Cargo.toml -- --check
clippy:
name: Clippy
runs-on: ubuntu-latest
@@ -80,3 +83,37 @@ jobs:
- name: Build
run: cargo build --release --verbose
wasm:
name: WebAssembly
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
components: clippy
- name: Cache cargo
uses: Swatinem/rust-cache@v2
with:
workspaces: |
wasm -> target
key: wasm
- name: Check WebAssembly bindings
run: cargo check --manifest-path wasm/Cargo.toml --target wasm32-unknown-unknown
- name: Check root package for WebAssembly
run: cargo check --target wasm32-unknown-unknown
- name: Lint WebAssembly bindings
run: cargo clippy --manifest-path wasm/Cargo.toml --target wasm32-unknown-unknown -- -D warnings
- name: Install wasm-pack
run: cargo install wasm-pack --version 0.15.0 --locked
- name: Test WebAssembly package
run: wasm-pack test --node --release wasm
+111
View File
@@ -0,0 +1,111 @@
name: Publish WebAssembly package
on:
push:
branches: [main]
paths: ['wasm/Cargo.toml']
workflow_dispatch:
permissions:
contents: read
id-token: write
env:
CARGO_TERM_COLOR: always
jobs:
check-version:
name: Check version change
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
outputs:
changed: ${{ steps.check.outputs.changed }}
package_exists: ${{ steps.check.outputs.package_exists }}
published: ${{ steps.check.outputs.published }}
version: ${{ steps.check.outputs.version }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Check package version
id: check
run: |
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("wasm/Cargo.toml").read_text())["package"]["version"])')
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "changed=true" >> "$GITHUB_OUTPUT"
elif git cat-file -e HEAD~1:wasm/Cargo.toml 2>/dev/null; then
OLD_VERSION=$(git show HEAD~1:wasm/Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "published=false" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "changed=true" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
if ! npm view "@firecrawl/pdf-inspector-wasm" name >/dev/null 2>&1; then
echo "package_exists=false" >> "$GITHUB_OUTPUT"
echo "published=false" >> "$GITHUB_OUTPUT"
echo "The initial package must be published once before trusted publishing can be configured."
exit 0
fi
echo "package_exists=true" >> "$GITHUB_OUTPUT"
if npm view "@firecrawl/pdf-inspector-wasm@$NEW_VERSION" version >/dev/null 2>&1; then
echo "published=true" >> "$GITHUB_OUTPUT"
else
echo "published=false" >> "$GITHUB_OUTPUT"
fi
publish:
name: Build and publish
needs: check-version
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.package_exists == 'true' && needs.check-version.outputs.published == 'false'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-unknown
- uses: actions/setup-node@v6
with:
node-version: '24'
registry-url: 'https://registry.npmjs.org'
- name: Install wasm-pack
run: cargo install wasm-pack --version 0.15.0 --locked
- name: Build browser package
run: wasm-pack build wasm --target web --scope firecrawl --out-dir pkg --release
- name: Prepare package metadata
run: |
node -e '
const fs = require("fs")
const path = "wasm/pkg/package.json"
const pkg = JSON.parse(fs.readFileSync(path, "utf8"))
pkg.name = "@firecrawl/pdf-inspector-wasm"
pkg.description = "Browser WebAssembly bindings for the pdf-inspector Rust PDF parser"
pkg.keywords = ["pdf", "pdf-parser", "webassembly", "wasm", "markdown", "rust", "firecrawl"]
pkg.repository = { type: "git", url: "https://github.com/firecrawl/pdf-inspector" }
pkg.homepage = "https://github.com/firecrawl/pdf-inspector/tree/main/wasm"
pkg.publishConfig = { access: "public" }
fs.writeFileSync(path, JSON.stringify(pkg, null, 2) + "\n")
'
- name: Inspect package contents
run: npm pack --dry-run ./wasm/pkg
- name: Publish package
run: npm publish ./wasm/pkg --provenance --access public
+3 -1
View File
@@ -1,10 +1,13 @@
# Rust build artifacts
/target/
/wasm/target/
/wasm/pkg/
debug/
*.pdb
# Cargo lock (optional for libraries)
Cargo.lock
!/wasm/Cargo.lock
# IDE
.idea/
@@ -39,4 +42,3 @@ test_output/
__pycache__/
*.pyc
.pytest_cache/
+18 -12
View File
@@ -12,13 +12,13 @@ readme = "docs/rust-api.md"
# alone exceeds that. external/bcmaps ships in the crate — tounicode.rs
# loads it at runtime relative to CARGO_MANIFEST_DIR.
include = [
"src/**",
"external/bcmaps/**",
"docs/rust-api.md",
"LICENSE",
"/src/**",
"/external/bcmaps/**",
"/docs/rust-api.md",
"/LICENSE",
# maturin derives the sdist file list from this allowlist; the stub must
# ship so wheels built from the sdist keep their type hints.
"pdf_inspector.pyi",
"/pdf_inspector.pyi",
]
[lib]
@@ -29,18 +29,11 @@ crate-type = ["lib", "cdylib"]
# Python bindings
pyo3 = { version = "0.25", features = ["extension-module", "abi3-py38"], optional = true }
# PDF parsing
lopdf = { version = "0.41.0", features = ["rayon"] }
# Error handling
thiserror = "2.0"
# Parallel processing
rayon = "1.10"
# Logging
log = "0.4"
env_logger = "0.11"
# Text processing
regex = "1.10"
@@ -50,6 +43,19 @@ unicode-normalization = "0.1"
# TrueType font parsing (for Identity-H CID font cmap extraction)
ttf-parser = "0.25"
# Native builds keep lopdf's parallel parser and CLI logging. Browser WASM is
# deliberately single-threaded so it works without cross-origin isolation.
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
lopdf = { version = "0.41.0", features = ["rayon"] }
rayon = "1.10"
env_logger = "0.11"
# Browser builds use JavaScript randomness for encrypted PDFs and embed the
# bundled CMaps because there is no filesystem at runtime.
[target.'cfg(target_arch = "wasm32")'.dependencies]
lopdf = { version = "0.41.0", default-features = false, features = ["wasm_js"] }
include_dir = "0.7"
[dev-dependencies]
tempfile = "3.3"
+23 -1
View File
@@ -5,7 +5,7 @@
[![PyPI](https://img.shields.io/pypi/v/pdf-inspector.svg)](https://pypi.org/project/pdf-inspector/)
[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md), [Node.js](napi/README.md), and [browser WebAssembly](wasm/README.md).
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
@@ -19,6 +19,7 @@ Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in
- **Multi-column layout** — Automatic detection of newspaper-style columns, sequential reading order, and RTL text support.
- **Encoding issue detection** — Automatically flags broken font encodings so callers can fall back to OCR.
- **Single document load** — The document is parsed once and shared between detection and extraction, avoiding redundant I/O.
- **Browser WebAssembly** — Run the same Rust parser locally in browsers and Web Workers, with embedded CMaps and no server round trip.
- **Lightweight** — Pure Rust, no ML models, no external services. Single dependency on `lopdf` for PDF parsing.
## Benchmark
@@ -77,6 +78,26 @@ console.log(result.markdown); // Markdown string or null
> Full API reference: [napi/README.md](napi/README.md)
### Browser WebAssembly
```bash
npm install @firecrawl/pdf-inspector-wasm
```
```javascript
import init, { processPdf } from '@firecrawl/pdf-inspector-wasm';
await init();
const response = await fetch('/document.pdf');
const pdf = new Uint8Array(await response.arrayBuffer());
const result = processPdf(pdf);
console.log(result.pdfType);
console.log(result.markdown);
```
> Full API reference: [wasm/README.md](wasm/README.md)
### Rust
Install from [crates.io](https://crates.io/crates/pdf-inspector):
@@ -188,6 +209,7 @@ src/
markdown/ — Markdown conversion and structure detection
bin/ — CLI tools (pdf2md, detect_pdf)
napi/ — Node.js/Bun bindings (napi-rs)
wasm/ — Browser bindings (wasm-bindgen)
```
## How classification works
+17
View File
@@ -19,3 +19,20 @@ The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
## Browser WebAssembly package
The browser package is published as `@firecrawl/pdf-inspector-wasm`. Its version lives in `wasm/Cargo.toml`, and `.github/workflows/publish-wasm.yml` builds the `web` target with `wasm-pack` before publishing the generated package.
The npm package must exist before a trusted publisher can be configured. For the first release only:
1. Build with `wasm-pack build wasm --target web --scope firecrawl --out-dir pkg --release`.
2. Inspect with `npm pack --dry-run ./wasm/pkg`.
3. Publish with `npm publish ./wasm/pkg --access public` from an authorized maintainer session.
4. In the package settings on npm, configure the GitHub Actions trusted publisher:
- Organization: `firecrawl`
- Repository: `pdf-inspector`
- Workflow: `publish-wasm.yml`
- Allowed action: `npm publish`
After that one-time bootstrap, bumping the version in `wasm/Cargo.toml` and merging it to `main` publishes through OIDC. Until the package exists, the workflow exits cleanly without attempting an unauthenticated first publish. See npm's [trusted publishing documentation](https://docs.npmjs.com/trusted-publishers/) for the registry-side setup.
+429 -4
View File
@@ -302,6 +302,171 @@
.text-link { color: var(--heat-dark); text-decoration: underline; text-decoration-color: var(--heat-16); text-underline-offset: 4px; }
.text-link:hover, .text-link:focus-visible { text-decoration-color: var(--heat); }
.demo-shell {
border: 1px solid var(--border);
border-radius: 12px;
background: var(--surface);
box-shadow: 0 22px 60px rgba(38,38,38,.07);
overflow: hidden;
}
.demo-toolbar {
min-height: 48px;
padding: 0 17px;
border-bottom: 1px solid var(--border);
display: flex;
align-items: center;
justify-content: space-between;
gap: 18px;
color: var(--ink-48);
font: 11px/1.3 var(--mono);
}
.demo-engine,
.demo-privacy { display: inline-flex; align-items: center; gap: 8px; }
.demo-engine strong { color: var(--ink); font-weight: 500; }
.demo-status-dot { width: 7px; height: 7px; border-radius: 50%; background: var(--heat); box-shadow: 0 0 0 4px var(--heat-8); }
.demo-lock { color: var(--heat); font-size: 12px; }
.demo-grid { display: grid; grid-template-columns: minmax(0, .86fr) minmax(0, 1.14fr); min-height: 500px; }
.demo-input {
padding: 28px;
border-right: 1px solid var(--border);
display: flex;
flex-direction: column;
background: var(--lighter);
}
.drop-zone {
min-height: 286px;
padding: 30px;
border: 1px dashed var(--ink-16);
border-radius: 10px;
display: flex;
flex: 1;
flex-direction: column;
align-items: center;
justify-content: center;
text-align: center;
background: var(--surface);
cursor: pointer;
transition: border-color .15s ease, background .15s ease, transform .15s ease;
}
.drop-zone:hover,
.drop-zone:focus-visible,
.drop-zone.is-dragging { border-color: var(--heat); background: var(--heat-4); }
.drop-zone.is-dragging { transform: scale(.995); }
.drop-mark {
width: 52px;
height: 58px;
margin-bottom: 21px;
border: 1px solid var(--ink-16);
border-radius: 5px 5px 9px 5px;
display: grid;
place-items: center;
position: relative;
color: var(--heat-dark);
background: var(--heat-4);
font: 600 11px/1 var(--mono);
letter-spacing: .03em;
}
.drop-mark::after {
content: "";
position: absolute;
top: -1px;
right: -1px;
width: 14px;
height: 14px;
border-left: 1px solid var(--ink-16);
border-bottom: 1px solid var(--ink-16);
background: var(--surface);
}
.drop-zone strong { font-size: 18px; font-weight: 500; letter-spacing: -.02em; }
.drop-zone > span:not(.drop-mark):not(.demo-choose) { margin-top: 7px; color: var(--ink-48); font-size: 13px; }
.demo-choose { margin-top: 20px; color: var(--ink); }
.demo-file {
margin-top: 14px;
padding: 13px 14px;
border: 1px solid var(--border);
border-radius: 8px;
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 4px 16px;
background: var(--surface);
}
.demo-file strong { overflow: hidden; text-overflow: ellipsis; font: 500 12px/1.35 var(--mono); white-space: nowrap; }
.demo-file span { grid-column: 1; color: var(--ink-32); font: 10px/1.3 var(--mono); }
.demo-file button {
grid-column: 2;
grid-row: 1 / 3;
padding: 0;
border: 0;
color: var(--ink-48);
background: transparent;
cursor: pointer;
font: 11px/1 var(--mono);
}
.demo-file button:hover,
.demo-file button:focus-visible { color: var(--heat-dark); }
.demo-message { margin: 13px 2px 0; min-height: 19px; color: var(--ink-48); font: 11px/1.55 var(--mono); }
.demo-message[data-tone="error"] { color: #a63520; }
.demo-message[data-tone="success"] { color: #34725a; }
.demo-output { min-width: 0; display: flex; flex-direction: column; background: var(--code); color: #f5f5f5; }
.demo-output-head {
min-height: 52px;
padding: 0 18px;
border-bottom: 1px solid rgba(255,255,255,.08);
display: flex;
align-items: center;
justify-content: space-between;
color: rgba(255,255,255,.42);
font: 11px/1 var(--mono);
}
.demo-copy {
height: 29px;
padding: 0 10px;
border: 1px solid rgba(255,255,255,.12);
border-radius: 6px;
color: rgba(255,255,255,.68);
background: rgba(255,255,255,.04);
cursor: pointer;
font: 10px/1 var(--mono);
}
.demo-copy:hover,
.demo-copy:focus-visible { border-color: var(--heat); color: #fff; }
.demo-copy[disabled] { opacity: .35; cursor: not-allowed; }
.demo-empty {
min-height: 448px;
padding: 40px;
display: grid;
place-items: center;
text-align: center;
color: rgba(255,255,255,.28);
font: 12px/1.7 var(--mono);
}
.demo-empty span { display: block; color: var(--heat); font-size: 22px; line-height: 1; }
.demo-result { min-height: 0; flex: 1; display: flex; flex-direction: column; }
.demo-result[hidden],
.demo-file[hidden],
.demo-empty[hidden] { display: none; }
.demo-metrics {
display: grid;
grid-template-columns: repeat(3, 1fr);
border-bottom: 1px solid rgba(255,255,255,.08);
}
.demo-metric { min-width: 0; padding: 14px 17px; border-right: 1px solid rgba(255,255,255,.08); }
.demo-metric:last-child { border-right: 0; }
.demo-metric span { display: block; margin-bottom: 6px; color: rgba(255,255,255,.32); font: 9px/1 var(--mono); letter-spacing: .07em; text-transform: uppercase; }
.demo-metric strong { display: block; overflow: hidden; text-overflow: ellipsis; color: #fff; font: 500 12px/1.25 var(--mono); white-space: nowrap; }
.demo-metric:first-child strong { color: #ff9c70; }
.markdown-output {
min-height: 0;
max-height: 364px;
margin: 0;
padding: 23px 20px 28px;
flex: 1;
overflow: auto;
color: rgba(255,255,255,.82);
font: 12px/1.7 var(--mono);
white-space: pre-wrap;
word-break: break-word;
}
.capability-grid { display: grid; grid-template-columns: repeat(3, 1fr); border: 1px solid var(--border); }
.capability {
min-height: 245px;
@@ -409,6 +574,9 @@
.hero-main { grid-template-columns: 1fr; gap: 44px; }
.terminal { max-width: 580px; }
.section-intro { grid-template-columns: 1fr; gap: 20px; }
.demo-grid { grid-template-columns: 1fr; }
.demo-input { border-right: 0; border-bottom: 1px solid var(--border); }
.demo-empty { min-height: 360px; }
.capability-grid { grid-template-columns: repeat(2, 1fr); }
.capability:nth-child(3n) { border-right: 1px solid var(--border); }
.capability:nth-child(2n) { border-right: 0; }
@@ -429,6 +597,13 @@
.package-row { grid-template-columns: 1fr; }
.package { border-right: 0; border-bottom: 1px solid var(--border); }
.package:last-child { border-bottom: 0; }
.demo-toolbar { padding: 12px 14px; align-items: flex-start; flex-direction: column; gap: 8px; }
.demo-input { padding: 18px; }
.drop-zone { min-height: 250px; padding: 24px 18px; }
.demo-metric { padding: 12px; }
.demo-metric strong { font-size: 11px; }
.demo-empty { min-height: 320px; padding: 28px; }
.markdown-output { max-height: 340px; padding: 20px 16px 24px; font-size: 11px; }
.capability-grid { grid-template-columns: 1fr; }
.capability, .capability:nth-child(3n), .capability:nth-child(2n) { min-height: 210px; border-right: 0; border-bottom: 1px solid var(--border); }
.capability:last-child { border-bottom: 0; }
@@ -463,6 +638,7 @@
</a>
<div class="nav-links">
<a href="#packages">Packages</a>
<a href="#demo">Demo</a>
<a href="#capabilities">Capabilities</a>
<a href="#architecture">Architecture</a>
<a href="#benchmark">Benchmark</a>
@@ -484,6 +660,7 @@
<p class="hero-copy">A Rust-powered, open-source parser that classifies PDFs and turns native text into clean, position-aware Markdown. Use it from Node.js or the bundled CLI, with packages also available from PyPI and crates.io.</p>
<div class="hero-actions">
<a class="button button-primary" href="https://github.com/firecrawl/pdf-inspector">View on GitHub <span class="arrow" aria-hidden="true"></span></a>
<a class="button" href="#demo">Try it locally <span class="arrow" aria-hidden="true"></span></a>
<a class="button" href="https://github.com/firecrawl/pdf-inspector#quick-start">Read the docs <span class="arrow" aria-hidden="true"></span></a>
</div>
</div>
@@ -538,9 +715,61 @@
</div>
</div>
<section id="demo" class="full-rule">
<div class="shell">
<div class="section-label pad"><span>[ <b>01</b> / 05 ]</span><span>Browser demo</span></div>
<div class="section-body pad">
<div class="section-intro">
<h2>Try it in<br>your browser.</h2>
<p>Drop in a native-text PDF to classify it and turn it into Markdown. The Rust core runs locally as WebAssembly in a background worker—your document never leaves this tab.</p>
</div>
<div class="demo-shell">
<div class="demo-toolbar">
<span class="demo-engine"><i class="demo-status-dot" aria-hidden="true"></i><strong>Rust core · WebAssembly</strong><span id="demo-engine-status">Loads on first run</span></span>
<span class="demo-privacy"><span class="demo-lock" aria-hidden="true"></span>PDF bytes stay in your browser</span>
</div>
<div class="demo-grid">
<div class="demo-input">
<input id="pdf-input" type="file" accept=".pdf,application/pdf" hidden>
<div class="drop-zone" id="drop-zone" role="button" tabindex="0" aria-controls="pdf-input" aria-label="Choose or drop a PDF file">
<span class="drop-mark" aria-hidden="true">PDF</span>
<strong>Drop a PDF here</strong>
<span>Native-text documents · up to 25 MB</span>
<span class="button demo-choose" aria-hidden="true">Choose PDF</span>
</div>
<div class="demo-file" id="demo-file" hidden>
<strong id="demo-file-name"></strong>
<span id="demo-file-size"></span>
<button id="demo-clear" type="button">Remove</button>
</div>
<p class="demo-message" id="demo-message" role="status" aria-live="polite">Select a PDF to begin.</p>
</div>
<div class="demo-output" aria-label="PDF parsing result">
<div class="demo-output-head">
<span>MARKDOWN OUTPUT</span>
<button class="demo-copy" id="demo-copy" type="button" disabled>Copy Markdown</button>
</div>
<div class="demo-empty" id="demo-empty">
<p><span aria-hidden="true">_</span><br>Your parsed Markdown will appear here.</p>
</div>
<div class="demo-result" id="demo-result" hidden>
<div class="demo-metrics">
<div class="demo-metric"><span>Document type</span><strong id="demo-type"></strong></div>
<div class="demo-metric"><span>Pages</span><strong id="demo-pages"></strong></div>
<div class="demo-metric"><span>Processing</span><strong id="demo-time"></strong></div>
</div>
<pre class="markdown-output" id="markdown-output"></pre>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<section id="capabilities" class="full-rule">
<div class="shell">
<div class="section-label pad"><span>[ <b>01</b> / 04 ]</span><span>Core capabilities</span></div>
<div class="section-label pad"><span>[ <b>02</b> / 05 ]</span><span>Core capabilities</span></div>
<div class="section-body pad">
<div class="section-intro">
<h2>A focused PDF<br>toolchain.</h2>
@@ -584,7 +813,7 @@
<section id="architecture" class="full-rule">
<div class="shell">
<div class="section-label pad"><span>[ <b>02</b> / 04 ]</span><span>Architecture</span></div>
<div class="section-label pad"><span>[ <b>03</b> / 05 ]</span><span>Architecture</span></div>
<div class="section-body pad">
<div class="section-intro">
<h2>One parse.<br>Clear stages.</h2>
@@ -622,7 +851,7 @@
<section id="benchmark" class="full-rule">
<div class="shell">
<div class="section-label pad"><span>[ <b>03</b> / 04 ]</span><span>Benchmark</span></div>
<div class="section-label pad"><span>[ <b>04</b> / 05 ]</span><span>Benchmark</span></div>
<div class="section-body pad">
<div class="section-intro">
<h2>Measured on<br>real documents.</h2>
@@ -656,7 +885,7 @@
<section id="usage" class="full-rule">
<div class="shell">
<div class="section-label pad"><span>[ <b>04</b> / 04 ]</span><span>Usage</span></div>
<div class="section-label pad"><span>[ <b>05</b> / 05 ]</span><span>Usage</span></div>
<div class="section-body pad">
<div class="section-intro">
<h2>Start with Node.<br>Use the CLI.</h2>
@@ -743,5 +972,201 @@ result = pdf_inspector.<span class="fn">process_pdf</span>(<span class="str">"do
</div>
</footer>
<script>
(() => {
const MAX_FILE_SIZE = 25 * 1024 * 1024;
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@0.1.1/pdf_inspector_wasm.js";
const input = document.querySelector("#pdf-input");
const dropZone = document.querySelector("#drop-zone");
const filePanel = document.querySelector("#demo-file");
const fileName = document.querySelector("#demo-file-name");
const fileSize = document.querySelector("#demo-file-size");
const clearButton = document.querySelector("#demo-clear");
const copyButton = document.querySelector("#demo-copy");
const message = document.querySelector("#demo-message");
const engineStatus = document.querySelector("#demo-engine-status");
const emptyOutput = document.querySelector("#demo-empty");
const resultPanel = document.querySelector("#demo-result");
const markdownOutput = document.querySelector("#markdown-output");
const typeOutput = document.querySelector("#demo-type");
const pagesOutput = document.querySelector("#demo-pages");
const timeOutput = document.querySelector("#demo-time");
let selectedFile = null;
let currentMarkdown = "";
let busy = false;
const formatBytes = (bytes) => {
if (bytes < 1024) return `${bytes} B`;
const units = ["KB", "MB", "GB"];
let value = bytes / 1024;
let unit = units[0];
for (let index = 1; value >= 1024 && index < units.length; index += 1) {
value /= 1024;
unit = units[index];
}
return `${value >= 10 ? value.toFixed(1) : value.toFixed(2)} ${unit}`;
};
const setMessage = (text, tone = "") => {
message.textContent = text;
message.dataset.tone = tone;
};
const setBusy = (isBusy) => {
busy = isBusy;
clearButton.disabled = isBusy;
input.disabled = isBusy;
};
const clearResult = () => {
currentMarkdown = "";
copyButton.disabled = true;
copyButton.textContent = "Copy Markdown";
markdownOutput.textContent = "";
resultPanel.hidden = true;
emptyOutput.hidden = false;
};
const clearFile = () => {
if (busy) return;
selectedFile = null;
input.value = "";
filePanel.hidden = true;
clearResult();
setMessage("Select a PDF to begin.");
};
const selectFile = (file) => {
if (!file) return;
const isPdf = file.type === "application/pdf" || file.name.toLowerCase().endsWith(".pdf");
if (!isPdf) {
clearFile();
setMessage("Choose a PDF file to run this demo.", "error");
return;
}
if (file.size > MAX_FILE_SIZE) {
clearFile();
setMessage("This demo accepts PDFs up to 25 MB.", "error");
return;
}
selectedFile = file;
fileName.textContent = file.name;
fileSize.textContent = formatBytes(file.size);
filePanel.hidden = false;
clearResult();
parseSelectedFile();
};
const processInWorker = (buffer) => new Promise((resolve, reject) => {
const source = `
import init, { processPdf, version } from "${WASM_MODULE_URL}";
self.onmessage = async ({ data }) => {
try {
await init();
const result = processPdf(new Uint8Array(data.buffer), { profile: "fidelity" });
self.postMessage({ ok: true, result, engineVersion: version() });
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
self.postMessage({ ok: false, error: detail });
}
};
`;
const workerUrl = URL.createObjectURL(new Blob([source], { type: "text/javascript" }));
let worker;
try {
worker = new Worker(workerUrl, { type: "module" });
} catch (error) {
URL.revokeObjectURL(workerUrl);
reject(error);
return;
}
URL.revokeObjectURL(workerUrl);
worker.addEventListener("message", ({ data }) => {
worker.terminate();
if (data.ok) resolve(data);
else reject(new Error(data.error));
}, { once: true });
worker.addEventListener("error", (event) => {
worker.terminate();
reject(new Error(event.message || "The WebAssembly engine could not be loaded."));
}, { once: true });
worker.postMessage({ buffer }, [buffer]);
});
const renderResult = ({ result, engineVersion }) => {
currentMarkdown = typeof result.markdown === "string" ? result.markdown : "";
typeOutput.textContent = result.pdfType || "Unknown";
pagesOutput.textContent = String(result.pageCount ?? "—");
timeOutput.textContent = Number.isFinite(result.processingTimeMs)
? `${Math.max(1, Math.round(result.processingTimeMs))} ms`
: "—";
markdownOutput.textContent = currentMarkdown || "No Markdown output was produced for this document.";
emptyOutput.hidden = true;
resultPanel.hidden = false;
copyButton.disabled = !currentMarkdown;
engineStatus.textContent = engineVersion ? `Ready · v${engineVersion}` : "Ready";
};
const parseSelectedFile = async () => {
if (!selectedFile || busy) return;
const file = selectedFile;
setBusy(true);
setMessage("Loading the Rust engine and parsing in this browser…");
engineStatus.textContent = "Loading…";
try {
const buffer = await file.arrayBuffer();
const response = await processInWorker(buffer);
renderResult(response);
setMessage(`Parsed ${file.name} locally.`, "success");
} catch (error) {
engineStatus.textContent = "Load failed";
clearResult();
const detail = error instanceof Error ? error.message : String(error);
setMessage(detail || "The PDF could not be parsed.", "error");
} finally {
setBusy(false);
}
};
dropZone.addEventListener("click", () => {
if (!busy) {
input.value = "";
input.click();
}
});
dropZone.addEventListener("keydown", (event) => {
if ((event.key === "Enter" || event.key === " ") && !busy) {
event.preventDefault();
input.value = "";
input.click();
}
});
dropZone.addEventListener("dragover", (event) => {
event.preventDefault();
if (!busy) dropZone.classList.add("is-dragging");
});
dropZone.addEventListener("dragleave", () => dropZone.classList.remove("is-dragging"));
dropZone.addEventListener("drop", (event) => {
event.preventDefault();
dropZone.classList.remove("is-dragging");
if (!busy) selectFile(event.dataTransfer.files[0]);
});
input.addEventListener("change", () => selectFile(input.files[0]));
clearButton.addEventListener("click", clearFile);
copyButton.addEventListener("click", async () => {
if (!currentMarkdown) return;
try {
await navigator.clipboard.writeText(currentMarkdown);
copyButton.textContent = "Copied";
window.setTimeout(() => { copyButton.textContent = "Copy Markdown"; }, 1600);
} catch {
setMessage("Copy is unavailable here. Select the Markdown output instead.", "error");
}
});
})();
</script>
</body>
</html>
+1
View File
@@ -63,6 +63,7 @@ fn json_escape(s: &str) -> String {
}
fn main() {
#[cfg(not(target_arch = "wasm32"))]
env_logger::init();
let args: Vec<String> = env::args().collect();
+1
View File
@@ -190,6 +190,7 @@ fn print_layout_info(layout: &LayoutComplexity) {
}
fn main() {
#[cfg(not(target_arch = "wasm32"))]
env_logger::init();
let args: Vec<String> = env::args().collect();
+1 -1
View File
@@ -162,7 +162,7 @@ pub(crate) fn extract_page_text_items(
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
// Build font encoding maps from Differences arrays
let (font_encodings, has_gid_fonts) = build_font_encodings(doc, &fonts);
let (font_encodings, has_gid_fonts) = build_font_encodings(doc, &fonts, font_cmaps);
// Build font width info for accurate text positioning
let font_widths = build_font_widths(doc, &fonts);
+154 -12
View File
@@ -497,9 +497,13 @@ pub(crate) fn get_operand_bytes(obj: &Object) -> Option<&[u8]> {
/// Build encoding maps for all fonts on a page.
/// Returns `(encodings, has_gid_fonts)` where `has_gid_fonts` is true when
/// any font uses raw glyph ID names (gidNNNNN) that can't be decoded.
/// Gid names whose codes the font's own ToUnicode CMap maps are decodable
/// and do not set the flag (LibreOffice subsets write /gidNNNN Differences
/// names alongside a complete ToUnicode CMap).
pub(crate) fn build_font_encodings(
doc: &Document,
fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
cmaps: &FontCMaps,
) -> (PageFontEncodings, bool) {
let mut encodings = PageFontEncodings::new();
let mut has_gid_fonts = false;
@@ -508,7 +512,9 @@ pub(crate) fn build_font_encodings(
let resource_name = String::from_utf8_lossy(font_name).to_string();
if let Some(result) = parse_font_encoding(doc, font_dict) {
if result.gid_glyph_count > 0 {
if !result.gid_codes.is_empty()
&& !tounicode_maps_codes(font_dict, cmaps, &result.gid_codes)
{
has_gid_fonts = true;
}
if !result.map.is_empty() {
@@ -520,6 +526,34 @@ pub(crate) fn build_font_encodings(
(encodings, has_gid_fonts)
}
/// True when the font's ToUnicode CMap maps the gid-named character codes,
/// so the Differences entries still decode through the CMap.
fn tounicode_maps_codes(font_dict: &lopdf::Dictionary, cmaps: &FontCMaps, codes: &[u8]) -> bool {
let Some(obj_ref) = font_dict
.get(b"ToUnicode")
.ok()
.and_then(|o| o.as_reference().ok())
else {
return false;
};
let Some(entry) = cmaps.get_by_obj(obj_ref.0) else {
return false;
};
// At least one gid code usably mapped means the CMap addresses these
// codes; remaining unmapped codes are subset leftovers (e.g. the
// component glyphs of an emoji ZWJ sequence mapped whole on its first
// code). A mapping is usable only when extraction would accept it —
// empty or U+FFFD results are rejected there as invalid. Fonts whose
// CMap ignores the gid codes entirely stay flagged, and the downstream
// garbage/encoding checks still catch partial damage.
codes.iter().any(|&code| {
entry
.primary
.lookup(code as u16)
.is_some_and(|s| !s.is_empty() && !s.contains('\u{FFFD}'))
})
}
/// Parse font encoding from a font dictionary
pub(crate) fn parse_font_encoding(
doc: &Document,
@@ -558,11 +592,10 @@ pub(crate) fn parse_font_encoding(
/// Result of parsing an encoding dictionary's Differences array.
pub(crate) struct EncodingResult {
pub map: FontEncodingMap,
/// Number of glyph names matching the `gidNNNNN` pattern (raw glyph IDs).
/// These indicate a font with unresolvable encoding — the glyph IDs
/// reference the original font's glyph table, but without the original
/// font's cmap there is no way to map them to Unicode.
pub gid_glyph_count: u32,
/// Character codes whose glyph names match the `gidNNNNN` pattern (raw
/// glyph IDs). These reference the original font's glyph table and are
/// only decodable when the font's ToUnicode CMap maps the code.
pub gid_codes: Vec<u8>,
}
/// Parse an encoding dictionary with Differences array
@@ -588,7 +621,7 @@ pub(crate) fn parse_encoding_dictionary(
let mut encoding_map = FontEncodingMap::new();
let mut current_code: u8 = 0;
let mut ligature_count = 0u32;
let mut gid_glyph_count = 0u32;
let mut gid_codes: Vec<u8> = Vec::new();
for item in diff_array {
match item {
@@ -614,7 +647,7 @@ pub(crate) fn parse_encoding_dictionary(
&& glyph_name.len() >= 4
&& glyph_name[3..].chars().all(|c| c.is_ascii_digit())
{
gid_glyph_count += 1;
gid_codes.push(current_code);
}
if let Some(ch) = mapped_char {
encoding_map.insert(current_code, ch);
@@ -638,16 +671,16 @@ pub(crate) fn parse_encoding_dictionary(
);
}
if gid_glyph_count > 0 {
if !gid_codes.is_empty() {
debug!(
" Differences: {} gid-encoded glyphs (unresolvable without original font)",
gid_glyph_count
" Differences: {} gid-encoded glyphs (decodable only via ToUnicode)",
gid_codes.len()
);
}
Some(EncodingResult {
map: encoding_map,
gid_glyph_count,
gid_codes,
})
}
@@ -1938,4 +1971,113 @@ mod tests {
false
));
}
fn gid_font_doc(bfchar: Option<&str>) -> (Document, lopdf::ObjectId) {
use lopdf::Stream;
let mut doc = Document::with_version("1.4");
let cmap = format!(
"/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
1 begincodespacerange
<00> <FF>
endcodespacerange
1 beginbfchar
{}
endbfchar
endcmap
CMapName currentdict /CMap defineresource pop
end
end",
bfchar.unwrap_or_default()
);
let tounicode_id = doc.add_object(Object::Stream(Stream::new(
dictionary! {},
cmap.into_bytes(),
)));
let enc_id = doc.add_object(dictionary! {
"Type" => "Encoding",
"Differences" => vec![
1.into(),
Object::Name(b"gid1283".to_vec()),
Object::Name(b"gid1464".to_vec()),
],
});
let mut font = dictionary! {
"Type" => "Font",
"Subtype" => "TrueType",
"BaseFont" => "ABCDEF+OpenSymbol",
"Encoding" => Object::Reference(enc_id),
};
if bfchar.is_some() {
font.set("ToUnicode", Object::Reference(tounicode_id));
}
let font_id = doc.add_object(font);
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Resources" => dictionary! {
"Font" => dictionary! { "F1" => Object::Reference(font_id) },
},
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
});
let pages_id = doc.add_object(dictionary! {
"Type" => "Pages",
"Count" => Object::Integer(1),
"Kids" => vec![Object::Reference(page_id)],
});
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => Object::Reference(pages_id),
});
doc.trailer.set("Root", Object::Reference(catalog_id));
(doc, page_id)
}
fn gid_flagged(bfchar: Option<&str>) -> bool {
let (doc, page_id) = gid_font_doc(bfchar);
let cmaps = FontCMaps::from_doc(&doc);
let fonts = doc.get_page_fonts(page_id).unwrap();
let (_, has_gid_fonts) = build_font_encodings(&doc, &fonts, &cmaps);
has_gid_fonts
}
#[test]
fn gid_differences_with_covering_tounicode_are_not_flagged() {
// LibreOffice subsets write /gidNNNN Differences names alongside a
// ToUnicode CMap that decodes those codes; the page must not be
// flagged as unresolvable (which would suppress the whole document's
// markdown when every page carries such a font).
assert!(!gid_flagged(Some("<01> <2022>\n<02> <25E6>")));
}
#[test]
fn gid_differences_with_partial_tounicode_are_not_flagged() {
// An emoji ZWJ sequence maps whole on its first code; the remaining
// component-glyph codes are subset leftovers, not damage.
assert!(!gid_flagged(Some(
"<01> <D83DDC68200DD83DDC69200DD83DDC67>"
)));
}
#[test]
fn gid_differences_without_tounicode_are_flagged() {
assert!(
gid_flagged(None),
"gid glyphs without ToUnicode are unresolvable"
);
}
#[test]
fn gid_differences_with_disjoint_tounicode_are_flagged() {
// A ToUnicode that never addresses the gid codes leaves them
// unresolvable.
assert!(gid_flagged(Some("<10> <0041>")));
}
#[test]
fn gid_differences_with_replacement_char_tounicode_are_flagged() {
// A mapping to U+FFFD is not usable — extraction rejects it as an
// invalid CMap result — so it must not clear the gid flag.
assert!(gid_flagged(Some("<01> <FFFD>\n<02> <FFFD>")));
}
}
+44 -5
View File
@@ -1153,6 +1153,22 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
group_into_lines_with_thresholds(items, &HashMap::new(), &HashSet::new())
}
/// Group text items into lines without removing numeric page headers or footers.
///
/// Plain-text extraction uses this path because every extracted item is part of
/// the API result. Markdown conversion keeps using [`group_into_lines`], where
/// page-number suppression is an intentional presentation cleanup.
pub fn group_into_lines_preserving_all_text(items: Vec<TextItem>) -> Vec<TextLine> {
group_into_lines_with_thresholds_and_regions_impl(
items,
&HashMap::new(),
&HashSet::new(),
&HashMap::new(),
&HashMap::new(),
false,
)
}
/// Group text items into lines, using pre-computed per-page adaptive thresholds
/// from Canva-style letter-spacing detection. Falls back to computing the
/// threshold from item gaps when no pre-computed value is available.
@@ -1195,16 +1211,39 @@ pub(crate) fn group_into_lines_with_thresholds_and_regions(
table_pages: &HashSet<u32>,
chart_regions: &HashMap<u32, Vec<(f32, f32, f32, f32)>>,
image_regions: &HashMap<u32, Vec<super::reading_order::ImageRegion>>,
) -> Vec<TextLine> {
group_into_lines_with_thresholds_and_regions_impl(
items,
page_thresholds,
table_pages,
chart_regions,
image_regions,
true,
)
}
fn group_into_lines_with_thresholds_and_regions_impl(
items: Vec<TextItem>,
page_thresholds: &HashMap<u32, f32>,
table_pages: &HashSet<u32>,
chart_regions: &HashMap<u32, Vec<(f32, f32, f32, f32)>>,
image_regions: &HashMap<u32, Vec<super::reading_order::ImageRegion>>,
filter_page_numbers: bool,
) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
// Filter out page numbers (standalone numbers at top/bottom of page)
let items: Vec<TextItem> = items
.into_iter()
.filter(|item| !is_page_number(item))
.collect();
// Markdown output omits standalone numeric headers/footers. Plain-text
// callers opt out because dropping extracted text violates that API.
let items = if filter_page_numbers {
items
.into_iter()
.filter(|item| !is_page_number(item))
.collect()
} else {
items
};
// Get unique pages
let mut pages: Vec<u32> = items.iter().map(|i| i.page).collect();
+13 -1
View File
@@ -27,12 +27,12 @@ pub use crate::text_utils::{is_bold_font, is_italic_font};
pub use crate::types::{ItemType, TextLine};
pub(crate) use fonts::FontStyleCache;
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;
pub(crate) use layout::group_into_lines_with_thresholds;
pub(crate) use layout::group_into_lines_with_thresholds_and_charts;
pub(crate) use layout::group_into_lines_with_thresholds_and_regions;
pub(crate) use layout::is_newspaper_layout;
pub(crate) use layout::ColumnRegion;
pub use layout::{group_into_lines, group_into_lines_preserving_all_text};
// ---------------------------------------------------------------------------
// Public API
@@ -1519,6 +1519,18 @@ mod tests {
assert_eq!(lines[1].text(), "Next line");
}
#[test]
fn preserving_all_text_keeps_numeric_page_footer() {
let mut page_number = make_merge_item("42", 100.0, 12.0);
page_number.y = 50.0;
assert!(group_into_lines(vec![page_number.clone()]).is_empty());
let lines = group_into_lines_preserving_all_text(vec![page_number]);
assert_eq!(lines.len(), 1);
assert_eq!(lines[0].text(), "42");
}
#[test]
fn test_bold_italic_detection() {
// Test bold detection
+1 -1
View File
@@ -162,7 +162,7 @@ fn extract_form_xobject_text_inner(
// Get fonts from the Form's Resources
let form_fonts = get_form_fonts(doc, &stream.dict);
let (font_encodings, _has_gid_fonts) = build_font_encodings(doc, &form_fonts);
let (font_encodings, _has_gid_fonts) = build_font_encodings(doc, &form_fonts, font_cmaps);
// Build font width info for the form
let font_widths = build_font_widths(doc, &form_fonts);
+40 -6
View File
@@ -68,6 +68,40 @@ use text_quality::{
};
use tounicode::FontCMaps;
#[cfg(not(target_arch = "wasm32"))]
struct ProcessingTimer(std::time::Instant);
#[cfg(target_arch = "wasm32")]
struct ProcessingTimer;
impl ProcessingTimer {
fn start() -> Self {
#[cfg(not(target_arch = "wasm32"))]
{
Self(std::time::Instant::now())
}
#[cfg(target_arch = "wasm32")]
{
Self
}
}
fn elapsed_ms(&self) -> u64 {
#[cfg(not(target_arch = "wasm32"))]
{
self.0.elapsed().as_millis() as u64
}
#[cfg(target_arch = "wasm32")]
{
// The wasm32-unknown-unknown standard library has no clock.
// Browser bindings measure with JavaScript's host clock.
0
}
}
}
/// OCR reason emitted when the extracted text layer appears garbled due to
/// broken font decoding or mojibake.
pub const OCR_REASON_SUSPECTED_GARBLED_TEXT: &str = "suspected_garbled_text";
@@ -250,7 +284,7 @@ pub fn process_pdf_with_options<P: AsRef<Path>>(
path: P,
options: PdfOptions,
) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
let start = ProcessingTimer::start();
validate_pdf_file(&path)?;
// Load the document once — shared by detection AND extraction.
@@ -277,7 +311,7 @@ pub fn process_pdf_mem_with_options(
buffer: &[u8],
options: PdfOptions,
) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
let start = ProcessingTimer::start();
validate_pdf_bytes(buffer)?;
let (doc, page_count) =
@@ -3526,7 +3560,7 @@ fn process_document(
doc: Document,
page_count: u32,
options: PdfOptions,
start: std::time::Instant,
start: ProcessingTimer,
) -> Result<PdfProcessResult, PdfError> {
// Step 1 — Detection (cheap: scans content streams for text operators)
let detection = detector::detect_from_document(&doc, page_count, &options.detection)?;
@@ -3542,7 +3576,7 @@ fn process_document(
pdf_type,
markdown: None,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
processing_time_ms: start.elapsed_ms(),
pages_needing_ocr,
ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons),
title,
@@ -3558,7 +3592,7 @@ fn process_document(
pdf_type,
markdown: None,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
processing_time_ms: start.elapsed_ms(),
pages_needing_ocr,
ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons),
title,
@@ -3824,7 +3858,7 @@ fn process_document(
pdf_type,
markdown,
page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
processing_time_ms: start.elapsed_ms(),
pages_needing_ocr,
ocr_reasons_by_page: {
// Detector reasons (scanned / no_text / vector_text / garbled) merged
+23 -10
View File
@@ -4,11 +4,17 @@
use log::{debug, warn};
use lopdf::{Document, Object, ObjectId};
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
#[cfg(not(target_arch = "wasm32"))]
use std::path::{Path, PathBuf};
use crate::glyph_names::glyph_to_char;
#[cfg(target_arch = "wasm32")]
static BUILTIN_CMAPS: include_dir::Dir<'_> =
include_dir::include_dir!("$CARGO_MANIFEST_DIR/external/bcmaps");
/// A parsed ToUnicode CMap mapping CIDs to Unicode strings
#[derive(Debug, Default, Clone)]
pub struct ToUnicodeCMap {
@@ -1149,9 +1155,7 @@ fn build_gid_to_unicode(face: &ttf_parser::Face<'_>) -> Option<HashMap<u16, char
/// Build a ToUnicodeCMap from pdf.js built-in binary CMaps (bcmaps).
fn build_cmap_from_builtin_cmap(ordering: &str) -> Option<ToUnicodeCMap> {
let name = format!("Adobe-{}-UCS2.bcmap", ordering);
let dir = find_bcmaps_dir()?;
let path = dir.join(name);
let data = std::fs::read(&path).ok()?;
let data = read_builtin_cmap_file(&name)?;
let mut cmap = parse_binary_cmap(&data).ok()?;
if cmap.char_map.is_empty() && cmap.ranges.is_empty() {
return None;
@@ -1159,13 +1163,14 @@ fn build_cmap_from_builtin_cmap(ordering: &str) -> Option<ToUnicodeCMap> {
cmap.code_byte_length = 2;
debug!(
"Built-in CMap {}: char_map={} ranges={}",
path.display(),
name,
cmap.char_map.len(),
cmap.ranges.len()
);
Some(cmap)
}
#[cfg(not(target_arch = "wasm32"))]
fn find_bcmaps_dir() -> Option<PathBuf> {
if let Ok(dir) = std::env::var("PDF_INSPECTOR_BCMAPS_DIR") {
let p = PathBuf::from(dir);
@@ -1182,6 +1187,18 @@ fn find_bcmaps_dir() -> Option<PathBuf> {
None
}
#[cfg(not(target_arch = "wasm32"))]
fn read_builtin_cmap_file(name: &str) -> Option<Cow<'static, [u8]>> {
let path = find_bcmaps_dir()?.join(name);
std::fs::read(path).ok().map(Cow::Owned)
}
#[cfg(target_arch = "wasm32")]
fn read_builtin_cmap_file(name: &str) -> Option<Cow<'static, [u8]>> {
let file = BUILTIN_CMAPS.get_file(name)?;
Some(Cow::Borrowed(file.contents()))
}
fn parse_binary_cmap(data: &[u8]) -> Result<ToUnicodeCMap, String> {
let mut stream = BinaryCMapStream::new(data);
let _header = stream.read_byte().ok_or("unexpected EOF in bcmap header")?;
@@ -1492,9 +1509,7 @@ fn parse_encoding_cmap_object(obj: &Object, doc: &Document) -> Option<EncodingCM
}
fn load_builtin_encoding_cmap(name: &str) -> Option<EncodingCMap> {
let dir = find_bcmaps_dir()?;
let path = dir.join(format!("{}.bcmap", name));
let data = std::fs::read(&path).ok()?;
let data = read_builtin_cmap_file(&format!("{}.bcmap", name))?;
parse_binary_cmap_encoding(&data).ok()
}
@@ -1779,9 +1794,7 @@ fn load_builtin_cmap_by_name(name: &str) -> Option<ToUnicodeCMap> {
if !name.ends_with("UCS2") {
return None;
}
let dir = find_bcmaps_dir()?;
let path = dir.join(format!("{}.bcmap", name));
let data = std::fs::read(&path).ok()?;
let data = read_builtin_cmap_file(&format!("{}.bcmap", name))?;
let mut cmap = parse_binary_cmap(&data).ok()?;
if cmap.char_map.is_empty() && cmap.ranges.is_empty() {
return None;
+1304
View File
File diff suppressed because it is too large Load Diff
+37
View File
@@ -0,0 +1,37 @@
[package]
name = "pdf-inspector-wasm"
version = "0.1.2"
edition = "2021"
authors = ["Firecrawl Team"]
description = "Browser WebAssembly bindings for pdf-inspector"
license = "MIT"
repository = "https://github.com/firecrawl/pdf-inspector"
homepage = "https://github.com/firecrawl/pdf-inspector"
readme = "README.md"
publish = false
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
console_error_panic_hook = "0.1"
js-sys = "0.3"
pdf-inspector = { path = ".." }
serde = { version = "1", features = ["derive"] }
serde-wasm-bindgen = "0.6"
wasm-bindgen = "0.2"
[dev-dependencies]
wasm-bindgen-test = "0.3"
[profile.release]
codegen-units = 1
lto = true
opt-level = "s"
strip = true
[package.metadata.wasm-pack.profile.release]
# Rust 1.95 emits bulk-memory instructions that the binaryen bundled with
# wasm-pack 0.15.0 does not yet validate. rustc still performs the release,
# size, and LTO optimizations above.
wasm-opt = false
+58
View File
@@ -0,0 +1,58 @@
MIT License
Copyright (c) 2026 Firecrawl
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Third-party notices
===================
Adobe CMaps
-----------
The WebAssembly binary embeds binary CMaps derived from Adobe CMap resources.
Copyright 1990-2009 Adobe Systems Incorporated.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
Neither the name of Adobe Systems Incorporated nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
+60
View File
@@ -0,0 +1,60 @@
# @firecrawl/pdf-inspector-wasm
Browser WebAssembly bindings for [pdf-inspector](https://github.com/firecrawl/pdf-inspector). Classify PDFs and extract structured Markdown locally from a `Uint8Array`, using the same Rust core as the native Node.js, Python, and Rust packages.
## Install
```bash
npm install @firecrawl/pdf-inspector-wasm
```
## Usage
```ts
import init, { processPdf } from "@firecrawl/pdf-inspector-wasm";
await init();
const response = await fetch("/annual-report.pdf");
const pdf = new Uint8Array(await response.arrayBuffer());
const result = processPdf(pdf);
console.log(result.pdfType);
console.log(result.markdown);
```
Pass options when you need selected pages or compact Markdown:
```ts
const result = processPdf(pdf, {
pages: [1, 3, 5],
profile: "compact",
includePageMarkers: true,
});
```
The package also exports:
- `detectPdf(pdf, options?)` for detection without extraction.
- `classifyPdf(pdf)` for the lightweight result shape shared with the native Node.js API.
- `extractText(pdf)` for plain text.
- `version()` for the WASM package version.
## Browser behavior
- Parsing runs locally. PDF bytes are not uploaded anywhere.
- The build is single-threaded and does not require cross-origin isolation.
- CMaps are embedded so CJK font decoding does not depend on a filesystem.
- Extraction is synchronous after `init()`. For large documents, call it from a Web Worker to keep the UI responsive.
- Image-only documents still require a separate OCR step.
## Build from source
```bash
cargo install wasm-pack --version 0.15.0 --locked
wasm-pack build wasm --target web --scope firecrawl --release
```
## License
MIT
+440
View File
@@ -0,0 +1,440 @@
use pdf_inspector::{
LayoutComplexity, MarkdownProfile, PageOcrReasons, PdfOptions, PdfProcessResult, PdfType,
ProcessMode,
};
use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
#[wasm_bindgen(typescript_custom_section)]
const TYPESCRIPT_TYPES: &str = r#"
export type PdfType = "TextBased" | "Scanned" | "ImageBased" | "Mixed";
export type MarkdownProfile = "fidelity" | "compact";
export interface ProcessOptions {
/** Restrict extraction to these 1-indexed page numbers. */
pages?: number[];
/** Password for an encrypted PDF. */
password?: string;
/** Source-faithful output by default, or compact output for fewer tokens. */
profile?: MarkdownProfile;
/** Insert `<!-- Page N -->` markers between pages. */
includePageMarkers?: boolean;
/** Include image placeholders in Markdown output. */
includeImages?: boolean;
}
export interface PageOcrReasons {
/** 1-indexed page number. */
page: number;
reasons: string[];
}
export interface LayoutComplexity {
isComplex: boolean;
/** 1-indexed page numbers. */
pagesWithTables: number[];
/** 1-indexed page numbers. */
pagesWithColumns: number[];
}
export interface PdfProcessResult {
pdfType: PdfType;
markdown?: string;
pageCount: number;
processingTimeMs: number;
/** 1-indexed page numbers. */
pagesNeedingOcr: number[];
ocrReasonsByPage: PageOcrReasons[];
title?: string;
confidence: number;
layout: LayoutComplexity;
hasEncodingIssues: boolean;
}
export interface PdfClassification {
pdfType: PdfType;
pageCount: number;
/** 0-indexed page numbers, matching the native Node.js API. */
pagesNeedingOcr: number[];
confidence: number;
}
export function processPdf(data: Uint8Array, options?: ProcessOptions): PdfProcessResult;
export function detectPdf(data: Uint8Array, options?: Pick<ProcessOptions, "password">): PdfProcessResult;
export function classifyPdf(data: Uint8Array): PdfClassification;
export function extractText(data: Uint8Array): string;
export function version(): string;
"#;
#[derive(Debug, Default, Deserialize)]
#[serde(default, rename_all = "camelCase", deny_unknown_fields)]
struct WasmProcessOptions {
pages: Option<Vec<u32>>,
password: Option<String>,
profile: Option<WasmMarkdownProfile>,
include_page_markers: Option<bool>,
include_images: Option<bool>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "lowercase")]
enum WasmMarkdownProfile {
Fidelity,
Compact,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WasmPageOcrReasons {
page: u32,
reasons: Vec<String>,
}
impl From<PageOcrReasons> for WasmPageOcrReasons {
fn from(value: PageOcrReasons) -> Self {
Self {
page: value.page,
reasons: value.reasons,
}
}
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WasmLayoutComplexity {
is_complex: bool,
pages_with_tables: Vec<u32>,
pages_with_columns: Vec<u32>,
}
impl From<LayoutComplexity> for WasmLayoutComplexity {
fn from(value: LayoutComplexity) -> Self {
Self {
is_complex: value.is_complex,
pages_with_tables: value.pages_with_tables,
pages_with_columns: value.pages_with_columns,
}
}
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WasmPdfProcessResult {
pdf_type: &'static str,
markdown: Option<String>,
page_count: u32,
processing_time_ms: f64,
pages_needing_ocr: Vec<u32>,
ocr_reasons_by_page: Vec<WasmPageOcrReasons>,
title: Option<String>,
confidence: f64,
layout: WasmLayoutComplexity,
has_encoding_issues: bool,
}
impl From<PdfProcessResult> for WasmPdfProcessResult {
fn from(value: PdfProcessResult) -> Self {
Self {
pdf_type: pdf_type_name(value.pdf_type),
markdown: value.markdown,
page_count: value.page_count,
processing_time_ms: value.processing_time_ms as f64,
pages_needing_ocr: value.pages_needing_ocr,
ocr_reasons_by_page: value
.ocr_reasons_by_page
.into_iter()
.map(Into::into)
.collect(),
title: value.title,
confidence: value.confidence as f64,
layout: value.layout.into(),
has_encoding_issues: value.has_encoding_issues,
}
}
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WasmPdfClassification {
pdf_type: &'static str,
page_count: u32,
pages_needing_ocr: Vec<u32>,
confidence: f64,
}
fn pdf_type_name(pdf_type: PdfType) -> &'static str {
match pdf_type {
PdfType::TextBased => "TextBased",
PdfType::Scanned => "Scanned",
PdfType::ImageBased => "ImageBased",
PdfType::Mixed => "Mixed",
}
}
fn js_error(context: &str, error: impl std::fmt::Display) -> JsValue {
js_sys::Error::new(&format!("{context}: {error}")).into()
}
fn deserialize_options(value: JsValue) -> Result<WasmProcessOptions, JsValue> {
if value.is_undefined() || value.is_null() {
return Ok(WasmProcessOptions::default());
}
serde_wasm_bindgen::from_value(value).map_err(|error| js_error("invalid options", error))
}
fn build_options(value: JsValue, mode: ProcessMode) -> Result<PdfOptions, JsValue> {
let options = deserialize_options(value)?;
if options
.pages
.as_ref()
.is_some_and(|pages| pages.contains(&0))
{
return Err(js_error(
"invalid options",
"pages are 1-indexed; page 0 is invalid",
));
}
let mut result = PdfOptions::new().mode(mode);
if let Some(pages) = options.pages {
result = result.pages(pages);
}
if let Some(password) = options.password {
result = result.password(password);
}
if let Some(profile) = options.profile {
result.markdown.profile = match profile {
WasmMarkdownProfile::Fidelity => MarkdownProfile::Fidelity,
WasmMarkdownProfile::Compact => MarkdownProfile::Compact,
};
}
if let Some(include_page_markers) = options.include_page_markers {
result.markdown.include_page_numbers = include_page_markers;
}
if let Some(include_images) = options.include_images {
result.markdown.include_images = include_images;
}
Ok(result)
}
fn serialize<T: Serialize>(value: &T) -> Result<JsValue, JsValue> {
serde_wasm_bindgen::to_value(value).map_err(|error| js_error("serialize result", error))
}
fn initialize() {
console_error_panic_hook::set_once();
}
/// Process PDF bytes entirely inside WebAssembly.
#[wasm_bindgen(js_name = processPdf, skip_typescript)]
pub fn process_pdf(data: &[u8], options: JsValue) -> Result<JsValue, JsValue> {
initialize();
let options = build_options(options, ProcessMode::Full)?;
let started = js_sys::Date::now();
let mut result = pdf_inspector::process_pdf_mem_with_options(data, options)
.map_err(|error| js_error("process PDF", error))?;
result.processing_time_ms = (js_sys::Date::now() - started).max(0.0) as u64;
serialize(&WasmPdfProcessResult::from(result))
}
/// Classify PDF bytes without extracting text or producing Markdown.
#[wasm_bindgen(js_name = detectPdf, skip_typescript)]
pub fn detect_pdf(data: &[u8], options: JsValue) -> Result<JsValue, JsValue> {
initialize();
let options = build_options(options, ProcessMode::DetectOnly)?;
let started = js_sys::Date::now();
let mut result = pdf_inspector::process_pdf_mem_with_options(data, options)
.map_err(|error| js_error("detect PDF", error))?;
result.processing_time_ms = (js_sys::Date::now() - started).max(0.0) as u64;
serialize(&WasmPdfProcessResult::from(result))
}
/// Return the lightweight classification shape used by the native Node API.
#[wasm_bindgen(js_name = classifyPdf, skip_typescript)]
pub fn classify_pdf(data: &[u8]) -> Result<JsValue, JsValue> {
initialize();
let result =
pdf_inspector::classify_pdf_mem(data).map_err(|error| js_error("classify PDF", error))?;
serialize(&WasmPdfClassification {
pdf_type: pdf_type_name(result.pdf_type),
page_count: result.page_count,
pages_needing_ocr: result.pages_needing_ocr,
confidence: result.confidence as f64,
})
}
/// Extract plain text from PDF bytes without Markdown conversion.
#[wasm_bindgen(js_name = extractText, skip_typescript)]
pub fn extract_text(data: &[u8]) -> Result<String, JsValue> {
initialize();
let items = pdf_inspector::extractor::extract_text_with_positions_mem(data)
.map_err(|error| js_error("extract text", error))?;
Ok(
pdf_inspector::extractor::group_into_lines_preserving_all_text(items)
.into_iter()
.map(|line| line.text())
.filter(|line| !line.trim().is_empty())
.collect::<Vec<_>>()
.join("\n"),
)
}
/// Return the WebAssembly package version.
#[wasm_bindgen(skip_typescript)]
pub fn version() -> String {
env!("CARGO_PKG_VERSION").to_string()
}
#[cfg(all(test, target_arch = "wasm32"))]
mod tests {
use super::*;
use js_sys::Reflect;
use wasm_bindgen_test::*;
const TEXT_PDF: &[u8] = include_bytes!("../../tests/fixtures/thermo-freon12.pdf");
const ENCRYPTED_PDF: &[u8] = include_bytes!("../../tests/fixtures/encrypted-secret123.pdf");
fn synthetic_korea1_pdf() -> Vec<u8> {
let mut pdf = b"%PDF-1.4\n".to_vec();
let mut offsets = vec![0usize];
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
offsets.push(pdf.len());
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
pdf.extend_from_slice(body.as_bytes());
pdf.extend_from_slice(b"\nendobj\n");
}
add_object(
&mut pdf,
&mut offsets,
1,
"<< /Type /Catalog /Pages 2 0 R >>",
);
add_object(
&mut pdf,
&mut offsets,
2,
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
);
add_object(
&mut pdf,
&mut offsets,
3,
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F0 5 0 R >> >> /Contents 4 0 R >>",
);
// Adobe-Korea1 CID 1086 (0x043E) maps to U+AC00 (Korean syllable GA).
// There is deliberately no ToUnicode stream: decoding must use the
// embedded predefined CMap rather than lopdf's plain-text fallback.
// Korea1 CIDs 21 and 19 map to ASCII "4" and "2". Place them near
// the bottom edge so they look exactly like a numeric page footer.
let content = "BT /F0 12 Tf 50 100 Td <043E> Tj 0 -60 Td <00150013> Tj ET";
add_object(
&mut pdf,
&mut offsets,
4,
&format!(
"<< /Length {} >>\nstream\n{}\nendstream",
content.len(),
content
),
);
add_object(
&mut pdf,
&mut offsets,
5,
"<< /Type /Font /Subtype /Type0 /BaseFont /SyntheticKorea1 /Encoding /Identity-H /DescendantFonts [6 0 R] >>",
);
add_object(
&mut pdf,
&mut offsets,
6,
"<< /Type /Font /Subtype /CIDFontType2 /BaseFont /SyntheticKorea1 /CIDSystemInfo << /Registry (Adobe) /Ordering (Korea1) /Supplement 2 >> /FontDescriptor 7 0 R /DW 1000 >>",
);
add_object(
&mut pdf,
&mut offsets,
7,
"<< /Type /FontDescriptor /FontName /SyntheticKorea1 /Flags 4 /FontBBox [-100 -200 1000 900] /ItalicAngle 0 /Ascent 800 /Descent -200 /CapHeight 700 /StemV 80 >>",
);
let xref_start = pdf.len();
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
pdf.extend_from_slice(b"0000000000 65535 f \n");
for offset in offsets.iter().skip(1) {
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
pdf.extend_from_slice(
format!(
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
offsets.len(),
xref_start
)
.as_bytes(),
);
pdf
}
#[wasm_bindgen_test]
fn processes_pdf_to_markdown() {
let result = process_pdf(TEXT_PDF, JsValue::UNDEFINED).expect("process PDF");
let pdf_type = Reflect::get(&result, &JsValue::from_str("pdfType"))
.expect("pdfType")
.as_string()
.expect("pdfType string");
let markdown = Reflect::get(&result, &JsValue::from_str("markdown"))
.expect("markdown")
.as_string()
.expect("markdown string");
assert_eq!(pdf_type, "TextBased");
assert!(!markdown.is_empty());
}
#[wasm_bindgen_test]
fn rejects_non_pdf_bytes() {
assert!(process_pdf(b"not a PDF", JsValue::UNDEFINED).is_err());
}
#[wasm_bindgen_test]
fn classifies_and_extracts_plain_text() {
let classification = classify_pdf(TEXT_PDF).expect("classify PDF");
let pdf_type = Reflect::get(&classification, &JsValue::from_str("pdfType"))
.expect("pdfType")
.as_string()
.expect("pdfType string");
let text = extract_text(TEXT_PDF).expect("extract text");
assert_eq!(pdf_type, "TextBased");
assert!(!text.is_empty());
}
#[wasm_bindgen_test]
fn extracts_cjk_and_preserves_numeric_page_footer() {
let text = extract_text(&synthetic_korea1_pdf()).expect("extract predefined CMap text");
assert_eq!(text, "\n42");
}
#[wasm_bindgen_test]
fn opens_encrypted_pdf_with_password() {
assert!(process_pdf(ENCRYPTED_PDF, JsValue::UNDEFINED).is_err());
let options = js_sys::Object::new();
Reflect::set(
&options,
&JsValue::from_str("password"),
&JsValue::from_str("secret123"),
)
.expect("set password");
let result = process_pdf(ENCRYPTED_PDF, options.into()).expect("process encrypted PDF");
let markdown = Reflect::get(&result, &JsValue::from_str("markdown"))
.expect("markdown")
.as_string()
.expect("markdown string");
assert!(!markdown.is_empty());
}
}