Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
062ede203e | ||
|
Abimael Martell
|
16fbe29b52 |
@@ -180,31 +180,18 @@ jobs:
|
||||
test -n "$ort_path"
|
||||
echo "ORT_DYLIB_PATH=$ort_path" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Configure isolated model cache
|
||||
shell: bash
|
||||
run: echo "PDF_INSPECTOR_MODEL_CACHE=$RUNNER_TEMP/pdf-inspector-models" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build OCR CLI
|
||||
run: cargo build --features ocr --bin pdf2md
|
||||
|
||||
- name: Test PDFium runtime
|
||||
run: cargo test --features ocr --test local_render_tests
|
||||
|
||||
- name: Provision OCR model cache
|
||||
shell: bash
|
||||
run: |
|
||||
target/debug/pdf2md \
|
||||
tests/fixtures/scan_with_native_header_text.pdf \
|
||||
--ocr force \
|
||||
--json > /dev/null
|
||||
|
||||
- name: Run OCR CLI
|
||||
shell: bash
|
||||
run: |
|
||||
target/debug/pdf2md \
|
||||
tests/fixtures/scan_with_native_header_text.pdf \
|
||||
--ocr auto \
|
||||
--ocr-offline \
|
||||
--json > "$RUNNER_TEMP/ocr-result.json"
|
||||
|
||||
- name: Validate OCR JSON contract
|
||||
@@ -226,12 +213,6 @@ jobs:
|
||||
assert "layout_ms" not in result["pages"][0]["timings"]
|
||||
PY
|
||||
|
||||
- name: Run OCR launch smoke set
|
||||
shell: bash
|
||||
run: |
|
||||
export PDF_INSPECTOR_OCR_TEST_MODELS="$PDF_INSPECTOR_MODEL_CACHE/pp-ocrv6-small/oar-ocr-v0.7.0"
|
||||
cargo test --features ocr --test ocr_tests -- --nocapture
|
||||
|
||||
- name: Build Node binding
|
||||
working-directory: napi
|
||||
run: |
|
||||
|
||||
@@ -86,14 +86,12 @@ jobs:
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
docker-options: -e CFLAGS_aarch64_unknown_linux_gnu=-D__ARM_ARCH=8
|
||||
# macos-13 was retired by GitHub; macos-15-intel is the remaining
|
||||
# Intel runner label (available through 2027).
|
||||
- os: macos-15-intel
|
||||
@@ -115,9 +113,6 @@ jobs:
|
||||
target: ${{ matrix.target }}
|
||||
args: --release --out dist
|
||||
manylinux: auto
|
||||
# The manylinux AArch64 GCC omits this macro while preprocessing
|
||||
# ring's assembly. AArch64 is ARMv8 by definition.
|
||||
docker-options: ${{ matrix.docker-options }}
|
||||
|
||||
- name: Upload wheel
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -60,7 +60,6 @@ jobs:
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
@@ -71,7 +70,6 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
build-flags: --use-napi-cross
|
||||
cflags: -D__ARM_ARCH=8
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-musl
|
||||
build-flags: -x
|
||||
@@ -128,10 +126,6 @@ jobs:
|
||||
|
||||
- name: Build native addon
|
||||
working-directory: napi
|
||||
env:
|
||||
# napi-cross's old AArch64 GCC omits this predefined macro while
|
||||
# preprocessing ring's assembly. AArch64 is ARMv8 by definition.
|
||||
CFLAGS_aarch64_unknown_linux_gnu: ${{ matrix.cflags }}
|
||||
run: bunx napi build --platform --release --target ${{ matrix.target }} ${{ matrix.build-flags }}
|
||||
|
||||
- name: Upload native binary
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
|
||||
@@ -48,7 +48,8 @@ Use the [paired benchmark harness](docs/benchmarking.md) to compare two local bu
|
||||
### Python
|
||||
|
||||
```bash
|
||||
pip install pdf-inspector
|
||||
pip install maturin
|
||||
maturin develop --release
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -181,9 +182,8 @@ and confidence, warnings, and pages recommended for the hosted document
|
||||
pipeline. Native Python and Node packages expose the same pipeline without a
|
||||
source-build feature. All native entry points still require separately
|
||||
installed PDFium and ONNX Runtime libraries only when OCR is routed. See the
|
||||
[OCR runtime setup guide](docs/ocr-runtime.md) for pinned downloads, platform
|
||||
support, model-cache behavior, and hosted-fallback integration. See the
|
||||
[Rust API guide](docs/rust-api.md#complete-ocr-api) for lower-level controls.
|
||||
[Rust API guide](docs/rust-api.md#complete-ocr-api) for model cache and offline
|
||||
configuration.
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# OCR runtime setup
|
||||
|
||||
Selective OCR is available from the Rust library and CLI, Python, and Node.js.
|
||||
Clean native-text documents do not load an OCR dependency or download a model.
|
||||
When `auto` routes at least one page, the process needs PDFium, ONNX Runtime,
|
||||
and the pinned PP-OCRv6 Small model set.
|
||||
|
||||
## Validated versions
|
||||
|
||||
The reproducible runtime path uses these builds:
|
||||
|
||||
- [Firecrawl PDFium `native-v7988`](https://github.com/firecrawl/pdfium-rs/releases/tag/native-v7988),
|
||||
containing PDFium `153.0.7988.0`
|
||||
- [ONNX Runtime `1.27.0`](https://github.com/microsoft/onnxruntime/releases/tag/v1.27.0)
|
||||
- PP-OCRv6 Small artifact revision `oar-ocr-v0.7.0`
|
||||
|
||||
Use these versions for the reproducible path. Other compatible shared-library
|
||||
builds may work, but are not part of the release smoke test.
|
||||
|
||||
## Install the shared libraries
|
||||
|
||||
Download and extract the matching archives:
|
||||
|
||||
| Platform | PDFium asset | ONNX Runtime asset |
|
||||
|---|---|---|
|
||||
| Linux x64 | `firecrawl-pdfium-linux-x64.tgz` | `onnxruntime-linux-x64-1.27.0.tgz` |
|
||||
| Linux ARM64 | `firecrawl-pdfium-linux-arm64.tgz` | `onnxruntime-linux-aarch64-1.27.0.tgz` |
|
||||
| macOS Apple Silicon | `firecrawl-pdfium-mac-arm64.tgz` | `onnxruntime-osx-arm64-1.27.0.tgz` |
|
||||
| Windows x64 | `firecrawl-pdfium-win-x64.tgz` | `onnxruntime-win-x64-1.27.0.zip` |
|
||||
|
||||
The PDFium release publishes `SHA256SUMS`, build provenance, license files,
|
||||
and an SPDX document for every platform archive. GitHub publishes a SHA-256
|
||||
digest with each ONNX Runtime asset.
|
||||
|
||||
Point pdf-inspector at the extracted shared libraries when they are not on the
|
||||
platform library search path:
|
||||
|
||||
```bash
|
||||
export PDFIUM_LIB_PATH=/absolute/path/to/libpdfium.so
|
||||
export ORT_DYLIB_PATH=/absolute/path/to/libonnxruntime.so
|
||||
pdf2md scan.pdf --ocr auto --json
|
||||
```
|
||||
|
||||
On macOS the filenames end in `.dylib`. On Windows, use PowerShell and point
|
||||
the variables at `pdfium.dll` and `onnxruntime.dll`:
|
||||
|
||||
```powershell
|
||||
$env:PDFIUM_LIB_PATH = "C:\absolute\path\to\pdfium.dll"
|
||||
$env:ORT_DYLIB_PATH = "C:\absolute\path\to\onnxruntime.dll"
|
||||
pdf2md scan.pdf --ocr auto --json
|
||||
```
|
||||
|
||||
The native extraction packages also support platforms without these exact
|
||||
runtime assets. In particular, the Python package has an Intel macOS wheel,
|
||||
but ONNX Runtime 1.27.0 does not publish an Intel macOS archive; local OCR on
|
||||
that target requires a compatible custom ONNX Runtime build.
|
||||
|
||||
The full OCR path is exercised end to end on Linux x64 in CI. macOS and
|
||||
Windows compile and run the feature's platform-independent tests, while their
|
||||
external-runtime paths should be treated as preview until equivalent smoke
|
||||
jobs are added.
|
||||
|
||||
## Model cache and offline mode
|
||||
|
||||
The first routed page downloads and SHA-256-verifies three pinned artifacts:
|
||||
the detection model, recognition model, and character dictionary. Together
|
||||
they are about 31 MB. They are stored below the platform cache directory.
|
||||
Set `PDF_INSPECTOR_MODEL_CACHE` to choose a managed cache root.
|
||||
|
||||
For hermetic deployments, populate the model directory ahead of time and use
|
||||
the language-specific offline option:
|
||||
|
||||
- CLI: `--ocr-offline --ocr-model-dir /models/pp-ocrv6-small`
|
||||
- Rust: `ModelDownloadPolicy::Offline` with `OcrOptions::model_directory`
|
||||
- Python: `offline=True, model_directory="/models/pp-ocrv6-small"`
|
||||
- Node.js: `offline: true, modelDirectory: "/models/pp-ocrv6-small"`
|
||||
|
||||
The model artifacts come from
|
||||
[`GreatV/oar-ocr`](https://github.com/GreatV/oar-ocr/releases/tag/v0.7.0),
|
||||
whose OCR implementation and upstream PaddleOCR project use Apache-2.0
|
||||
licensing. Models are downloaded at runtime and are not embedded in any
|
||||
pdf-inspector package.
|
||||
|
||||
## Hosted fallback boundary
|
||||
|
||||
`pages_recommending_hosted` is available after the local pipeline completes.
|
||||
It marks pages whose completed OCR result is empty, low-confidence, or still
|
||||
appears incomplete.
|
||||
|
||||
Setup and execution failures happen before that result exists. A missing or
|
||||
incompatible PDFium/ONNX Runtime library, failed model acquisition, or OCR
|
||||
execution error is returned as an error. A downstream integration that has a
|
||||
hosted parser should catch that error and route the document to the hosted
|
||||
path. This keeps deployment problems distinct from page-quality judgments.
|
||||
|
||||
In `auto`, documents with no routed pages return successfully without touching
|
||||
PDFium, ONNX Runtime, the model cache, or the network.
|
||||
+1
-3
@@ -44,9 +44,7 @@ OCR calls that route work require compatible PDFium and ONNX Runtime shared
|
||||
libraries. Set `PDFIUM_LIB_PATH` and `ORT_DYLIB_PATH` when they are not on the
|
||||
platform library search path. The pinned OCR model set is downloaded and
|
||||
checksum-verified on the first routed page; use `offline=True` with a warm
|
||||
cache or `model_directory` to prohibit network access. See the
|
||||
[OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
for pinned downloads, supported platforms, and hosted-fallback behavior.
|
||||
cache or `model_directory` to prohibit network access.
|
||||
|
||||
## Usage
|
||||
|
||||
|
||||
@@ -383,10 +383,6 @@ shape; `Force` renders every selected page. OCR uses the existing deterministic
|
||||
table, column, reading-order, and Markdown assembly path; no learned layout
|
||||
model is included.
|
||||
|
||||
The [OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
lists the pinned PDFium and ONNX Runtime builds, environment variables, model
|
||||
cache behavior, and the error boundary downstream hosted fallbacks should use.
|
||||
|
||||
For ambiguous mixed pages, `Auto` privately retains clean native fragments
|
||||
instead of discarding them when OCR is selected. After recognition it compares
|
||||
script-agnostic text quality, OCR confidence, character overlap, and material
|
||||
|
||||
Generated
+2
-2
@@ -2089,7 +2089,7 @@ checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"dirs",
|
||||
"env_logger",
|
||||
@@ -2114,7 +2114,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+1
-3
@@ -41,9 +41,7 @@ OCR calls that route work require compatible PDFium and ONNX Runtime shared
|
||||
libraries. Set `PDFIUM_LIB_PATH` and `ORT_DYLIB_PATH` when they are not on the
|
||||
platform library search path. The pinned OCR model set is downloaded and
|
||||
checksum-verified on the first routed page; use `offline: true` with a warm
|
||||
cache or `modelDirectory` to prohibit network access. See the
|
||||
[OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
for pinned downloads, supported platforms, and hosted-fallback behavior.
|
||||
cache or `modelDirectory` to prohibit network access.
|
||||
|
||||
## API
|
||||
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.15.0",
|
||||
"version": "1.14.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -52,11 +52,11 @@
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.15.0"
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
||||
name = "pdf-inspector"
|
||||
# Keep package versions in sync with `python3 scripts/version.py <version>`.
|
||||
# CI publishes automatically when the synchronized change lands on main.
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
+1
-1
@@ -975,7 +975,7 @@ result = pdf_inspector.<span class="fn">process_pdf</span>(<span class="str">"do
|
||||
<script>
|
||||
(() => {
|
||||
const MAX_FILE_SIZE = 25 * 1024 * 1024;
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.15.0/pdf_inspector_wasm.js";
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.2/pdf_inspector_wasm.js";
|
||||
const input = document.querySelector("#pdf-input");
|
||||
const dropZone = document.querySelector("#drop-zone");
|
||||
const filePanel = document.querySelector("#demo-file");
|
||||
|
||||
+12
-150
@@ -403,15 +403,8 @@ pub(crate) fn detect_from_document(
|
||||
&& !(analysis.has_decodable_text_fonts && analysis.text_operator_count >= 10);
|
||||
let looks_like_scan =
|
||||
analysis.image_count <= 1 && analysis.text_operator_count < 50 && alphanum_low;
|
||||
// A template-image page below the `pages_with_text` floor is
|
||||
// a scan with incidental chrome (masthead, stamp, date line)
|
||||
// even when that chrome is diverse, decodable text — keep
|
||||
// this in sync with `page_ocr_signals`.
|
||||
let sparse_text_over_scan = analysis.has_template_image
|
||||
&& analysis.text_operator_count < config.min_text_ops_per_page.max(10);
|
||||
if (analysis.has_template_image && looks_like_scan)
|
||||
|| analysis.has_vector_text
|
||||
|| sparse_text_over_scan
|
||||
|| (analysis.text_operator_count < config.min_text_ops_per_page
|
||||
&& analysis.has_images)
|
||||
{
|
||||
@@ -1802,24 +1795,17 @@ pub(crate) fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u
|
||||
/// low alphanumeric diversity in raw string operands (unless decodable
|
||||
/// CID/ToUnicode fonts explain that away) — the gate used for
|
||||
/// `pages_with_template_images` and Mixed-type per-page routing.
|
||||
/// 2. Insufficient real text volume, using the same `effective_min_ops`
|
||||
/// floor (`min_text_ops_per_page.max(10)`) that `pages_with_text`
|
||||
/// applies to image-bearing pages. That floor is a per-page judgment,
|
||||
/// not part of the cross-page aggregate: classification counts a
|
||||
/// template-image page with fewer ops as textless and routes it to OCR,
|
||||
/// so this function must agree. The lower bare threshold (3) let a
|
||||
/// full-page scan carrying a small native masthead — a newspaper
|
||||
/// header, stamp, or date line of ~4 diverse, decodable text ops —
|
||||
/// extract as "a text page" here while whole-document classification
|
||||
/// called the same page scanned, silently dropping the page body from
|
||||
/// OCR routing. `alphanum_low` can't catch that case: masthead chrome
|
||||
/// is real text, so its byte diversity is high.
|
||||
///
|
||||
/// This function always evaluates against `DetectionConfig::default()` —
|
||||
/// it has no config parameter, and the per-page extraction path that calls
|
||||
/// it never carries one. A caller passing a custom `min_text_ops_per_page`
|
||||
/// to `detect_from_document` affects whole-document detection only; the
|
||||
/// two paths agree under the default configuration.
|
||||
/// 2. Insufficient real text volume, using `DetectionConfig::default()`'s
|
||||
/// `min_text_ops_per_page` (3) — the same threshold Mixed-type per-page
|
||||
/// routing applies via `text_operator_count < config.min_text_ops_per_page
|
||||
/// && has_images` (simplified here since a template image implies
|
||||
/// `has_images`). Deliberately *not* the higher `effective_min_ops`
|
||||
/// floor (`min_text_ops_per_page.max(10)`) that whole-document
|
||||
/// `PdfType::ImageBased`/`Scanned` classification uses for
|
||||
/// `pages_with_text` — that's a cross-page aggregate decision this
|
||||
/// per-page function has no way to replicate exactly, and the lower
|
||||
/// per-page threshold is the one a single page's own signals can
|
||||
/// actually agree with.
|
||||
///
|
||||
/// `has_vector_text` is true when a page has vector-outlined text (glyphs
|
||||
/// drawn as paths rather than shown via text-showing operators) —
|
||||
@@ -1841,7 +1827,7 @@ pub(crate) fn page_ocr_signals(doc: &Document, page_id: ObjectId) -> (bool, bool
|
||||
let looks_like_scan =
|
||||
analysis.image_count <= 1 && analysis.text_operator_count < 50 && alphanum_low;
|
||||
let insufficient_text =
|
||||
analysis.text_operator_count < DetectionConfig::default().min_text_ops_per_page.max(10);
|
||||
analysis.text_operator_count < DetectionConfig::default().min_text_ops_per_page;
|
||||
looks_like_scan || insufficient_text
|
||||
};
|
||||
|
||||
@@ -3009,130 +2995,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- masthead-over-scan tests: template image + sparse chrome ----------
|
||||
|
||||
/// Builds a page whose only image is a full-page scan inside a Form
|
||||
/// XObject, plus `masthead_ops` native text-show ops of diverse,
|
||||
/// decodable chrome (newspaper masthead / date line style).
|
||||
fn masthead_scan_page(masthead_lines: &[&str]) -> (Document, ObjectId) {
|
||||
use lopdf::dictionary;
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
|
||||
let image_id = doc.add_object(Object::Stream(lopdf::Stream::new(
|
||||
dictionary! {
|
||||
"Type" => "XObject",
|
||||
"Subtype" => Object::Name(b"Image".to_vec()),
|
||||
"Width" => Object::Integer(1500),
|
||||
"Height" => Object::Integer(2383),
|
||||
},
|
||||
Vec::new(),
|
||||
)));
|
||||
let form_id = doc.add_object(Object::Stream(lopdf::Stream::new(
|
||||
dictionary! {
|
||||
"Type" => "XObject",
|
||||
"Subtype" => Object::Name(b"Form".to_vec()),
|
||||
"Resources" => dictionary! {
|
||||
"XObject" => dictionary! {
|
||||
"Im0" => Object::Reference(image_id),
|
||||
},
|
||||
},
|
||||
},
|
||||
b"1500 0 0 2383 0 0 cm /Im0 Do".to_vec(),
|
||||
)));
|
||||
let font_id = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => Object::Name(b"Type1".to_vec()),
|
||||
"BaseFont" => Object::Name(b"Helvetica".to_vec()),
|
||||
});
|
||||
|
||||
let mut content = b"q /Fm0 Do Q BT /F1 12 Tf ".to_vec();
|
||||
for line in masthead_lines {
|
||||
content.extend_from_slice(format!("({line}) Tj ").as_bytes());
|
||||
}
|
||||
content.extend_from_slice(b"ET");
|
||||
let content_id =
|
||||
doc.add_object(Object::Stream(lopdf::Stream::new(dictionary! {}, content)));
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
Object::Dictionary(dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => Object::Reference(pages_id),
|
||||
"MediaBox" => vec![0.into(), 0.into(), 1500.into(), 2383.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! { "F1" => Object::Reference(font_id) },
|
||||
"XObject" => dictionary! { "Fm0" => Object::Reference(form_id) },
|
||||
},
|
||||
"Contents" => Object::Reference(content_id),
|
||||
}),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
Object::Dictionary(dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![Object::Reference(page_id)],
|
||||
"Count" => Object::Integer(1),
|
||||
}),
|
||||
);
|
||||
(doc, page_id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_masthead_over_form_wrapped_scan_needs_ocr() {
|
||||
// A full-page scan wrapped in a Form XObject with ~4 ops of real,
|
||||
// diverse masthead text. `alphanum_low` can't flag it (the chrome is
|
||||
// genuine text), so the sparse-text floor must: without OCR the page
|
||||
// body is silently lost while classification calls the page scanned.
|
||||
let (doc, page_id) = masthead_scan_page(&[
|
||||
"18",
|
||||
"FINANCIAL EXPRESS",
|
||||
"WWW.FINANCIALEXPRESS.COM",
|
||||
"FRIDAY, DECEMBER 13, 2024",
|
||||
]);
|
||||
let analysis = analyze_page_content(&doc, page_id);
|
||||
assert!(
|
||||
analysis.has_template_image,
|
||||
"sanity: full-page image inside the form must be found"
|
||||
);
|
||||
assert!(
|
||||
analysis.unique_alphanum_chars >= 10,
|
||||
"sanity: masthead text is diverse, alphanum_low cannot fire"
|
||||
);
|
||||
let (needs_ocr, _) = page_ocr_signals(&doc, page_id);
|
||||
assert!(
|
||||
needs_ocr,
|
||||
"template image + text below the pages_with_text floor is a scan"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_page_over_background_image_stays_native() {
|
||||
// Counterpart: a real text page over a full-page background image
|
||||
// (letterhead/watermark) has enough text ops to clear the
|
||||
// `pages_with_text` floor and must NOT be routed to OCR.
|
||||
let lines: Vec<String> = (0..12)
|
||||
.map(|i| format!("Paragraph line {i} with ordinary body text"))
|
||||
.collect();
|
||||
let refs: Vec<&str> = lines.iter().map(String::as_str).collect();
|
||||
let (doc, page_id) = masthead_scan_page(&refs);
|
||||
let analysis = analyze_page_content(&doc, page_id);
|
||||
assert!(
|
||||
analysis.has_template_image,
|
||||
"sanity: background image found"
|
||||
);
|
||||
assert!(
|
||||
analysis.text_operator_count >= 10,
|
||||
"sanity: body text clears the floor"
|
||||
);
|
||||
let (needs_ocr, _) = page_ocr_signals(&doc, page_id);
|
||||
assert!(
|
||||
!needs_ocr,
|
||||
"a text page with a background image must stay native"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------- P2 tests: Form XObject font traversal ----------
|
||||
|
||||
#[test]
|
||||
|
||||
Generated
+2
-2
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Browser WebAssembly bindings for pdf-inspector"
|
||||
|
||||
Reference in New Issue
Block a user