fix(extractor): require a usable ToUnicode mapping to clear the gid flag

A mapping to U+FFFD (or an empty string) is rejected by extraction as an invalid CMap result, so it must not count as decodable when deciding whether gid-named Differences codes are resolvable.
fix(extractor): don't flag gid Differences names covered by ToUnicode
2026-07-31 17:21:54 +01:00 · 2026-07-31 16:50:39 +01:00
11 changed files with 41 additions and 44 deletions
@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector"
-version = "0.1.7"
+version = "0.1.6"
 edition = "2021"
 autobins = false
 authors = ["Firecrawl Team"]
@@ -28,17 +28,17 @@ Evaluated on the [opendataloader-bench](https://github.com/opendataloader-projec

 | Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
 |---|---|---|---|---|---|
-| pdf-inspector | **0.875** | **0.915** | **0.814** | 0.788 | **0.470s** |
-| liteparse | 0.873 | 0.913 | 0.693 | **0.811** | 0.750s |
-| opendataloader | 0.831 | 0.902 | 0.489 | 0.739 | 2.569s |
-| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 17.117s |
-| markitdown | 0.589 | 0.844 | 0.273 | 0.000 | 16.165s |
+| pdf-inspector | **0.875** | **0.915** | **0.814** | 0.788 | **2.8s** |
+| liteparse | 0.870 | 0.908 | 0.693 | **0.811** | 13.9s |
+| opendataloader | 0.843 | 0.912 | 0.489 | 0.760 | 9.8s |
+| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 15.5s |
+| markitdown | 0.583 | 0.879 | 0.000 | 0.000 | 6.7s |

-Results were refreshed on July 31, 2026, on an Apple M4 Pro. Engine versions were pdf-inspector 0.2.6, LiteParse 2.10.1, OpenDataLoader 2.2.1, PyMuPDF4LLM 0.2.0, and MarkItDown 0.1.5. Speed is the median of five alternating or rotating complete corpus runs after an excluded warm-up run, with each parser processing documents sequentially in a single process.
+Results were refreshed on July 16, 2026, on an Apple M4 Pro. Engine versions were pdf-inspector 0.1.6, LiteParse 2.6.0, OpenDataLoader 2.1.1, PyMuPDF4LLM 0.2.0, and MarkItDown 0.1.4. Speed is the median of three complete corpus runs.

-The complete parser configuration, per-document predictions, evaluator output, and generated charts are available in the [reproducible results branch](https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results).
+For context, engines that use OCR or model-based document parsing (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus — pdf-inspector reaches the top of that range without either, in 2.8 seconds.

-**Best fit:** Native-text PDFs where speed, reading order, and table structure matter. In this comparison, pdf-inspector delivered the higher overall, reading-order, and table scores, along with the fastest complete run. That makes it a strong local default for reports, research papers, financial documents, invoices, and legal PDFs that need clean, structured Markdown without adding OCR latency or infrastructure.
+**Best fit:** Native-text PDFs where speed, reading order, and table structure matter. pdf-inspector delivered the highest overall, reading-order, and table scores, along with the fastest complete run in this benchmark. That makes it a strong local default for reports, research papers, financial documents, invoices, and legal PDFs that need clean, structured Markdown without adding OCR latency or infrastructure.

 Use the [paired benchmark harness](docs/benchmarking.md) to compare two local builds against the exact same corpus and evaluator revision.

@@ -30,14 +30,11 @@ overwrite one another.

 ## Published comparison protocol

-The public benchmark table was refreshed on July 31, 2026, on an Apple M4 Pro
-using pdf-inspector 0.2.6, LiteParse 2.10.1, OpenDataLoader 2.2.1,
-PyMuPDF4LLM 0.2.0, and MarkItDown 0.1.5. Every engine processed the same 200
-PDFs sequentially in a single process with OCR disabled. Reported speed is the
-median of five alternating or rotating complete corpus runs after an excluded
-warm-up run; quality scores come from the benchmark evaluator over all 200
-outputs. Raw timings, predictions, evaluations, and charts are available in the
-[results branch](https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results).
+The public benchmark table was refreshed on July 16, 2026, on an Apple M4 Pro
+using pdf-inspector 0.1.6, LiteParse 2.6.0, OpenDataLoader 2.1.1,
+PyMuPDF4LLM 0.2.0, and MarkItDown 0.1.4. Every engine processed the same 200
+PDFs with OCR disabled. Reported speed is the median of three complete corpus
+runs; quality scores come from the benchmark evaluator over all 200 outputs.

 ## Optional backend evidence probe

@@ -18,13 +18,13 @@ Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in

 | Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
 |---|---|---|---|---|---|
-| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **0.470s** |
-| liteparse | 0.873 | 0.913 | 0.693 | **0.811** | 0.750s |
-| opendataloader | 0.831 | 0.902 | 0.489 | 0.739 | 2.569s |
-| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 17.117s |
-| markitdown | 0.589 | 0.844 | 0.273 | 0.000 | 16.165s |
+| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **2.8s** |
+| liteparse | 0.870 | 0.908 | 0.693 | **0.811** | 13.9s |
+| opendataloader | 0.843 | 0.912 | 0.489 | 0.760 | 9.8s |
+| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 15.5s |
+| markitdown | 0.583 | 0.879 | 0.000 | 0.000 | 6.7s |

-Refreshed July 31, 2026, on Apple M4 Pro; speed is the median of five complete corpus runs after an excluded warm-up. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark), with raw timings and artifacts in the [results branch](https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results).
+Refreshed July 16, 2026, on Apple M4 Pro; speed is the median of three complete corpus runs. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).

 ## Install

@@ -18,13 +18,13 @@ Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in

 | Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
 |---|---|---|---|---|---|
-| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **0.470s** |
-| liteparse | 0.873 | 0.913 | 0.693 | **0.811** | 0.750s |
-| opendataloader | 0.831 | 0.902 | 0.489 | 0.739 | 2.569s |
-| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 17.117s |
-| markitdown | 0.589 | 0.844 | 0.273 | 0.000 | 16.165s |
+| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **2.8s** |
+| liteparse | 0.870 | 0.908 | 0.693 | **0.811** | 13.9s |
+| opendataloader | 0.843 | 0.912 | 0.489 | 0.760 | 9.8s |
+| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 15.5s |
+| markitdown | 0.583 | 0.879 | 0.000 | 0.000 | 6.7s |

-Refreshed July 31, 2026, on Apple M4 Pro; speed is the median of five complete corpus runs after an excluded warm-up. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark), with raw timings and artifacts in the [results branch](https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results).
+Refreshed July 16, 2026, on Apple M4 Pro; speed is the median of three complete corpus runs. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).

 ## Install

@@ -830,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"

 [[package]]
 name = "pdf-inspector"
-version = "0.1.7"
+version = "0.1.6"
 dependencies = [
 "env_logger",
 "log",
@@ -18,13 +18,13 @@ Built by [Firecrawl](https://firecrawl.dev) for hybrid OCR pipelines — extract

 | Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
 |---|---|---|---|---|---|
-| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **0.470s** |
-| liteparse | 0.873 | 0.913 | 0.693 | **0.811** | 0.750s |
-| opendataloader | 0.831 | 0.902 | 0.489 | 0.739 | 2.569s |
-| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 17.117s |
-| markitdown | 0.589 | 0.844 | 0.273 | 0.000 | 16.165s |
+| **pdf-inspector** | **0.875** | **0.915** | **0.814** | 0.788 | **2.8s** |
+| liteparse | 0.870 | 0.908 | 0.693 | **0.811** | 13.9s |
+| opendataloader | 0.843 | 0.912 | 0.489 | 0.760 | 9.8s |
+| pymupdf4llm | 0.735 | 0.886 | 0.401 | 0.424 | 15.5s |
+| markitdown | 0.583 | 0.879 | 0.000 | 0.000 | 6.7s |

-Refreshed July 31, 2026, on Apple M4 Pro; speed is the median of five complete corpus runs after an excluded warm-up. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark), with raw timings and artifacts in the [results branch](https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results).
+Refreshed July 16, 2026, on Apple M4 Pro; speed is the median of three complete corpus runs. Full methodology and versions are in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).

 ## Install

@@ -1,6 +1,6 @@
 {
  "name": "@firecrawl/pdf-inspector",
-  "version": "1.11.2",
+  "version": "1.11.1",
  "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
  "main": "index.js",
  "types": "index.d.ts",
@@ -49,8 +49,8 @@
    "@napi-rs/cli": "^3.4.1"
  },
  "optionalDependencies": {
-    "@firecrawl/pdf-inspector-linux-x64-gnu": "1.11.2",
-    "@firecrawl/pdf-inspector-darwin-arm64": "1.11.2",
-    "@firecrawl/pdf-inspector-win32-x64-msvc": "1.11.2"
+    "@firecrawl/pdf-inspector-linux-x64-gnu": "1.11.1",
+    "@firecrawl/pdf-inspector-darwin-arm64": "1.11.1",
+    "@firecrawl/pdf-inspector-win32-x64-msvc": "1.11.1"
  }
 }
@@ -6,7 +6,7 @@ build-backend = "maturin"
 name = "pdf-inspector"
 # Bump this to publish to PyPI — CI publishes automatically when the version
 # changes on main (same flow as napi/package.json for npm).
-version = "0.2.6"
+version = "0.2.5"
 description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
 readme = "docs/python.md"
 license = { text = "MIT" }
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"

 [[package]]
 name = "pdf-inspector"
-version = "0.1.7"
+version = "0.1.6"
 dependencies = [
 "env_logger",
 "include_dir",
@@ -740,7 +740,7 @@ dependencies = [

 [[package]]
 name = "pdf-inspector-wasm"
-version = "0.1.3"
+version = "0.1.2"
 dependencies = [
 "console_error_panic_hook",
 "js-sys",
@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector-wasm"
-version = "0.1.3"
+version = "0.1.2"
 edition = "2021"
 authors = ["Firecrawl Team"]
 description = "Browser WebAssembly bindings for pdf-inspector"