Compare commits
20
Commits
@@ -4,6 +4,9 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['Cargo.toml']
|
||||
# Manual fallback: retry a publish that failed after the version was
|
||||
# already merged (a plain re-push won't register as a version change).
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -14,6 +17,10 @@ env:
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
# Guard manual dispatches: crates.io trusted publishing matches
|
||||
# repo+workflow+environment but NOT branch, so without this a
|
||||
# workflow_dispatch from any branch could publish unmerged code.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
@@ -28,17 +35,26 @@ jobs:
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
# Manual dispatch publishes the current version regardless of the
|
||||
# previous commit; the crates.io check below still prevents
|
||||
# double-publishing an already-released version.
|
||||
echo "manual dispatch: publishing v$NEW_VERSION"
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
|
||||
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
name: Publish Python package
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['pyproject.toml']
|
||||
# Manual fallback: re-publish the current version without a version bump
|
||||
# (e.g. first run after PyPI trusted publishing is configured).
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
# Guard manual dispatches too: PyPI trusted publishing matches
|
||||
# repo+workflow+environment but NOT branch, so without this a
|
||||
# workflow_dispatch from any branch could publish unmerged code.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
# Manual dispatch always rebuilds and publishes. Combined with
|
||||
# skip-existing on the publish step, this repairs partial releases
|
||||
# (PyPI's version endpoint returns 200 even when only some of the
|
||||
# expected wheels were uploaded).
|
||||
echo "manual dispatch: publishing v$NEW_VERSION (skip-existing handles uploaded files)"
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# .get(): the parent commit may predate the static version field
|
||||
# (pyproject.toml used dynamic = ["version"]) — treat that as a change
|
||||
# so the very first merge of this workflow publishes.
|
||||
OLD_VERSION=$(git show HEAD~1:pyproject.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"].get("version", ""))')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/pypi-version.json --write-out "%{http_code}" \
|
||||
"https://pypi.org/pypi/pdf-inspector/$NEW_VERSION/json")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published to PyPI"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/pypi-version.json
|
||||
echo "Unexpected PyPI response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
build:
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
# macos-13 was retired by GitHub; macos-15-intel is the remaining
|
||||
# Intel runner label (available through 2027).
|
||||
- os: macos-15-intel
|
||||
target: x86_64-apple-darwin
|
||||
- os: macos-14
|
||||
target: aarch64-apple-darwin
|
||||
- os: windows-latest
|
||||
target: x86_64-pc-windows-msvc
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Build wheel
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
args: --release --out dist
|
||||
manylinux: auto
|
||||
|
||||
- name: Upload wheel
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: wheels-${{ matrix.target }}
|
||||
path: dist/*.whl
|
||||
if-no-files-found: error
|
||||
|
||||
sdist:
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
name: Build sdist
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build sdist
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
command: sdist
|
||||
args: --out dist
|
||||
|
||||
- name: Upload sdist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sdist
|
||||
path: dist/*.tar.gz
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
name: Publish to PyPI
|
||||
needs: [check-version, build, sdist]
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: List artifacts
|
||||
run: ls -la dist/
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: dist
|
||||
# Tolerate already-uploaded files so a manual re-run can complete
|
||||
# a release that previously failed partway through.
|
||||
skip-existing: true
|
||||
@@ -4,6 +4,9 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['napi/package.json']
|
||||
# Manual fallback: retry a publish that failed partway (per-package
|
||||
# already-published checks make re-runs idempotent).
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -12,6 +15,10 @@ permissions:
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
# Guard manual dispatches: npm trusted publishing matches
|
||||
# repo+workflow+environment but NOT branch, so without this a
|
||||
# workflow_dispatch from any branch could publish unmerged code.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
@@ -25,11 +32,21 @@ jobs:
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(node -p "require('./napi/package.json').version")
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
# Manual dispatch rebuilds and publishes the current version; the
|
||||
# per-package already-published checks in the publish job skip
|
||||
# anything that made it out in a previous partial run.
|
||||
echo "manual dispatch: publishing v$NEW_VERSION"
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OLD_VERSION=$(git show HEAD~1:napi/package.json | node -p "JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')).version")
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
if [ "$NEW_VERSION" != "$OLD_VERSION" ]; then
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
@@ -115,14 +132,81 @@ jobs:
|
||||
with:
|
||||
path: napi/artifacts
|
||||
|
||||
- name: Collect binaries and publish
|
||||
- name: Publish platform packages
|
||||
working-directory: napi
|
||||
run: |
|
||||
cp artifacts/bindings-*/*.node .
|
||||
VERSION="${{ needs.check-version.outputs.version }}"
|
||||
|
||||
for node_file in artifacts/bindings-*/pdf-inspector.*.node; do
|
||||
base=$(basename "$node_file")
|
||||
suffix=${base#pdf-inspector.}
|
||||
suffix=${suffix%.node}
|
||||
pkg="@firecrawl/pdf-inspector-$suffix"
|
||||
|
||||
if npm view "$pkg@$VERSION" version >/dev/null 2>&1; then
|
||||
echo "$pkg@$VERSION already published — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
dir="npm-dist/$suffix"
|
||||
mkdir -p "$dir"
|
||||
cp "$node_file" "$dir/"
|
||||
node -e '
|
||||
const [suffix, version] = process.argv.slice(1)
|
||||
const meta = {
|
||||
"linux-x64-gnu": { os: ["linux"], cpu: ["x64"], libc: ["glibc"] },
|
||||
"darwin-arm64": { os: ["darwin"], cpu: ["arm64"] },
|
||||
"win32-x64-msvc": { os: ["win32"], cpu: ["x64"] },
|
||||
}[suffix]
|
||||
if (!meta) {
|
||||
console.error(`unknown platform suffix: ${suffix} — add it to the meta map`)
|
||||
process.exit(1)
|
||||
}
|
||||
const pkg = {
|
||||
name: `@firecrawl/pdf-inspector-${suffix}`,
|
||||
version,
|
||||
description: `Prebuilt ${suffix} binary for @firecrawl/pdf-inspector`,
|
||||
main: `pdf-inspector.${suffix}.node`,
|
||||
files: [`pdf-inspector.${suffix}.node`],
|
||||
license: "MIT",
|
||||
engines: { node: ">= 10" },
|
||||
repository: { type: "git", url: "https://github.com/firecrawl/pdf-inspector" },
|
||||
publishConfig: { access: "public" },
|
||||
...meta,
|
||||
}
|
||||
require("fs").writeFileSync(`npm-dist/${suffix}/package.json`, JSON.stringify(pkg, null, 2) + "\n")
|
||||
' "$suffix" "$VERSION"
|
||||
|
||||
echo "=== $pkg@$VERSION ==="
|
||||
ls -la "$dir"
|
||||
(cd "$dir" && npm publish --provenance --access public)
|
||||
done
|
||||
|
||||
- name: Publish main package
|
||||
working-directory: napi
|
||||
run: |
|
||||
VERSION="${{ needs.check-version.outputs.version }}"
|
||||
|
||||
if npm view "@firecrawl/pdf-inspector@$VERSION" version >/dev/null 2>&1; then
|
||||
echo "@firecrawl/pdf-inspector@$VERSION already published — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cp artifacts/js-bindings/index.js .
|
||||
cp artifacts/js-bindings/index.d.ts .
|
||||
|
||||
echo "=== Package contents ==="
|
||||
ls -la *.node index.js index.d.ts
|
||||
# Stamp optionalDependencies to this exact version so the platform
|
||||
# pins can never drift from the main package version.
|
||||
node -e '
|
||||
const fs = require("fs")
|
||||
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"))
|
||||
for (const dep of Object.keys(pkg.optionalDependencies ?? {})) {
|
||||
pkg.optionalDependencies[dep] = pkg.version
|
||||
}
|
||||
fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n")
|
||||
'
|
||||
|
||||
echo "=== Main package contents ==="
|
||||
npm pack --dry-run
|
||||
|
||||
npm publish --provenance --access public
|
||||
|
||||
+15
-2
@@ -1,12 +1,25 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.4"
|
||||
version = "0.1.6"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/firecrawl/pdf-inspector"
|
||||
readme = "docs/rust-api.md"
|
||||
# Explicit allowlist: crates.io caps uploads at 10 MiB and tests/fixtures
|
||||
# alone exceeds that. external/bcmaps ships in the crate — tounicode.rs
|
||||
# loads it at runtime relative to CARGO_MANIFEST_DIR.
|
||||
include = [
|
||||
"src/**",
|
||||
"external/bcmaps/**",
|
||||
"docs/rust-api.md",
|
||||
"LICENSE",
|
||||
# maturin derives the sdist file list from this allowlist; the stub must
|
||||
# ship so wheels built from the sdist keep their type hints.
|
||||
"pdf_inspector.pyi",
|
||||
]
|
||||
|
||||
[lib]
|
||||
name = "pdf_inspector"
|
||||
@@ -14,7 +27,7 @@ crate-type = ["lib", "cdylib"]
|
||||
|
||||
[dependencies]
|
||||
# Python bindings
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
pyo3 = { version = "0.25", features = ["extension-module", "abi3-py38"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
[](https://pypi.org/project/pdf-inspector/)
|
||||
[](LICENSE)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
@@ -26,7 +27,7 @@ Evaluated on the [opendataloader-bench](https://github.com/opendataloader-projec
|
||||
|
||||
| Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
|
||||
|---|---|---|---|---|---|
|
||||
| pdf-inspector | 0.83 | 0.88 | 0.66 | 0.74 | 4s |
|
||||
| pdf-inspector | 0.83 | 0.89 | 0.66 | 0.74 | 4s |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
| markitdown | 0.58 | 0.88 | 0.00 | 0.00 | 8s |
|
||||
|
||||
+73
-10
@@ -1,9 +1,37 @@
|
||||
# Python API
|
||||
# pdf-inspector
|
||||
|
||||
Python bindings via [PyO3](https://pyo3.rs). Requires Rust toolchain for building from source.
|
||||
Fast PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Python bindings via [PyO3](https://pyo3.rs) for the [pdf-inspector](https://github.com/firecrawl/pdf-inspector) Rust library.
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
|
||||
## Features
|
||||
|
||||
- **Smart classification** — `text_based` / `scanned` / `image_based` / `mixed` in ~10–50ms, with a confidence score and per-page OCR routing.
|
||||
- **Markdown conversion** — headings, lists, code blocks, bold/italic, URL linking, and dual-mode table detection (PDF drawing ops + text-alignment heuristics).
|
||||
- **Layout-aware extraction** — multi-column reading order, position and font info per text item, RTL support.
|
||||
- **Robust text decoding** — CID/Type0 fonts via ToUnicode CMaps, plus automatic flagging of broken encodings so callers can fall back to OCR.
|
||||
- **Lightweight** — native Rust core, no ML models, no external services; ships type stubs.
|
||||
|
||||
## Benchmark
|
||||
|
||||
[opendataloader-bench](https://github.com/opendataloader-project/opendataloader-bench) corpus (200 PDFs), direct-extraction engines only — no OCR, no ML. Scores 0–1, higher is better:
|
||||
|
||||
| Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
|
||||
|---|---|---|---|---|---|
|
||||
| **pdf-inspector** | 0.83 | 0.88 | **0.66** | 0.74 | **4s** |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
|
||||
OCR/ML engines (docling, marker, mineru) score 0.83–0.88 overall but take 2–180 minutes on the same corpus. Full numbers in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install pdf-inspector
|
||||
```
|
||||
|
||||
Prebuilt wheels cover CPython ≥3.8 on Linux (x86_64, aarch64), macOS (Intel, Apple Silicon), and Windows (x64). Other platforms build from source, which requires a Rust toolchain. For local development in a repo checkout:
|
||||
|
||||
```bash
|
||||
pip install maturin
|
||||
maturin develop --release
|
||||
@@ -73,16 +101,51 @@ result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
|
||||
|
||||
## Types
|
||||
|
||||
**`PdfResult` fields:** `pdf_type`, `markdown`, `page_count`, `processing_time_ms`, `pages_needing_ocr`, `title`, `confidence`, `is_complex_layout`, `pages_with_tables`, `pages_with_columns`, `has_encoding_issues`
|
||||
Type stubs (`pdf_inspector.pyi`) ship with the package. Result types at a glance:
|
||||
|
||||
**`PdfClassification` fields:** `pdf_type`, `page_count`, `pages_needing_ocr` (0-indexed), `confidence`
|
||||
```python
|
||||
class PdfResult: # process_pdf / detect_pdf
|
||||
pdf_type: str # "text_based" | "scanned" | "image_based" | "mixed"
|
||||
markdown: str | None # extracted Markdown (None for detect_pdf)
|
||||
page_count: int
|
||||
processing_time_ms: int
|
||||
pages_needing_ocr: list[int]
|
||||
title: str | None
|
||||
confidence: float # 0.0 - 1.0
|
||||
is_complex_layout: bool
|
||||
pages_with_tables: list[int]
|
||||
pages_with_columns: list[int]
|
||||
has_encoding_issues: bool # broken font encodings — consider OCR fallback
|
||||
|
||||
**`TextItem` fields:** `text`, `x`, `y`, `width`, `height`, `font`, `font_size`, `page`, `is_bold`, `is_italic`, `item_type`
|
||||
class PdfClassification: # classify_pdf
|
||||
pdf_type: str
|
||||
page_count: int
|
||||
pages_needing_ocr: list[int] # 0-indexed
|
||||
confidence: float
|
||||
|
||||
**`RegionText` fields:** `text`, `needs_ocr`
|
||||
class TextItem: # extract_text_with_positions
|
||||
text: str
|
||||
x: float
|
||||
y: float
|
||||
width: float
|
||||
height: float
|
||||
font: str
|
||||
font_size: float
|
||||
page: int
|
||||
is_bold: bool
|
||||
is_italic: bool
|
||||
is_underline: bool
|
||||
is_strikeout: bool
|
||||
item_type: str
|
||||
|
||||
**`PageRegionTexts` fields:** `page` (0-indexed), `regions` (list of RegionText)
|
||||
class PageRegionTexts: # extract_text_in_regions
|
||||
page: int # 0-indexed
|
||||
regions: list[RegionText] # RegionText: text: str, needs_ocr: bool
|
||||
|
||||
**`PageMarkdown` fields:** `page` (0-indexed), `markdown`, `needs_ocr`
|
||||
|
||||
**`PagesExtractionResult` fields:** `pages` (list of PageMarkdown), `pages_with_tables` (1-indexed), `pages_with_columns` (1-indexed), `pages_needing_ocr` (1-indexed), `is_complex`
|
||||
class PagesExtractionResult: # extract_pages_markdown
|
||||
pages: list[PageMarkdown] # PageMarkdown: page (0-indexed), markdown, needs_ocr
|
||||
pages_with_tables: list[int] # 1-indexed
|
||||
pages_with_columns: list[int] # 1-indexed
|
||||
pages_needing_ocr: list[int] # 1-indexed
|
||||
is_complex: bool # any page has tables or multi-column layout
|
||||
```
|
||||
|
||||
+38
-2
@@ -1,12 +1,48 @@
|
||||
# Rust API
|
||||
# pdf-inspector
|
||||
|
||||
Add to your `Cargo.toml`:
|
||||
Fast PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Pure Rust, no ML models, no external services; the only PDF dependency is [lopdf](https://crates.io/crates/lopdf). Also available for [Python](https://pypi.org/project/pdf-inspector/) and [Node.js](https://www.npmjs.com/package/@firecrawl/pdf-inspector).
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
|
||||
## Features
|
||||
|
||||
- **Smart classification** — TextBased / Scanned / ImageBased / Mixed in ~10–50ms, with a confidence score and per-page OCR routing.
|
||||
- **Markdown conversion** — headings, lists, code blocks, bold/italic, URL linking, and dual-mode table detection (PDF drawing ops + text-alignment heuristics).
|
||||
- **Layout-aware extraction** — multi-column reading order, position and font info per text item, RTL support.
|
||||
- **Robust text decoding** — CID/Type0 fonts via ToUnicode CMaps, plus automatic flagging of broken encodings so callers can fall back to OCR.
|
||||
- **Lightweight** — pure Rust, no ML models, no external services; single PDF dependency ([lopdf](https://crates.io/crates/lopdf)).
|
||||
|
||||
## Benchmark
|
||||
|
||||
[opendataloader-bench](https://github.com/opendataloader-project/opendataloader-bench) corpus (200 PDFs), direct-extraction engines only — no OCR, no ML. Scores 0–1, higher is better:
|
||||
|
||||
| Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
|
||||
|---|---|---|---|---|---|
|
||||
| **pdf-inspector** | 0.83 | 0.88 | **0.66** | 0.74 | **4s** |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
|
||||
OCR/ML engines (docling, marker, mineru) score 0.83–0.88 overall but take 2–180 minutes on the same corpus. Full numbers in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
cargo add pdf-inspector
|
||||
```
|
||||
|
||||
For the latest unreleased changes, use the git dependency instead:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
```
|
||||
|
||||
The crate also ships CLI binaries — `pdf2md` (PDF → Markdown, with `--json`, `--pages`, `--select-pages`) and `detect-pdf` (classification, with `--analyze --json`):
|
||||
|
||||
```bash
|
||||
cargo install pdf-inspector
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Detect and extract in one call:
|
||||
|
||||
Generated
+1
-1
@@ -830,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.4"
|
||||
version = "0.1.6"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"log",
|
||||
|
||||
+28
-5
@@ -4,6 +4,26 @@ Fast PDF classification and region-based text extraction for Node.js/Bun. Native
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) for hybrid OCR pipelines — extract text from PDF structure where possible, fall back to OCR only when needed.
|
||||
|
||||
## Features
|
||||
|
||||
- **Smart classification** — text-based / scanned / image-based / mixed in ~10–50ms, with a confidence score and per-page OCR routing.
|
||||
- **Region-based extraction** — pull text from bounding boxes with per-region quality checks (`needsOcr`).
|
||||
- **Layout-aware** — multi-column reading order, position and font info per text item, RTL support.
|
||||
- **Robust text decoding** — CID/Type0 fonts via ToUnicode CMaps, plus automatic flagging of broken encodings so callers can fall back to OCR.
|
||||
- **Lightweight** — native Rust core via napi-rs, no ML models, no external services; ~5–6 MB platform binary, TypeScript definitions included.
|
||||
|
||||
## Benchmark
|
||||
|
||||
[opendataloader-bench](https://github.com/opendataloader-project/opendataloader-bench) corpus (200 PDFs), direct-extraction engines only — no OCR, no ML. Scores 0–1, higher is better:
|
||||
|
||||
| Engine | Overall | Reading order | Tables (TEDS) | Headings | Speed |
|
||||
|---|---|---|---|---|---|
|
||||
| **pdf-inspector** | 0.83 | 0.88 | **0.66** | 0.74 | **4s** |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
|
||||
OCR/ML engines (docling, marker, mineru) score 0.83–0.88 overall but take 2–180 minutes on the same corpus. Full numbers in the [repo README](https://github.com/firecrawl/pdf-inspector#benchmark).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
@@ -12,7 +32,7 @@ npm install @firecrawl/pdf-inspector
|
||||
bun add @firecrawl/pdf-inspector
|
||||
```
|
||||
|
||||
Prebuilt binaries included for **linux-x64** and **macOS ARM64**. No Rust toolchain needed.
|
||||
Prebuilt binaries for **Linux x64**, **macOS ARM64**, and **Windows x64** — npm installs only the one matching your platform. No Rust toolchain needed.
|
||||
|
||||
## API
|
||||
|
||||
@@ -90,10 +110,13 @@ interface RegionText {
|
||||
|
||||
## Platforms
|
||||
|
||||
| Platform | Architecture | Supported |
|
||||
|----------|-------------|-----------|
|
||||
| Linux | x64 | Yes |
|
||||
| macOS | ARM64 | Yes |
|
||||
Prebuilt binaries ship as platform-specific packages installed automatically via `optionalDependencies`:
|
||||
|
||||
| Platform | Architecture | Package |
|
||||
|----------|-------------|---------|
|
||||
| Linux | x64 (glibc) | `@firecrawl/pdf-inspector-linux-x64-gnu` |
|
||||
| macOS | ARM64 | `@firecrawl/pdf-inspector-darwin-arm64` |
|
||||
| Windows | x64 | `@firecrawl/pdf-inspector-win32-x64-msvc` |
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
"devDependencies": {
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.11.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.11.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.11.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
|
||||
+7
-6
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.10.2",
|
||||
"version": "1.11.1",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -22,7 +22,6 @@
|
||||
"files": [
|
||||
"index.js",
|
||||
"index.d.ts",
|
||||
"*.node",
|
||||
"bin/",
|
||||
"README.md"
|
||||
],
|
||||
@@ -40,10 +39,7 @@
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"aarch64-apple-darwin",
|
||||
"x86_64-pc-windows-msvc"
|
||||
],
|
||||
"package": {
|
||||
"name": "@firecrawl/pdf-inspector-js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"scripts": {
|
||||
"build": "napi build --platform --release",
|
||||
@@ -51,5 +47,10 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.11.1",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.11.1",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.11.1"
|
||||
}
|
||||
}
|
||||
|
||||
+9
-3
@@ -4,10 +4,11 @@ build-backend = "maturin"
|
||||
|
||||
[project]
|
||||
name = "pdf-inspector"
|
||||
# Version is sourced from Cargo.toml [package] version by maturin so the Python
|
||||
# artifact always tracks the crate release instead of drifting on its own.
|
||||
dynamic = ["version"]
|
||||
# Bump this to publish to PyPI — CI publishes automatically when the version
|
||||
# changes on main (same flow as napi/package.json for npm).
|
||||
version = "0.2.5"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
requires-python = ">=3.8"
|
||||
classifiers = [
|
||||
@@ -19,5 +20,10 @@ classifiers = [
|
||||
"Topic :: Text Processing",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/firecrawl/pdf-inspector"
|
||||
Repository = "https://github.com/firecrawl/pdf-inspector"
|
||||
Documentation = "https://github.com/firecrawl/pdf-inspector/blob/main/docs/python.md"
|
||||
|
||||
[tool.maturin]
|
||||
features = ["python"]
|
||||
|
||||
+1
-1
@@ -310,7 +310,7 @@
|
||||
<tr><th>Engine</th><th>Overall</th><th>Reading order</th><th>Tables</th><th>Headings</th><th>200 docs</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="us"><td>pdf-inspector</td><td>0.83</td><td>0.88</td><td>0.66</td><td>0.74</td><td>4s</td></tr>
|
||||
<tr class="us"><td>pdf-inspector</td><td>0.83</td><td>0.89</td><td>0.66</td><td>0.74</td><td>4s</td></tr>
|
||||
<tr><td>opendataloader</td><td>0.84</td><td>0.91</td><td>0.49</td><td>0.74</td><td>11s</td></tr>
|
||||
<tr><td>pymupdf4llm</td><td>0.73</td><td>0.89</td><td>0.40</td><td>0.41</td><td>18s</td></tr>
|
||||
<tr><td>markitdown</td><td>0.58</td><td>0.88</td><td>0.00</td><td>0.00</td><td>8s</td></tr>
|
||||
|
||||
+135
-9
@@ -176,14 +176,89 @@ fn extract_positioned_text_impl(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let ((mut items, mut rects, mut lines), has_gid_fonts, coords_rotated) =
|
||||
extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
// Clip to the visible page box: single-page extracts and imposed
|
||||
// spreads keep neighboring pages' content in the stream, positioned
|
||||
// outside the CropBox. Extracting it interleaves invisible text into
|
||||
// the page and poisons font statistics. Rotated pages are left alone
|
||||
// — their item coordinates are already transformed out of box space.
|
||||
let mut clipped_box: Option<(f32, f32, f32, f32)> = None;
|
||||
if !coords_rotated {
|
||||
if let Some((bx0, by0, bx1, by1)) = get_page_box(doc, page_id) {
|
||||
const TOL: f32 = 6.0;
|
||||
let outside = |it: &TextItem| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
!(cx >= bx0 - TOL && cx <= bx1 + TOL && it.y >= by0 - TOL && it.y <= by1 + TOL)
|
||||
};
|
||||
// Only clip when the off-page material reads as coherent text
|
||||
// (neighboring-page paragraphs). Curved/rotated display text
|
||||
// leaves short glyph fragments with artifact coordinates
|
||||
// outside the box, and those must stay.
|
||||
let off: Vec<&TextItem> = items.iter().filter(|it| outside(it)).collect();
|
||||
// Judge by character mass: paragraphs are dominated by long
|
||||
// word runs even when interleaved with short math fragments,
|
||||
// while glyph-confetti is short items through and through.
|
||||
let total_chars: usize = off.iter().map(|it| it.text.trim().chars().count()).sum();
|
||||
let wordy_chars: usize = off
|
||||
.iter()
|
||||
.map(|it| it.text.trim().chars().count())
|
||||
.filter(|&n| n >= 4)
|
||||
.sum();
|
||||
// Genuine neighboring-page content is cleanly separated from
|
||||
// on-page text. When an off-page item continues an on-page
|
||||
// line (same baseline, near-adjacent x), the coordinates are
|
||||
// artifacts of transforms we mis-model — don't clip those.
|
||||
let straddles = off.iter().any(|o| {
|
||||
items.iter().any(|i| {
|
||||
!outside(i)
|
||||
&& (i.y - o.y).abs() <= 2.0
|
||||
&& (o.x - (i.x + i.width)).abs() <= 10.0
|
||||
})
|
||||
});
|
||||
let coherent =
|
||||
off.len() >= 10 && wordy_chars * 2 >= total_chars.max(1) && !straddles;
|
||||
if bx1 - bx0 >= 72.0 && by1 - by0 >= 72.0 && coherent {
|
||||
let before = items.len();
|
||||
items.retain(|it| !outside(it));
|
||||
if items.len() < before {
|
||||
debug!(
|
||||
"page {}: clipped {} items outside page box ({:.0},{:.0})-({:.0},{:.0})",
|
||||
page_num,
|
||||
before - items.len(),
|
||||
bx0,
|
||||
by0,
|
||||
bx1,
|
||||
by1
|
||||
);
|
||||
// Only prune off-page geometry when off-page text
|
||||
// existed — same neighboring-page content.
|
||||
let overlaps = |x: f32, y: f32, w: f32, h: f32| {
|
||||
let (x0, x1) = if w < 0.0 { (x + w, x) } else { (x, x + w) };
|
||||
let (y0, y1) = if h < 0.0 { (y + h, y) } else { (y, y + h) };
|
||||
x0 < bx1 + TOL && x1 > bx0 - TOL && y0 < by1 + TOL && y1 > by0 - TOL
|
||||
};
|
||||
rects.retain(|r| overlaps(r.x, r.y, r.width, r.height));
|
||||
clipped_box = Some((bx0, by0, bx1, by1));
|
||||
lines.retain(|l| {
|
||||
overlaps(
|
||||
l.x1.min(l.x2),
|
||||
l.y1.min(l.y2),
|
||||
(l.x2 - l.x1).abs(),
|
||||
(l.y2 - l.y1).abs(),
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_gid_fonts {
|
||||
gid_encoded_pages.insert(*page_num);
|
||||
}
|
||||
@@ -223,7 +298,18 @@ fn extract_positioned_text_impl(
|
||||
all_lines.extend(lines);
|
||||
|
||||
// Extract hyperlinks from page annotations
|
||||
let links = extract_page_links(doc, page_id, *page_num);
|
||||
let mut links = extract_page_links(doc, page_id, *page_num);
|
||||
// Annotations from the neighboring page are off-box too.
|
||||
if let Some((bx0, by0, bx1, by1)) = clipped_box {
|
||||
links.retain(|it| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
// Center-y, not it.y: link items carry an annotation rect,
|
||||
// so y is a box edge — unlike text items, where y is a
|
||||
// baseline and testing it directly is the natural semantics.
|
||||
let cy = it.y + it.height / 2.0;
|
||||
cx >= bx0 - 6.0 && cx <= bx1 + 6.0 && cy >= by0 - 6.0 && cy <= by1 + 6.0
|
||||
});
|
||||
}
|
||||
all_items.extend(links);
|
||||
}
|
||||
|
||||
@@ -967,6 +1053,46 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Visible page box: CropBox if present, else MediaBox, walking page-tree
|
||||
/// inheritance (both attributes are inheritable). Returns normalized
|
||||
/// (x0, y0, x1, y1) in PDF space.
|
||||
fn get_page_box(doc: &Document, page_id: ObjectId) -> Option<(f32, f32, f32, f32)> {
|
||||
fn find_box(doc: &Document, page_id: ObjectId, key: &[u8]) -> Option<Vec<f32>> {
|
||||
let mut id = page_id;
|
||||
for _ in 0..32 {
|
||||
let dict = doc.get_dictionary(id).ok()?;
|
||||
if let Ok(obj) = dict.get(key) {
|
||||
let arr = match obj {
|
||||
Object::Array(a) => Some(a.clone()),
|
||||
Object::Reference(r) => match doc.get_object(*r) {
|
||||
Ok(Object::Array(a)) => Some(a.clone()),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
};
|
||||
if let Some(arr) = arr {
|
||||
let vals: Vec<f32> = arr.iter().filter_map(get_number).collect();
|
||||
if vals.len() >= 4 {
|
||||
return Some(vals);
|
||||
}
|
||||
}
|
||||
}
|
||||
match dict.get(b"Parent") {
|
||||
Ok(Object::Reference(p)) => id = *p,
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
let v = find_box(doc, page_id, b"CropBox").or_else(|| find_box(doc, page_id, b"MediaBox"))?;
|
||||
Some((
|
||||
v[0].min(v[2]),
|
||||
v[1].min(v[3]),
|
||||
v[0].max(v[2]),
|
||||
v[1].max(v[3]),
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
+78
-16
@@ -673,18 +673,51 @@ pub fn extract_text_in_regions_mem(
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
// Exclusive item->region assignment: overlapping layout regions used
|
||||
// to extract shared items into EVERY region they touched (the
|
||||
// 1.5pt inclusion margin makes borders generous), duplicating whole
|
||||
// lines in the final markdown on 21% of bench docs — and downstream
|
||||
// duplicate-handling sometimes dropped the variant holding a
|
||||
// sentence tail, turning duplication into content LOSS. Each item
|
||||
// now belongs to the single region with the largest overlap area;
|
||||
// items are partitioned, never suppressed, so no content can vanish.
|
||||
let all_bounds: Vec<RegionBounds> = regions
|
||||
.iter()
|
||||
.map(|rect| {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
region_bounds(rx1, ry1, rx2, ry2, page_h, coords)
|
||||
})
|
||||
.collect();
|
||||
// Single pass over items: assign each to the best-overlap region and
|
||||
// bucket the clone directly (review: avoid a second O(items x
|
||||
// regions) traversal). `had_candidates` marks regions that touched
|
||||
// at least one item even if every one was assigned elsewhere.
|
||||
let mut region_items: Vec<Vec<TextItem>> = vec![Vec::new(); regions.len()];
|
||||
let mut had_candidates: Vec<bool> = vec![false; regions.len()];
|
||||
if let Some(items) = items {
|
||||
for item in items {
|
||||
let mut best: Option<usize> = None;
|
||||
let mut best_area = 0.0_f32;
|
||||
for (ri, b) in all_bounds.iter().enumerate() {
|
||||
if !region_overlaps_item(item, *b) {
|
||||
continue;
|
||||
}
|
||||
had_candidates[ri] = true;
|
||||
let area = region_item_overlap_area(item, *b);
|
||||
if area > best_area {
|
||||
best_area = area;
|
||||
best = Some(ri);
|
||||
}
|
||||
}
|
||||
if let Some(ri) = best {
|
||||
region_items[ri].push(item.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
let matched: Vec<TextItem> = match items {
|
||||
Some(items) => items
|
||||
.iter()
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect(),
|
||||
None => Vec::new(),
|
||||
};
|
||||
for (region_idx, _rect) in regions.iter().enumerate() {
|
||||
let matched: Vec<TextItem> = std::mem::take(&mut region_items[region_idx]);
|
||||
let assigned_count = matched.len();
|
||||
let has_text_quality_issue = region_items_have_decoding_issue(&matched);
|
||||
let text = collect_text_from_matched_items(matched, adaptive_threshold);
|
||||
let has_cid_issue = is_cid_garbage(&text);
|
||||
@@ -698,8 +731,21 @@ pub fn extract_text_in_regions_mem(
|
||||
// Check per-region text quality instead of blanket page-level
|
||||
// GID rejection. A GID font in a logo elsewhere on the page
|
||||
// shouldn't force GPU OCR for clean text regions.
|
||||
let needs_ocr =
|
||||
ocr_reason.is_some() || text.trim().is_empty() || is_garbage_text(&text);
|
||||
// A region whose ONLY overlapping items were assigned to a
|
||||
// better-overlapping neighbor must not fall back to OCR: the
|
||||
// pixels it would re-read belong to that neighbor, and OCR
|
||||
// would reintroduce the duplication exclusivity removed.
|
||||
// Before exclusive assignment these regions were non-empty
|
||||
// native (no OCR), so this preserves the old OCR load too.
|
||||
// Requires ZERO items assigned HERE: a region whose own
|
||||
// assigned items materialize to empty text (whitespace-only,
|
||||
// collector-filtered) keeps its OCR fallback.
|
||||
let lost_to_neighbor = text.trim().is_empty()
|
||||
&& ocr_reason.is_none()
|
||||
&& assigned_count == 0
|
||||
&& had_candidates[region_idx];
|
||||
let needs_ocr = !lost_to_neighbor
|
||||
&& (ocr_reason.is_some() || text.trim().is_empty() || is_garbage_text(&text));
|
||||
|
||||
page_results.push(RegionText {
|
||||
text,
|
||||
@@ -3215,8 +3261,26 @@ fn region_bounds(
|
||||
}
|
||||
}
|
||||
|
||||
/// Inclusion margin shared by the region/item overlap predicates and the
|
||||
/// exclusive-assignment area score — these MUST stay in sync: an item that
|
||||
/// passes the boolean guard must always have positive overlap area.
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
|
||||
/// Overlap area between an item and region bounds (same margin as the
|
||||
/// boolean test) — the exclusive-assignment score.
|
||||
fn region_item_overlap_area(item: &TextItem, bounds: RegionBounds) -> f32 {
|
||||
let item_x_max = item.x + text_utils::effective_width(item);
|
||||
let item_y_max = item.y + item.height;
|
||||
let x_overlap = (item_x_max.min(bounds.x_max + REGION_MARGIN)
|
||||
- item.x.max(bounds.x_min - REGION_MARGIN))
|
||||
.max(0.0);
|
||||
let y_overlap = (item_y_max.min(bounds.y_max + REGION_MARGIN)
|
||||
- item.y.max(bounds.y_min - REGION_MARGIN))
|
||||
.max(0.0);
|
||||
x_overlap * y_overlap
|
||||
}
|
||||
|
||||
fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let item_x_min = item.x;
|
||||
let item_x_max = item.x + text_utils::effective_width(item);
|
||||
let item_y_min = item.y;
|
||||
@@ -3232,7 +3296,6 @@ fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
}
|
||||
|
||||
fn region_overlaps_rect(rect: &PdfRect, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let (x_min, y_min, x_max, y_max) = normalized_rect_edges(rect);
|
||||
ranges_overlap(
|
||||
x_min,
|
||||
@@ -3248,7 +3311,6 @@ fn region_overlaps_rect(rect: &PdfRect, bounds: RegionBounds) -> bool {
|
||||
}
|
||||
|
||||
fn region_overlaps_line(line: &PdfLine, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let x_min = line.x1.min(line.x2);
|
||||
let x_max = line.x1.max(line.x2);
|
||||
let y_min = line.y1.min(line.y2);
|
||||
|
||||
+307
-7
@@ -179,6 +179,128 @@ pub(crate) fn split_side_by_side(items: &[TextItem]) -> Vec<(f32, f32)> {
|
||||
/// zone layout (calendar months, form sections). This function checks if hint
|
||||
/// regions pair up at the same Y bands and returns `[(x_min, split), (split,
|
||||
/// x_max)]` if a consistent split exists.
|
||||
/// True when a table-shaped rect cluster (≥6 rects) ends at an interior band
|
||||
/// boundary and its rows visibly continue on the far side: cell-like text
|
||||
/// across the boundary is y-aligned with most cluster rows, and nearly all
|
||||
/// far-side text in the cluster's y-range participates in that alignment.
|
||||
/// Tables often rule only their leading columns, so the text gap before the
|
||||
/// borderless columns masquerades as a page-layout gutter — a real second
|
||||
/// layout column would instead be dense prose that doesn't track table rows.
|
||||
fn rect_cluster_spans_band_boundary(
|
||||
items: &[TextItem],
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
bands: &[(f32, f32)],
|
||||
) -> bool {
|
||||
if bands.len() < 2 {
|
||||
return false;
|
||||
}
|
||||
// Normalize: raw PDF rects can carry negative extents.
|
||||
let page_rects: Vec<(f32, f32, f32, f32)> = rects
|
||||
.iter()
|
||||
.filter(|r| r.page == page)
|
||||
.map(|r| {
|
||||
let (x, w) = if r.width < 0.0 {
|
||||
(r.x + r.width, -r.width)
|
||||
} else {
|
||||
(r.x, r.width)
|
||||
};
|
||||
let (y, h) = if r.height < 0.0 {
|
||||
(r.y + r.height, -r.height)
|
||||
} else {
|
||||
(r.y, r.height)
|
||||
};
|
||||
(x, y, w, h)
|
||||
})
|
||||
.collect();
|
||||
if page_rects.len() < 6 {
|
||||
return false;
|
||||
}
|
||||
let clusters = crate::tables::detect_rects::cluster_rects(&page_rects, 3.0, 6);
|
||||
let boundaries: Vec<f32> = bands[..bands.len() - 1].iter().map(|&(_, hi)| hi).collect();
|
||||
|
||||
boundaries.iter().any(|&b| {
|
||||
// Y-ranges of clusters that individually indicate the split cuts a
|
||||
// table: either ruled on both sides of the boundary, or ending at
|
||||
// the boundary with cell-like text row-aligned beyond it.
|
||||
let mut table_y_ranges: Vec<(f32, f32)> = Vec::new();
|
||||
for cluster in &clusters {
|
||||
let bbox = cluster.iter().fold(
|
||||
(
|
||||
f32::INFINITY,
|
||||
f32::INFINITY,
|
||||
f32::NEG_INFINITY,
|
||||
f32::NEG_INFINITY,
|
||||
),
|
||||
|(x0, y0, x1, y1), &i| {
|
||||
let (x, y, w, h) = page_rects[i];
|
||||
(x0.min(x), y0.min(y), x1.max(x + w), y1.max(y + h))
|
||||
},
|
||||
);
|
||||
let spans = bbox.0 < b - 20.0 && bbox.2 > b + 20.0;
|
||||
let ends_at = bbox.2 >= b - 60.0 && bbox.2 <= b + 10.0 && bbox.0 <= b;
|
||||
if !spans && !ends_at {
|
||||
continue;
|
||||
}
|
||||
// Distinct row baselines of items inside the cluster bbox.
|
||||
let mut row_ys: Vec<f32> = Vec::new();
|
||||
for it in items {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
if it.page == page
|
||||
&& cx > bbox.0
|
||||
&& cx < bbox.2
|
||||
&& it.y >= bbox.1 - 2.0
|
||||
&& it.y <= bbox.3 + 2.0
|
||||
&& !row_ys.iter().any(|&y| (y - it.y).abs() <= 2.0)
|
||||
{
|
||||
row_ys.push(it.y);
|
||||
}
|
||||
}
|
||||
if row_ys.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
// Cell-like far-side items row-aligned with the cluster.
|
||||
let cell_like = |it: &&TextItem| it.width <= 150.0;
|
||||
let far_aligned_rows = row_ys
|
||||
.iter()
|
||||
.filter(|&&y| {
|
||||
items.iter().any(|it| {
|
||||
it.page == page
|
||||
&& it.x + it.width / 2.0 > b
|
||||
&& cell_like(&it)
|
||||
&& (it.y - y).abs() <= 2.0
|
||||
})
|
||||
})
|
||||
.count();
|
||||
if far_aligned_rows >= 2 && far_aligned_rows * 2 >= row_ys.len() {
|
||||
table_y_ranges.push((bbox.1, bbox.3));
|
||||
}
|
||||
}
|
||||
if table_y_ranges.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// The split is only wrong if the table rows account for most of the
|
||||
// far side. A figure legitimately spanning two text columns leaves
|
||||
// the majority of far-side text (column prose) outside its y-range.
|
||||
let far: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|it| it.page == page && it.x + it.width / 2.0 > b)
|
||||
.collect();
|
||||
if far.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let inside = far
|
||||
.iter()
|
||||
.filter(|it| {
|
||||
table_y_ranges
|
||||
.iter()
|
||||
.any(|&(lo, hi)| it.y >= lo - 2.0 && it.y <= hi + 2.0)
|
||||
})
|
||||
.count();
|
||||
inside * 10 >= far.len() * 6
|
||||
})
|
||||
}
|
||||
|
||||
fn split_from_hint_regions(items: &[TextItem], rects: &[PdfRect], page: u32) -> Vec<(f32, f32)> {
|
||||
use crate::tables::{cluster_rects, RectHintRegion};
|
||||
|
||||
@@ -622,8 +744,46 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
cols.len() >= 2
|
||||
};
|
||||
|
||||
// Chart-bar regions: bar charts drawn as filled rects read as cell
|
||||
// rects or aligned text and get gridded into phantom tables. Their
|
||||
// items are excluded from every table detector below and flow through
|
||||
// as plain text instead.
|
||||
let page_rect_vec: Vec<PdfRect> =
|
||||
rects.iter().filter(|r| r.page == page).cloned().collect();
|
||||
let chart_regions = crate::tables::detect_chart_regions(&page_items, &page_rect_vec, page);
|
||||
// Pad the claim region: axis/category labels sit just outside the
|
||||
// bar rects (below the axis, left of the scale) and belong to the
|
||||
// chart as much as the bars do.
|
||||
const CHART_PAD: f32 = 20.0;
|
||||
let in_chart = |it: &TextItem| {
|
||||
chart_regions.iter().any(|&(x0, y0, x1, y1)| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
cx >= x0 - CHART_PAD
|
||||
&& cx <= x1 + CHART_PAD
|
||||
&& it.y >= y0 - CHART_PAD
|
||||
&& it.y <= y1 + CHART_PAD
|
||||
})
|
||||
};
|
||||
if !chart_regions.is_empty() {
|
||||
log::debug!(
|
||||
"page {}: {} chart region(s) masked from table detection",
|
||||
page,
|
||||
chart_regions.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Check for side-by-side layout (e.g. two tables placed left and right)
|
||||
let mut bands = split_side_by_side(&page_items);
|
||||
// A rect table crossing a proposed split boundary means the "gutter"
|
||||
// is really the gap between ruled and borderless table columns —
|
||||
// splitting there cleaves the table in half. Veto the split.
|
||||
if !bands.is_empty() && rect_cluster_spans_band_boundary(&page_items, rects, page, &bands) {
|
||||
log::debug!(
|
||||
"page {}: side-by-side split vetoed by spanning rect cluster",
|
||||
page
|
||||
);
|
||||
bands.clear();
|
||||
}
|
||||
// Fallback: use rect hint regions to detect side-by-side layout
|
||||
// when the text gap is too narrow for split_side_by_side to detect
|
||||
// (e.g. calendars with left/right month columns ~10pt apart).
|
||||
@@ -705,6 +865,16 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
// Track which band-local indices are claimed by structural detection
|
||||
let mut rect_claimed: HashSet<usize> = HashSet::new();
|
||||
|
||||
// Pre-claim chart items: every detector below skips claimed
|
||||
// indices, and unclaimed-by-tables text flows out as plain lines.
|
||||
if !chart_regions.is_empty() {
|
||||
for (idx, item) in band_items.iter().enumerate() {
|
||||
if in_chart(item) {
|
||||
rect_claimed.insert(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 0. Structure-tree detection (highest priority — semantic PDF tagging)
|
||||
// Only use struct-tree tables when they capture a majority (≥50%) of
|
||||
// band items. Incomplete struct trees (partial tagging) should fall
|
||||
@@ -960,15 +1130,20 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
}
|
||||
}
|
||||
if synth_lines.len() >= 10 {
|
||||
let page_text: Vec<TextItem> = text_items
|
||||
// Chart text stays out of the thin-rect fallback too — a
|
||||
// chart's thin grid rules would otherwise re-grid it.
|
||||
let (page_text, page_text_map): (Vec<TextItem>, Vec<usize>) = text_items
|
||||
.iter()
|
||||
.filter(|i| i.page == page)
|
||||
.cloned()
|
||||
.collect();
|
||||
.enumerate()
|
||||
.filter(|(_, i)| i.page == page && !in_chart(i))
|
||||
.map(|(idx, i)| (i.clone(), idx))
|
||||
.unzip();
|
||||
let line_tables = detect_tables_from_lines(&page_text, &synth_lines, page);
|
||||
for table in &line_tables {
|
||||
for &idx in &table.item_indices {
|
||||
table_items.insert(idx);
|
||||
if let Some(&global_idx) = page_text_map.get(idx) {
|
||||
table_items.insert(global_idx);
|
||||
}
|
||||
}
|
||||
let table_y = table.rows.first().copied().unwrap_or(0.0);
|
||||
let table_md = table_to_markdown(table);
|
||||
@@ -992,10 +1167,20 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
band_items.len(),
|
||||
was_split
|
||||
);
|
||||
let heuristic_tables = detect_tables(band_items, base_size, page_has_columns);
|
||||
// Chart text stays out of the retry as well.
|
||||
let (chart_free, chart_free_map): (Vec<TextItem>, Vec<usize>) = band_items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, it)| !in_chart(it))
|
||||
.map(|(i, it)| (it.clone(), i))
|
||||
.unzip();
|
||||
let heuristic_tables = detect_tables(&chart_free, base_size, page_has_columns);
|
||||
for table in &heuristic_tables {
|
||||
for &idx in &table.item_indices {
|
||||
if let Some(&page_idx) = band_index_map.get(idx) {
|
||||
if let Some(&page_idx) = chart_free_map
|
||||
.get(idx)
|
||||
.and_then(|&band_idx| band_index_map.get(band_idx))
|
||||
{
|
||||
if let Some(&(global_idx, _)) = group.get(page_idx) {
|
||||
table_items.insert(global_idx);
|
||||
}
|
||||
@@ -1227,6 +1412,121 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn make_item_w(x: f32, y: f32, width: f32, page: u32) -> TextItem {
|
||||
let mut it = make_item(x, y, page);
|
||||
it.width = width;
|
||||
it
|
||||
}
|
||||
|
||||
/// 4-row × 2-col ruled grid from x=100..300 (rows every 20pt from y=600).
|
||||
fn ruled_cluster_rects() -> Vec<PdfRect> {
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..4 {
|
||||
for col in 0..2 {
|
||||
rects.push(PdfRect {
|
||||
x: 100.0 + col as f32 * 100.0,
|
||||
y: 600.0 + row as f32 * 20.0,
|
||||
width: 100.0,
|
||||
height: 20.0,
|
||||
page: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
rects
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn band_veto_cluster_ruled_across_boundary() {
|
||||
// Rects on both sides of the boundary and cell text on both sides,
|
||||
// row-aligned → the split cuts straight through a drawn table.
|
||||
let mut rects = ruled_cluster_rects();
|
||||
for r in &mut rects {
|
||||
r.width = 150.0; // right column now spans 250..400, past b=320
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
for row in 0..4 {
|
||||
let y = 610.0 + row as f32 * 20.0;
|
||||
items.push(make_item_w(110.0, y, 80.0, 1)); // left cells
|
||||
items.push(make_item_w(330.0, y, 30.0, 1)); // right cells past b
|
||||
}
|
||||
assert!(rect_cluster_spans_band_boundary(
|
||||
&items,
|
||||
&rects,
|
||||
1,
|
||||
&[(90.0, 320.0), (320.0, 500.0)]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn band_veto_ignores_spanning_figure() {
|
||||
// A figure's rects span the boundary at the top of the page, but the
|
||||
// far side is dominated by column prose below it → keep the split.
|
||||
let mut rects = ruled_cluster_rects(); // y 600..680
|
||||
for r in &mut rects {
|
||||
r.width = 150.0; // spans past b=320
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
// A few figure labels inside the cluster, aligned rows.
|
||||
for row in 0..4 {
|
||||
let y = 610.0 + row as f32 * 20.0;
|
||||
items.push(make_item_w(110.0, y, 40.0, 1));
|
||||
items.push(make_item_w(330.0, y, 20.0, 1));
|
||||
}
|
||||
// Dense prose column far below the figure (outside cluster y-range).
|
||||
let mut y = 100.0;
|
||||
while y < 560.0 {
|
||||
items.push(make_item_w(330.0, y, 140.0, 1));
|
||||
y += 12.0;
|
||||
}
|
||||
assert!(!rect_cluster_spans_band_boundary(
|
||||
&items,
|
||||
&rects,
|
||||
1,
|
||||
&[(90.0, 320.0), (320.0, 500.0)]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn band_veto_borderless_columns_continue_rows() {
|
||||
// Rects end at x=300 (just short of b=320); cell-like text at x=340
|
||||
// aligns with every grid row → the "gutter" is inside the table.
|
||||
let rects = ruled_cluster_rects();
|
||||
let mut items = Vec::new();
|
||||
for row in 0..4 {
|
||||
let y = 610.0 + row as f32 * 20.0;
|
||||
items.push(make_item_w(110.0, y, 80.0, 1)); // label cells
|
||||
items.push(make_item_w(340.0, y, 30.0, 1)); // borderless column
|
||||
}
|
||||
assert!(rect_cluster_spans_band_boundary(
|
||||
&items,
|
||||
&rects,
|
||||
1,
|
||||
&[(90.0, 320.0), (320.0, 500.0)]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn band_veto_ignores_prose_column() {
|
||||
// Dense prose right of the boundary: wide lines, three per grid row,
|
||||
// mostly not row-aligned → keep the side-by-side split.
|
||||
let rects = ruled_cluster_rects();
|
||||
let mut items = Vec::new();
|
||||
for row in 0..4 {
|
||||
items.push(make_item_w(110.0, 610.0 + row as f32 * 20.0, 80.0, 1));
|
||||
}
|
||||
let mut y = 602.0;
|
||||
while y < 680.0 {
|
||||
items.push(make_item_w(340.0, y, 200.0, 1)); // full-width prose lines
|
||||
y += 7.0;
|
||||
}
|
||||
assert!(!rect_cluster_spans_band_boundary(
|
||||
&items,
|
||||
&rects,
|
||||
1,
|
||||
&[(90.0, 320.0), (320.0, 500.0)]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_from_hint_regions_too_few_rects() {
|
||||
// Fewer than 60 rects → no split
|
||||
|
||||
@@ -914,7 +914,126 @@ fn looks_like_number(s: &str) -> bool {
|
||||
///
|
||||
/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
|
||||
pub fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||
is_dot_leader_toc(cells) || is_tabular_toc(cells)
|
||||
is_dot_leader_toc(cells) || is_tabular_toc(cells) || is_page_number_toc(cells)
|
||||
}
|
||||
|
||||
/// Parse a page-number-like token: a short arabic integer (≤4 digits) or a
|
||||
/// canonical roman numeral (front-matter pages: i, ii, …, xxxviii). Roman
|
||||
/// parsing is shared with the formatter via `super::canonical_roman_value` so
|
||||
/// the two stay in sync.
|
||||
fn page_number_value(token: &str) -> Option<u32> {
|
||||
let t = token.trim();
|
||||
if t.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if t.chars().all(|c| c.is_ascii_digit()) && t.len() <= 4 {
|
||||
return t.parse().ok();
|
||||
}
|
||||
super::canonical_roman_value(t)
|
||||
}
|
||||
|
||||
/// Page-number-column TOC: title-based contents with no dot leaders and no
|
||||
/// section numbers (e.g. "About the Publisher vii", "Experiment #1 … 3").
|
||||
/// The signature is a text-title first column and a last column that is almost
|
||||
/// entirely page numbers whose values are *mostly non-decreasing* — the
|
||||
/// monotonic run is what separates a real TOC from an incidental 2-column
|
||||
/// numeric data table.
|
||||
pub(super) fn is_page_number_toc(cells: &[Vec<String>]) -> bool {
|
||||
let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
|
||||
// A page-number TOC is a narrow list (title + page, optionally a leader
|
||||
// column). Wider grids are data tables, not contents.
|
||||
if !(2..=3).contains(&num_cols) || cells.len() < 5 {
|
||||
return false;
|
||||
}
|
||||
let last = num_cols - 1;
|
||||
|
||||
// No header row: a TOC's first row is already an entry, so its last cell is
|
||||
// a page number. A data table's first row is a column header (non-numeric,
|
||||
// or an empty units cell like "Category | ") — the tell that separates
|
||||
// "Mineral | CEC" tables from real contents. Check the actual first row,
|
||||
// not the first non-empty one, so a blank header cell still rejects.
|
||||
let first_last = cells[0].get(last).map(|s| s.trim()).unwrap_or("");
|
||||
if page_number_value(first_last).is_none() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Last column: page numbers on ≥70% of filled rows; collect their values.
|
||||
let mut filled = 0u32;
|
||||
let mut page_vals: Vec<u32> = Vec::new();
|
||||
for row in cells {
|
||||
let cell = row.get(last).map(|s| s.trim()).unwrap_or("");
|
||||
if cell.is_empty() {
|
||||
continue;
|
||||
}
|
||||
filled += 1;
|
||||
if let Some(v) = page_number_value(cell) {
|
||||
page_vals.push(v);
|
||||
}
|
||||
}
|
||||
if filled < 4 || (page_vals.len() as f32) < 0.7 * filled as f32 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// First column: mostly text titles (has alphabetic content). This rejects
|
||||
// numeric-vs-numeric grids.
|
||||
let text_first = cells
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
row.first()
|
||||
.is_some_and(|c| c.chars().any(|ch| ch.is_alphabetic()))
|
||||
})
|
||||
.count();
|
||||
if (text_first as f32) < 0.6 * cells.len() as f32 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Page numbers mostly ascend (allow front-matter→body resets and noise).
|
||||
if page_vals.len() < 2 {
|
||||
return false;
|
||||
}
|
||||
let non_decreasing = page_vals.windows(2).filter(|w| w[1] >= w[0]).count();
|
||||
if (non_decreasing as f32) < 0.7 * (page_vals.len() - 1) as f32 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Stronger TOC signal. Real page numbers SPAN the document — entries skip
|
||||
// (3, 6, 13, 24, …) so their range exceeds the entry count. A rank / ID /
|
||||
// ordinal column is instead a *perfectly dense* consecutive run (1,2,3,… or
|
||||
// 100,101,102,…). Accept anything with page gaps; for a dense run — which a
|
||||
// one-page-per-entry TOC can also produce — fall back to a title signal:
|
||||
// real contents entries are multi-word headings, rank labels are short.
|
||||
let min = *page_vals.iter().min().unwrap();
|
||||
let max = *page_vals.iter().max().unwrap();
|
||||
let span = max.saturating_sub(min);
|
||||
if span > page_vals.len() as u32 {
|
||||
return true;
|
||||
}
|
||||
let dense_consecutive = (span as usize) + 1 == page_vals.len() && {
|
||||
let mut sorted = page_vals.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
sorted.len() == page_vals.len()
|
||||
};
|
||||
if !dense_consecutive {
|
||||
// Narrow range but with a gap or repeat — still contents-like.
|
||||
return true;
|
||||
}
|
||||
// Dense counter: only a TOC if the titles read like headings, not the
|
||||
// short single-word labels typical of rank/leaderboard/ID tables.
|
||||
let (total_words, titled_rows) = cells
|
||||
.iter()
|
||||
.filter_map(|row| row.first())
|
||||
.filter(|c| c.chars().any(|ch| ch.is_alphabetic()))
|
||||
.fold((0usize, 0usize), |(w, n), c| {
|
||||
(
|
||||
w + c
|
||||
.split_whitespace()
|
||||
.filter(|t| t.chars().any(|ch| ch.is_alphabetic()))
|
||||
.count(),
|
||||
n + 1,
|
||||
)
|
||||
});
|
||||
titled_rows > 0 && (total_words as f32) / titled_rows as f32 >= 1.8
|
||||
}
|
||||
|
||||
/// Dot-leader TOC: any "Chapter 1 ........ 42" style with explicit leader
|
||||
@@ -1886,4 +2005,166 @@ mod tests {
|
||||
assert!(!starts_with_section_number(""));
|
||||
assert!(!starts_with_section_number("Hello world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_value_rejects_roman_lookalike_words() {
|
||||
// Ordinary words made only of {i,v,x,l,c} are not page numbers.
|
||||
assert!(page_number_value("civil").is_none());
|
||||
assert!(page_number_value("mix").is_none());
|
||||
assert!(page_number_value("ill").is_none());
|
||||
assert!(page_number_value("lil").is_none());
|
||||
// Canonical roman numerals still parse.
|
||||
assert_eq!(page_number_value("vii"), Some(7));
|
||||
assert_eq!(page_number_value("ix"), Some(9));
|
||||
assert_eq!(page_number_value("xii"), Some(12));
|
||||
assert_eq!(page_number_value("42"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_matches_consecutive_pages_with_titles() {
|
||||
// A short chapter-per-page contents: pages are a dense 1..n run, but
|
||||
// the multi-word titles mark it as a real TOC (recovered by the title
|
||||
// signal rather than rejected for lacking page gaps).
|
||||
let cells: Vec<Vec<String>> = vec![
|
||||
vec!["Introduction to the Study".into(), "1".into()],
|
||||
vec!["Materials and Methods".into(), "2".into()],
|
||||
vec!["Results and Discussion".into(), "3".into()],
|
||||
vec!["Summary of Findings".into(), "4".into()],
|
||||
vec!["References and Notes".into(), "5".into()],
|
||||
];
|
||||
assert!(is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_dense_ordinal_column() {
|
||||
// Headerless title | rank table: values are a consecutive 1..n
|
||||
// sequence (monotonic, no header, text first column) but their range
|
||||
// ~= the row count, so it is data, not a table of contents.
|
||||
let cells: Vec<Vec<String>> = vec![
|
||||
vec!["Alice".into(), "1".into()],
|
||||
vec!["Bob".into(), "2".into()],
|
||||
vec!["Carol".into(), "3".into()],
|
||||
vec!["Dave".into(), "4".into()],
|
||||
vec!["Erin".into(), "5".into()],
|
||||
vec!["Frank".into(), "6".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_blank_header_cell() {
|
||||
// First row is a header whose last cell is blank ("Category | ");
|
||||
// must not be flattened even though later rows look TOC-like.
|
||||
let cells = vec![
|
||||
vec!["Category".into(), "".into()],
|
||||
vec!["Alpha".into(), "3".into()],
|
||||
vec!["Beta".into(), "9".into()],
|
||||
vec!["Gamma".into(), "14".into()],
|
||||
vec!["Delta".into(), "20".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_matches_title_based_contents() {
|
||||
// Title-left, page-number-right, no dot leaders, no section numbers.
|
||||
let cells = vec![
|
||||
vec!["About the Publisher".into(), "vii".into()],
|
||||
vec!["About This Project".into(), "ix".into()],
|
||||
vec!["Acknowledgments".into(), "xi".into()],
|
||||
vec!["Experiment #1: Hydrostatic Pressure".into(), "3".into()],
|
||||
vec!["Experiment #2: Bernoulli's Theorem".into(), "13".into()],
|
||||
vec![
|
||||
"Experiment #3: Energy Loss in Pipe Fittings".into(),
|
||||
"24".into(),
|
||||
],
|
||||
];
|
||||
assert!(is_page_number_toc(&cells));
|
||||
assert!(is_table_of_contents(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_numeric_data_table() {
|
||||
// Real 2-col data table: numeric first column, non-monotonic values.
|
||||
let cells = vec![
|
||||
vec!["101".into(), "45".into()],
|
||||
vec!["102".into(), "12".into()],
|
||||
vec!["103".into(), "88".into()],
|
||||
vec!["104".into(), "7".into()],
|
||||
vec!["105".into(), "63".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_non_monotonic_pages() {
|
||||
// Text labels but the "page" column jumps around — a small data table,
|
||||
// not a contents listing. 5 rows so the row-count guard passes and the
|
||||
// monotonicity check is what does the rejecting.
|
||||
let cells: Vec<Vec<String>> = vec![
|
||||
vec!["Apples".into(), "42".into()],
|
||||
vec!["Oranges".into(), "7".into()],
|
||||
vec!["Pears".into(), "91".into()],
|
||||
vec!["Plums".into(), "3".into()],
|
||||
vec!["Grapes".into(), "60".into()],
|
||||
];
|
||||
// Sanity: this input clears the row-count and header guards, so a
|
||||
// failure here is genuinely the monotonicity check.
|
||||
assert!(cells.len() >= 5 && page_number_value(cells[0][1].trim()).is_some());
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_header_row_data_table() {
|
||||
// Real 2-col data table with a header row ("Mineral | CEC") and
|
||||
// ascending values that mimic page numbers — the header tells us it
|
||||
// is data, not contents.
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Mineral or colloid type".into(),
|
||||
"CEC of pure colloid".into(),
|
||||
],
|
||||
vec!["kaolinite".into(), "10".into()],
|
||||
vec!["illite".into(), "30".into()],
|
||||
vec!["montmorillonite".into(), "100".into()],
|
||||
vec!["vermiculite".into(), "150".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_rejects_wide_data_grid() {
|
||||
// A 4-column regional data table must not be read as a TOC even with a
|
||||
// text first column and integer last column.
|
||||
let cells = vec![
|
||||
vec![
|
||||
"REGIONS".into(),
|
||||
"2007".into(),
|
||||
"2010".into(),
|
||||
"2016".into(),
|
||||
],
|
||||
vec![
|
||||
"National Capital Region".into(),
|
||||
"9".into(),
|
||||
"8".into(),
|
||||
"5".into(),
|
||||
],
|
||||
vec!["Cordillera".into(), "1".into(), "2".into(), "1".into()],
|
||||
vec!["Ilocos Region".into(), "1".into(), "5".into(), "4".into()],
|
||||
vec!["Cagayan Valley".into(), "1".into(), "3".into(), "5".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_number_toc_needs_page_number_last_column() {
|
||||
// Last column is prose, not page numbers.
|
||||
let cells = vec![
|
||||
vec!["Section A".into(), "see appendix".into()],
|
||||
vec!["Section B".into(), "see notes".into()],
|
||||
vec!["Section C".into(), "later".into()],
|
||||
vec!["Section D".into(), "TBD".into()],
|
||||
];
|
||||
assert!(!is_page_number_toc(&cells));
|
||||
}
|
||||
}
|
||||
|
||||
+653
-5
@@ -226,6 +226,67 @@ pub struct RectHintRegion {
|
||||
/// Also returns hint regions: bounding boxes of cell-sized rects from clusters
|
||||
/// that failed full grid validation. These can be used to scope heuristic
|
||||
/// detection and prevent unrelated items from being merged into tables.
|
||||
/// Bounding boxes of chart-bar clusters on the page. Text inside these
|
||||
/// regions (axis labels, data values, legends) belongs to a figure and must
|
||||
/// not be gridded into a table by any detection strategy.
|
||||
pub fn detect_chart_regions(
|
||||
items: &[TextItem],
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
) -> Vec<(f32, f32, f32, f32)> {
|
||||
// Match detect_tables_from_rects: image placeholders are not text and
|
||||
// would defeat the bar-content check.
|
||||
let items_owned: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| crate::extractor::is_text_layout_item(i))
|
||||
.cloned()
|
||||
.collect();
|
||||
let items = items_owned.as_slice();
|
||||
let page_rects: Vec<(f32, f32, f32, f32)> = rects
|
||||
.iter()
|
||||
.filter(|r| r.page == page)
|
||||
.map(|r| {
|
||||
let (x, w) = if r.width < 0.0 {
|
||||
(r.x + r.width, -r.width)
|
||||
} else {
|
||||
(r.x, r.width)
|
||||
};
|
||||
let (y, h) = if r.height < 0.0 {
|
||||
(r.y + r.height, -r.height)
|
||||
} else {
|
||||
(r.y, r.height)
|
||||
};
|
||||
(x, y, w, h)
|
||||
})
|
||||
// Origin-anchored page backgrounds/clipping paths are never chart
|
||||
// geometry, and letting one bridge into a bar cluster would inflate
|
||||
// the region to the whole page.
|
||||
.filter(|&(x, y, w, h)| w >= 5.0 && h >= 5.0 && !(x < 5.0 && y < 5.0))
|
||||
.collect();
|
||||
if page_rects.len() < 6 {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut regions = Vec::new();
|
||||
for cluster in &cluster_rects(&page_rects, 3.0, 6) {
|
||||
let group: Vec<(f32, f32, f32, f32)> = cluster.iter().map(|&i| page_rects[i]).collect();
|
||||
if is_chart_bar_cluster(items, &group, page) {
|
||||
let bbox = group.iter().fold(
|
||||
(
|
||||
f32::INFINITY,
|
||||
f32::INFINITY,
|
||||
f32::NEG_INFINITY,
|
||||
f32::NEG_INFINITY,
|
||||
),
|
||||
|(x0, y0, x1, y1), &(x, y, w, h)| {
|
||||
(x0.min(x), y0.min(y), x1.max(x + w), y1.max(y + h))
|
||||
},
|
||||
);
|
||||
regions.push(bbox);
|
||||
}
|
||||
}
|
||||
regions
|
||||
}
|
||||
|
||||
pub fn detect_tables_from_rects(
|
||||
items: &[TextItem],
|
||||
rects: &[PdfRect],
|
||||
@@ -379,13 +440,29 @@ pub fn detect_tables_from_rects(
|
||||
.collect();
|
||||
|
||||
debug!("page {}: {} clusters with >= 6 rects", page, clusters.len());
|
||||
for cluster_indices in &clusters {
|
||||
let mut chart_cluster_ids: Vec<usize> = Vec::new();
|
||||
for (cluster_id, cluster_indices) in clusters.iter().enumerate() {
|
||||
let group_rects: Vec<(f32, f32, f32, f32)> =
|
||||
cluster_indices.iter().map(|&i| page_rects[i]).collect();
|
||||
// Chart bars are neither table cells nor a hint region — gridding
|
||||
// a chart's axis labels scrambles the page. Skip the cluster
|
||||
// entirely so it can't reach any detector, the merged fallback,
|
||||
// or the hint fallback.
|
||||
if is_chart_bar_cluster(items, &group_rects, page) {
|
||||
debug!(
|
||||
"page {}: skipping chart-bar cluster ({} rects)",
|
||||
page,
|
||||
group_rects.len()
|
||||
);
|
||||
chart_cluster_ids.push(cluster_id);
|
||||
continue;
|
||||
}
|
||||
if let Some(table) = detect_table_from_rect_group(items, &group_rects, page) {
|
||||
tables.push(table);
|
||||
} else if let Some(table) = detect_row_stripe_table(items, &group_rects, page) {
|
||||
tables.push(table);
|
||||
} else if let Some(table) = detect_stacked_box_table(items, &group_rects, page) {
|
||||
tables.push(table);
|
||||
} else if let Some((left, right)) = split_wide_cluster(&group_rects, 15.0, 6) {
|
||||
// Cluster was too wide — retry each half independently
|
||||
debug!(
|
||||
@@ -419,12 +496,19 @@ pub fn detect_tables_from_rects(
|
||||
// text-based column detection.
|
||||
let only_narrow = !tables.is_empty() && tables.iter().all(|t| t.columns.len() <= 3);
|
||||
if tables.is_empty() || only_narrow {
|
||||
let total_clustered: usize = clusters.iter().map(|c| c.len()).sum();
|
||||
if clusters.len() >= 3 && total_clustered >= 50 {
|
||||
// Chart clusters stay out of the merge as well.
|
||||
let table_clusters: Vec<&Vec<usize>> = clusters
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(id, _)| !chart_cluster_ids.contains(id))
|
||||
.map(|(_, c)| c)
|
||||
.collect();
|
||||
let total_clustered: usize = table_clusters.iter().map(|c| c.len()).sum();
|
||||
if table_clusters.len() >= 3 && total_clustered >= 50 {
|
||||
debug!(
|
||||
"page {}: trying merged-cluster fallback ({} clusters, {} rects{})",
|
||||
page,
|
||||
clusters.len(),
|
||||
table_clusters.len(),
|
||||
total_clustered,
|
||||
if only_narrow {
|
||||
", replacing narrow tables"
|
||||
@@ -432,7 +516,7 @@ pub fn detect_tables_from_rects(
|
||||
""
|
||||
}
|
||||
);
|
||||
let all_cluster_rects: Vec<(f32, f32, f32, f32)> = clusters
|
||||
let all_cluster_rects: Vec<(f32, f32, f32, f32)> = table_clusters
|
||||
.iter()
|
||||
.flat_map(|idxs| idxs.iter().map(|&i| page_rects[i]))
|
||||
.collect();
|
||||
@@ -491,6 +575,14 @@ pub fn detect_tables_from_rects(
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: 3-5 box stacks never reach detect_stacked_box_table — the main
|
||||
// loop requires >=6-rect clusters (and a >=6-rect page). This is a
|
||||
// deliberate precision gate: routing smaller clusters through the
|
||||
// detector was tried and regressed four pdf-evals documents (striped
|
||||
// bullet lists, wrapped regulation text, stats-table columns) while
|
||||
// improving nothing — with so few boxes the anti-prose guards have too
|
||||
// little signal to discriminate. See stacked_box_three_rows_below_
|
||||
// cluster_minimum for the pinned behavior.
|
||||
if tables.is_empty() {
|
||||
// When no tables detected but clusters exist, generate XY hint regions
|
||||
// from cluster bounding boxes to scope heuristic table detection.
|
||||
@@ -631,6 +723,240 @@ pub fn detect_tables_from_rects(
|
||||
/// overlap or are close (gap < 50pt). This handles calendar-style layouts where a
|
||||
/// month zone's decorative rects split into 2-3 adjacent clusters with small X gaps.
|
||||
/// Runs iteratively until no more merges occur.
|
||||
/// Detect a single-column table drawn as a vertical stack of boxes, each
|
||||
/// holding one short line of text (framework/step lists on slide-style
|
||||
/// pages). The normal grid path rejects these — one column means only two
|
||||
/// x-edges — so the rows would otherwise flow into surrounding prose as a
|
||||
/// run-on paragraph.
|
||||
fn detect_stacked_box_table(
|
||||
items: &[TextItem],
|
||||
group_rects: &[(f32, f32, f32, f32)],
|
||||
page: u32,
|
||||
) -> Option<Table> {
|
||||
// Candidate row boxes: single-text-line height, substantial width.
|
||||
let cands: Vec<(f32, f32, f32, f32)> = group_rects
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&(_, _, w, h)| w >= 100.0 && (8.0..=80.0).contains(&h))
|
||||
.collect();
|
||||
// The row boxes form the largest family of same-width, x-aligned rects
|
||||
// (backgrounds and decor have their own geometry and stay out).
|
||||
let mut boxes: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||
for &anchor in &cands {
|
||||
let family: Vec<(f32, f32, f32, f32)> = cands
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&(x, _, w, h)| {
|
||||
(x - anchor.0).abs() <= 12.0
|
||||
&& (w - anchor.2).abs() <= anchor.2 * 0.15
|
||||
&& (h - anchor.3).abs() <= anchor.3 * 0.3
|
||||
})
|
||||
.collect();
|
||||
if family.len() > boxes.len() {
|
||||
boxes = family;
|
||||
}
|
||||
}
|
||||
if boxes.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
// Boxes flanked at the same y-level — by other rects or by text outside
|
||||
// the family's x-range — are one column of a wider structure. Leave
|
||||
// those to the grid/cell-rect paths instead of collapsing to one column.
|
||||
let flanked = boxes
|
||||
.iter()
|
||||
.filter(|&&(bx, by, bw, bh)| {
|
||||
let rect_sibling = group_rects.iter().any(|&(ox, oy, ow, oh)| {
|
||||
let y_overlap = (by + bh).min(oy + oh) - by.max(oy);
|
||||
oh >= 8.0
|
||||
&& y_overlap > bh * 0.5
|
||||
&& (ox + ow <= bx + 2.0 || ox >= bx + bw - 2.0)
|
||||
&& ow >= 30.0
|
||||
});
|
||||
let text_sibling = items.iter().any(|it| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
it.page == page
|
||||
&& it.y >= by - 2.0
|
||||
&& it.y <= by + bh + 2.0
|
||||
&& (cx < bx - 5.0 || cx > bx + bw + 5.0)
|
||||
&& it.width >= 10.0
|
||||
});
|
||||
rect_sibling || text_sibling
|
||||
})
|
||||
.count();
|
||||
if flanked * 3 >= boxes.len() {
|
||||
debug!(
|
||||
" stacked-box rejected: {}/{} boxes flanked by rects or text",
|
||||
flanked,
|
||||
boxes.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
boxes.sort_by(|a, b| b.1.total_cmp(&a.1)); // top to bottom (descending y)
|
||||
|
||||
// Merge duplicates (border + fill pairs draw the same box twice), then
|
||||
// require a clean vertical stack: no overlaps beyond a small tolerance.
|
||||
boxes.dedup_by(|a, b| (a.1 - b.1).abs() <= 3.0 && (a.3 - b.3).abs() <= 6.0);
|
||||
if boxes.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
for w in boxes.windows(2) {
|
||||
let (upper, lower) = (w[0], w[1]);
|
||||
let upper_bottom = upper.1;
|
||||
let lower_top = lower.1 + lower.3;
|
||||
if lower_top > upper_bottom + 4.0 {
|
||||
return None; // vertical overlap — not a stack
|
||||
}
|
||||
if upper_bottom - lower_top > upper.3.max(lower.3) {
|
||||
return None; // gap larger than a row — unrelated boxes
|
||||
}
|
||||
}
|
||||
|
||||
// Assign items to boxes; every box needs text and cells must stay short
|
||||
// (prose paragraphs inside stacked frames are page decor, not a table).
|
||||
let mut cells: Vec<Vec<String>> = Vec::with_capacity(boxes.len());
|
||||
let mut item_indices: Vec<usize> = Vec::new();
|
||||
let mut multi_run_boxes = 0usize;
|
||||
for &(bx, by, bw, bh) in &boxes {
|
||||
let mut in_box: Vec<(usize, &TextItem)> = items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, it)| {
|
||||
it.page == page
|
||||
&& it.y >= by - 2.0
|
||||
&& it.y <= by + bh + 2.0
|
||||
&& it.x + it.width / 2.0 >= bx
|
||||
&& it.x + it.width / 2.0 <= bx + bw
|
||||
})
|
||||
.collect();
|
||||
if in_box.is_empty() {
|
||||
return None;
|
||||
}
|
||||
in_box.sort_by(|a, b| {
|
||||
b.1.y
|
||||
.partial_cmp(&a.1.y)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| {
|
||||
a.1.x
|
||||
.partial_cmp(&b.1.x)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
});
|
||||
// Count horizontally separated text runs inside the box. A single
|
||||
// list row flows as one run; two-plus runs across most boxes means
|
||||
// multi-column content (striped prose or a real grid) that must not
|
||||
// collapse into a one-column table. Same-baseline only: boxed
|
||||
// display/diagram rows legitimately scatter segments at mixed
|
||||
// baselines, and those must stay one row.
|
||||
let mut runs = 1usize;
|
||||
for pair in in_box.windows(2) {
|
||||
let (prev, item) = (pair[0].1, pair[1].1);
|
||||
if (prev.y - item.y).abs() <= 2.0 && item.x - (prev.x + prev.width) > 15.0 {
|
||||
runs += 1;
|
||||
}
|
||||
}
|
||||
if runs >= 2 {
|
||||
multi_run_boxes += 1;
|
||||
}
|
||||
let text = in_box
|
||||
.iter()
|
||||
.map(|(_, it)| it.text.trim())
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
if text.is_empty() || text.chars().count() > 120 {
|
||||
return None;
|
||||
}
|
||||
item_indices.extend(in_box.iter().map(|(i, _)| *i));
|
||||
cells.push(vec![text]);
|
||||
}
|
||||
if multi_run_boxes * 2 >= boxes.len() {
|
||||
debug!(
|
||||
" stacked-box rejected: {}/{} boxes hold multiple text runs",
|
||||
multi_run_boxes,
|
||||
boxes.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject prose behind per-line stripe rects: sentence fragments flowing
|
||||
// across rows read as long, function-word-dense cells, while genuine
|
||||
// list-table rows are short labels/titles.
|
||||
const PROSE_WORDS: &[&str] = &[
|
||||
"a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on", "at",
|
||||
"with", "for", "by", "as", "and", "or", "but", "this", "that", "these", "those", "from",
|
||||
"into", "has", "have", "had", "not", "it", "its", "their", "such", "shall", "which",
|
||||
];
|
||||
let total_chars: usize = cells.iter().map(|r| r[0].chars().count()).sum();
|
||||
let mean_chars = total_chars / cells.len().max(1);
|
||||
let prose_cells = cells
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
r[0].to_ascii_lowercase()
|
||||
.split(|c: char| !c.is_ascii_alphabetic() && c != '\'')
|
||||
.any(|w| PROSE_WORDS.contains(&w))
|
||||
})
|
||||
.count();
|
||||
if mean_chars > 60 && prose_cells * 5 >= cells.len() * 2 {
|
||||
debug!(
|
||||
" stacked-box rejected: prose rows (mean {} chars, prose words {}/{})",
|
||||
mean_chars,
|
||||
prose_cells,
|
||||
cells.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
// Sentences wrapping across stripe rects: a row ending with a comma, or
|
||||
// a row without terminal punctuation followed by a row starting
|
||||
// lowercase, is mid-sentence flow — not list rows. Genuine label/title
|
||||
// rows produce none of these, so even a small share is disqualifying.
|
||||
let continuations = cells
|
||||
.windows(2)
|
||||
.filter(|pair| {
|
||||
let prev = pair[0][0].trim_end();
|
||||
let next = pair[1][0].trim_start();
|
||||
let prev_open = !prev.ends_with(['.', ':', ';', '!', '?', ')', '"', '%']);
|
||||
let next_lower = next.chars().next().is_some_and(|c| c.is_lowercase());
|
||||
prev.ends_with(',') || (prev_open && next_lower)
|
||||
})
|
||||
.count();
|
||||
if cells.len() >= 2 && (continuations >= 2 || continuations * 4 >= cells.len() - 1) {
|
||||
debug!(
|
||||
" stacked-box rejected: {}/{} row pairs continue a sentence",
|
||||
continuations,
|
||||
cells.len() - 1
|
||||
);
|
||||
return None;
|
||||
}
|
||||
// Numbered/lettered list items behind decorative stripes stay lists:
|
||||
// "1) content..." / "(ii) content..." / "a. content...".
|
||||
let list_marker = |t: &str| {
|
||||
let t = t.trim_start().strip_prefix('(').unwrap_or(t.trim_start());
|
||||
let marker_len = t.chars().take_while(|c| c.is_ascii_alphanumeric()).count();
|
||||
(1..=3).contains(&marker_len)
|
||||
&& t.chars()
|
||||
.nth(marker_len)
|
||||
.is_some_and(|c| c == ')' || c == '.')
|
||||
};
|
||||
let list_rows = cells.iter().filter(|r| list_marker(&r[0])).count();
|
||||
if list_rows * 2 >= cells.len() {
|
||||
debug!(
|
||||
" stacked-box rejected: {}/{} rows are numbered list items",
|
||||
list_rows,
|
||||
cells.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"page {}: stacked-box table: {} single-column rows",
|
||||
page,
|
||||
cells.len()
|
||||
);
|
||||
let columns = vec![boxes[0].0 + boxes[0].2 / 2.0];
|
||||
let rows: Vec<f32> = boxes.iter().map(|b| b.1 + b.3 / 2.0).collect();
|
||||
Some(Table::new(columns, rows, cells, item_indices))
|
||||
}
|
||||
|
||||
fn merge_overlapping_hints(mut hints: Vec<RectHintRegion>) -> Vec<RectHintRegion> {
|
||||
if hints.len() <= 1 {
|
||||
return hints;
|
||||
@@ -1583,6 +1909,118 @@ fn row_stripe_is_sparse_prose_outline(cells: &[Vec<String>]) -> bool {
|
||||
/// Uses rect Y-edges for row boundaries and text X-position clustering for
|
||||
/// columns. Handles tables with cell backgrounds that don't form a clean
|
||||
/// X-edge grid (variable column widths, decorative fills).
|
||||
/// Chart-bar signature: ≥3 rects sharing an aligned bottom edge (the axis),
|
||||
/// with similar widths (bars) but strongly varying heights (data-driven),
|
||||
/// holding at most a single numeric data label each. Bar charts drawn as
|
||||
/// filled rects otherwise read as cell rects and grid their axis labels
|
||||
/// into a phantom table. The mirrored check catches horizontal bar charts.
|
||||
fn is_chart_bar_cluster(
|
||||
items: &[TextItem],
|
||||
group_rects: &[(f32, f32, f32, f32)],
|
||||
page: u32,
|
||||
) -> bool {
|
||||
let numeric_or_empty = |(rx, ry, rw, rh): (f32, f32, f32, f32)| {
|
||||
let inside: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|it| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
it.page == page && cx >= rx && cx <= rx + rw && it.y >= ry && it.y <= ry + rh
|
||||
})
|
||||
.collect();
|
||||
// Any number of numeric data labels is chart-like; a single run of
|
||||
// word text inside means a table cell.
|
||||
inside.iter().all(|it| {
|
||||
let t = it.text.trim();
|
||||
let data = t
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit() || ",.%-".contains(*c))
|
||||
.count();
|
||||
t.is_empty() || data * 2 >= t.chars().count()
|
||||
})
|
||||
};
|
||||
|
||||
// Bars: the dominant equal-width family, arranged in >=2 spaced columns
|
||||
// (inter-column gap >= half a bar width — table cell rects touch), with
|
||||
// data-driven height variation (checkbox/cell grids are uniform).
|
||||
// Mirrored predicate catches horizontal bar charts.
|
||||
let bar_family = |pos: fn(&(f32, f32, f32, f32)) -> f32,
|
||||
breadth: fn(&(f32, f32, f32, f32)) -> f32,
|
||||
length: fn(&(f32, f32, f32, f32)) -> f32,
|
||||
along: fn(&(f32, f32, f32, f32)) -> f32| {
|
||||
group_rects.iter().any(|anchor| {
|
||||
let bw = breadth(anchor);
|
||||
if bw <= 0.0 {
|
||||
return false;
|
||||
}
|
||||
let family: Vec<&(f32, f32, f32, f32)> = group_rects
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
(breadth(r) - bw).abs() <= (bw * 0.1).max(2.0)
|
||||
&& length(r) > 0.0
|
||||
&& length(r) < bw * 20.0
|
||||
})
|
||||
.collect();
|
||||
if family.len() < 4 {
|
||||
return false;
|
||||
}
|
||||
// Distinct positions along the axis (bar columns).
|
||||
let mut positions: Vec<f32> = Vec::new();
|
||||
for r in &family {
|
||||
let p = pos(r);
|
||||
if !positions.iter().any(|&q| (q - p).abs() <= 2.0) {
|
||||
positions.push(p);
|
||||
}
|
||||
}
|
||||
if positions.len() < 2 {
|
||||
return false;
|
||||
}
|
||||
positions.sort_by(|a, b| a.total_cmp(b));
|
||||
let min_gap = positions
|
||||
.windows(2)
|
||||
.map(|w| w[1] - w[0] - bw)
|
||||
.fold(f32::INFINITY, f32::min);
|
||||
if min_gap < bw * 0.5 {
|
||||
return false;
|
||||
}
|
||||
// Data-driven variation along the bar direction.
|
||||
let len_min = family
|
||||
.iter()
|
||||
.map(|r| length(r))
|
||||
.fold(f32::INFINITY, f32::min);
|
||||
let len_max = family
|
||||
.iter()
|
||||
.map(|r| length(r))
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
if len_max < len_min * 1.3 {
|
||||
return false;
|
||||
}
|
||||
// Grid rows disguise as bars: a table's cell rects have same-y,
|
||||
// same-height partners in other columns (uniform row heights).
|
||||
// Chart segments start where the previous datum ended, so their
|
||||
// extents rarely pair up across positions.
|
||||
let matched = family
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
family.iter().any(|s| {
|
||||
(pos(s) - pos(r)).abs() > 2.0
|
||||
&& (along(s) - along(r)).abs() <= 3.0
|
||||
&& (length(s) - length(r)).abs() <= 3.0
|
||||
})
|
||||
})
|
||||
.count();
|
||||
if matched * 5 >= family.len() * 3 {
|
||||
return false;
|
||||
}
|
||||
family.iter().filter(|r| numeric_or_empty(***r)).count() * 3 >= family.len() * 2
|
||||
})
|
||||
};
|
||||
|
||||
// vertical bars: position/breadth = x/width, length = height, along = y
|
||||
bar_family(|r| r.0, |r| r.2, |r| r.3, |r| r.1)
|
||||
// horizontal bars: position/breadth = y/height, length = width, along = x
|
||||
|| bar_family(|r| r.1, |r| r.3, |r| r.2, |r| r.0)
|
||||
}
|
||||
|
||||
fn detect_row_stripe_table_from_cell_rects(
|
||||
items: &[TextItem],
|
||||
group_rects: &[(f32, f32, f32, f32)],
|
||||
@@ -2458,6 +2896,173 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// --- is_chart_bar_cluster / detect_chart_regions ---
|
||||
|
||||
/// Stacked bar chart: frame + 3 columns of equal-width segments with
|
||||
/// data-driven heights, holding numeric labels.
|
||||
fn chart_rects() -> Vec<PdfRect> {
|
||||
let mut rects = vec![PdfRect {
|
||||
x: 126.0,
|
||||
y: 548.0,
|
||||
width: 396.0,
|
||||
height: 216.0,
|
||||
page: 1,
|
||||
}];
|
||||
let bars = [
|
||||
(208.0, 618.0, 59.0),
|
||||
(208.0, 661.0, 39.0),
|
||||
(208.0, 696.0, 37.0),
|
||||
(313.0, 618.0, 67.0),
|
||||
(313.0, 670.0, 49.0),
|
||||
(313.0, 691.0, 42.0),
|
||||
(419.0, 618.0, 73.0),
|
||||
(419.0, 684.0, 37.0),
|
||||
(419.0, 708.0, 25.0),
|
||||
];
|
||||
for (x, y, h) in bars {
|
||||
rects.push(PdfRect {
|
||||
x,
|
||||
y,
|
||||
width: 46.0,
|
||||
height: h,
|
||||
page: 1,
|
||||
});
|
||||
}
|
||||
rects
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chart_bars_produce_region_not_table() {
|
||||
let items: Vec<TextItem> = [
|
||||
("38", 228.0, 638.0),
|
||||
("30", 228.0, 676.0),
|
||||
("46", 333.0, 643.0),
|
||||
("17", 333.0, 679.0),
|
||||
("57", 438.0, 650.0),
|
||||
("20", 438.0, 694.0),
|
||||
]
|
||||
.iter()
|
||||
.map(|&(t, x, y)| make_item(t, x, y, 9.0))
|
||||
.collect();
|
||||
let rects = chart_rects();
|
||||
let regions = detect_chart_regions(&items, &rects, 1);
|
||||
assert_eq!(regions.len(), 1, "expected one chart region");
|
||||
let (tables, hints) = detect_tables_from_rects(&items, &rects, 1);
|
||||
assert!(tables.is_empty(), "chart bars must not become a table");
|
||||
assert!(hints.is_empty(), "chart bars must not become a hint region");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uniform_cell_grid_is_not_a_chart() {
|
||||
// Touching, uniform-height cell rects (a real table) must not match:
|
||||
// no inter-column gap and no bar-length variation.
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..4 {
|
||||
for col in 0..3 {
|
||||
rects.push(PdfRect {
|
||||
x: 100.0 + col as f32 * 80.0,
|
||||
y: 600.0 - row as f32 * 20.0,
|
||||
width: 80.0,
|
||||
height: 20.0,
|
||||
page: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
let items: Vec<TextItem> = (0..4)
|
||||
.flat_map(|r| {
|
||||
(0..3).map(move |c| (100.0 + c as f32 * 80.0 + 10.0, 605.0 - r as f32 * 20.0))
|
||||
})
|
||||
.map(|(x, y)| make_item("42", x, y, 9.0))
|
||||
.collect();
|
||||
assert!(detect_chart_regions(&items, &rects, 1).is_empty());
|
||||
}
|
||||
|
||||
// --- detect_stacked_box_table ---
|
||||
|
||||
/// N stacked boxes at x=100, w=300, h=22, top-to-bottom from y=600.
|
||||
fn stacked_boxes(n: usize) -> Vec<(f32, f32, f32, f32)> {
|
||||
(0..n)
|
||||
.map(|i| (100.0, 600.0 - i as f32 * 22.0, 300.0, 22.0))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_list_becomes_single_column_table() {
|
||||
let rects = stacked_boxes(5);
|
||||
let items: Vec<TextItem> = (0..5)
|
||||
.map(|i| make_item("#1: Recycling Basics", 120.0, 605.0 - i as f32 * 22.0, 10.0))
|
||||
.collect();
|
||||
let table = detect_stacked_box_table(&items, &rects, 1).expect("stacked-box table");
|
||||
assert_eq!(table.cells.len(), 5);
|
||||
assert_eq!(table.cells[0].len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_rejects_wrapped_sentences() {
|
||||
// Line stripes behind flowing prose: rows continue mid-sentence.
|
||||
let rects = stacked_boxes(4);
|
||||
let texts = [
|
||||
"the provisions of this section apply to",
|
||||
"companies subject to tax under those",
|
||||
"sections, except that the copy of the",
|
||||
"annual statement must be retained.",
|
||||
];
|
||||
let items: Vec<TextItem> = texts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, t)| make_item(t, 120.0, 605.0 - i as f32 * 22.0, 10.0))
|
||||
.collect();
|
||||
assert!(detect_stacked_box_table(&items, &rects, 1).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_rejects_flanking_text() {
|
||||
// A ruled label column with plain-text data columns beside it is one
|
||||
// column of a wider table, not a single-column list.
|
||||
let rects = stacked_boxes(4);
|
||||
let mut items = Vec::new();
|
||||
for i in 0..4 {
|
||||
let y = 605.0 - i as f32 * 22.0;
|
||||
items.push(make_item("Section 1.382", 120.0, y, 10.0));
|
||||
items.push(make_item("removed text", 450.0, y, 10.0)); // beside the box
|
||||
}
|
||||
assert!(detect_stacked_box_table(&items, &rects, 1).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_rejects_two_column_content() {
|
||||
// Boxes holding two separated runs are striped multi-column content.
|
||||
let rects = stacked_boxes(4);
|
||||
let mut items = Vec::new();
|
||||
for i in 0..4 {
|
||||
let y = 605.0 - i as f32 * 22.0;
|
||||
let mut left = make_item("left words", 110.0, y, 10.0);
|
||||
left.width = 60.0;
|
||||
let mut right = make_item("right words", 250.0, y, 10.0);
|
||||
right.width = 60.0;
|
||||
items.push(left);
|
||||
items.push(right);
|
||||
}
|
||||
assert!(detect_stacked_box_table(&items, &rects, 1).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_rejects_mixed_height_stripes() {
|
||||
// Mixed 13/27pt stripes (redline markup) — height uniformity splits
|
||||
// the family and the gap check rejects the remainder.
|
||||
let mut rects = Vec::new();
|
||||
let mut y = 600.0;
|
||||
for i in 0..8 {
|
||||
let h = if i % 3 == 0 { 27.0 } else { 13.5 };
|
||||
y -= h;
|
||||
rects.push((100.0, y, 300.0, h));
|
||||
}
|
||||
let items: Vec<TextItem> = (0..8)
|
||||
.map(|i| make_item("PART 602 OMB CONTROL", 120.0, 590.0 - i as f32 * 18.0, 10.0))
|
||||
.collect();
|
||||
assert!(detect_stacked_box_table(&items, &rects, 1).is_none());
|
||||
}
|
||||
|
||||
// --- has_dominant_prose_cell ---
|
||||
|
||||
fn cells_of(rows: &[&[&str]]) -> Vec<Vec<String>> {
|
||||
@@ -3480,6 +4085,49 @@ mod tests {
|
||||
assert!((merged[0].x_right - 340.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stacked_box_three_rows_below_cluster_minimum() {
|
||||
// Pins a deliberate precision gate: a 3-box stack stays below the
|
||||
// main loop's 6-rect cluster minimum and is NOT detected end-to-end.
|
||||
// Routing smaller clusters through detect_stacked_box_table was
|
||||
// tried and regressed four pdf-evals documents (striped bullet
|
||||
// lists, wrapped regulation text, stats-table columns) with no
|
||||
// corpus gains — too few boxes for the anti-prose guards to work.
|
||||
// If this ever becomes worth revisiting, the guards need stronger
|
||||
// signals first; flipping this assertion is the entry point.
|
||||
let mut rects: Vec<PdfRect> = (0..3)
|
||||
.map(|i| PdfRect {
|
||||
x: 100.0,
|
||||
y: 600.0 - i as f32 * 22.0,
|
||||
width: 300.0,
|
||||
height: 22.0,
|
||||
page: 1,
|
||||
})
|
||||
.collect();
|
||||
// Unrelated scattered rects push the page past the 6-rect page gate
|
||||
// so the run reaches clustering, while the 3-box stack itself stays
|
||||
// below the 6-rect cluster minimum.
|
||||
for i in 0..4 {
|
||||
rects.push(PdfRect {
|
||||
x: 100.0 + i as f32 * 120.0,
|
||||
y: 100.0,
|
||||
width: 40.0,
|
||||
height: 15.0,
|
||||
page: 1,
|
||||
});
|
||||
}
|
||||
let items: Vec<TextItem> = ["Step One: Plan", "Step Two: Build", "Step Three: Ship"]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, t)| make_item(t, 120.0, 605.0 - i as f32 * 22.0, 10.0))
|
||||
.collect();
|
||||
let (tables, _) = detect_tables_from_rects(&items, &rects, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"3-box stacks are intentionally below the detection floor"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cluster_generates_hint_with_items() {
|
||||
// A cluster of rects forming an outer border (2 x-edges after snapping)
|
||||
|
||||
@@ -123,6 +123,7 @@ fn format_toc_as_list(cells: &[Vec<String>], footnotes: &[String]) -> String {
|
||||
|
||||
/// True when the cell looks like a page number. Accepts:
|
||||
/// - plain digit tokens: "42", "86 86"
|
||||
/// - canonical roman numerals (front-matter pages): "vii", "ix", "xii"
|
||||
/// - dashed section-page IDs: "5-21", "A-1", "B--3", "TC-2" (common in
|
||||
/// technical manuals)
|
||||
fn is_page_number_cell(cell: &str) -> bool {
|
||||
@@ -138,6 +139,9 @@ fn is_page_number_cell(cell: &str) -> bool {
|
||||
if all_digits {
|
||||
return t.len() <= 4;
|
||||
}
|
||||
if super::canonical_roman_value(t).is_some() {
|
||||
return true;
|
||||
}
|
||||
// Section-page form: uppercase letters, digits, dashes; at least
|
||||
// one digit present.
|
||||
t.chars()
|
||||
|
||||
+56
-2
@@ -4,7 +4,7 @@
|
||||
|
||||
mod detect_heuristic;
|
||||
mod detect_lines;
|
||||
mod detect_rects;
|
||||
pub(crate) mod detect_rects;
|
||||
mod detect_struct;
|
||||
mod financial;
|
||||
mod format;
|
||||
@@ -15,7 +15,7 @@ pub use detect_heuristic::detect_tables;
|
||||
pub(crate) use detect_heuristic::is_table_of_contents;
|
||||
pub use detect_lines::detect_tables_from_lines;
|
||||
pub(crate) use detect_rects::cluster_rects;
|
||||
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
||||
pub use detect_rects::{detect_chart_regions, detect_tables_from_rects, RectHintRegion};
|
||||
pub use detect_struct::detect_tables_from_struct_tree;
|
||||
pub use format::table_to_markdown;
|
||||
pub use structured::{cells_to_markdown, StructuredCell};
|
||||
@@ -177,6 +177,60 @@ pub(crate) fn try_build_rect_guided_table(
|
||||
))
|
||||
}
|
||||
|
||||
/// Canonical lowercase roman numeral for `n` (the i/v/x/l/c range).
|
||||
pub(super) fn to_roman_lower(mut n: u32) -> String {
|
||||
const TABLE: [(u32, &str); 9] = [
|
||||
(100, "c"),
|
||||
(90, "xc"),
|
||||
(50, "l"),
|
||||
(40, "xl"),
|
||||
(10, "x"),
|
||||
(9, "ix"),
|
||||
(5, "v"),
|
||||
(4, "iv"),
|
||||
(1, "i"),
|
||||
];
|
||||
let mut out = String::new();
|
||||
for (val, sym) in TABLE {
|
||||
while n >= val {
|
||||
out.push_str(sym);
|
||||
n -= val;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse a *canonical* roman numeral (i/v/x/l/c range, ≤8 chars) to its value.
|
||||
/// Returns `None` for non-canonical strings, so ordinary words made of those
|
||||
/// letters — "civil", "mix", "ill" — are not mistaken for numbers. Shared by
|
||||
/// the TOC detector and the TOC formatter so the two stay in sync.
|
||||
pub(super) fn canonical_roman_value(token: &str) -> Option<u32> {
|
||||
let lower = token.trim().to_ascii_lowercase();
|
||||
if lower.is_empty() || lower.len() > 8 || !lower.chars().all(|c| "ivxlc".contains(c)) {
|
||||
return None;
|
||||
}
|
||||
let mut total = 0i32;
|
||||
let mut prev = 0i32;
|
||||
for c in lower.chars().rev() {
|
||||
let v = match c {
|
||||
'i' => 1,
|
||||
'v' => 5,
|
||||
'x' => 10,
|
||||
'l' => 50,
|
||||
'c' => 100,
|
||||
_ => return None,
|
||||
};
|
||||
if v < prev {
|
||||
total -= v;
|
||||
} else {
|
||||
total += v;
|
||||
prev = v;
|
||||
}
|
||||
}
|
||||
let value = u32::try_from(total).ok().filter(|&n| n > 0)?;
|
||||
(to_roman_lower(value) == lower).then_some(value)
|
||||
}
|
||||
|
||||
/// Split a TextItem whose text contains multiple whitespace-separated tokens
|
||||
/// (like "10 11 12 ... 31") into individual TextItems, each assigned to the
|
||||
/// nearest column boundary.
|
||||
|
||||
Reference in New Issue
Block a user