Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
2b6d78e972 |
@@ -20,7 +20,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --verbose
|
||||
@@ -51,9 +61,17 @@ jobs:
|
||||
components: clippy
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: clippy
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-clippy-
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy -- -D warnings
|
||||
@@ -71,9 +89,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: build
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-build-
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --verbose
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
name: Publish Rust crate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['Cargo.toml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
|
||||
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
|
||||
"https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/crate-version.json
|
||||
echo "Unexpected crates.io response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish:
|
||||
name: Publish to crates.io
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
environment: crates-io
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Verify package
|
||||
run: cargo publish --dry-run
|
||||
|
||||
- name: Authenticate with crates.io
|
||||
id: auth
|
||||
uses: rust-lang/crates-io-auth-action@v1
|
||||
|
||||
- name: Publish crate
|
||||
run: cargo publish
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.3"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2.0"
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# pdf-inspector
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
@@ -74,17 +71,9 @@ console.log(result.markdown); // Markdown string or null
|
||||
|
||||
### Rust
|
||||
|
||||
Install from [crates.io](https://crates.io/crates/pdf-inspector):
|
||||
|
||||
```bash
|
||||
cargo add pdf-inspector
|
||||
```
|
||||
|
||||
Or add it manually:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-inspector = "0.1"
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
```
|
||||
|
||||
```rust
|
||||
@@ -102,37 +91,29 @@ if let Some(markdown) = &result.markdown {
|
||||
### CLI
|
||||
|
||||
```bash
|
||||
# Install the CLI tools
|
||||
cargo install pdf-inspector
|
||||
|
||||
# Convert PDF to Markdown
|
||||
pdf2md document.pdf
|
||||
cargo run --bin pdf2md -- document.pdf
|
||||
|
||||
# JSON output (for piping)
|
||||
pdf2md document.pdf --json
|
||||
|
||||
# Positioned TextItem JSON, including is_underline metadata
|
||||
pdf2md document.pdf --items-json
|
||||
cargo run --bin pdf2md -- document.pdf --json
|
||||
|
||||
# Raw markdown only (no headers)
|
||||
pdf2md document.pdf --raw
|
||||
cargo run --bin pdf2md -- document.pdf --raw
|
||||
|
||||
# Insert page break markers (<!-- Page N -->)
|
||||
pdf2md document.pdf --pages
|
||||
cargo run --bin pdf2md -- document.pdf --pages
|
||||
|
||||
# Process only specific pages
|
||||
pdf2md document.pdf --select-pages 1,3,5-10
|
||||
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
|
||||
|
||||
# Detection only (no extraction)
|
||||
detect-pdf document.pdf
|
||||
detect-pdf document.pdf --json
|
||||
cargo run --bin detect-pdf -- document.pdf
|
||||
cargo run --bin detect-pdf -- document.pdf --json
|
||||
|
||||
# Detection + layout analysis (tables, columns)
|
||||
detect-pdf document.pdf --analyze --json
|
||||
cargo run --bin detect-pdf -- document.pdf --analyze --json
|
||||
```
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you believe you've found a security vulnerability in pdf-inspector, please
|
||||
report it privately so we can fix it before public disclosure.
|
||||
|
||||
**Preferred:** Email **help@firecrawl.dev** with:
|
||||
|
||||
- A description of the issue and its impact
|
||||
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
|
||||
- The version or commit hash of pdf-inspector you tested against
|
||||
|
||||
**Alternative:** Use GitHub's private vulnerability reporting under the
|
||||
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
|
||||
|
||||
We'll acknowledge your report in a timely manner and keep you updated on
|
||||
remediation progress. Please do not open a public GitHub issue for security
|
||||
bugs.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
- Memory-safety issues (panics, OOB reads, UB) reachable from a crafted PDF
|
||||
- Denial-of-service vectors (unbounded allocation, infinite loops) on
|
||||
reasonably-sized inputs
|
||||
- Bugs in the `pdf2md` / `detect-pdf` binaries or the `pdf-inspector` crate
|
||||
that affect downstream consumers
|
||||
|
||||
Out of scope:
|
||||
- Bugs in upstream dependencies (`lopdf`, etc.) — please report those upstream
|
||||
- Extraction quality issues (wrong text, missing tables) — open a regular
|
||||
GitHub issue instead
|
||||
@@ -1,21 +0,0 @@
|
||||
# Publishing
|
||||
|
||||
The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
|
||||
|
||||
## crates.io Trusted Publisher
|
||||
|
||||
Configure the trusted publisher for the `pdf-inspector` crate with:
|
||||
|
||||
- Repository: `firecrawl/pdf-inspector`
|
||||
- Workflow: `publish-crate.yml`
|
||||
- Environment: `crates-io`
|
||||
|
||||
The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
|
||||
|
||||
## Release Steps
|
||||
|
||||
1. Update `version` in `Cargo.toml`.
|
||||
2. Merge the version bump to `main`.
|
||||
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
|
||||
|
||||
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
|
||||
Generated
+4
-5
@@ -672,9 +672,8 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d"
|
||||
version = "0.40.0"
|
||||
source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"bitflags",
|
||||
@@ -830,7 +829,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.3"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"log",
|
||||
@@ -845,7 +844,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.2"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.2"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+1
-2
@@ -37,7 +37,7 @@ console.log(result.confidence) // 0.875
|
||||
|
||||
Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pipelines where a layout model detects regions in rendered page images, and this function extracts text from the PDF structure for text-based pages — skipping GPU OCR.
|
||||
|
||||
Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). When the cause is a suspected garbled text layer, `ocrReason` is set to `"suspected_garbled_text"`.
|
||||
Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues).
|
||||
|
||||
```typescript
|
||||
import { extractTextInRegions } from '@firecrawl/pdf-inspector'
|
||||
@@ -84,7 +84,6 @@ interface PageRegionTexts {
|
||||
interface RegionText {
|
||||
text: string
|
||||
needsOcr: boolean // true when text is unreliable
|
||||
ocrReason?: string // "suspected_garbled_text" when known
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.9",
|
||||
"version": "1.8.7",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -40,8 +40,6 @@ pub struct PdfResult {
|
||||
pub processing_time_ms: u32,
|
||||
/// 1-indexed page numbers that need OCR.
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Machine-readable OCR reasons by 1-indexed page.
|
||||
pub ocr_reasons_by_page: Vec<PageOcrReasons>,
|
||||
pub title: Option<String>,
|
||||
pub confidence: f64,
|
||||
pub is_complex_layout: bool,
|
||||
@@ -50,13 +48,6 @@ pub struct PdfResult {
|
||||
pub has_encoding_issues: bool,
|
||||
}
|
||||
|
||||
/// OCR reasons for a single 1-indexed page.
|
||||
#[napi(object)]
|
||||
pub struct PageOcrReasons {
|
||||
pub page: u32,
|
||||
pub reasons: Vec<String>,
|
||||
}
|
||||
|
||||
/// Lightweight PDF classification result.
|
||||
#[napi(object)]
|
||||
pub struct PdfClassification {
|
||||
@@ -80,9 +71,6 @@ pub struct TextItem {
|
||||
pub page: u32,
|
||||
pub is_bold: bool,
|
||||
pub is_italic: bool,
|
||||
/// Underline detected geometrically (drawn rule/thin rect under the
|
||||
/// baseline) — PDFs carry no underline font flag.
|
||||
pub is_underline: bool,
|
||||
pub item_type: ItemType,
|
||||
/// URL for link items, `None` for other types.
|
||||
pub link_url: Option<String>,
|
||||
@@ -102,8 +90,6 @@ pub struct RegionText {
|
||||
pub text: String,
|
||||
/// `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues).
|
||||
pub needs_ocr: bool,
|
||||
/// Machine-readable OCR reason when the cause is known.
|
||||
pub ocr_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Extracted text for one page's regions.
|
||||
@@ -140,7 +126,6 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
page_count: r.page_count,
|
||||
processing_time_ms: r.processing_time_ms as u32,
|
||||
pages_needing_ocr: r.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(r.ocr_reasons_by_page),
|
||||
title: r.title,
|
||||
confidence: r.confidence as f64,
|
||||
is_complex_layout: r.layout.is_complex,
|
||||
@@ -150,18 +135,6 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_napi_page_ocr_reasons(
|
||||
reasons: Vec<pdf_inspector::PageOcrReasons>,
|
||||
) -> Vec<PageOcrReasons> {
|
||||
reasons
|
||||
.into_iter()
|
||||
.map(|reason| PageOcrReasons {
|
||||
page: reason.page,
|
||||
reasons: reason.reasons,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
|
||||
match t {
|
||||
pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
|
||||
@@ -293,7 +266,6 @@ pub fn extract_text_with_positions(
|
||||
page: item.page,
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
item_type,
|
||||
link_url,
|
||||
}
|
||||
@@ -591,8 +563,6 @@ pub struct PageMarkdownResult {
|
||||
pub markdown: String,
|
||||
/// `true` when text on this page is unreliable.
|
||||
pub needs_ocr: bool,
|
||||
/// Machine-readable OCR reason when the cause is known.
|
||||
pub ocr_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Combined per-page markdown extraction and layout classification result.
|
||||
@@ -606,8 +576,6 @@ pub struct PagesExtractionResult {
|
||||
pub pages_with_columns: Vec<u32>,
|
||||
/// 1-indexed pages that need OCR (scanned/image-based).
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Machine-readable OCR reasons by 1-indexed page.
|
||||
pub ocr_reasons_by_page: Vec<PageOcrReasons>,
|
||||
/// True if any page has tables or columns.
|
||||
pub is_complex: bool,
|
||||
}
|
||||
@@ -639,13 +607,11 @@ pub fn extract_pages_markdown(
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
})
|
||||
@@ -682,7 +648,6 @@ fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<Pa
|
||||
.map(|r| RegionText {
|
||||
text: r.text,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
|
||||
@@ -38,7 +38,6 @@ class TextItem:
|
||||
page: int
|
||||
is_bold: bool
|
||||
is_italic: bool
|
||||
is_underline: bool
|
||||
item_type: str
|
||||
|
||||
class RegionText:
|
||||
|
||||
+3
-127
@@ -1,10 +1,6 @@
|
||||
//! CLI tool for PDF to Markdown conversion
|
||||
|
||||
use pdf_inspector::extractor::ItemType;
|
||||
use pdf_inspector::{
|
||||
extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions,
|
||||
PdfType, ProcessMode, TextItem,
|
||||
};
|
||||
use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode};
|
||||
use std::collections::HashSet;
|
||||
use std::env;
|
||||
use std::fmt::Write;
|
||||
@@ -35,108 +31,6 @@ fn json_escape(s: &str) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> String {
|
||||
reasons
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let reasons_json = entry
|
||||
.reasons
|
||||
.iter()
|
||||
.map(|reason| format!(r#""{}""#, json_escape(reason)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(r#"{{"page":{},"reasons":[{}]}}"#, entry.page, reasons_json)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
fn item_type_label(item_type: &ItemType) -> &'static str {
|
||||
match item_type {
|
||||
ItemType::Text => "text",
|
||||
ItemType::Image => "image",
|
||||
ItemType::Link(_) => "link",
|
||||
ItemType::FormField => "form_field",
|
||||
}
|
||||
}
|
||||
|
||||
fn format_items_json(items: &[TextItem]) -> String {
|
||||
let underlined_count = items.iter().filter(|item| item.is_underline).count();
|
||||
let items_json = items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let mcid = item
|
||||
.mcid
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "null".to_string());
|
||||
let link_url = match &item.item_type {
|
||||
ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)),
|
||||
_ => String::new(),
|
||||
};
|
||||
format!(
|
||||
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
|
||||
json_escape(&item.text),
|
||||
item.page,
|
||||
item.x,
|
||||
item.y,
|
||||
item.width,
|
||||
item.height,
|
||||
json_escape(&item.font),
|
||||
item.font_size,
|
||||
item.is_bold,
|
||||
item.is_italic,
|
||||
item.is_underline,
|
||||
item_type_label(&item.item_type),
|
||||
mcid,
|
||||
link_url,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
|
||||
format!(
|
||||
r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#,
|
||||
items.len(),
|
||||
underlined_count,
|
||||
items_json
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::format_items_json;
|
||||
use pdf_inspector::extractor::ItemType;
|
||||
use pdf_inspector::TextItem;
|
||||
|
||||
#[test]
|
||||
fn items_json_includes_position_and_underline_metadata() {
|
||||
let items = vec![TextItem {
|
||||
text: "A \"quoted\" item".to_string(),
|
||||
x: 12.345,
|
||||
y: 67.891,
|
||||
width: 23.456,
|
||||
height: 9.876,
|
||||
font: "F1".to_string(),
|
||||
font_size: 10.0,
|
||||
page: 2,
|
||||
is_bold: false,
|
||||
is_italic: true,
|
||||
is_underline: true,
|
||||
item_type: ItemType::Text,
|
||||
mcid: Some(7),
|
||||
}];
|
||||
|
||||
let json = format_items_json(&items);
|
||||
|
||||
assert!(json.contains(r#""text":"A \"quoted\" item""#));
|
||||
assert!(json.contains(r#""page":2"#));
|
||||
assert!(json.contains(r#""x":12.35"#));
|
||||
assert!(json.contains(r#""is_underline":true"#));
|
||||
assert!(json.contains(r#""item_type":"text""#));
|
||||
assert!(json.contains(r#""mcid":7"#));
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
|
||||
fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
|
||||
let mut pages = HashSet::new();
|
||||
@@ -194,7 +88,6 @@ fn main() {
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
|
||||
eprintln!(" {} <pdf_file> --json", args[0]);
|
||||
eprintln!(" {} <pdf_file> --items-json", args[0]);
|
||||
eprintln!(" {} <pdf_file> --raw", args[0]);
|
||||
eprintln!();
|
||||
eprintln!("Converts PDF to Markdown with smart type detection.");
|
||||
@@ -202,7 +95,6 @@ fn main() {
|
||||
eprintln!();
|
||||
eprintln!("Options:");
|
||||
eprintln!(" --json Output result as JSON");
|
||||
eprintln!(" --items-json Output positioned TextItem JSON");
|
||||
eprintln!(" --raw Output only markdown (no headers)");
|
||||
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
|
||||
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
|
||||
@@ -213,7 +105,6 @@ fn main() {
|
||||
|
||||
let pdf_path = &args[1];
|
||||
let json_output = args.iter().any(|a| a == "--json");
|
||||
let items_json_output = args.iter().any(|a| a == "--items-json");
|
||||
let raw_output = args.iter().any(|a| a == "--raw");
|
||||
let page_numbers = args.iter().any(|a| a == "--pages");
|
||||
let detect_only = args.iter().any(|a| a == "--detect-only");
|
||||
@@ -238,17 +129,6 @@ fn main() {
|
||||
})
|
||||
});
|
||||
|
||||
if items_json_output {
|
||||
match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) {
|
||||
Ok(items) => println!("{}", format_items_json(&items)),
|
||||
Err(e) => {
|
||||
println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let output_file = args
|
||||
.get(2)
|
||||
.filter(|a| !a.starts_with("--"))
|
||||
@@ -297,14 +177,12 @@ fn main() {
|
||||
.iter()
|
||||
.map(|p| p.to_string())
|
||||
.collect();
|
||||
let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page);
|
||||
println!(
|
||||
r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{}}}"#,
|
||||
r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{}}}"#,
|
||||
pdf_type_str,
|
||||
result.page_count,
|
||||
result.processing_time_ms,
|
||||
ocr_pages.join(","),
|
||||
ocr_reasons,
|
||||
result.layout.is_complex,
|
||||
table_pages.join(","),
|
||||
col_pages.join(","),
|
||||
@@ -345,9 +223,8 @@ fn main() {
|
||||
.iter()
|
||||
.map(|p| p.to_string())
|
||||
.collect();
|
||||
let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page);
|
||||
println!(
|
||||
r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{},"markdown":"{}"}}"#,
|
||||
r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{},"markdown":"{}"}}"#,
|
||||
match result.pdf_type {
|
||||
PdfType::TextBased => "text_based",
|
||||
PdfType::Scanned => "scanned",
|
||||
@@ -359,7 +236,6 @@ fn main() {
|
||||
result.processing_time_ms,
|
||||
result.markdown.as_ref().map(|m| m.len()).unwrap_or(0),
|
||||
ocr_pages.join(","),
|
||||
ocr_reasons,
|
||||
result.layout.is_complex,
|
||||
table_pages.join(","),
|
||||
col_pages.join(","),
|
||||
|
||||
+21
-345
@@ -17,9 +17,8 @@ use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::underline::UnderlineLine;
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
/// Strip PDF comments (% to end of line) from content stream bytes.
|
||||
///
|
||||
@@ -75,37 +74,6 @@ fn strip_pdf_comments(data: &[u8]) -> Vec<u8> {
|
||||
result
|
||||
}
|
||||
|
||||
fn transform_path_point(x: f32, y: f32, ctm: &[f32; 6]) -> (f32, f32) {
|
||||
(
|
||||
x * ctm[0] + y * ctm[2] + ctm[4],
|
||||
x * ctm[1] + y * ctm[3] + ctm[5],
|
||||
)
|
||||
}
|
||||
|
||||
fn transformed_stroke_width(
|
||||
line_width: f32,
|
||||
ctm: &[f32; 6],
|
||||
x1: f32,
|
||||
y1: f32,
|
||||
x2: f32,
|
||||
y2: f32,
|
||||
) -> f32 {
|
||||
let user_width = line_width.abs();
|
||||
let dx = x2 - x1;
|
||||
let dy = y2 - y1;
|
||||
let len = (dx * dx + dy * dy).sqrt();
|
||||
if len <= f32::EPSILON {
|
||||
return user_width;
|
||||
}
|
||||
|
||||
// PDF stroke width scales perpendicular to the path direction.
|
||||
let nx = -dy / len;
|
||||
let ny = dx / len;
|
||||
let ndx = nx * ctm[0] + ny * ctm[2];
|
||||
let ndy = nx * ctm[1] + ny * ctm[3];
|
||||
user_width * (ndx * ndx + ndy * ndy).sqrt()
|
||||
}
|
||||
|
||||
/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
|
||||
/// the page uses fonts with unresolvable gid-encoded glyphs.
|
||||
pub(crate) fn extract_page_text_items(
|
||||
@@ -121,7 +89,6 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut rects: Vec<PdfRect> = Vec::new();
|
||||
let mut clip_rects: Vec<PdfRect> = Vec::new();
|
||||
let mut lines: Vec<PdfLine> = Vec::new();
|
||||
let mut underline_lines: Vec<UnderlineLine> = Vec::new();
|
||||
|
||||
// Path construction state for m/l/h → S/s line extraction
|
||||
let mut path_subpath_start: Option<(f32, f32)> = None;
|
||||
@@ -130,12 +97,6 @@ pub(crate) fn extract_page_text_items(
|
||||
// Completed subpaths (each a vec of line segments) for f/f* rect extraction
|
||||
let mut pending_subpaths: Vec<Vec<(f32, f32, f32, f32)>> = Vec::new();
|
||||
let mut fill_rects: Vec<PdfRect> = Vec::new();
|
||||
// `re` rects awaiting a paint operator. Underline detection must only
|
||||
// see painted rects: a `re W n` clip path or `re n` no-op draws nothing
|
||||
// on the page, so treating every `re` as ink would underline text that
|
||||
// merely sits near an invisible clip boundary.
|
||||
let mut pending_re_rects: Vec<PdfRect> = Vec::new();
|
||||
let mut painted_rects: Vec<PdfRect> = Vec::new();
|
||||
|
||||
// Get fonts for encoding
|
||||
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
|
||||
@@ -227,19 +188,7 @@ pub(crate) fn extract_page_text_items(
|
||||
// Graphics state tracking
|
||||
let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
|
||||
let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
|
||||
let mut line_width: f32 = 1.0;
|
||||
#[derive(Clone)]
|
||||
struct SavedGraphicsState {
|
||||
ctm: [f32; 6],
|
||||
text_rendering_mode: i32,
|
||||
line_width: f32,
|
||||
char_spacing: f32,
|
||||
word_spacing: f32,
|
||||
text_leading: f32,
|
||||
current_font: String,
|
||||
current_font_size: f32,
|
||||
}
|
||||
let mut gstate_stack: Vec<SavedGraphicsState> = Vec::new();
|
||||
let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();
|
||||
|
||||
// Text state tracking
|
||||
let mut current_font = String::new();
|
||||
@@ -278,28 +227,15 @@ pub(crate) fn extract_page_text_items(
|
||||
match op.operator.as_str() {
|
||||
"q" => {
|
||||
// Save graphics state
|
||||
gstate_stack.push(SavedGraphicsState {
|
||||
ctm,
|
||||
text_rendering_mode,
|
||||
line_width,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
text_leading,
|
||||
current_font: current_font.clone(),
|
||||
current_font_size,
|
||||
});
|
||||
gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
|
||||
}
|
||||
"Q" => {
|
||||
// Restore graphics state
|
||||
if let Some(saved) = gstate_stack.pop() {
|
||||
ctm = saved.ctm;
|
||||
text_rendering_mode = saved.text_rendering_mode;
|
||||
line_width = saved.line_width;
|
||||
char_spacing = saved.char_spacing;
|
||||
word_spacing = saved.word_spacing;
|
||||
text_leading = saved.text_leading;
|
||||
current_font = saved.current_font;
|
||||
current_font_size = saved.current_font_size;
|
||||
if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
|
||||
ctm = saved_ctm;
|
||||
text_rendering_mode = saved_tr;
|
||||
char_spacing = saved_tc;
|
||||
word_spacing = saved_tw;
|
||||
}
|
||||
}
|
||||
"cm" => {
|
||||
@@ -316,11 +252,6 @@ pub(crate) fn extract_page_text_items(
|
||||
ctm = multiply_matrices(&new_matrix, &ctm);
|
||||
}
|
||||
}
|
||||
"w" => {
|
||||
if let Some(width) = op.operands.first().and_then(get_number) {
|
||||
line_width = width;
|
||||
}
|
||||
}
|
||||
"BT" => {
|
||||
// Begin text block
|
||||
in_text_block = true;
|
||||
@@ -489,7 +420,6 @@ pub(crate) fn extract_page_text_items(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -655,7 +585,6 @@ pub(crate) fn extract_page_text_items(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -719,7 +648,6 @@ pub(crate) fn extract_page_text_items(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -736,30 +664,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if let Some(xobj_type) = xobjects.get(&xobj_name) {
|
||||
match xobj_type {
|
||||
XObjectType::Image => {
|
||||
// Emit a positional placeholder for the image
|
||||
// so downstream consumers (layout-aware
|
||||
// pipelines, figure-OCR routers) can locate
|
||||
// raster figures without parsing the PDF
|
||||
// again. The text field carries the
|
||||
// XObject resource name in the legacy
|
||||
// `[Image: Im0]` format that the markdown
|
||||
// emitter already recognizes.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
// Skip images — text extraction only
|
||||
}
|
||||
XObjectType::Form(form_id) => {
|
||||
// Extract text from Form XObject
|
||||
@@ -853,7 +758,6 @@ pub(crate) fn extract_page_text_items(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: entry
|
||||
.mcid
|
||||
@@ -878,19 +782,13 @@ pub(crate) fn extract_page_text_items(
|
||||
let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5];
|
||||
let w_dev = rw * ctm[0];
|
||||
let h_dev = rh * ctm[3];
|
||||
let rect = PdfRect {
|
||||
rects.push(PdfRect {
|
||||
x: x_dev,
|
||||
y: y_dev,
|
||||
width: w_dev,
|
||||
height: h_dev,
|
||||
page: page_num,
|
||||
};
|
||||
// Underline detection must only see rects that are
|
||||
// actually painted — a `re` used purely as a clip path
|
||||
// (`re W n`) or discarded (`re n`) draws nothing. Hold
|
||||
// the rect as pending until a paint operator confirms it.
|
||||
pending_re_rects.push(rect.clone());
|
||||
rects.push(rect);
|
||||
});
|
||||
}
|
||||
}
|
||||
// ── Path construction operators ──────────────────────
|
||||
@@ -940,8 +838,10 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
}
|
||||
for (x1, y1, x2, y2) in pending_lines.drain(..) {
|
||||
let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
|
||||
let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
|
||||
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
|
||||
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
|
||||
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
|
||||
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
|
||||
lines.push(PdfLine {
|
||||
x1: x1d,
|
||||
y1: y1d,
|
||||
@@ -949,16 +849,7 @@ pub(crate) fn extract_page_text_items(
|
||||
y2: y2d,
|
||||
page: page_num,
|
||||
});
|
||||
underline_lines.push(UnderlineLine {
|
||||
x1: x1d,
|
||||
y1: y1d,
|
||||
x2: x2d,
|
||||
y2: y2d,
|
||||
stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
|
||||
page: page_num,
|
||||
});
|
||||
}
|
||||
painted_rects.append(&mut pending_re_rects);
|
||||
pending_subpaths.clear();
|
||||
path_subpath_start = None;
|
||||
path_current = None;
|
||||
@@ -974,8 +865,10 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
}
|
||||
for (x1, y1, x2, y2) in pending_lines.drain(..) {
|
||||
let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
|
||||
let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
|
||||
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
|
||||
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
|
||||
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
|
||||
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
|
||||
lines.push(PdfLine {
|
||||
x1: x1d,
|
||||
y1: y1d,
|
||||
@@ -983,16 +876,7 @@ pub(crate) fn extract_page_text_items(
|
||||
y2: y2d,
|
||||
page: page_num,
|
||||
});
|
||||
underline_lines.push(UnderlineLine {
|
||||
x1: x1d,
|
||||
y1: y1d,
|
||||
x2: x2d,
|
||||
y2: y2d,
|
||||
stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
|
||||
page: page_num,
|
||||
});
|
||||
}
|
||||
painted_rects.append(&mut pending_re_rects);
|
||||
pending_subpaths.clear();
|
||||
path_subpath_start = None;
|
||||
path_current = None;
|
||||
@@ -1050,7 +934,6 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
}
|
||||
}
|
||||
painted_rects.append(&mut pending_re_rects);
|
||||
pending_lines.clear();
|
||||
path_subpath_start = None;
|
||||
path_current = None;
|
||||
@@ -1116,10 +999,7 @@ pub(crate) fn extract_page_text_items(
|
||||
// Do NOT clear pending_lines — the following `n` does that
|
||||
}
|
||||
"n" => {
|
||||
// end path (no-op): discard — including any `re` rects that
|
||||
// were only ever part of a clip path (`re W n`), which draw
|
||||
// no ink and must not feed underline detection.
|
||||
pending_re_rects.clear();
|
||||
// end path (no-op): discard
|
||||
pending_lines.clear();
|
||||
pending_subpaths.clear();
|
||||
path_subpath_start = None;
|
||||
@@ -1129,12 +1009,6 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
}
|
||||
|
||||
// Underline detection reads only painted ink: `re` rects confirmed by
|
||||
// a paint operator plus filled-subpath rects — never clip-only rects,
|
||||
// which draw nothing.
|
||||
let mut underline_rects = painted_rects;
|
||||
underline_rects.extend(fill_rects.iter().cloned());
|
||||
|
||||
// Only use clip/fill rects when no `re` rects exist on this page.
|
||||
// Clip rects take priority over fill rects, but first we deduplicate
|
||||
// them: some PDFs wrap every text block in a full-page W* clip path,
|
||||
@@ -1164,17 +1038,8 @@ pub(crate) fn extract_page_text_items(
|
||||
// Some PDFs embed landscape content in portrait pages using a rotated text
|
||||
// matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW). The layout engine
|
||||
// assumes x=horizontal, y=vertical — so we swap coordinates to match.
|
||||
let (mut items, rects, lines, coords_rotated) =
|
||||
let (items, rects, lines, coords_rotated) =
|
||||
correct_rotated_page(items, rects, lines, &rotation_votes);
|
||||
if coords_rotated {
|
||||
rotate_underline_graphics(&mut underline_rects, &mut underline_lines);
|
||||
}
|
||||
super::underline::mark_underlined_items(
|
||||
&mut items,
|
||||
&underline_rects,
|
||||
&underline_lines,
|
||||
page_num,
|
||||
);
|
||||
|
||||
let items = super::merge_text_items(items);
|
||||
let items = super::merge_subscript_items(items);
|
||||
@@ -1260,27 +1125,6 @@ fn correct_rotated_page(
|
||||
(items, rects, lines, true)
|
||||
}
|
||||
|
||||
fn rotate_underline_graphics(rects: &mut [PdfRect], lines: &mut [UnderlineLine]) {
|
||||
for rect in rects {
|
||||
let new_x = rect.y;
|
||||
let new_y = -(rect.x + rect.width.abs());
|
||||
rect.x = new_x;
|
||||
rect.y = new_y;
|
||||
std::mem::swap(&mut rect.width, &mut rect.height);
|
||||
}
|
||||
|
||||
for line in lines {
|
||||
let new_x1 = line.y1;
|
||||
let new_y1 = -line.x1;
|
||||
let new_x2 = line.y2;
|
||||
let new_y2 = -line.x2;
|
||||
line.x1 = new_x1;
|
||||
line.y1 = new_y1;
|
||||
line.x2 = new_x2;
|
||||
line.y2 = new_y2;
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance).
|
||||
/// Some PDFs emit a full-page clip path for every text block, producing
|
||||
/// thousands of identical rects. After dedup these collapse to one rect,
|
||||
@@ -1330,57 +1174,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn simple_doc_with_content(content: &[u8]) -> (lopdf::Document, lopdf::ObjectId) {
|
||||
use lopdf::{dictionary, Object, Stream};
|
||||
|
||||
let mut doc = lopdf::Document::new();
|
||||
let widths: Vec<Object> = (0..=255).map(|_| 600.into()).collect();
|
||||
let font_id = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
"FirstChar" => 0,
|
||||
"LastChar" => 255,
|
||||
"Widths" => Object::Array(widths),
|
||||
});
|
||||
let content_id = doc.add_object(Object::Stream(Stream::new(
|
||||
dictionary! {},
|
||||
content.to_vec(),
|
||||
)));
|
||||
let page_id = doc.add_object(dictionary! {
|
||||
"Type" => "Page",
|
||||
"Contents" => Object::Reference(content_id),
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => Object::Reference(font_id),
|
||||
},
|
||||
},
|
||||
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
|
||||
});
|
||||
let pages_id = doc.add_object(dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Count" => Object::Integer(1),
|
||||
"Kids" => vec![Object::Reference(page_id)],
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => Object::Reference(pages_id),
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
(doc, page_id)
|
||||
}
|
||||
|
||||
fn extract_simple_items(content: &[u8]) -> Vec<TextItem> {
|
||||
use crate::tounicode::FontCMaps;
|
||||
|
||||
let (doc, page_id) = simple_doc_with_content(content);
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
items
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dedup_rects_identical() {
|
||||
let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759];
|
||||
@@ -1430,38 +1223,6 @@ mod tests {
|
||||
assert_eq!(single.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thick_stroked_rule_does_not_mark_underline() {
|
||||
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (THICK) Tj ET
|
||||
4 w
|
||||
100 498 m 170 498 l S
|
||||
BT /F1 12 Tf 1 0 0 1 100 480 Tm (THIN) Tj ET
|
||||
1 w
|
||||
100 478 m 160 478 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let thick = items.iter().find(|item| item.text == "THICK").unwrap();
|
||||
let thin = items.iter().find(|item| item.text == "THIN").unwrap();
|
||||
|
||||
assert!(!thick.is_underline);
|
||||
assert!(thin.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotated_page_underline_is_detected_after_coordinate_correction() {
|
||||
let content = b"BT /F1 12 Tf 0 1 -1 0 200 100 Tm (HELLO) Tj ET
|
||||
BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
1 w
|
||||
202 100 m 202 170 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let hello = items.iter().find(|item| item.text == "HELLO").unwrap();
|
||||
let world = items.iter().find(|item| item.text == "WORLD").unwrap();
|
||||
|
||||
assert!(hello.is_underline);
|
||||
assert!(!world.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skip_excessive_operations() {
|
||||
use crate::tounicode::FontCMaps;
|
||||
@@ -1503,91 +1264,6 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
assert!(lines.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_q_restores_current_font_for_text_decoding() {
|
||||
use crate::tounicode::FontCMaps;
|
||||
use lopdf::{dictionary, Object, Stream};
|
||||
|
||||
fn cmap_stream(dst_hex: &str) -> Stream {
|
||||
let cmap = format!(
|
||||
r#"/CIDInit /ProcSet findresource begin
|
||||
12 dict begin
|
||||
begincmap
|
||||
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
|
||||
/CMapName /Test-UCS def
|
||||
/CMapType 2 def
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
1 beginbfchar
|
||||
<41> <{dst_hex}>
|
||||
endbfchar
|
||||
endcmap
|
||||
CMapName currentdict /CMap defineresource pop
|
||||
end
|
||||
end"#
|
||||
);
|
||||
Stream::new(dictionary! {}, cmap.into_bytes())
|
||||
}
|
||||
|
||||
let mut doc = lopdf::Document::new();
|
||||
let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X
|
||||
let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y
|
||||
let f1 = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
"ToUnicode" => Object::Reference(f1_cmap),
|
||||
});
|
||||
let f2 = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
"ToUnicode" => Object::Reference(f2_cmap),
|
||||
});
|
||||
|
||||
let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET
|
||||
q
|
||||
BT /F2 12 Tf 20 700 Tm <41> Tj ET
|
||||
Q
|
||||
BT 30 700 Tm <41> Tj ET";
|
||||
let content_id = doc.add_object(Object::Stream(Stream::new(
|
||||
dictionary! {},
|
||||
content.to_vec(),
|
||||
)));
|
||||
let page_id = doc.add_object(dictionary! {
|
||||
"Type" => "Page",
|
||||
"Contents" => Object::Reference(content_id),
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => Object::Reference(f1),
|
||||
"F2" => Object::Reference(f2),
|
||||
},
|
||||
},
|
||||
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
|
||||
});
|
||||
let pages_id = doc.add_object(dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Count" => Object::Integer(1),
|
||||
"Kids" => vec![Object::Reference(page_id)],
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => Object::Reference(pages_id),
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let text = items
|
||||
.iter()
|
||||
.map(|item| item.text.as_str())
|
||||
.collect::<String>();
|
||||
|
||||
assert_eq!(text, "XYX");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_strip_pdf_comments() {
|
||||
// Basic comment stripping
|
||||
|
||||
+46
-277
@@ -526,11 +526,6 @@ pub(crate) fn parse_font_encoding(
|
||||
font_dict: &lopdf::Dictionary,
|
||||
) -> Option<EncodingResult> {
|
||||
let encoding_obj = font_dict.get(b"Encoding").ok()?;
|
||||
let base_font_name = font_dict
|
||||
.get(b"BaseFont")
|
||||
.ok()
|
||||
.and_then(|o| o.as_name().ok())
|
||||
.map(|n| String::from_utf8_lossy(n).to_string());
|
||||
|
||||
// Encoding can be a name or a dictionary
|
||||
match encoding_obj {
|
||||
@@ -543,14 +538,12 @@ pub(crate) fn parse_font_encoding(
|
||||
Object::Reference(obj_ref) => {
|
||||
// Reference to encoding dictionary
|
||||
if let Ok(enc_dict) = doc.get_dictionary(*obj_ref) {
|
||||
parse_encoding_dictionary(doc, enc_dict, base_font_name.as_deref())
|
||||
parse_encoding_dictionary(doc, enc_dict)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Object::Dictionary(enc_dict) => {
|
||||
parse_encoding_dictionary(doc, enc_dict, base_font_name.as_deref())
|
||||
}
|
||||
Object::Dictionary(enc_dict) => parse_encoding_dictionary(doc, enc_dict),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -569,7 +562,6 @@ pub(crate) struct EncodingResult {
|
||||
pub(crate) fn parse_encoding_dictionary(
|
||||
doc: &Document,
|
||||
enc_dict: &lopdf::Dictionary,
|
||||
base_font_name: Option<&str>,
|
||||
) -> Option<EncodingResult> {
|
||||
let differences = enc_dict.get(b"Differences").ok()?;
|
||||
|
||||
@@ -599,9 +591,11 @@ pub(crate) fn parse_encoding_dictionary(
|
||||
Object::Name(name) => {
|
||||
// Map current code to glyph name -> Unicode
|
||||
let glyph_name = String::from_utf8_lossy(&name).to_string();
|
||||
let mapped_char = glyph_to_char(&glyph_name)
|
||||
.or_else(|| private_glyph_to_char(&glyph_name, base_font_name));
|
||||
if mapped_char.is_some_and(is_ligature_char) {
|
||||
if glyph_name == "fi"
|
||||
|| glyph_name == "fl"
|
||||
|| glyph_name == "ffi"
|
||||
|| glyph_name == "ffl"
|
||||
{
|
||||
debug!(
|
||||
" Differences: code=0x{:02X} glyph={:?} (ligature)",
|
||||
current_code, glyph_name
|
||||
@@ -616,7 +610,7 @@ pub(crate) fn parse_encoding_dictionary(
|
||||
{
|
||||
gid_glyph_count += 1;
|
||||
}
|
||||
if let Some(ch) = mapped_char {
|
||||
if let Some(ch) = glyph_to_char(&glyph_name) {
|
||||
encoding_map.insert(current_code, ch);
|
||||
} else {
|
||||
debug!(
|
||||
@@ -651,31 +645,6 @@ pub(crate) fn parse_encoding_dictionary(
|
||||
})
|
||||
}
|
||||
|
||||
fn private_glyph_to_char(glyph_name: &str, base_font_name: Option<&str>) -> Option<char> {
|
||||
let base_font_name = strip_subset_prefix(base_font_name?);
|
||||
|
||||
// Aptos CFF subsets from Office PDFs can expose the ff ligature as /g431
|
||||
// without a ToUnicode map. Keep this font-scoped because /gNNN names are private.
|
||||
if base_font_name.eq_ignore_ascii_case("Aptos") && glyph_name == "g431" {
|
||||
Some('\u{FB00}')
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_subset_prefix(font_name: &str) -> &str {
|
||||
font_name
|
||||
.split_once('+')
|
||||
.map_or(font_name, |(_, stripped)| stripped)
|
||||
}
|
||||
|
||||
fn is_ligature_char(ch: char) -> bool {
|
||||
matches!(
|
||||
ch,
|
||||
'\u{FB00}' | '\u{FB01}' | '\u{FB02}' | '\u{FB03}' | '\u{FB04}'
|
||||
)
|
||||
}
|
||||
|
||||
/// Get the CMap lookup key for an Identity-H/V CID font without ToUnicode.
|
||||
/// Returns the object number used by `collect_cmaps_from_fonts` to store the CMap:
|
||||
/// - FontFile2 or FontFile3 obj_num (for embedded font cmap)
|
||||
@@ -756,8 +725,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
let is_type0_cid_font = font_widths
|
||||
.get(current_font)
|
||||
.is_some_and(|info| info.is_cid);
|
||||
let use_cp1252_fallback =
|
||||
should_use_cp1252_single_byte_fallback(base_font_name, is_type0_cid_font);
|
||||
let result = (|| -> Option<String> {
|
||||
if let Object::String(bytes, _) = obj {
|
||||
let mut decode_with_entry = |entry: &crate::tounicode::CMapEntry| -> Option<String> {
|
||||
@@ -788,12 +755,9 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(ch.to_string());
|
||||
}
|
||||
}
|
||||
// 4. Printable single-byte fallback
|
||||
// 4. Printable ASCII/Latin-1 fallback
|
||||
if b >= 0x20 {
|
||||
return Some(
|
||||
decode_single_byte_fallback_char(b, use_cp1252_fallback)
|
||||
.to_string(),
|
||||
);
|
||||
return Some((b as char).to_string());
|
||||
}
|
||||
None
|
||||
})
|
||||
@@ -894,13 +858,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
// unmapped. Don't fall through to text-interpretation fallbacks
|
||||
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
|
||||
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
|
||||
// Try our custom encoding map from Differences arrays.
|
||||
// The Differences array overrides specific codes in a base encoding (typically
|
||||
@@ -916,9 +873,8 @@ pub(crate) fn extract_text_from_operand(
|
||||
Some(ch)
|
||||
} else if b >= 0x20 {
|
||||
// Base encoding fallback for printable bytes.
|
||||
// Most PDFs with simple fonts use WinAnsi/PDFDocEncoding
|
||||
// semantics, not ISO-8859-1 C1 controls.
|
||||
Some(decode_single_byte_fallback_char(b, use_cp1252_fallback))
|
||||
// For codes 0x20-0x7E this matches all standard PDF encodings.
|
||||
Some(b as char)
|
||||
} else {
|
||||
None // Skip unmapped control characters
|
||||
}
|
||||
@@ -974,7 +930,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
// Try to decode using cached font encoding from lopdf
|
||||
if let Some(encoding) = encoding_cache.get(current_font) {
|
||||
if let Ok(text) = Document::decode_text(encoding, bytes) {
|
||||
let text = normalize_cp1252_controls(text, use_cp1252_fallback);
|
||||
if text.contains('\u{FFFD}') {
|
||||
debug!(
|
||||
"decode_text produced replacement for font={} bytes_len={}",
|
||||
@@ -1011,119 +966,37 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Non-CID (Type1 / TrueType / Type3) fonts use single-byte
|
||||
// encodings. In practice the fallback should follow WinAnsi for
|
||||
// 0x80..=0x9F so bytes like 0x92 become smart punctuation instead
|
||||
// of C1 controls that look like CID mojibake.
|
||||
Some(decode_single_byte_fallback(bytes, use_cp1252_fallback))
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
// canonical interpretation.
|
||||
Some(bytes.iter().map(|&b| b as char).collect())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})();
|
||||
result.map(|text| {
|
||||
let text = clean_symbol_pua(text);
|
||||
normalize_cp1252_controls(text, use_cp1252_fallback)
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_single_byte_fallback(bytes: &[u8], use_cp1252_fallback: bool) -> String {
|
||||
bytes
|
||||
.iter()
|
||||
.map(|&b| decode_single_byte_fallback_char(b, use_cp1252_fallback))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn decode_single_byte_fallback_char(byte: u8, use_cp1252_fallback: bool) -> char {
|
||||
if !use_cp1252_fallback {
|
||||
return byte as char;
|
||||
}
|
||||
|
||||
match byte {
|
||||
0x80 => '\u{20AC}',
|
||||
0x82 => '\u{201A}',
|
||||
0x83 => '\u{0192}',
|
||||
0x84 => '\u{201E}',
|
||||
0x85 => '\u{2026}',
|
||||
0x86 => '\u{2020}',
|
||||
0x87 => '\u{2021}',
|
||||
0x88 => '\u{02C6}',
|
||||
0x89 => '\u{2030}',
|
||||
0x8A => '\u{0160}',
|
||||
0x8B => '\u{2039}',
|
||||
0x8C => '\u{0152}',
|
||||
0x8E => '\u{017D}',
|
||||
0x91 => '\u{2018}',
|
||||
0x92 => '\u{2019}',
|
||||
0x93 => '\u{201C}',
|
||||
0x94 => '\u{201D}',
|
||||
0x95 => '\u{2022}',
|
||||
0x96 => '\u{2013}',
|
||||
0x97 => '\u{2014}',
|
||||
0x98 => '\u{02DC}',
|
||||
0x99 => '\u{2122}',
|
||||
0x9A => '\u{0161}',
|
||||
0x9B => '\u{203A}',
|
||||
0x9C => '\u{0153}',
|
||||
0x9E => '\u{017E}',
|
||||
0x9F => '\u{0178}',
|
||||
_ => byte as char,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_cp1252_controls(text: String, use_cp1252_fallback: bool) -> String {
|
||||
if !use_cp1252_fallback {
|
||||
return text;
|
||||
}
|
||||
if !text
|
||||
.chars()
|
||||
.any(|ch| ('\u{0080}'..='\u{009F}').contains(&ch))
|
||||
{
|
||||
return text;
|
||||
}
|
||||
|
||||
text.chars()
|
||||
.map(|ch| {
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
decode_single_byte_fallback_char(ch as u8, true)
|
||||
} else {
|
||||
ch
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn should_use_cp1252_single_byte_fallback(
|
||||
base_font_name: Option<&str>,
|
||||
is_type0_cid_font: bool,
|
||||
) -> bool {
|
||||
if is_type0_cid_font {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(base_font_name) = base_font_name else {
|
||||
return true;
|
||||
};
|
||||
let font_name = base_font_name
|
||||
.rsplit_once('+')
|
||||
.map_or(base_font_name, |(_, stripped)| stripped)
|
||||
.to_ascii_lowercase();
|
||||
|
||||
// TeX/Computer Modern and math/symbol fonts often place ligatures or
|
||||
// symbols in the C1 byte range. Treating those bytes as Windows-1252 makes
|
||||
// words like "deficiente" become "de…ciente" and "fluid" become "‡uid".
|
||||
let non_cp1252_prefixes = [
|
||||
"cmr", "cmb", "cmmi", "cmsy", "cmex", "cmtt", "cmss", "cmti", "ecrm", "ecbx", "ecti",
|
||||
"tcrm", "tctt", "msam", "msbm", "ttdc",
|
||||
];
|
||||
if non_cp1252_prefixes
|
||||
.iter()
|
||||
.any(|prefix| font_name.starts_with(prefix))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let non_cp1252_names = ["math", "symbol", "dingbat", "emoji"];
|
||||
!non_cp1252_names.iter().any(|name| font_name.contains(name))
|
||||
result.map(clean_symbol_pua)
|
||||
}
|
||||
|
||||
/// Replace PUA characters in the F000-F0FF range with standard Unicode equivalents.
|
||||
@@ -1245,7 +1118,6 @@ fn score_text(text: &str) -> i32 {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use lopdf::dictionary;
|
||||
|
||||
fn make_font_info(widths: &[(u16, u16)], default_width: u16, is_cid: bool) -> FontWidthInfo {
|
||||
FontWidthInfo {
|
||||
@@ -1370,51 +1242,6 @@ mod tests {
|
||||
assert!(score_text(good) > score_text(bad));
|
||||
}
|
||||
|
||||
fn doc_with_private_differences() -> (Document, lopdf::ObjectId) {
|
||||
let mut doc = Document::with_version("1.7");
|
||||
let encoding_id = doc.add_object(dictionary! {
|
||||
"Differences" => Object::Array(vec![
|
||||
Object::Integer(0x88),
|
||||
Object::Name(b"g431".to_vec()),
|
||||
Object::Name(b"fi".to_vec()),
|
||||
Object::Integer(0xAD),
|
||||
Object::Name(b"fl".to_vec()),
|
||||
]),
|
||||
});
|
||||
|
||||
(doc, encoding_id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aptos_private_g431_maps_to_ff_ligature() {
|
||||
let (doc, encoding_id) = doc_with_private_differences();
|
||||
let font_dict = dictionary! {
|
||||
"BaseFont" => Object::Name(b"NJEQOD+Aptos".to_vec()),
|
||||
"Encoding" => Object::Reference(encoding_id),
|
||||
};
|
||||
|
||||
let result = parse_font_encoding(&doc, &font_dict).expect("encoding should parse");
|
||||
|
||||
assert_eq!(result.map.get(&0x88u8), Some(&'\u{FB00}'));
|
||||
assert_eq!(result.map.get(&0x89u8), Some(&'\u{FB01}'));
|
||||
assert_eq!(result.map.get(&0xADu8), Some(&'\u{FB02}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_g431_does_not_map_for_unrelated_fonts() {
|
||||
let (doc, encoding_id) = doc_with_private_differences();
|
||||
let font_dict = dictionary! {
|
||||
"BaseFont" => Object::Name(b"ABCDEF+OtherFont".to_vec()),
|
||||
"Encoding" => Object::Reference(encoding_id),
|
||||
};
|
||||
|
||||
let result = parse_font_encoding(&doc, &font_dict).expect("encoding should parse");
|
||||
|
||||
assert!(!result.map.contains_key(&0x88u8));
|
||||
assert_eq!(result.map.get(&0x89u8), Some(&'\u{FB01}'));
|
||||
assert_eq!(result.map.get(&0xADu8), Some(&'\u{FB02}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cid_font_with_unparseable_cmap_does_not_emit_latin1_mojibake() {
|
||||
// Type0/CID font (font_widths reports `is_cid=true`) where the
|
||||
@@ -1466,15 +1293,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_font_single_byte_fallback_passes_high_bytes_through() {
|
||||
fn simple_font_latin1_fallback_passes_high_bytes_through() {
|
||||
// A Type1/TrueType simple font (is_cid=false) with a `/ToUnicode`
|
||||
// reference but no usable CMap and no `/Differences` map.
|
||||
// Per-byte fallback is the canonical interpretation here — these
|
||||
// bytes are character codes, not CIDs. The CID guard must NOT strip
|
||||
// them. Reproduces the false positive that an earlier version of the
|
||||
// guard introduced for fonts in PDFs like pdf-evals/Navigating-
|
||||
// Artificial-Intelligence-..., where bytes like 0xB6 are legitimate
|
||||
// single-byte character codes.
|
||||
// Per-byte Latin-1 IS the canonical interpretation here — these
|
||||
// bytes are character codes, not CIDs. The CID guard must NOT
|
||||
// strip them. Reproduces the false positive that an earlier
|
||||
// version of the guard introduced for fonts in PDFs like
|
||||
// pdf-evals/Navigating-Artificial-Intelligence-..., where bytes
|
||||
// like 0xB6 are legitimate Latin-1 character codes.
|
||||
let bytes = vec![0x24_u8, 0x47, 0xB6, 0x56]; // "$G¶V"
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
@@ -1507,62 +1334,4 @@ mod tests {
|
||||
"simple font fallback must not stamp FFFD over legitimate bytes: {text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_font_single_byte_fallback_maps_cp1252_punctuation() {
|
||||
let bytes = vec![b'l', 0x92_u8, b'a', b'c', b'a', b'd'];
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
let font_cmaps = FontCMaps::default();
|
||||
let font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
let inline_cmaps = HashMap::new();
|
||||
let font_encodings: PageFontEncodings = HashMap::new();
|
||||
let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
|
||||
let mut decisions = CMapDecisionCache::new();
|
||||
let font_widths: PageFontWidths = HashMap::new();
|
||||
|
||||
let text = extract_text_from_operand(
|
||||
&obj,
|
||||
"F1",
|
||||
None,
|
||||
&font_cmaps,
|
||||
&font_tounicode_refs,
|
||||
&inline_cmaps,
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut decisions,
|
||||
&font_widths,
|
||||
)
|
||||
.expect("simple font should decode CP1252 punctuation");
|
||||
|
||||
assert_eq!(text, "l’acad");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_encoding_decode_normalizes_cp1252_controls() {
|
||||
let text = normalize_cp1252_controls("d\u{92}un \u{96} test".to_string(), true);
|
||||
assert_eq!(text, "d’un – test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tex_font_decode_keeps_c1_ligature_bytes_unmodified() {
|
||||
let text = normalize_cp1252_controls("de\u{85}ciente \u{87}uid".to_string(), false);
|
||||
assert_eq!(text, "de\u{85}ciente \u{87}uid");
|
||||
assert!(!should_use_cp1252_single_byte_fallback(
|
||||
Some("TTdcr10"),
|
||||
false
|
||||
));
|
||||
assert!(!should_use_cp1252_single_byte_fallback(
|
||||
Some("cmr10"),
|
||||
false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn winansi_text_font_uses_cp1252_fallback() {
|
||||
assert!(should_use_cp1252_single_byte_fallback(
|
||||
Some("BJPQNQ+Times-Roman"),
|
||||
false
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,12 +29,8 @@ pub(crate) fn detect_columns(
|
||||
const MIN_ITEMS_PER_COLUMN: usize = 10;
|
||||
const NOISE_FRACTION: f32 = 0.15;
|
||||
|
||||
// Get items for this page. Strip Image placeholders — an image's left edge
|
||||
// would otherwise count toward the column projection profile.
|
||||
let page_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.page == page && crate::extractor::is_text_layout_item(i))
|
||||
.collect();
|
||||
// Get items for this page
|
||||
let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect();
|
||||
|
||||
if page_items.is_empty() {
|
||||
return vec![];
|
||||
@@ -1234,7 +1230,11 @@ pub(crate) fn group_into_lines_with_thresholds(
|
||||
ci,
|
||||
item.x,
|
||||
item.y,
|
||||
super::trace_text_preview(&item.text, 60)
|
||||
if item.text.len() > 60 {
|
||||
&item.text[..60]
|
||||
} else {
|
||||
&item.text
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1494,7 +1494,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1624,7 +1623,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -78,7 +78,6 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Link(url),
|
||||
mcid: None,
|
||||
});
|
||||
@@ -317,7 +316,6 @@ pub(crate) fn walk_form_fields(
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::FormField,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+16
-433
@@ -6,12 +6,11 @@ pub(crate) mod content_stream;
|
||||
mod fonts;
|
||||
mod layout;
|
||||
mod links;
|
||||
pub(crate) mod underline;
|
||||
mod xobjects;
|
||||
|
||||
use crate::text_utils::is_rtl_text;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
|
||||
use crate::types::{PageExtraction, TextItem};
|
||||
use crate::PdfError;
|
||||
use log::debug;
|
||||
use lopdf::{Document, Object, ObjectId};
|
||||
@@ -34,13 +33,6 @@ pub(crate) use layout::ColumnRegion;
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub(crate) fn trace_text_preview(text: &str, max_chars: usize) -> &str {
|
||||
match text.char_indices().nth(max_chars) {
|
||||
Some((idx, _)) => &text[..idx],
|
||||
None => text,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract text from PDF file as plain string
|
||||
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
@@ -181,7 +173,6 @@ fn extract_positioned_text_impl(
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
suppress_table_underlines(&mut items, &rects, &lines, *page_num);
|
||||
debug!(
|
||||
"page {}: {} text items, {} rects, {} lines{}",
|
||||
page_num,
|
||||
@@ -204,7 +195,11 @@ fn extract_positioned_text_impl(
|
||||
item.width,
|
||||
item.font_size,
|
||||
item.font,
|
||||
trace_text_preview(&item.text, 80)
|
||||
if item.text.len() > 80 {
|
||||
&item.text[..80]
|
||||
} else {
|
||||
&item.text
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -228,105 +223,10 @@ fn extract_positioned_text_impl(
|
||||
))
|
||||
}
|
||||
|
||||
fn suppress_table_underlines(
|
||||
items: &mut [TextItem],
|
||||
rects: &[PdfRect],
|
||||
lines: &[PdfLine],
|
||||
page: u32,
|
||||
) {
|
||||
if !items.iter().any(|item| item.is_underline) {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut table_item_indices: HashSet<usize> = HashSet::new();
|
||||
|
||||
if !rects.is_empty() {
|
||||
let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page);
|
||||
for table in rect_tables {
|
||||
table_item_indices.extend(table.item_indices);
|
||||
}
|
||||
}
|
||||
|
||||
if !lines.is_empty() {
|
||||
for table in crate::tables::detect_tables_from_lines(items, lines, page) {
|
||||
table_item_indices.extend(table.item_indices);
|
||||
}
|
||||
}
|
||||
|
||||
for index in table_item_indices {
|
||||
if let Some(item) = items.get_mut(index) {
|
||||
item.is_underline = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (used by submodules via `super::`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Return true when this item should participate in text-layout
|
||||
/// heuristics (column detection, table grid detection, line grouping).
|
||||
///
|
||||
/// Image XObjects emit a positional placeholder via
|
||||
/// `extract_text_with_positions` (so layout-aware callers can crop +
|
||||
/// caption figures), but their bboxes don't carry text glyphs and would
|
||||
/// skew column/row clustering if they reached the heuristics. Hyperlinks
|
||||
/// and form fields *do* participate — the existing logic treats them as
|
||||
/// text-like and we keep that.
|
||||
pub(crate) fn is_text_layout_item(item: &crate::types::TextItem) -> bool {
|
||||
!matches!(item.item_type, crate::types::ItemType::Image)
|
||||
}
|
||||
|
||||
/// Map a (u, v) point in unit-square coordinates through the 6-element CTM
|
||||
/// to page-space. CTM format is `[a, b, c, d, e, f]` per
|
||||
/// [`multiply_matrices`].
|
||||
fn apply_ctm_point(ctm: &[f32; 6], u: f32, v: f32) -> (f32, f32) {
|
||||
(
|
||||
u * ctm[0] + v * ctm[2] + ctm[4],
|
||||
u * ctm[1] + v * ctm[3] + ctm[5],
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute the page-space axis-aligned bounding box of an Image XObject
|
||||
/// invoked under the given CTM.
|
||||
///
|
||||
/// Per the PDF spec, an image XObject is always rendered into a unit
|
||||
/// square `(0,0)–(1,1)` in its local coordinate system, and the `Do`
|
||||
/// operator applies the current CTM to position/scale/rotate that square
|
||||
/// onto the page. For the common axis-aligned case (no rotation/shear),
|
||||
/// the CTM reduces to `[w, 0, 0, h, x, y]` and the bbox is just
|
||||
/// `(x, y, w, h)`. For rotated/sheared images we transform all four
|
||||
/// corners and return their axis-aligned bbox so the caller always gets
|
||||
/// an upright rectangle.
|
||||
///
|
||||
/// Coordinates are PDF user space (origin at bottom-left, y-up). Width
|
||||
/// and height are non-negative.
|
||||
pub(crate) fn image_bbox_from_ctm(ctm: &[f32; 6]) -> (f32, f32, f32, f32) {
|
||||
let corners = [
|
||||
apply_ctm_point(ctm, 0.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 1.0),
|
||||
apply_ctm_point(ctm, 0.0, 1.0),
|
||||
];
|
||||
let (mut x_min, mut x_max) = (corners[0].0, corners[0].0);
|
||||
let (mut y_min, mut y_max) = (corners[0].1, corners[0].1);
|
||||
for (cx, cy) in corners.iter().skip(1) {
|
||||
if *cx < x_min {
|
||||
x_min = *cx;
|
||||
}
|
||||
if *cx > x_max {
|
||||
x_max = *cx;
|
||||
}
|
||||
if *cy < y_min {
|
||||
y_min = *cy;
|
||||
}
|
||||
if *cy > y_max {
|
||||
y_max = *cy;
|
||||
}
|
||||
}
|
||||
(x_min, y_min, x_max - x_min, y_max - y_min)
|
||||
}
|
||||
|
||||
/// Multiply two 2D transformation matrices
|
||||
/// Matrix format: [a, b, c, d, e, f] representing:
|
||||
/// | a b 0 |
|
||||
@@ -386,133 +286,6 @@ fn effective_merge_width(item: &TextItem) -> f32 {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_standalone_bullet_text(text: &str) -> bool {
|
||||
matches!(text.trim(), "•" | "○" | "●" | "◦")
|
||||
}
|
||||
|
||||
fn first_text_char(text: &str) -> Option<char> {
|
||||
text.trim_start().chars().next()
|
||||
}
|
||||
|
||||
fn is_short_alpha_fragment(text: &str) -> bool {
|
||||
let trimmed = text.trim();
|
||||
let char_count = trimmed.chars().count();
|
||||
(1..=4).contains(&char_count) && trimmed.chars().all(char::is_alphabetic)
|
||||
}
|
||||
|
||||
fn has_phrase_continuation_shape(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
trimmed
|
||||
.chars()
|
||||
.take(24)
|
||||
.any(|ch| ch.is_whitespace() || matches!(ch, '-'))
|
||||
}
|
||||
|
||||
fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
|
||||
if group.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(first) = group.iter().find(|item| !item.text.trim().is_empty()) else {
|
||||
return false;
|
||||
};
|
||||
if group.iter().all(|item| item.mcid.is_none()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut nonempty_count = 0;
|
||||
let mut saw_backtrack = false;
|
||||
let mut nonspace_chars = 0;
|
||||
let mut math_symbol_chars = 0;
|
||||
let mut max_font_size = first.font_size;
|
||||
|
||||
for item in group {
|
||||
if !item.text.trim().is_empty() {
|
||||
nonempty_count += 1;
|
||||
}
|
||||
if (item.font_size - first.font_size).abs() > first.font_size * 0.25 {
|
||||
return false;
|
||||
}
|
||||
max_font_size = max_font_size.max(item.font_size);
|
||||
for ch in item.text.chars().filter(|ch| !ch.is_whitespace()) {
|
||||
nonspace_chars += 1;
|
||||
if matches!(
|
||||
ch,
|
||||
'*' | 'ˆ' | '^' | '=' | '+' | '_' | '[' | ']' | '{' | '}' | '|' | '<' | '>'
|
||||
) {
|
||||
math_symbol_chars += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if nonempty_count < 2 {
|
||||
return false;
|
||||
}
|
||||
if nonspace_chars > 0 && math_symbol_chars * 4 > nonspace_chars {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut sorted_by_x = group.to_vec();
|
||||
sorted_by_x.sort_by(|a, b| a.x.total_cmp(&b.x));
|
||||
let cluster_start = sorted_by_x[0].x;
|
||||
let mut cluster_end = cluster_start + effective_merge_width(sorted_by_x[0]);
|
||||
for item in sorted_by_x.iter().skip(1) {
|
||||
let gap = item.x - cluster_end;
|
||||
if gap > max_font_size * 2.5 {
|
||||
return false;
|
||||
}
|
||||
cluster_end = cluster_end.max(item.x + effective_merge_width(item));
|
||||
}
|
||||
if cluster_end - cluster_start > max_font_size * 36.0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
for index in 0..group.len() - 1 {
|
||||
let previous = group[index];
|
||||
let next = group[index + 1];
|
||||
let font_size = previous.font_size.max(next.font_size);
|
||||
let backtrack_threshold = font_size * 0.25;
|
||||
let previous_start = previous.x;
|
||||
let next_start = next.x;
|
||||
let next_end = next.x + effective_merge_width(next);
|
||||
if next_start < previous_start - backtrack_threshold
|
||||
&& next_end > previous_start + backtrack_threshold
|
||||
{
|
||||
let has_near_prefix = group[..=index].iter().rev().take(4).any(|item| {
|
||||
is_short_alpha_fragment(&item.text)
|
||||
&& item.x >= next_start - font_size * 0.5
|
||||
&& item.x <= next_start + font_size * 4.0
|
||||
});
|
||||
let starts_lowercase = first_text_char(&next.text).is_some_and(char::is_lowercase);
|
||||
let phrase_continuation = has_phrase_continuation_shape(&next.text);
|
||||
let has_near_bullet = group[..=index]
|
||||
.iter()
|
||||
.position(|item| {
|
||||
is_standalone_bullet_text(&item.text) && next_start <= item.x + font_size * 3.0
|
||||
})
|
||||
.is_some_and(|bullet_index| {
|
||||
if bullet_index >= index {
|
||||
return false;
|
||||
}
|
||||
group[bullet_index + 1..=index]
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|item| !item.text.trim().is_empty())
|
||||
.is_some_and(|item| {
|
||||
item.text.trim().chars().count() <= 8
|
||||
&& has_phrase_continuation_shape(&next.text)
|
||||
})
|
||||
});
|
||||
if (has_near_prefix && starts_lowercase && phrase_continuation) || has_near_bullet {
|
||||
saw_backtrack = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
saw_backtrack
|
||||
}
|
||||
|
||||
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if items.is_empty() {
|
||||
return items;
|
||||
@@ -533,33 +306,28 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
}
|
||||
}
|
||||
|
||||
let mut ordered_line_groups: Vec<(u32, f32, Vec<&TextItem>, bool)> = Vec::new();
|
||||
|
||||
// Sort each group by X position (direction-aware), except for lines whose
|
||||
// content stream intentionally backtracks to overlay ActualText fragments.
|
||||
for (page, y, mut group) in line_groups {
|
||||
// Sort each group by X position (direction-aware)
|
||||
for (_, _, group) in &mut line_groups {
|
||||
let rtl = is_rtl_text(group.iter().map(|i| &i.text));
|
||||
let preserve_stream_order = !rtl && should_preserve_overlapping_stream_order(&group);
|
||||
if rtl {
|
||||
group.sort_by(|a, b| b.x.total_cmp(&a.x));
|
||||
} else if !preserve_stream_order {
|
||||
} else {
|
||||
group.sort_by(|a, b| a.x.total_cmp(&b.x));
|
||||
}
|
||||
ordered_line_groups.push((page, y, group, preserve_stream_order));
|
||||
}
|
||||
|
||||
// Sort groups by page then Y descending (top of page first)
|
||||
ordered_line_groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.total_cmp(&a.1)));
|
||||
line_groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.total_cmp(&a.1)));
|
||||
|
||||
let mut merged = Vec::new();
|
||||
|
||||
for (_, _, group, preserve_stream_order) in &ordered_line_groups {
|
||||
for (_, _, group) in &line_groups {
|
||||
let mut i = 0;
|
||||
while i < group.len() {
|
||||
let first = group[i];
|
||||
let mut text = first.text.clone();
|
||||
let mut end_x = first.x + effective_merge_width(first);
|
||||
let mut is_underline = first.is_underline;
|
||||
let x_gap_max = first.font_size * 0.5;
|
||||
|
||||
let mut j = i + 1;
|
||||
while j < group.len() {
|
||||
@@ -569,15 +337,10 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
break;
|
||||
}
|
||||
let gap = next.x - end_x;
|
||||
let x_gap_max = if *preserve_stream_order && is_standalone_bullet_text(&text) {
|
||||
first.font_size * 1.2
|
||||
} else {
|
||||
first.font_size * 0.5
|
||||
};
|
||||
if gap > x_gap_max {
|
||||
break;
|
||||
}
|
||||
if gap < -first.font_size * 0.5 && !preserve_stream_order {
|
||||
if gap < -first.font_size * 0.5 {
|
||||
break;
|
||||
}
|
||||
// Insert space at word boundaries.
|
||||
@@ -599,20 +362,11 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
first.font_size * 0.08
|
||||
}
|
||||
};
|
||||
let needs_bullet_space = *preserve_stream_order
|
||||
&& is_standalone_bullet_text(&text)
|
||||
&& !next.text.trim().is_empty();
|
||||
if needs_bullet_space || gap > threshold {
|
||||
if gap > threshold {
|
||||
text.push(' ');
|
||||
}
|
||||
text.push_str(&next.text);
|
||||
is_underline |= next.is_underline;
|
||||
let next_end = next.x + effective_merge_width(next);
|
||||
end_x = if *preserve_stream_order {
|
||||
end_x.max(next_end)
|
||||
} else {
|
||||
next_end
|
||||
};
|
||||
end_x = next.x + effective_merge_width(next);
|
||||
j += 1;
|
||||
}
|
||||
|
||||
@@ -627,7 +381,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
page: first.page,
|
||||
is_bold: first.is_bold,
|
||||
is_italic: first.is_italic,
|
||||
is_underline,
|
||||
item_type: first.item_type.clone(),
|
||||
mcid: first.mcid,
|
||||
});
|
||||
@@ -738,7 +491,7 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items};
|
||||
use crate::types::{ItemType, PdfLine, TextLine};
|
||||
use crate::types::{ItemType, TextLine};
|
||||
use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
|
||||
|
||||
fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
|
||||
@@ -753,37 +506,11 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_mcid(mut item: TextItem) -> TextItem {
|
||||
item.mcid = Some(1);
|
||||
item
|
||||
}
|
||||
|
||||
fn make_line(x1: f32, y1: f32, x2: f32, y2: f32) -> PdfLine {
|
||||
PdfLine {
|
||||
x1,
|
||||
y1,
|
||||
x2,
|
||||
y2,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_text_preview_truncates_on_char_boundary() {
|
||||
let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}');
|
||||
let preview = trace_text_preview(&text, 80);
|
||||
|
||||
assert_eq!(preview.chars().count(), 80);
|
||||
assert!(text.is_char_boundary(preview.len()));
|
||||
assert!(preview.ends_with('\u{FFFD}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_no_space_before_period() {
|
||||
// Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap
|
||||
@@ -822,127 +549,6 @@ mod tests {
|
||||
assert_eq!(merged[0].text, "hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_preserves_underline_from_later_fragment() {
|
||||
let mut items = vec![
|
||||
make_merge_item("pre", 100.0, 18.0),
|
||||
make_merge_item("fix", 119.0, 18.0),
|
||||
];
|
||||
items[1].is_underline = true;
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "prefix");
|
||||
assert!(merged[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_preserves_stream_order_for_backtracking_heading() {
|
||||
// Some tagged PDFs emit first-letter ActualText fragments, then reset
|
||||
// the text matrix and draw the rest of the word from the line start.
|
||||
let items = vec![
|
||||
with_mcid(make_merge_item("F", 79.4, 4.5)),
|
||||
with_mcid(make_merge_item("r", 83.9, 3.3)),
|
||||
with_mcid(make_merge_item("om tables to data-", 79.4, 89.7)),
|
||||
with_mcid(make_merge_item("", 168.9, 33.9)),
|
||||
with_mcid(make_merge_item("analytics-", 168.9, 75.5)),
|
||||
with_mcid(make_merge_item("ready content", 210.5, 60.8)),
|
||||
];
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(
|
||||
merged[0].text,
|
||||
"From tables to data-analytics-ready content"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_preserves_stream_order_for_reset_word_prefix() {
|
||||
let items = vec![
|
||||
with_mcid(make_merge_item("N", 68.0, 7.0)),
|
||||
with_mcid(make_merge_item("e", 75.1, 4.0)),
|
||||
with_mcid(make_merge_item("w fields created", 68.0, 82.0)),
|
||||
];
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "New fields created");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_uses_x_order_for_untagged_backtracking_text() {
|
||||
let items = vec![
|
||||
make_merge_item("N", 68.0, 7.0),
|
||||
make_merge_item("e", 75.1, 4.0),
|
||||
make_merge_item("w fields created", 68.2, 82.0),
|
||||
];
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
let texts: Vec<_> = merged.iter().map(|item| item.text.as_str()).collect();
|
||||
assert_eq!(texts, vec!["N", "w fields created", "e"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_preserves_bullet_stream_order_with_backtracking() {
|
||||
let items = vec![
|
||||
with_mcid(make_merge_item("•", 79.4, 5.0)),
|
||||
with_mcid(make_merge_item("The MS", 91.0, 32.6)),
|
||||
with_mcid(make_merge_item("A LoS project", 84.4, 70.0)),
|
||||
];
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "• The MSA LoS project");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_keeps_normal_bullet_gap_limit_without_stream_order() {
|
||||
let items = vec![
|
||||
make_merge_item("•", 79.4, 5.0),
|
||||
make_merge_item("Distant item", 91.0, 60.0),
|
||||
];
|
||||
|
||||
let merged = merge_text_items(items);
|
||||
|
||||
let texts: Vec<_> = merged.iter().map(|item| item.text.as_str()).collect();
|
||||
assert_eq!(texts, vec!["•", "Distant item"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suppress_table_underlines_clears_line_detected_table_items() {
|
||||
let mut items = vec![
|
||||
make_merge_item("H1", 125.0, 20.0),
|
||||
make_merge_item("H2", 225.0, 20.0),
|
||||
make_merge_item("A", 125.0, 20.0),
|
||||
make_merge_item("B", 225.0, 20.0),
|
||||
];
|
||||
items[0].y = 490.0;
|
||||
items[1].y = 490.0;
|
||||
items[2].y = 470.0;
|
||||
items[3].y = 470.0;
|
||||
for item in &mut items {
|
||||
item.is_underline = true;
|
||||
}
|
||||
let lines = vec![
|
||||
make_line(100.0, 500.0, 300.0, 500.0),
|
||||
make_line(100.0, 480.0, 300.0, 480.0),
|
||||
make_line(100.0, 460.0, 300.0, 460.0),
|
||||
make_line(100.0, 460.0, 100.0, 500.0),
|
||||
make_line(200.0, 460.0, 200.0, 500.0),
|
||||
make_line(300.0, 460.0, 300.0, 500.0),
|
||||
];
|
||||
|
||||
suppress_table_underlines(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_into_lines() {
|
||||
let items = vec![
|
||||
@@ -957,7 +563,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -972,7 +577,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -987,7 +591,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1042,7 +645,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1057,7 +659,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1072,7 +673,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1098,7 +698,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1113,7 +712,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1128,7 +726,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1156,7 +753,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: true,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1191,7 +787,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1227,7 +822,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1242,7 +836,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1257,7 +850,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1280,7 +872,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1393,7 +984,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1408,7 +998,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1433,7 +1022,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1448,7 +1036,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1489,7 +1076,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1534,7 +1120,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1579,7 +1164,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1617,7 +1201,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -1,500 +0,0 @@
|
||||
//! Geometric underline detection.
|
||||
//!
|
||||
//! PDFs have no underline font flag — underlines are drawn as separate
|
||||
//! graphics: stroked horizontal lines (`l`/`S` operators) or thin filled
|
||||
//! rectangles (`re`/`f`). This pass correlates those graphics with text
|
||||
//! items after extraction: an item is underlined when a horizontal
|
||||
//! line/thin rect sits just below its baseline and covers most of its
|
||||
//! horizontal extent.
|
||||
//!
|
||||
//! Repeated same-span rules are treated as table/form rulings rather than
|
||||
//! underlines, which avoids marking every cell in ruled tables.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::types::{ItemType, PdfRect, TextItem};
|
||||
|
||||
/// Max thickness (pt) for a stroked line / filled rect to count as an
|
||||
/// underline rule rather than a border or decorative band.
|
||||
const MAX_RULE_THICKNESS: f32 = 2.0;
|
||||
|
||||
/// Fraction of the item's width that the rule must cover horizontally.
|
||||
const MIN_X_OVERLAP: f32 = 0.6;
|
||||
|
||||
/// Same-span rules repeated at this many y-levels are usually table/form
|
||||
/// rulings, not semantic underlines.
|
||||
const MIN_REPEATED_RULE_LEVELS: usize = 3;
|
||||
|
||||
/// Vertical tolerance for considering two rules to be on the same row edge.
|
||||
const RULE_Y_DEDUP_EPS: f32 = 2.0;
|
||||
|
||||
/// Horizontal span similarity required when clustering repeated rulings.
|
||||
const RULE_SPAN_OVERLAP_RATIO: f32 = 0.8;
|
||||
const RULE_SPAN_WIDTH_RATIO: f32 = 1.5;
|
||||
|
||||
/// Multiple separated rule segments on one row are usually per-column table
|
||||
/// header/body separators.
|
||||
const MIN_SEGMENTED_ROW_RULES: usize = 3;
|
||||
const MIN_SEGMENTED_ROW_GAPS: usize = 2;
|
||||
const SEGMENTED_ROW_GAP_MIN: f32 = 12.0;
|
||||
|
||||
/// A single rule under several widely separated items is usually a table
|
||||
/// header/body separator, not a sentence underline.
|
||||
const MIN_TABULAR_RULE_ITEMS: usize = 3;
|
||||
const MIN_TABULAR_RULE_GAPS: usize = 2;
|
||||
const TABULAR_RULE_GAP_EM: f32 = 2.0;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct UnderlineLine {
|
||||
pub(crate) x1: f32,
|
||||
pub(crate) y1: f32,
|
||||
pub(crate) x2: f32,
|
||||
pub(crate) y2: f32,
|
||||
pub(crate) stroke_width: f32,
|
||||
pub(crate) page: u32,
|
||||
}
|
||||
|
||||
/// A horizontal rule candidate in page coordinates (PDF y-up).
|
||||
#[derive(Clone)]
|
||||
struct Rule {
|
||||
x1: f32,
|
||||
x2: f32,
|
||||
y: f32,
|
||||
}
|
||||
|
||||
impl Rule {
|
||||
fn width(&self) -> f32 {
|
||||
self.x2 - self.x1
|
||||
}
|
||||
}
|
||||
|
||||
fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) -> Vec<Rule> {
|
||||
let mut rules: Vec<Rule> = Vec::new();
|
||||
for l in lines {
|
||||
if l.page != page {
|
||||
continue;
|
||||
}
|
||||
// Horizontal stroked line (tolerate slight skew).
|
||||
if l.stroke_width <= MAX_RULE_THICKNESS && (l.y1 - l.y2).abs() <= MAX_RULE_THICKNESS {
|
||||
let (x1, x2) = if l.x1 <= l.x2 {
|
||||
(l.x1, l.x2)
|
||||
} else {
|
||||
(l.x2, l.x1)
|
||||
};
|
||||
if x2 - x1 > 1.0 {
|
||||
rules.push(Rule {
|
||||
x1,
|
||||
x2,
|
||||
y: (l.y1 + l.y2) / 2.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for r in rects {
|
||||
if r.page != page {
|
||||
continue;
|
||||
}
|
||||
// Thin filled rect used as an underline rule. Extents are
|
||||
// normalized first: `re` operands pass through the CTM, so
|
||||
// width/height can be negative (flipped axes / negative scale) —
|
||||
// without normalization negative-width rules are missed and
|
||||
// negative-height bands sneak past the thickness check.
|
||||
let (x1, x2) = if r.width >= 0.0 {
|
||||
(r.x, r.x + r.width)
|
||||
} else {
|
||||
(r.x + r.width, r.x)
|
||||
};
|
||||
if r.height.abs() <= MAX_RULE_THICKNESS && x2 - x1 > 1.0 {
|
||||
rules.push(Rule {
|
||||
x1,
|
||||
x2,
|
||||
y: r.y + r.height / 2.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
rules
|
||||
}
|
||||
|
||||
fn discard_repeated_ruling_rules(rules: Vec<Rule>) -> Vec<Rule> {
|
||||
if rules.len() < MIN_REPEATED_RULE_LEVELS {
|
||||
return rules;
|
||||
}
|
||||
|
||||
rules
|
||||
.iter()
|
||||
.filter(|rule| {
|
||||
!is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules)
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
|
||||
let mut y_levels: Vec<f32> = rules
|
||||
.iter()
|
||||
.filter(|other| has_similar_span(rule, other))
|
||||
.map(|other| other.y)
|
||||
.collect();
|
||||
|
||||
y_levels.sort_by(|a, b| a.total_cmp(b));
|
||||
y_levels.dedup_by(|a, b| (*a - *b).abs() <= RULE_Y_DEDUP_EPS);
|
||||
y_levels.len() >= MIN_REPEATED_RULE_LEVELS
|
||||
}
|
||||
|
||||
fn is_segmented_row_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
|
||||
let mut row_rules: Vec<&Rule> = rules
|
||||
.iter()
|
||||
.filter(|other| (other.y - rule.y).abs() <= RULE_Y_DEDUP_EPS)
|
||||
.collect();
|
||||
|
||||
if row_rules.len() < MIN_SEGMENTED_ROW_RULES {
|
||||
return false;
|
||||
}
|
||||
|
||||
row_rules.sort_by(|a, b| a.x1.total_cmp(&b.x1));
|
||||
let large_gaps = row_rules
|
||||
.windows(2)
|
||||
.filter(|pair| pair[1].x1 - pair[0].x2 > SEGMENTED_ROW_GAP_MIN)
|
||||
.count();
|
||||
|
||||
large_gaps >= MIN_SEGMENTED_ROW_GAPS
|
||||
}
|
||||
|
||||
fn has_similar_span(a: &Rule, b: &Rule) -> bool {
|
||||
let a_width = a.width();
|
||||
let b_width = b.width();
|
||||
if a_width <= 1.0 || b_width <= 1.0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let width_ratio = a_width.max(b_width) / a_width.min(b_width);
|
||||
if width_ratio > RULE_SPAN_WIDTH_RATIO {
|
||||
return false;
|
||||
}
|
||||
|
||||
let overlap = a.x2.min(b.x2) - a.x1.max(b.x1);
|
||||
overlap >= a_width.min(b_width) * RULE_SPAN_OVERLAP_RATIO
|
||||
}
|
||||
|
||||
fn tabular_row_separator_rule_indices(rules: &[Rule], items: &[TextItem]) -> HashSet<usize> {
|
||||
let mut tabular_rules = HashSet::new();
|
||||
|
||||
for (rule_idx, rule) in rules.iter().enumerate() {
|
||||
let mut matched_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item))
|
||||
.collect();
|
||||
|
||||
if matched_items.len() < MIN_TABULAR_RULE_ITEMS {
|
||||
continue;
|
||||
}
|
||||
|
||||
matched_items.sort_by(|a, b| a.x.total_cmp(&b.x));
|
||||
let large_gaps = matched_items
|
||||
.windows(2)
|
||||
.filter(|pair| {
|
||||
let left = pair[0];
|
||||
let right = pair[1];
|
||||
let gap = right.x - (left.x + left.width);
|
||||
let font_size = left.font_size.max(right.font_size).max(1.0);
|
||||
gap > font_size * TABULAR_RULE_GAP_EM
|
||||
})
|
||||
.count();
|
||||
|
||||
if large_gaps >= MIN_TABULAR_RULE_GAPS {
|
||||
tabular_rules.insert(rule_idx);
|
||||
}
|
||||
}
|
||||
|
||||
tabular_rules
|
||||
}
|
||||
|
||||
fn is_underline_candidate(item: &TextItem) -> bool {
|
||||
matches!(item.item_type, ItemType::Text) && !item.text.trim().is_empty() && item.width > 0.0
|
||||
}
|
||||
|
||||
fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
// Vertical window: underlines sit at or slightly below the baseline.
|
||||
// Fonts draw them at roughly 5-15% of the em below; allow up to 35%
|
||||
// (min 3pt) below and 1pt above for rounding.
|
||||
let below = (item.font_size * 0.35).max(3.0);
|
||||
let y_min = item.y - below;
|
||||
let y_max = item.y + 1.0;
|
||||
if rule.y < y_min || rule.y > y_max {
|
||||
return false;
|
||||
}
|
||||
|
||||
let ix1 = item.x;
|
||||
let ix2 = item.x + item.width;
|
||||
let min_overlap = item.width * MIN_X_OVERLAP;
|
||||
let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
|
||||
overlap >= min_overlap
|
||||
}
|
||||
|
||||
/// Mark `is_underline` on text items that have a horizontal rule just
|
||||
/// below their baseline. `items`, `rects`, and `lines` are a single
|
||||
/// page's extraction output (all in PDF coordinates, y-up, where
|
||||
/// `TextItem::y` is the text baseline).
|
||||
pub(crate) fn mark_underlined_items(
|
||||
items: &mut [TextItem],
|
||||
rects: &[PdfRect],
|
||||
lines: &[UnderlineLine],
|
||||
page: u32,
|
||||
) {
|
||||
let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page));
|
||||
if rules.is_empty() {
|
||||
return;
|
||||
}
|
||||
let tabular_rules = tabular_row_separator_rule_indices(&rules, items);
|
||||
|
||||
for item in items.iter_mut() {
|
||||
if !is_underline_candidate(item) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (rule_idx, rule) in rules.iter().enumerate() {
|
||||
if tabular_rules.contains(&rule_idx) {
|
||||
continue;
|
||||
}
|
||||
if rule_matches_item(rule, item) {
|
||||
item.is_underline = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::ItemType;
|
||||
|
||||
fn item(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.to_string(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: font_size,
|
||||
font: "F1".to_string(),
|
||||
font_size,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn hline(x1: f32, x2: f32, y: f32) -> UnderlineLine {
|
||||
UnderlineLine {
|
||||
x1,
|
||||
y1: y,
|
||||
x2,
|
||||
y2: y,
|
||||
stroke_width: 1.0,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect {
|
||||
PdfRect {
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: 0.8,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stroked_line_under_baseline_marks_underline() {
|
||||
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 498.5)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thin_filled_rect_under_baseline_marks_underline() {
|
||||
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![thin_rect(100.0, 497.8, 60.0)];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_rule_under_multiple_items_marks_each() {
|
||||
// One underline drawn under a whole sentence: every overlapped
|
||||
// item gets the flag.
|
||||
let mut items = vec![
|
||||
item("first", 100.0, 500.0, 40.0, 10.0),
|
||||
item("second", 145.0, 500.0, 50.0, 10.0),
|
||||
];
|
||||
let lines = vec![hline(98.0, 200.0, 498.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_underline);
|
||||
assert!(items[1].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_far_below_baseline_is_not_an_underline() {
|
||||
// A horizontal rule 30pt below (section divider) must not mark.
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(90.0, 300.0, 470.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thick_stroked_line_is_not_an_underline() {
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let mut line = hline(99.0, 161.0, 498.5);
|
||||
line.stroke_width = 4.0;
|
||||
|
||||
mark_underlined_items(&mut items, &[], &[line], 1);
|
||||
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_above_baseline_is_not_an_underline() {
|
||||
// Strikethrough / overline geometry must not mark.
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(90.0, 300.0, 505.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insufficient_horizontal_overlap_is_not_an_underline() {
|
||||
// Rule under only a quarter of the item (e.g. neighboring cell
|
||||
// border) must not mark.
|
||||
let mut items = vec![item("wide text item", 100.0, 500.0, 100.0, 10.0)];
|
||||
let lines = vec![hline(100.0, 125.0, 498.5)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_width_rect_is_normalized_and_marks_underline() {
|
||||
// A CTM with negative x-scale (or negative `re` operands) produces
|
||||
// rects whose width is negative; the rule extents must normalize.
|
||||
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![PdfRect {
|
||||
x: 160.0,
|
||||
y: 497.8,
|
||||
width: -60.0,
|
||||
height: 0.8,
|
||||
page: 1,
|
||||
}];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_height_band_is_not_an_underline() {
|
||||
// A 14pt band expressed with negative height must not pass the
|
||||
// thickness check via sign trickery.
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![PdfRect {
|
||||
x: 95.0,
|
||||
y: 509.0,
|
||||
width: 80.0,
|
||||
height: -14.0,
|
||||
page: 1,
|
||||
}];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thick_band_is_not_an_underline() {
|
||||
// A highlight bar / filled cell background (tall rect) must not mark.
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![PdfRect {
|
||||
x: 95.0,
|
||||
y: 495.0,
|
||||
width: 80.0,
|
||||
height: 14.0,
|
||||
page: 1,
|
||||
}];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vertical_line_is_not_an_underline() {
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![UnderlineLine {
|
||||
x1: 120.0,
|
||||
y1: 498.0,
|
||||
x2: 120.0,
|
||||
y2: 400.0,
|
||||
stroke_width: 1.0,
|
||||
page: 1,
|
||||
}];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_pages_graphics_do_not_mark() {
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let mut line = hline(99.0, 161.0, 498.5);
|
||||
line.page = 2;
|
||||
mark_underlined_items(&mut items, &[], &[line], 1);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_table_row_rules_do_not_mark_cell_text() {
|
||||
let mut items = vec![
|
||||
item("A", 110.0, 500.0, 20.0, 10.0),
|
||||
item("B", 110.0, 480.0, 20.0, 10.0),
|
||||
item("C", 110.0, 460.0, 20.0, 10.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(100.0, 150.0, 498.0),
|
||||
hline(100.0, 150.0, 478.0),
|
||||
hline(100.0, 150.0, 458.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn row_separator_under_spaced_column_labels_is_not_an_underline() {
|
||||
let mut items = vec![
|
||||
item("Date", 100.0, 500.0, 25.0, 10.0),
|
||||
item("Rate", 200.0, 500.0, 25.0, 10.0),
|
||||
item("Yield", 300.0, 500.0, 30.0, 10.0),
|
||||
];
|
||||
let lines = vec![hline(90.0, 340.0, 498.0)];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_row_spaced_rule_segments_do_not_mark_column_labels() {
|
||||
let mut items = vec![
|
||||
item("Date", 100.0, 500.0, 25.0, 10.0),
|
||||
item("Rate", 200.0, 500.0, 25.0, 10.0),
|
||||
item("Yield", 300.0, 500.0, 30.0, 10.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(98.0, 128.0, 498.0),
|
||||
hline(198.0, 228.0, 498.0),
|
||||
hline(298.0, 333.0, 498.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
}
|
||||
+13
-40
@@ -10,7 +10,7 @@ use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
const MAX_FORM_XOBJECT_DEPTH: u8 = 5;
|
||||
|
||||
@@ -262,44 +262,19 @@ fn extract_form_xobject_text_inner(
|
||||
if !op.operands.is_empty() {
|
||||
if let Ok(name) = op.operands[0].as_name() {
|
||||
let xobj_name = String::from_utf8_lossy(name).to_string();
|
||||
match form_xobjects.get(&xobj_name) {
|
||||
Some(XObjectType::Form(nested_id)) => {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
if let Some(XObjectType::Form(nested_id)) = form_xobjects.get(&xobj_name) {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
Some(XObjectType::Image) => {
|
||||
// Mirror the top-level Image-XObject emission
|
||||
// in content_stream.rs so figures embedded
|
||||
// inside Form XObjects (common in print-to-PDF
|
||||
// workflows) aren't silently dropped.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -439,7 +414,6 @@ fn extract_form_xobject_text_inner(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -588,7 +562,6 @@ fn extract_form_xobject_text_inner(
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+97
-2018
File diff suppressed because it is too large
Load Diff
+2
-233
@@ -149,79 +149,6 @@ fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32)
|
||||
set
|
||||
}
|
||||
|
||||
/// Pre-scan body-size all-bold runs that are too long to be headings.
|
||||
///
|
||||
/// Some academic PDFs use an all-bold abstract/summary paragraph immediately
|
||||
/// after the author block. A line-local bold heading heuristic sees each
|
||||
/// wrapped visual line as "standalone" once the first line is misclassified,
|
||||
/// producing a stack of `##` headings. Multi-line body-size bold runs with a
|
||||
/// paragraph-sized word count should stay paragraph text.
|
||||
fn find_wrapped_bold_paragraph_lines(
|
||||
lines: &[TextLine],
|
||||
base_size: f32,
|
||||
para_threshold: f32,
|
||||
) -> HashSet<usize> {
|
||||
let mut set = HashSet::new();
|
||||
let mut i = 0usize;
|
||||
|
||||
while i < lines.len() {
|
||||
if !is_body_size_all_bold_line(&lines[i], base_size) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = i;
|
||||
let mut end = i;
|
||||
let mut word_count = lines[i].text().split_whitespace().count();
|
||||
|
||||
while end + 1 < lines.len()
|
||||
&& is_body_size_all_bold_line(&lines[end + 1], base_size)
|
||||
&& is_wrapped_same_style_line(&lines[end], &lines[end + 1], para_threshold)
|
||||
{
|
||||
end += 1;
|
||||
word_count += lines[end].text().split_whitespace().count();
|
||||
}
|
||||
|
||||
let line_count = end - start + 1;
|
||||
if line_count >= 3 && word_count > 20 {
|
||||
for idx in start..=end {
|
||||
set.insert(idx);
|
||||
}
|
||||
}
|
||||
|
||||
i = end + 1;
|
||||
}
|
||||
|
||||
set
|
||||
}
|
||||
|
||||
fn is_body_size_all_bold_line(line: &TextLine, base_size: f32) -> bool {
|
||||
let Some(first) = line.items.first() else {
|
||||
return false;
|
||||
};
|
||||
first.font_size >= base_size * 0.95
|
||||
&& first.font_size < base_size * 1.2
|
||||
&& line
|
||||
.items
|
||||
.iter()
|
||||
.all(|item| item.is_bold && (item.font_size - first.font_size).abs() < 0.5)
|
||||
}
|
||||
|
||||
fn is_wrapped_same_style_line(prev: &TextLine, next: &TextLine, para_threshold: f32) -> bool {
|
||||
if prev.page != next.page {
|
||||
return false;
|
||||
}
|
||||
|
||||
let y_gap = prev.y - next.y;
|
||||
if !(y_gap > 0.0 && y_gap <= para_threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let prev_x = prev.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
let next_x = next.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
(prev_x - next_x).abs() <= 40.0
|
||||
}
|
||||
|
||||
/// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
|
||||
///
|
||||
/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
|
||||
@@ -470,8 +397,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// between paragraphs at body font size. Inspired by opendataloader's
|
||||
// lookahead in HeadingProcessor (prevNode/nextNode context).
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
// Detect struct heading levels that are overused (body text mistagged as headings)
|
||||
let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
|
||||
@@ -485,7 +410,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
|
||||
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
|
||||
|
||||
@@ -551,7 +475,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
current_page = line.page;
|
||||
prev_y = f32::MAX;
|
||||
prev_x = 0.0;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -566,7 +489,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(table_md);
|
||||
@@ -584,7 +506,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(image_md);
|
||||
@@ -606,18 +527,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& y_gap.abs() <= para_threshold
|
||||
&& (prev_x - line_x).abs() > 50.0
|
||||
&& prev_y < f32::MAX;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_band_switch || is_bold_to_regular_break) && in_paragraph {
|
||||
if (is_para_break || is_band_switch) && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -660,7 +572,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -714,9 +625,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -748,7 +656,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -771,7 +678,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("- {}", trimmed));
|
||||
output.push('\n');
|
||||
@@ -785,7 +691,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -832,7 +737,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("> {}\n", trimmed));
|
||||
continue;
|
||||
@@ -843,7 +747,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
if !in_code_block {
|
||||
output.push_str("```\n");
|
||||
@@ -864,11 +767,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -938,8 +836,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
||||
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
let mut output = String::new();
|
||||
let mut current_page = 0u32;
|
||||
@@ -948,7 +844,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let mut in_paragraph = false;
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
for (line_idx, line) in lines.iter().enumerate() {
|
||||
// Page break
|
||||
@@ -965,7 +860,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
in_list = false;
|
||||
last_list_x = None;
|
||||
prev_had_dot_leaders = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -976,18 +870,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
// (newspaper columns emitted sequentially on the same page).
|
||||
let y_gap = prev_y - line.y;
|
||||
let is_para_break = y_gap.abs() > para_threshold;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_bold_to_regular_break) && in_paragraph {
|
||||
if is_para_break && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -1011,7 +896,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -1034,9 +918,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -1054,7 +935,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(header_level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -1069,7 +949,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -1114,7 +993,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Use plain text for code blocks
|
||||
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
||||
@@ -1132,11 +1010,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -1169,7 +1042,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
@@ -1484,109 +1356,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrapped_bold_abstract_is_not_split_into_headings() {
|
||||
// Regression for arXiv 1107.1353: the opening abstract paragraph is
|
||||
// entirely bold at body size. The first wrapped lines used to become
|
||||
// separate H2 headings, and the following body paragraph was joined to
|
||||
// the bold abstract because the paragraph gap is modest.
|
||||
let make = |text: &str, y: f32, font_size: f32, bold: bool| {
|
||||
let mut item = make_item(text, 1, None);
|
||||
item.y = y;
|
||||
item.font_size = font_size;
|
||||
item.height = font_size;
|
||||
item.is_bold = bold;
|
||||
item
|
||||
};
|
||||
|
||||
let lines = vec![
|
||||
make_line(vec![make(
|
||||
"Quantum Nature of Light Measured With a Single Detector",
|
||||
747.7,
|
||||
25.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Gesine A. Steudle1*, Stefan Schietinger1, David Höckel1",
|
||||
651.1,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Zwiller2, and Oliver Benson1",
|
||||
638.5,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"The introduction of light quanta by Einstein in 1905 triggered strong efforts to",
|
||||
607.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"demonstrate the quantum properties of light directly, without involving matter",
|
||||
594.8,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"quantization. It however took more than seven decades for the quantum granularity",
|
||||
582.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"of light to be observed in the fluorescence of single atoms. Single atoms emit",
|
||||
569.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"photons one at a time, this is typically demonstrated with a Hanbury-Brown-Twiss",
|
||||
556.9,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Our work significantly simplifies a widely used photon-correlation technique.",
|
||||
544.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"A photon is a single excitation of a mode of the electromagnetic field.",
|
||||
528.7,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
];
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(
|
||||
md.contains("# Quantum Nature of Light Measured With a Single Detector"),
|
||||
"title should remain a heading: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("## The introduction")
|
||||
&& !md.contains("## demonstrate")
|
||||
&& !md.contains("## quantization"),
|
||||
"bold abstract lines should not become headings: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("technique.**\n\nA photon is a single excitation"),
|
||||
"body paragraph should be separated from bold abstract: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_code_multiline_accumulation() {
|
||||
let mut line1 = make_item("fn main() {", 1, Some(0));
|
||||
|
||||
+1
-11
@@ -422,16 +422,7 @@ impl Default for MarkdownOptions {
|
||||
fix_hyphenation: true,
|
||||
detect_bold: true,
|
||||
detect_italic: true,
|
||||
// `include_images: false` is intentional. The content-stream walker
|
||||
// now emits `ItemType::Image` `TextItem`s for every Image XObject
|
||||
// it encounters (see `extractor/content_stream.rs`). If we rendered
|
||||
// those into markdown by default, every existing caller would
|
||||
// suddenly see `` placeholders inserted
|
||||
// throughout their output — a silent regression for anyone who
|
||||
// upgrades. Image bboxes are still available via
|
||||
// `extract_text_with_positions` for callers (e.g. layout-aware
|
||||
// pipelines) that want to crop + caption figures themselves.
|
||||
include_images: false,
|
||||
include_images: true,
|
||||
include_links: true,
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: true,
|
||||
@@ -1217,7 +1208,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
|
||||
// text item, which combine with gap-based space insertion to produce
|
||||
// double spaces ("Vice President" instead of "Vice President").
|
||||
collapse_consecutive_spaces(&mut text);
|
||||
remove_spaces_before_closing_brackets(&mut text);
|
||||
|
||||
// Remove excessive newlines (more than 2 in a row)
|
||||
while text.contains("\n\n\n") {
|
||||
@@ -72,20 +71,6 @@ fn collapse_consecutive_spaces(text: &mut String) {
|
||||
*text = result;
|
||||
}
|
||||
|
||||
/// Remove spaces before closing square brackets.
|
||||
/// Unit markers and markdown links occasionally pick up a gap-inserted space
|
||||
/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding.
|
||||
fn remove_spaces_before_closing_brackets(text: &mut String) {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
for ch in text.chars() {
|
||||
if ch == ']' && result.ends_with(' ') {
|
||||
result.pop();
|
||||
}
|
||||
result.push(ch);
|
||||
}
|
||||
*text = result;
|
||||
}
|
||||
|
||||
/// Collapse dot leaders (runs of 4+ dots) into " ... "
|
||||
/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
|
||||
fn collapse_dot_leaders(text: &str) -> String {
|
||||
@@ -357,18 +342,6 @@ mod tests {
|
||||
assert!(result.contains("Chapter 2 ... 20"));
|
||||
}
|
||||
|
||||
// --- remove_spaces_before_closing_brackets ---
|
||||
|
||||
#[test]
|
||||
fn test_remove_spaces_before_closing_brackets() {
|
||||
let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string();
|
||||
remove_spaces_before_closing_brackets(&mut input);
|
||||
assert_eq!(
|
||||
input,
|
||||
"Density [kg/m3] and [linked text](https://example.com)"
|
||||
);
|
||||
}
|
||||
|
||||
// --- fix_hyphenation ---
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -542,7 +542,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -30,9 +30,6 @@ pub struct PyPdfResult {
|
||||
/// 1-indexed page numbers that need OCR.
|
||||
#[pyo3(get)]
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Machine-readable OCR reasons by 1-indexed page.
|
||||
#[pyo3(get)]
|
||||
pub ocr_reasons_by_page: Vec<PyPageOcrReasons>,
|
||||
/// Title from PDF metadata.
|
||||
#[pyo3(get)]
|
||||
pub title: Option<String>,
|
||||
@@ -63,28 +60,6 @@ impl PyPdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// OCR reasons for a single 1-indexed page.
|
||||
#[pyclass(name = "PageOcrReasons")]
|
||||
#[derive(Clone)]
|
||||
pub struct PyPageOcrReasons {
|
||||
/// 1-indexed page number.
|
||||
#[pyo3(get)]
|
||||
pub page: u32,
|
||||
/// Machine-readable OCR reason identifiers.
|
||||
#[pyo3(get)]
|
||||
pub reasons: Vec<String>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyPageOcrReasons {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"PageOcrReasons(page={}, reasons={:?})",
|
||||
self.page, self.reasons
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Classification wrapper (lightweight)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -131,9 +106,6 @@ pub struct PyRegionText {
|
||||
/// True when the text should not be trusted (empty, GID fonts, garbage, encoding issues).
|
||||
#[pyo3(get)]
|
||||
pub needs_ocr: bool,
|
||||
/// Machine-readable OCR reason when the cause is known.
|
||||
#[pyo3(get)]
|
||||
pub ocr_reason: Option<String>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
@@ -188,9 +160,6 @@ pub struct PyPageMarkdown {
|
||||
/// encoding issues, garbage text, or empty extraction).
|
||||
#[pyo3(get)]
|
||||
pub needs_ocr: bool,
|
||||
/// Machine-readable OCR reason when the cause is known.
|
||||
#[pyo3(get)]
|
||||
pub ocr_reason: Option<String>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
@@ -221,9 +190,6 @@ pub struct PyPagesExtractionResult {
|
||||
/// 1-indexed pages that need OCR (scanned/image-based or unreliable text).
|
||||
#[pyo3(get)]
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Machine-readable OCR reasons by 1-indexed page.
|
||||
#[pyo3(get)]
|
||||
pub ocr_reasons_by_page: Vec<PyPageOcrReasons>,
|
||||
/// True if any page has tables or columns.
|
||||
#[pyo3(get)]
|
||||
pub is_complex: bool,
|
||||
@@ -266,8 +232,6 @@ pub struct PyTextItem {
|
||||
#[pyo3(get)]
|
||||
pub is_italic: bool,
|
||||
#[pyo3(get)]
|
||||
pub is_underline: bool,
|
||||
#[pyo3(get)]
|
||||
pub item_type: String,
|
||||
}
|
||||
|
||||
@@ -304,7 +268,6 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult {
|
||||
page_count: r.page_count,
|
||||
processing_time_ms: r.processing_time_ms,
|
||||
pages_needing_ocr: r.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_py_page_ocr_reasons(r.ocr_reasons_by_page),
|
||||
title: r.title,
|
||||
confidence: r.confidence,
|
||||
is_complex_layout: r.layout.is_complex,
|
||||
@@ -314,16 +277,6 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_py_page_ocr_reasons(reasons: Vec<crate::PageOcrReasons>) -> Vec<PyPageOcrReasons> {
|
||||
reasons
|
||||
.into_iter()
|
||||
.map(|reason| PyPageOcrReasons {
|
||||
page: reason.page,
|
||||
reasons: reason.reasons,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_py_err(e: crate::PdfError) -> PyErr {
|
||||
PyValueError::new_err(e.to_string())
|
||||
}
|
||||
@@ -351,7 +304,6 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
|
||||
page: item.page,
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
item_type: item_type_str(&item.item_type),
|
||||
})
|
||||
.collect()
|
||||
@@ -398,13 +350,11 @@ fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResul
|
||||
page: p.page,
|
||||
markdown: p.markdown,
|
||||
needs_ocr: p.needs_ocr,
|
||||
ocr_reason: p.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: r.pages_with_tables,
|
||||
pages_with_columns: r.pages_with_columns,
|
||||
pages_needing_ocr: r.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_py_page_ocr_reasons(r.ocr_reasons_by_page),
|
||||
is_complex: r.is_complex,
|
||||
}
|
||||
}
|
||||
@@ -420,7 +370,6 @@ fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRe
|
||||
.map(|r| PyRegionText {
|
||||
text: r.text,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
@@ -614,7 +563,6 @@ fn extract_pages_markdown_bytes(
|
||||
#[pymodule]
|
||||
fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<PyPdfResult>()?;
|
||||
m.add_class::<PyPageOcrReasons>()?;
|
||||
m.add_class::<PyPdfClassification>()?;
|
||||
m.add_class::<PyTextItem>()?;
|
||||
m.add_class::<PyRegionText>()?;
|
||||
|
||||
@@ -104,7 +104,6 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
|
||||
page: first_item.page,
|
||||
is_bold: first_item.is_bold,
|
||||
is_italic: first_item.is_italic,
|
||||
is_underline: first_item.is_underline,
|
||||
item_type: first_item.item_type.clone(),
|
||||
mcid: first_item.mcid,
|
||||
});
|
||||
|
||||
+29
-262
@@ -4,74 +4,11 @@
|
||||
//! gridlines. Many IRS forms and government PDFs use these instead of
|
||||
//! `re` (rectangle) operators.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::tables::Table;
|
||||
use crate::types::{PdfLine, TextItem};
|
||||
|
||||
use super::detect_rects::{assign_items_to_grid, snap_edges};
|
||||
|
||||
/// Derive column edges from the x-endpoints of horizontal-rule
|
||||
/// segments when no vertical lines were drawn.
|
||||
///
|
||||
/// Catalog and archival-finding-aid tables are commonly drawn with
|
||||
/// per-row horizontal rules broken into N segments (one segment per
|
||||
/// cell), with no vertical dividers at all. The segment break points
|
||||
/// (e.g. `[50, 127], [127, 485], [485, 562]` per row) implicitly
|
||||
/// encode the column boundaries.
|
||||
///
|
||||
/// Returns column edges if ≥3 distinct x-positions each show up as a
|
||||
/// segment endpoint on ≥50% of the unique horizontal-line rows.
|
||||
/// Returns `None` otherwise — decorative rules with varying widths
|
||||
/// shouldn't be mistaken for a table.
|
||||
fn derive_columns_from_horizontal_segments(horizontals: &[(f32, f32, f32)]) -> Option<Vec<f32>> {
|
||||
if horizontals.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut endpoints: Vec<f32> = Vec::with_capacity(horizontals.len() * 2);
|
||||
for &(_, x_min, x_max) in horizontals {
|
||||
endpoints.push(x_min);
|
||||
endpoints.push(x_max);
|
||||
}
|
||||
let clusters = snap_edges(&endpoints, 5.0);
|
||||
if clusters.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Bucket y-values to count unique rows. Tolerance ~0.1pt (×10
|
||||
// rounding) tolerates the snap_edges 3pt clustering used later
|
||||
// for row edges.
|
||||
let unique_rows: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
if unique_rows.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let min_rows = (unique_rows.len() as f32 * 0.5).ceil() as usize;
|
||||
|
||||
let qualifying: Vec<f32> = clusters
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&cluster_x| {
|
||||
let rows_touched: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.filter(|&&(_, x_min, x_max)| {
|
||||
(x_min - cluster_x).abs() < 5.0 || (x_max - cluster_x).abs() < 5.0
|
||||
})
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
rows_touched.len() >= min_rows
|
||||
})
|
||||
.collect();
|
||||
|
||||
if qualifying.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
Some(qualifying)
|
||||
}
|
||||
|
||||
/// Detect tables from line segments on a given page.
|
||||
///
|
||||
/// Lines are classified as horizontal or vertical, snapped into grid edges,
|
||||
@@ -115,50 +52,25 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
// Diagonal lines are ignored
|
||||
}
|
||||
|
||||
if horizontals.len() < 3 {
|
||||
if horizontals.len() < 3 || verticals.len() < 2 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// If no/very-few vertical lines are drawn, try to derive column edges
|
||||
// from the x-endpoints of the horizontal-rule segments. Catalog and
|
||||
// archival-finding-aid layouts commonly draw each row's horizontal
|
||||
// rule as N segments (one per cell), with no vertical dividers at
|
||||
// all — the segment break points encode the column boundaries.
|
||||
let implicit_col_edges: Option<Vec<f32>> = if verticals.len() < 2 {
|
||||
derive_columns_from_horizontal_segments(&horizontals)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if verticals.len() < 2 && implicit_col_edges.is_none() {
|
||||
return Vec::new();
|
||||
}
|
||||
let cols_from_segments = implicit_col_edges.is_some();
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page){}",
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
|
||||
page,
|
||||
horizontals.len(),
|
||||
verticals.len(),
|
||||
page_lines.len(),
|
||||
if cols_from_segments {
|
||||
" — columns from horizontal segments"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
page_lines.len()
|
||||
);
|
||||
|
||||
// Snap Y-values of horizontal lines → row edges
|
||||
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
|
||||
let row_edges = snap_edges(&h_ys, 3.0);
|
||||
|
||||
// Column edges from drawn verticals when present, else from the
|
||||
// horizontal-segment endpoints derived above.
|
||||
let col_edges = if let Some(c) = implicit_col_edges {
|
||||
c
|
||||
} else {
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
snap_edges(&v_xs, 3.0)
|
||||
};
|
||||
// Snap X-values of vertical lines → column edges
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
let col_edges = snap_edges(&v_xs, 3.0);
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} row edges, {} col edges after snap",
|
||||
@@ -198,21 +110,15 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Reject page-spanning frames: a decorative outer border has just 4
|
||||
// edges (top/bottom/left/right). Real full-page tables — common in
|
||||
// governmental ledgers, financial reports, etc. — span the same A4 /
|
||||
// Letter dimensions but have many internal row/column rules. Only
|
||||
// reject when the line set looks like a bare frame, not a grid.
|
||||
// Reject page-spanning frames: if the grid covers >90% of a standard page
|
||||
// dimension in both axes, it's a border frame, not a table.
|
||||
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
||||
if table_width > 500.0 && table_height > 700.0 && horizontals.len() <= 4 && verticals.len() <= 4
|
||||
{
|
||||
if table_width > 500.0 && table_height > 700.0 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0}, {} h + {} v)",
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})",
|
||||
page,
|
||||
table_width,
|
||||
table_height,
|
||||
horizontals.len(),
|
||||
verticals.len()
|
||||
table_height
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -240,33 +146,24 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
|
||||
// Validate vertical lines: at least 2 should span a meaningful height.
|
||||
// Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
|
||||
// for tables with partial column separators. Skipped entirely when
|
||||
// columns came from horizontal-segment endpoints — there are no
|
||||
// vertical lines to validate against, and the segment-endpoint
|
||||
// consistency check in `derive_columns_from_horizontal_segments`
|
||||
// is the equivalent guard.
|
||||
let spanning_v = if cols_from_segments {
|
||||
0
|
||||
} else {
|
||||
let s = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let p = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if s < 2 && p < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
s,
|
||||
p
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
s
|
||||
};
|
||||
// for tables with partial column separators.
|
||||
let spanning_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let partial_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if spanning_v < 2 && partial_v < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
spanning_v,
|
||||
partial_v
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Row edges need to be in descending order (top of page = higher Y first)
|
||||
let mut row_edges_desc = row_edges;
|
||||
@@ -393,7 +290,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -514,135 +410,6 @@ mod tests {
|
||||
assert!(tables.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_only_implicit_columns_accepted() {
|
||||
// Catalog/finding-aid pattern: each row's horizontal rule is
|
||||
// drawn as 3 segments at consistent x-endpoints (50, 127, 485,
|
||||
// 562), with no vertical lines anywhere. The segment break
|
||||
// points must be inferred as column edges.
|
||||
let mut lines = Vec::new();
|
||||
// Slightly uneven row spacing so the chart-gridline rejector
|
||||
// (CV < 0.02) doesn't fire.
|
||||
let row_ys = [80.0_f32, 145.0, 215.0, 280.0, 350.0, 415.0, 485.0];
|
||||
for &y in &row_ys {
|
||||
lines.push(make_hline(y, 50.0, 127.0, 1));
|
||||
lines.push(make_hline(y, 127.0, 485.0, 1));
|
||||
lines.push(make_hline(y, 485.0, 562.0, 1));
|
||||
}
|
||||
// Populate every cell so capture / density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for w in row_ys.windows(2) {
|
||||
let row_y = (w[0] + w[1]) / 2.0;
|
||||
items.push(make_item("id", 80.0, row_y, 1));
|
||||
items.push(make_item("description here", 200.0, row_y, 1));
|
||||
items.push(make_item("date", 510.0, row_y, 1));
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"horizontal-segment-only grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 3, "expected 3 columns");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_with_inconsistent_endpoints_rejected() {
|
||||
// Decorative rules of varying widths shouldn't be detected as a
|
||||
// table — each line has its own x-endpoints, no consistent
|
||||
// column boundary survives the 50%-of-rows threshold.
|
||||
let lines = vec![
|
||||
make_hline(100.0, 50.0, 150.0, 1),
|
||||
make_hline(200.0, 50.0, 220.0, 1),
|
||||
make_hline(300.0, 50.0, 310.0, 1),
|
||||
make_hline(400.0, 50.0, 470.0, 1),
|
||||
];
|
||||
let items = vec![
|
||||
make_item("decorative", 100.0, 150.0, 1),
|
||||
make_item("text", 100.0, 250.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"varying-width decorative rules should not be detected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_bare_frame_rejected() {
|
||||
// Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.
|
||||
// No internal structure → decorative border, not a table.
|
||||
let lines = vec![
|
||||
make_hline(20.0, 20.0, 575.0, 1), // top
|
||||
make_hline(820.0, 20.0, 575.0, 1), // bottom
|
||||
make_vline(20.0, 20.0, 820.0, 1), // left
|
||||
make_vline(575.0, 20.0, 820.0, 1), // right
|
||||
];
|
||||
let items = vec![
|
||||
make_item("title", 100.0, 100.0, 1),
|
||||
make_item("body", 100.0, 200.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"Page-sized 4-edge frame should be rejected as decoration"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_grid_with_internal_lines_accepted() {
|
||||
// Full-page table (governmental-ledger pattern): A4-sized grid
|
||||
// that previously hit the "page-spanning frame" early reject
|
||||
// before downstream validation could even look at it.
|
||||
// Verticals span the full table height so we isolate the
|
||||
// frame-vs-grid decision under test.
|
||||
let mut lines = Vec::new();
|
||||
// 13 horizontal rules: header + 12 row separators
|
||||
let h_ys = [
|
||||
22.5, 37.9, 95.5, 144.5, 184.9, 233.9, 291.7, 340.7, 415.8, 499.6, 574.7, 623.7, 698.8,
|
||||
];
|
||||
for &y in &h_ys {
|
||||
lines.push(make_hline(y, 22.6, 566.6, 1));
|
||||
}
|
||||
// 7 column dividers spanning full table height.
|
||||
let v_xs = [22.6, 66.3, 116.3, 186.6, 263.1, 493.5, 566.5];
|
||||
for &x in &v_xs {
|
||||
lines.push(make_vline(x, 22.5, 698.8, 1));
|
||||
}
|
||||
// Populate every cell so the capture-ratio + density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for r in 0..(h_ys.len() - 1) {
|
||||
let row_y = (h_ys[r] + h_ys[r + 1]) / 2.0;
|
||||
for c in 0..(v_xs.len() - 1) {
|
||||
let col_x = (v_xs[c] + v_xs[c + 1]) / 2.0;
|
||||
items.push(make_item("x", col_x, row_y, 1));
|
||||
}
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"Full-page table with internal grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 6,
|
||||
"expected ≥6 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert!(
|
||||
t.cells[0].len() >= 3,
|
||||
"expected ≥3 columns, got {}",
|
||||
t.cells[0].len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_rejected() {
|
||||
// Only 2 col edges (1 column) — not a table even with verticals
|
||||
|
||||
+14
-198
@@ -231,15 +231,6 @@ pub fn detect_tables_from_rects(
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
) -> (Vec<Table>, Vec<RectHintRegion>) {
|
||||
// Strip Image placeholders before column/row clustering — an image's bbox
|
||||
// would otherwise show up as a spurious column edge. See `is_text_layout_item`.
|
||||
let items_owned: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| crate::extractor::is_text_layout_item(i))
|
||||
.cloned()
|
||||
.collect();
|
||||
let items = items_owned.as_slice();
|
||||
|
||||
// Filter rects on this page; normalize negative widths/heights; skip tiny rects.
|
||||
let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized
|
||||
for r in rects {
|
||||
@@ -1403,15 +1394,13 @@ fn detect_row_stripe_table(
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
// Allow longer cells for multi-column tables (descriptions in one column
|
||||
// are common). Narrow grids with giant cells are usually layout
|
||||
// backgrounds — but only when the row count is also small. A 4+-row
|
||||
// key/value table with one descriptive column reads as a real table
|
||||
// on every other gate, so don't reject it on cell length alone.
|
||||
// are common). Single-column or 2-column "tables" with giant cells are
|
||||
// almost always layout backgrounds.
|
||||
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
||||
if max_cell_len > max_allowed && non_empty_rows < 4 {
|
||||
if max_cell_len > max_allowed {
|
||||
debug!(
|
||||
" row-stripe rejected: max cell length {} > {} (layout background, {} rows)",
|
||||
max_cell_len, max_allowed, non_empty_rows
|
||||
" row-stripe rejected: max cell length {} > {} (layout background)",
|
||||
max_cell_len, max_allowed
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1643,49 +1632,7 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
}
|
||||
};
|
||||
|
||||
// For wired-grid tables whose header text is centered/right-aligned but
|
||||
// whose data is left-aligned, cluster_x_positions can drop the header-only
|
||||
// x-cluster in its singleton-filter pass and merge adjacent data clusters
|
||||
// when the gap is below threshold, losing a column. Rect borders are
|
||||
// ground truth in that case — but only when each rect column actually
|
||||
// holds text. Decorative or background rects (prose laid out in a frame,
|
||||
// cell-fill rects with extra borders) can produce more rect-derived
|
||||
// columns than the text supports; preferring rects there would split a
|
||||
// logical column into spurious sub-columns.
|
||||
let rect_cols_match_text = match (&rect_col_edges, &text_col_edges) {
|
||||
(Some(rect_edges), _) if rect_edges.len() >= 4 => {
|
||||
let num_rect_cols = rect_edges.len() - 1;
|
||||
let mut col_item_counts = vec![0usize; num_rect_cols];
|
||||
for (_, item) in &page_items {
|
||||
let cx = item.x + item.width / 2.0;
|
||||
for c in 0..num_rect_cols {
|
||||
if cx >= rect_edges[c] - 2.0 && cx <= rect_edges[c + 1] + 2.0 {
|
||||
col_item_counts[c] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Require every rect column to hold multiple text items. A rect
|
||||
// column with no (or only one) item is decorative or the rect grid
|
||||
// is detecting a spurious column the data does not need; in those
|
||||
// cases the old text-cluster preference is the safer fallback.
|
||||
col_item_counts.iter().all(|&n| n >= 2)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||
(Some(rect_edges), text_edges_opt) if rect_cols_match_text => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns (text clusters: {}; rect cols well-distributed)",
|
||||
rect_edges.len() - 1,
|
||||
text_edges_opt
|
||||
.as_ref()
|
||||
.map(|e| (e.len() - 1) as i32)
|
||||
.unwrap_or(-1)
|
||||
);
|
||||
(rect_edges, false)
|
||||
}
|
||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||
@@ -1772,21 +1719,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject tables with paragraph-length cells — typically layout
|
||||
// backgrounds (sidebars, banners) where a single big rectangle
|
||||
// contains a wall of prose. Spare multi-row key/value tables where
|
||||
// the value column is a multi-bullet description: those pass every
|
||||
// other gate and shouldn't get killed on cell length alone.
|
||||
// Reject tables with paragraph-length cells (layout backgrounds, not tables)
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" cell-rect rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" cell-rect rejected: max cell length {} > 500",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2200,20 +2143,18 @@ fn detect_merged_cluster_table(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject if any cell has excessive text — layout background rects
|
||||
// produce "cells" containing paragraphs, not short data-table values.
|
||||
// Multi-row key/value tables can legitimately have one column of
|
||||
// long descriptive text, so only reject narrow-row layouts here.
|
||||
// Reject if any cell has excessive text — layout background rects produce
|
||||
// "cells" containing paragraphs, not short data-table values.
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" merged-cluster rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" merged-cluster rejected: max cell length {} > 500 (layout background)",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2314,7 +2255,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -2644,46 +2584,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_row_stripe_accepts_multi_row_key_value_long_cells() {
|
||||
// Multi-row 2-column key/value table where one value cell holds
|
||||
// a paragraph (>500 chars). The old `max_cell_len > 500` check
|
||||
// rejected this shape as a "layout background"; with the
|
||||
// multi-row guard, it should be accepted.
|
||||
let mut rects = Vec::new();
|
||||
let row_h = 25.0_f32;
|
||||
let y_top = 700.0_f32;
|
||||
for i in 0..8 {
|
||||
let y = y_top - (i as f32) * row_h;
|
||||
rects.push((40.0, y, 510.0, row_h));
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
for i in 0..8 {
|
||||
let row_center_y = y_top - (i as f32) * row_h + row_h / 2.0;
|
||||
// Left column: short label
|
||||
items.push(make_item(&format!("Field {}", i), 45.0, row_center_y, 10.0));
|
||||
// Right column: short value, except the last row which is a paragraph
|
||||
let value = if i == 7 {
|
||||
"X".repeat(800)
|
||||
} else {
|
||||
"value".to_string()
|
||||
};
|
||||
items.push(make_item(&value, 300.0, row_center_y, 10.0));
|
||||
}
|
||||
let result = detect_row_stripe_table(&items, &rects, 1);
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"multi-row key/value table with one long cell should be accepted"
|
||||
);
|
||||
let t = result.unwrap();
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 2, "expected 2 columns");
|
||||
}
|
||||
|
||||
// --- propagate_merged_cells ---
|
||||
|
||||
#[test]
|
||||
@@ -3273,7 +3173,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -3443,88 +3342,6 @@ mod tests {
|
||||
assert!(table.cells[2][1].contains("deny unauthorized"));
|
||||
}
|
||||
|
||||
/// Wire-bordered 4-column table whose header text is centered/right-aligned
|
||||
/// inside each cell while the data is left-aligned: cluster_x_positions
|
||||
/// merges adjacent columns (data Item→EAN gap is below threshold) and
|
||||
/// drops the header-only x-clusters in the filter pass, leaving only 3
|
||||
/// text-derived columns. Rect borders are 4 columns of ground truth.
|
||||
/// Before the fix the cell-rect path preferred text edges when they were
|
||||
/// the smaller set — losing a column. After the fix, 3+ rect columns
|
||||
/// always win.
|
||||
#[test]
|
||||
fn wired_header_data_misaligned_keeps_all_columns_from_rects() {
|
||||
let page = 1;
|
||||
// 4 cols: Item | EAN | Nombre | Cant
|
||||
let col_xs = [380.0_f32, 410.0, 470.0, 660.0, 700.0];
|
||||
// Header + 9 data rows at 15pt tall each (y descending).
|
||||
let row_ys: Vec<f32> = (0..=10).map(|r| 400.0 - 15.0 * r as f32).collect();
|
||||
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||
for r in 0..10 {
|
||||
let y_top = row_ys[r];
|
||||
let y_bot = row_ys[r + 1];
|
||||
for c in 0..4 {
|
||||
rects.push((col_xs[c], y_bot, col_xs[c + 1] - col_xs[c], y_top - y_bot));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items: Vec<TextItem> = Vec::new();
|
||||
// Header row (y ≈ 392.5): headers sit further to the right than data
|
||||
// because they are centered/right-aligned in the cells.
|
||||
items.push(make_item("Item", 389.0, 392.5, 9.0));
|
||||
items.push(make_item("EAN", 432.0, 392.5, 9.0));
|
||||
items.push(make_item("Nombre", 552.0, 392.5, 9.0));
|
||||
items.push(make_item("Cant", 672.0, 392.5, 9.0));
|
||||
|
||||
let names = [
|
||||
"Arnes Frontal",
|
||||
"Arnes Motor",
|
||||
"Arnes Piso",
|
||||
"Arnes Techo",
|
||||
"Arnes Puerta",
|
||||
"Arnes Tablero",
|
||||
"Arnes Trasero",
|
||||
"Arnes Lateral",
|
||||
"Arnes Sensor",
|
||||
];
|
||||
for r in 0..9 {
|
||||
let y = 377.5 - 15.0 * r as f32;
|
||||
items.push(make_item(&(r + 1).to_string(), 396.0, y, 9.0));
|
||||
items.push(make_item("7701023403016", 410.0, y, 9.0));
|
||||
items.push(make_item(names[r], 480.0, y, 9.0));
|
||||
items.push(make_item("1", 680.0, y, 9.0));
|
||||
}
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("wired 4-column table with header/data x-misalignment must detect");
|
||||
assert_eq!(
|
||||
table.columns.len(),
|
||||
4,
|
||||
"expected 4 columns from rect borders; cells: {:?}",
|
||||
table.cells
|
||||
);
|
||||
for c in 0..4 {
|
||||
let any_populated = table.cells.iter().any(|row| !row[c].trim().is_empty());
|
||||
assert!(
|
||||
any_populated,
|
||||
"column {} empty across all rows; cells: {:?}",
|
||||
c, table.cells
|
||||
);
|
||||
}
|
||||
// Header row populated in all 4 cells.
|
||||
let header = &table.cells[0];
|
||||
assert_eq!(header[0].trim(), "Item");
|
||||
assert_eq!(header[1].trim(), "EAN");
|
||||
assert_eq!(header[2].trim(), "Nombre");
|
||||
assert_eq!(header[3].trim(), "Cant");
|
||||
// First data row: Item="1", EAN, name, count="1" — no Item↔EAN merge.
|
||||
let data1 = &table.cells[1];
|
||||
assert_eq!(data1[0].trim(), "1");
|
||||
assert_eq!(data1[1].trim(), "7701023403016");
|
||||
assert!(data1[2].trim().contains("Arnes"));
|
||||
assert_eq!(data1[3].trim(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cluster_no_hint_without_items() {
|
||||
// Rects with no text items inside → no failed-cluster hint generated.
|
||||
@@ -3583,7 +3400,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -586,7 +586,6 @@ mod tests {
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -108,7 +108,6 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
|
||||
page: item.page,
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
|
||||
+7
-269
@@ -160,81 +160,6 @@ fn starts_with_uppercase_word(cell: &str) -> bool {
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_uppercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_lowercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_lowercase())
|
||||
}
|
||||
|
||||
fn starts_with_numbered_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim_start();
|
||||
let digit_count = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
|
||||
|
||||
digit_count > 0
|
||||
&& digit_count <= 3
|
||||
&& trimmed
|
||||
.chars()
|
||||
.nth(digit_count)
|
||||
.is_some_and(|c| matches!(c, '.' | ')' | '-' | ':'))
|
||||
}
|
||||
|
||||
fn alpha_word_count(cell: &str) -> usize {
|
||||
cell.split_whitespace()
|
||||
.filter(|word| word.chars().any(|c| c.is_alphabetic()))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn looks_like_compact_entry_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 3 || trimmed.len() > 80 {
|
||||
return false;
|
||||
}
|
||||
|
||||
if !starts_with_uppercase_alpha(trimmed) && !starts_with_numbered_label(trimmed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let words = alpha_word_count(trimmed);
|
||||
(1..=6).contains(&words)
|
||||
}
|
||||
|
||||
fn looks_like_plain_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 4 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-'))
|
||||
&& starts_with_uppercase_alpha(trimmed)
|
||||
&& (1..=4).contains(&alpha_word_count(trimmed))
|
||||
}
|
||||
|
||||
fn ends_like_incomplete_phrase(cell: &str) -> bool {
|
||||
let lower = cell.trim_end().to_ascii_lowercase();
|
||||
lower.ends_with(" and")
|
||||
|| lower.ends_with(" or")
|
||||
|| lower.ends_with(',')
|
||||
|| lower.ends_with('-')
|
||||
|| lower.ends_with('/')
|
||||
}
|
||||
|
||||
/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows
|
||||
fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
let mut cleaned: Vec<Vec<String>> = Vec::new();
|
||||
@@ -260,9 +185,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
|
||||
// Check if this is a continuation row (first column is empty but others have content).
|
||||
// A row with only 1 short non-empty cell (besides the first) is more likely a
|
||||
// section sub-header (e.g. "JAN", "FEB") than overflow text — don't merge it.
|
||||
@@ -300,76 +222,31 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.iter()
|
||||
.filter(|cell| starts_with_uppercase_word(cell))
|
||||
.count();
|
||||
let first_non_empty_col = row.iter().position(|c| !c.trim().is_empty());
|
||||
let first_non_empty_cell = first_non_empty_col
|
||||
.and_then(|idx| row.get(idx))
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let title_like_later_cells = first_non_empty_col
|
||||
.map(|idx| {
|
||||
row.iter()
|
||||
.skip(idx + 1)
|
||||
.map(|c| c.trim())
|
||||
.filter(|c| !c.is_empty() && starts_with_uppercase_alpha(c))
|
||||
.count()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let prev_first_cell_empty = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.is_some_and(|c| c.trim().is_empty());
|
||||
let prev_first_cell = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
&& uppercase_leading_cells >= non_first_cells.len().saturating_sub(1);
|
||||
// Hierarchical tables often use a row-spanned first column: sub-rows
|
||||
// leave column 0 blank, then start a compact title-like label in
|
||||
// column 1. Wrapped continuations in the existing fixtures start
|
||||
// mid-sentence/lowercase ("continued text here", "with 3.5%...") or
|
||||
// carry lowercase fragments in the later cells, so keep those mergeable.
|
||||
let looks_like_hierarchical_subrow = first_cell.is_empty()
|
||||
&& row.len() >= 3
|
||||
&& first_non_empty_col == Some(1)
|
||||
&& looks_like_compact_entry_label(first_non_empty_cell)
|
||||
&& ((non_first_cells.len() >= 2 && title_like_later_cells > 0)
|
||||
|| (non_first_cells.len() == 1
|
||||
&& prev_first_cell_empty
|
||||
&& alpha_word_count(first_non_empty_cell) >= 2));
|
||||
let looks_like_new_first_column_entry = !first_cell.is_empty()
|
||||
&& (starts_with_numbered_label(first_cell) || starts_with_uppercase_alpha(first_cell))
|
||||
&& filled_cells >= 2
|
||||
&& non_first_cells
|
||||
.iter()
|
||||
.any(|cell| looks_like_compact_entry_label(cell));
|
||||
let looks_like_section_label_row = !first_cell.is_empty()
|
||||
&& filled_cells == 1
|
||||
&& header_filled >= 3
|
||||
&& looks_like_plain_section_label(first_cell);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
&& !is_short_subheader
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& cleaned.len() > 1;
|
||||
|
||||
// Wrapped-cell continuation: row has fewer filled cells than the header
|
||||
// row, suggesting it's overflow text from the previous row's cells.
|
||||
// Only trigger when the previous row has significantly more filled cells.
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
let prev_filled = cleaned
|
||||
.last()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(0);
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
// Merge when the row has significantly fewer filled cells than header.
|
||||
// For wide tables (5+ cols), require ≤50% of header cells.
|
||||
// For narrow tables (2-4 cols), require fewer than header cells.
|
||||
@@ -380,18 +257,11 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
} else {
|
||||
header_filled.saturating_sub(1)
|
||||
};
|
||||
let continues_wrapped_first_column_label = !first_cell.is_empty()
|
||||
&& starts_with_lowercase_alpha(first_cell)
|
||||
&& ends_like_incomplete_phrase(prev_first_cell);
|
||||
let is_wrapped_continuation = cleaned.len() > 1
|
||||
&& filled_cells <= max_filled_for_merge
|
||||
&& (prev_filled > filled_cells
|
||||
|| (continues_wrapped_first_column_label && prev_filled >= filled_cells))
|
||||
&& prev_filled > filled_cells
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& !looks_like_new_first_column_entry
|
||||
&& !looks_like_section_label_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -537,44 +407,6 @@ mod tests {
|
||||
assert!(cleaned[1][1].contains("continued text here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_first_column_section_label_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Properties".into(),
|
||||
"Conditions".into(),
|
||||
"Method".into(),
|
||||
"Typical values".into(),
|
||||
"Units".into(),
|
||||
],
|
||||
vec![
|
||||
"Melt Flow Rate".into(),
|
||||
"230 C/2.16 kg".into(),
|
||||
"ASTM D1238".into(),
|
||||
"3.0".into(),
|
||||
"g/10 min".into(),
|
||||
],
|
||||
vec![
|
||||
"Mechanical".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"Tensile Stress at Yield".into(),
|
||||
"50 mm/min".into(),
|
||||
"ASTM D638".into(),
|
||||
"31".into(),
|
||||
"MPa".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 4);
|
||||
assert_eq!(cleaned[2][0], "Mechanical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_short_subheader_not_merged() {
|
||||
let cells = vec![
|
||||
@@ -627,100 +459,6 @@ mod tests {
|
||||
assert_eq!(cleaned[2][1], "Uncertainty around other copies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_numbered_hierarchy_rows_not_overmerged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"1. Group alpha".into(),
|
||||
"Task setup and".into(),
|
||||
"Begin setup".into(),
|
||||
"Faster start".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"management".into(),
|
||||
"recommended profile".into(),
|
||||
"with saved defaults".into(),
|
||||
],
|
||||
vec![
|
||||
"2. Group beta and".into(),
|
||||
"Storage setup".into(),
|
||||
"Provides upload tools".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"fine-tuning".into(),
|
||||
"".into(),
|
||||
"for filtered inputs".into(),
|
||||
"service".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Label workspace".into(),
|
||||
"Creates review sets".into(),
|
||||
"Lets teams review".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Model training".into(),
|
||||
"".into(),
|
||||
"Supports custom model".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 5);
|
||||
assert_eq!(cleaned[1][0], "1. Group alpha");
|
||||
assert_eq!(cleaned[1][1], "Task setup and management");
|
||||
assert_eq!(cleaned[2][0], "2. Group beta and fine-tuning");
|
||||
assert_eq!(cleaned[2][1], "Storage setup");
|
||||
assert_eq!(cleaned[3][1], "Label workspace");
|
||||
assert_eq!(cleaned[4][1], "Model training");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_partial_hierarchical_subrow_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"Group A".into(),
|
||||
"Alpha task".into(),
|
||||
"Initial detail".into(),
|
||||
"Initial benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Beta task".into(),
|
||||
"Parallel detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"second line".into(),
|
||||
"additional detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 3);
|
||||
assert_eq!(cleaned[1][1], "Alpha task");
|
||||
assert_eq!(cleaned[2][0], "");
|
||||
assert_eq!(cleaned[2][1], "Beta task second line");
|
||||
assert_eq!(cleaned[2][2], "Parallel detail additional detail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_full_width_continuation_row_still_merges_when_lowercase() {
|
||||
let cells = vec![
|
||||
|
||||
@@ -514,7 +514,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -867,7 +866,6 @@ mod tests {
|
||||
font: String::new(),
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
@@ -904,7 +902,6 @@ mod tests {
|
||||
font: String::new(),
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
|
||||
-1237
File diff suppressed because it is too large
Load Diff
@@ -883,7 +883,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1003,7 +1002,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -1080,7 +1078,6 @@ mod tests {
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
+20
-165
@@ -329,7 +329,7 @@ impl ToUnicodeCMap {
|
||||
if let (Some(start), Some(end), Some(base)) = (
|
||||
parse_hex_u16(&start_hex),
|
||||
parse_hex_u16(&end_hex),
|
||||
hex_to_unicode_scalar(&base_hex),
|
||||
parse_hex_u32(&base_hex),
|
||||
) {
|
||||
self.ranges.push((start, end, base));
|
||||
}
|
||||
@@ -575,86 +575,32 @@ fn parse_hex_u16(hex: &str) -> Option<u16> {
|
||||
u16::from_str_radix(hex.trim(), 16).ok()
|
||||
}
|
||||
|
||||
/// Convert a ToUnicode destination hex string to Unicode.
|
||||
///
|
||||
/// PDF ToUnicode destinations are UTF-16BE strings. Supplementary-plane
|
||||
/// characters are encoded as surrogate pairs, so treating each 4-hex chunk as
|
||||
/// a scalar drops emoji like D83CDF1F.
|
||||
/// Parse a hex string to u32
|
||||
fn parse_hex_u32(hex: &str) -> Option<u32> {
|
||||
u32::from_str_radix(hex.trim(), 16).ok()
|
||||
}
|
||||
|
||||
/// Convert a hex string to a Unicode string
|
||||
/// Handles both 2-byte (BMP) and 4-byte (supplementary) codepoints
|
||||
fn hex_to_unicode_string(hex: &str) -> Option<String> {
|
||||
let hex: String = hex.chars().filter(|ch| !ch.is_ascii_whitespace()).collect();
|
||||
if hex.is_empty() || !hex.len().is_multiple_of(2) {
|
||||
return None;
|
||||
}
|
||||
let hex = hex.trim();
|
||||
let mut result = String::new();
|
||||
|
||||
let bytes: Option<Vec<u8>> = (0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).ok())
|
||||
.collect();
|
||||
let bytes = bytes?;
|
||||
|
||||
if bytes.len().is_multiple_of(2) {
|
||||
let units: Vec<u16> = bytes
|
||||
.chunks_exact(2)
|
||||
.map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
|
||||
.collect();
|
||||
if let Ok(result) = String::from_utf16(&units) {
|
||||
if !result.is_empty() {
|
||||
return Some(normalize_tounicode_destination(result));
|
||||
// Process 4 hex digits at a time
|
||||
let mut i = 0;
|
||||
while i + 4 <= hex.len() {
|
||||
if let Ok(cp) = u32::from_str_radix(&hex[i..i + 4], 16) {
|
||||
if let Some(c) = char::from_u32(cp) {
|
||||
result.push(c);
|
||||
}
|
||||
}
|
||||
i += 4;
|
||||
}
|
||||
|
||||
// Be permissive for non-standard one-byte destinations.
|
||||
if bytes.len() == 1 {
|
||||
let ch = bytes[0] as char;
|
||||
if !ch.is_control() || ch == '\t' || ch == '\n' {
|
||||
return Some(ch.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn normalize_tounicode_destination(text: String) -> String {
|
||||
let is_multi_char = text.chars().nth(1).is_some();
|
||||
|
||||
// Some malformed producer CMaps put a list of alternative whitespace or
|
||||
// hyphen codepoints into one destination. Keep ordinary multi-character
|
||||
// mappings intact unless that malformed signature is present.
|
||||
if is_multi_char
|
||||
&& text.chars().all(char::is_whitespace)
|
||||
&& text.chars().any(|ch| matches!(ch, '\t' | '\n' | '\r'))
|
||||
{
|
||||
return if text.contains('\t') {
|
||||
"\t".to_string()
|
||||
} else {
|
||||
" ".to_string()
|
||||
};
|
||||
}
|
||||
|
||||
if is_multi_char
|
||||
&& text.contains('\u{00ad}')
|
||||
&& text.chars().all(|ch| {
|
||||
matches!(
|
||||
ch,
|
||||
'-' | '\u{00ad}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2212}'
|
||||
)
|
||||
})
|
||||
{
|
||||
return "-".to_string();
|
||||
}
|
||||
|
||||
text
|
||||
}
|
||||
|
||||
fn hex_to_unicode_scalar(hex: &str) -> Option<u32> {
|
||||
let text = hex_to_unicode_string(hex)?;
|
||||
let mut chars = text.chars();
|
||||
let ch = chars.next()?;
|
||||
if chars.next().is_none() {
|
||||
Some(ch as u32)
|
||||
} else {
|
||||
if result.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2661,97 +2607,6 @@ endbfrange
|
||||
assert_eq!(cmap.lookup(0x0005), Some("C".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bfchar_surrogate_pair_emoji() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
2 beginbfchar
|
||||
<16> <D83CDF1F>
|
||||
<9D> <D83CDFAD>
|
||||
endbfchar
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
assert_eq!(cmap.code_byte_length, 1);
|
||||
assert_eq!(cmap.lookup(0x16), Some("🌟".to_string()));
|
||||
assert_eq!(cmap.lookup(0x9D), Some("🎭".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bfrange_surrogate_pair_base() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
1 beginbfrange
|
||||
<C8> <C9> <D83CDFD8>
|
||||
endbfrange
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
assert_eq!(cmap.code_byte_length, 1);
|
||||
assert_eq!(cmap.lookup(0xC8), Some("🏘".to_string()));
|
||||
assert_eq!(cmap.lookup(0xC9), Some("🏙".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bfrange_preserves_single_hyphen_like_base() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
1 beginbfrange
|
||||
<21> <22> <2013>
|
||||
endbfrange
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
assert_eq!(cmap.lookup(0x21), Some("–".to_string()));
|
||||
assert_eq!(cmap.lookup(0x22), Some("—".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_spaced_destination_hex_without_control_noise() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
3 beginbfchar
|
||||
<21> < 0009 000d 0020 00a0 >
|
||||
<22> < 002d 00ad 2010 >
|
||||
<23> <00a0>
|
||||
endbfchar
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
assert_eq!(cmap.lookup(0x21), Some("\t".to_string()));
|
||||
assert_eq!(cmap.lookup(0x22), Some("-".to_string()));
|
||||
assert_eq!(cmap.lookup(0x23), Some("\u{00a0}".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_preserves_valid_multi_character_destinations() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
4 beginbfchar
|
||||
<21> <002d002d>
|
||||
<22> <20132013>
|
||||
<23> <002000a0>
|
||||
<24> <00660069>
|
||||
endbfchar
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
assert_eq!(cmap.lookup(0x21), Some("--".to_string()));
|
||||
assert_eq!(cmap.lookup(0x22), Some("––".to_string()));
|
||||
assert_eq!(cmap.lookup(0x23), Some(" \u{00a0}".to_string()));
|
||||
assert_eq!(cmap.lookup(0x24), Some("fi".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remap_to_sequential() {
|
||||
// Simulate a broken CMap where GIDs are from pre-subsetting:
|
||||
|
||||
@@ -116,10 +116,6 @@ pub struct TextItem {
|
||||
pub is_bold: bool,
|
||||
/// Whether the font is italic
|
||||
pub is_italic: bool,
|
||||
/// Whether the text is underlined (drawn rule/thin rect under the
|
||||
/// baseline — PDFs have no underline font flag, so this is detected
|
||||
/// geometrically after extraction; see `extractor::underline`).
|
||||
pub is_underline: bool,
|
||||
/// Type of item (text, image, link)
|
||||
pub item_type: ItemType,
|
||||
/// Marked Content ID from the content stream's BDC/BMC operator.
|
||||
|
||||
BIN
Binary file not shown.
+3
-217
@@ -2,14 +2,13 @@
|
||||
|
||||
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::ItemType;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
|
||||
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, extract_text_with_positions_mem,
|
||||
process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
|
||||
PdfType, TextItem,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -104,7 +103,6 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -130,7 +128,6 @@ fn make_text_item_with_font(
|
||||
page,
|
||||
is_bold: is_bold_font(font),
|
||||
is_italic: is_italic_font(font),
|
||||
is_underline: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1109,7 +1106,6 @@ fn test_pages_needing_ocr_field_accessible() {
|
||||
page_count: 1,
|
||||
processing_time_ms: 0,
|
||||
pages_needing_ocr: vec![1, 3],
|
||||
ocr_reasons_by_page: Vec::new(),
|
||||
title: None,
|
||||
confidence: 1.0,
|
||||
layout: pdf_inspector::LayoutComplexity::default(),
|
||||
@@ -1652,33 +1648,6 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_in_regions_uses_line_grid() {
|
||||
// Stroked-grid table (m/l/S path operators forming a 2x2 grid).
|
||||
// The heuristic text-only detector handles the same cells already,
|
||||
// so this guards that the line-backed path doesn't regress: the
|
||||
// markdown still contains all four data cells.
|
||||
let buf = synthetic_vector_grid_pdf(false);
|
||||
let results =
|
||||
extract_tables_in_regions_mem(&buf, &[(0, vec![[40.0, 50.0, 220.0, 760.0]])]).unwrap();
|
||||
let region = &results[0].regions[0];
|
||||
assert!(
|
||||
!region.needs_ocr,
|
||||
"stroked-grid table should be extracted, got needs_ocr=true"
|
||||
);
|
||||
for tok in ["A1", "B1", "A2", "B2"] {
|
||||
assert!(
|
||||
region.text.contains(tok),
|
||||
"expected '{tok}' in output, got: {}",
|
||||
region.text
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
region.text.contains('|'),
|
||||
"expected pipe-delimited markdown"
|
||||
);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_mem tests (TSR-aware path)
|
||||
// =========================================================================
|
||||
@@ -3395,186 +3364,3 @@ fn test_synthetic_type0_broken_tounicode_emits_fffd_not_latin1_mojibake() {
|
||||
result.pages_needing_ocr
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Image XObject emission
|
||||
// ============================================================================
|
||||
|
||||
/// Build a minimal PDF containing one Image XObject placed at a known CTM.
|
||||
/// `image_ctm` is the 6-element matrix applied to the unit square by the
|
||||
/// `Do` operator (per PDF spec section 8.9.5 "Image Coordinate System").
|
||||
/// For an axis-aligned image at `(x, y)` with size `w × h`, that's
|
||||
/// `[w, 0, 0, h, x, y]`.
|
||||
fn make_pdf_with_image(image_ctm: [f32; 6]) -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize];
|
||||
|
||||
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(body.as_bytes());
|
||||
pdf.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
fn add_stream_object(
|
||||
pdf: &mut Vec<u8>,
|
||||
offsets: &mut Vec<usize>,
|
||||
id: usize,
|
||||
dict: &str,
|
||||
stream_bytes: &[u8],
|
||||
) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(
|
||||
format!("<< {} /Length {} >>\nstream\n", dict, stream_bytes.len()).as_bytes(),
|
||||
);
|
||||
pdf.extend_from_slice(stream_bytes);
|
||||
pdf.extend_from_slice(b"\nendstream\nendobj\n");
|
||||
}
|
||||
|
||||
// 1: catalog → 2: pages → 3: page with XObject /Im0 → 4: content stream
|
||||
// 5: font → 6: image XObject (1×1 grayscale)
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
1,
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
2,
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
3,
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
|
||||
/Resources << /Font << /F1 5 0 R >> /XObject << /Im0 6 0 R >> >> \
|
||||
/Contents 4 0 R >>",
|
||||
);
|
||||
let [a, b, c, d, e, f] = image_ctm;
|
||||
// BT/ET around a small text item just so the page isn't classified as
|
||||
// image-only (which would route to a different code path). Then save
|
||||
// graphics state, apply the image CTM, invoke Im0, restore.
|
||||
let content = format!(
|
||||
"BT /F1 12 Tf 100 700 Td (Hi) Tj ET\nq {} {} {} {} {} {} cm /Im0 Do Q",
|
||||
a, b, c, d, e, f
|
||||
);
|
||||
add_stream_object(&mut pdf, &mut offsets, 4, "", content.as_bytes());
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
5,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
);
|
||||
// 1×1 grayscale image; the single byte is mid-gray. Contents don't
|
||||
// matter to the extractor — it only cares about the XObject's
|
||||
// /Subtype and the CTM at the `Do` operator.
|
||||
let image_pixel = [128u8];
|
||||
add_stream_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
6,
|
||||
"/Type /XObject /Subtype /Image /Width 1 /Height 1 \
|
||||
/ColorSpace /DeviceGray /BitsPerComponent 8",
|
||||
&image_pixel,
|
||||
);
|
||||
|
||||
let xref_start = pdf.len();
|
||||
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
|
||||
pdf.extend_from_slice(b"0000000000 65535 f \n");
|
||||
for offset in offsets.iter().skip(1) {
|
||||
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
pdf.extend_from_slice(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
|
||||
offsets.len(),
|
||||
xref_start
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
pdf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_text_with_positions_emits_image_bboxes() {
|
||||
// Place a 200×100 image at (50, 600) in PDF user space (origin
|
||||
// bottom-left). The Do operator applies the CTM to a unit square,
|
||||
// so for an axis-aligned image, CTM = [w, 0, 0, h, x, y].
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
|
||||
let images: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| matches!(i.item_type, ItemType::Image))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
images.len(),
|
||||
1,
|
||||
"expected exactly one Image item, got items: {:?}",
|
||||
items
|
||||
.iter()
|
||||
.map(|i| (&i.text, &i.item_type))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let img = images[0];
|
||||
assert!((img.x - 50.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 600.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 200.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
assert_eq!(img.page, 1);
|
||||
// text field carries the legacy `[Image: <resource-name>]` form that
|
||||
// the markdown emitter already knows how to parse.
|
||||
assert_eq!(img.text, "[Image: Im0]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_xobject_bbox_handles_rotated_ctm() {
|
||||
// 90° rotation CTM: a unit square at the origin maps to a square
|
||||
// rotated counter-clockwise about (0,0), then translated to (200, 300).
|
||||
// For a 100×100 image, that's CTM = [0, 100, -100, 0, 200, 300]
|
||||
// (apply the rotation: (1,0) → (0,100); (0,1) → (-100,0)).
|
||||
// The page-space corners are:
|
||||
// (0,0) → (200, 300)
|
||||
// (1,0) → (200, 400)
|
||||
// (1,1) → (100, 400)
|
||||
// (0,1) → (100, 300)
|
||||
// → AABB: x=100..200 (w=100), y=300..400 (h=100).
|
||||
let pdf = make_pdf_with_image([0.0, 100.0, -100.0, 0.0, 200.0, 300.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
let img = items
|
||||
.iter()
|
||||
.find(|i| matches!(i.item_type, ItemType::Image))
|
||||
.expect("image item");
|
||||
assert!((img.x - 100.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 300.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 100.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_emission_does_not_change_default_markdown() {
|
||||
// Default `MarkdownOptions::include_images = false` — adding image
|
||||
// emission MUST NOT make `extract_pages_markdown` start producing
|
||||
// `![Image: …]` placeholders for everyone. Existing callers that
|
||||
// upgrade should see no diff in their markdown.
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let result = extract_pages_markdown_mem(&pdf, None).expect("extract");
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(
|
||||
!result.pages[0].markdown.contains("Image:"),
|
||||
"default markdown leaked an image placeholder: {:?}",
|
||||
result.pages[0].markdown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_options_default_has_include_images_false() {
|
||||
// Explicit assertion so anyone flipping this back catches it in CI.
|
||||
// See `MarkdownOptions::default` in src/markdown/mod.rs for the
|
||||
// long-form rationale.
|
||||
let opts = MarkdownOptions::default();
|
||||
assert!(!opts.include_images);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user