Compare commits

..
Author SHA1 Message Date
Abimael Martell 70cc34ee82 fix staggered newspaper column ordering 2026-06-23 15:05:06 -07:00
Abimael Martell 57b98c6a5d fix(extractor): flag garbled text spans for OCR (#108)
* fix(extractor): flag garbled text spans for OCR

* fix(extractor): apply text quality checks to regions

* chore(napi): bump npm package version
2026-06-23 13:12:10 -07:00
Abimael Martell f25808e0a7 fix(extractor): restore CID font state for Chinese text (#106)
* fix Chinese CID text decoding

* bump package versions
2026-06-20 19:43:27 -06:00
Abimael Martell 9360c8464d ci: add crates trusted publishing (#103) 2026-06-05 11:21:55 -07:00
Abimael Martell 252d87ac58 docs(readme): add package badges and install docs (#102)
* docs: add crates.io install instructions

* docs: add npm badge
2026-06-05 11:01:56 -07:00
Abimael Martell 85890648c9 chore: use crates.io lopdf (#101) 2026-06-05 10:50:05 -07:00
Abimael Martell 6e55e38b55 fix(markdown): handle wrapped bold abstracts (#100)
* fix(markdown): handle wrapped bold abstracts

* chore: bump napi package version
2026-06-01 14:59:18 -07:00
Abimael Martell 42befcea57 fix(pdf-inspector): recover wrapped key-value tables (#99)
* fix(pdf-inspector): recover wrapped key-value tables

* fix(pdf-inspector): satisfy clippy
2026-06-01 10:10:55 -07:00
Abimael Martell e547f616f9 fix(pdf-inspector): recover key-value region tables (#98) 2026-05-29 18:55:38 -07:00
14 changed files with 1398 additions and 142 deletions
+87
View File
@@ -0,0 +1,87 @@
name: Publish Rust crate
on:
push:
branches: [main]
paths: ['Cargo.toml']
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
jobs:
check-version:
name: Check version change
runs-on: ubuntu-latest
outputs:
changed: ${{ steps.check.outputs.changed }}
published: ${{ steps.check.outputs.published }}
version: ${{ steps.check.outputs.version }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Check if version changed
id: check
run: |
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
echo "old=$OLD_VERSION new=$NEW_VERSION"
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "published=false" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "changed=true" >> "$GITHUB_OUTPUT"
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
"https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
case "$HTTP_STATUS" in
200)
echo "published=true" >> "$GITHUB_OUTPUT"
echo "pdf-inspector v$NEW_VERSION is already published"
;;
404)
echo "published=false" >> "$GITHUB_OUTPUT"
;;
*)
cat /tmp/crate-version.json
echo "Unexpected crates.io response: $HTTP_STATUS" >&2
exit 1
;;
esac
publish:
name: Publish to crates.io
needs: check-version
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
runs-on: ubuntu-latest
environment: crates-io
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Verify package
run: cargo publish --dry-run
- name: Authenticate with crates.io
id: auth
uses: rust-lang/crates-io-auth-action@v1
- name: Publish crate
run: cargo publish
env:
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "pdf-inspector"
version = "0.1.0"
version = "0.1.2"
edition = "2021"
autobins = false
authors = ["Firecrawl Team"]
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
# PDF parsing
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
lopdf = { version = "0.41.0", features = ["rayon"] }
# Error handling
thiserror = "2.0"
+25 -9
View File
@@ -1,5 +1,8 @@
# pdf-inspector
[![Crates.io](https://img.shields.io/crates/v/pdf-inspector.svg)](https://crates.io/crates/pdf-inspector)
[![npm](https://img.shields.io/npm/v/@firecrawl/pdf-inspector.svg)](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
@@ -71,9 +74,17 @@ console.log(result.markdown); // Markdown string or null
### Rust
Install from [crates.io](https://crates.io/crates/pdf-inspector):
```bash
cargo add pdf-inspector
```
Or add it manually:
```toml
[dependencies]
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
pdf-inspector = "0.1"
```
```rust
@@ -91,29 +102,34 @@ if let Some(markdown) = &result.markdown {
### CLI
```bash
# Install the CLI tools
cargo install pdf-inspector
# Convert PDF to Markdown
cargo run --bin pdf2md -- document.pdf
pdf2md document.pdf
# JSON output (for piping)
cargo run --bin pdf2md -- document.pdf --json
pdf2md document.pdf --json
# Raw markdown only (no headers)
cargo run --bin pdf2md -- document.pdf --raw
pdf2md document.pdf --raw
# Insert page break markers (<!-- Page N -->)
cargo run --bin pdf2md -- document.pdf --pages
pdf2md document.pdf --pages
# Process only specific pages
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
pdf2md document.pdf --select-pages 1,3,5-10
# Detection only (no extraction)
cargo run --bin detect-pdf -- document.pdf
cargo run --bin detect-pdf -- document.pdf --json
detect-pdf document.pdf
detect-pdf document.pdf --json
# Detection + layout analysis (tables, columns)
cargo run --bin detect-pdf -- document.pdf --analyze --json
detect-pdf document.pdf --analyze --json
```
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
## Architecture
```
+21
View File
@@ -0,0 +1,21 @@
# Publishing
The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
## crates.io Trusted Publisher
Configure the trusted publisher for the `pdf-inspector` crate with:
- Repository: `firecrawl/pdf-inspector`
- Workflow: `publish-crate.yml`
- Environment: `crates-io`
The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
## Release Steps
1. Update `version` in `Cargo.toml`.
2. Merge the version bump to `main`.
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
+5 -4
View File
@@ -672,8 +672,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lopdf"
version = "0.40.0"
source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d"
dependencies = [
"aes",
"bitflags",
@@ -829,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "pdf-inspector"
version = "0.1.0"
version = "0.1.1"
dependencies = [
"env_logger",
"log",
@@ -844,7 +845,7 @@ dependencies = [
[[package]]
name = "pdf-inspector-napi"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"napi",
"napi-build",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "pdf-inspector-napi"
version = "0.2.0"
version = "0.2.1"
edition = "2021"
[lib]
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.9.3",
"version": "1.9.7",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
+113 -7
View File
@@ -188,7 +188,17 @@ pub(crate) fn extract_page_text_items(
// Graphics state tracking
let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();
#[derive(Clone)]
struct SavedGraphicsState {
ctm: [f32; 6],
text_rendering_mode: i32,
char_spacing: f32,
word_spacing: f32,
text_leading: f32,
current_font: String,
current_font_size: f32,
}
let mut gstate_stack: Vec<SavedGraphicsState> = Vec::new();
// Text state tracking
let mut current_font = String::new();
@@ -227,15 +237,26 @@ pub(crate) fn extract_page_text_items(
match op.operator.as_str() {
"q" => {
// Save graphics state
gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
gstate_stack.push(SavedGraphicsState {
ctm,
text_rendering_mode,
char_spacing,
word_spacing,
text_leading,
current_font: current_font.clone(),
current_font_size,
});
}
"Q" => {
// Restore graphics state
if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
ctm = saved_ctm;
text_rendering_mode = saved_tr;
char_spacing = saved_tc;
word_spacing = saved_tw;
if let Some(saved) = gstate_stack.pop() {
ctm = saved.ctm;
text_rendering_mode = saved.text_rendering_mode;
char_spacing = saved.char_spacing;
word_spacing = saved.word_spacing;
text_leading = saved.text_leading;
current_font = saved.current_font;
current_font_size = saved.current_font_size;
}
}
"cm" => {
@@ -1286,6 +1307,91 @@ mod tests {
assert!(lines.is_empty());
}
#[test]
fn test_q_restores_current_font_for_text_decoding() {
use crate::tounicode::FontCMaps;
use lopdf::{dictionary, Object, Stream};
fn cmap_stream(dst_hex: &str) -> Stream {
let cmap = format!(
r#"/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
/CMapName /Test-UCS def
/CMapType 2 def
1 begincodespacerange
<00> <FF>
endcodespacerange
1 beginbfchar
<41> <{dst_hex}>
endbfchar
endcmap
CMapName currentdict /CMap defineresource pop
end
end"#
);
Stream::new(dictionary! {}, cmap.into_bytes())
}
let mut doc = lopdf::Document::new();
let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X
let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y
let f1 = doc.add_object(dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
"ToUnicode" => Object::Reference(f1_cmap),
});
let f2 = doc.add_object(dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
"ToUnicode" => Object::Reference(f2_cmap),
});
let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET
q
BT /F2 12 Tf 20 700 Tm <41> Tj ET
Q
BT 30 700 Tm <41> Tj ET";
let content_id = doc.add_object(Object::Stream(Stream::new(
dictionary! {},
content.to_vec(),
)));
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Contents" => Object::Reference(content_id),
"Resources" => dictionary! {
"Font" => dictionary! {
"F1" => Object::Reference(f1),
"F2" => Object::Reference(f2),
},
},
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
});
let pages_id = doc.add_object(dictionary! {
"Type" => "Pages",
"Count" => Object::Integer(1),
"Kids" => vec![Object::Reference(page_id)],
});
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => Object::Reference(pages_id),
});
doc.trailer.set("Root", Object::Reference(catalog_id));
let font_cmaps = FontCMaps::from_doc(&doc);
let ((items, _, _), _, _) =
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
let text = items
.iter()
.map(|item| item.text.as_str())
.collect::<String>();
assert_eq!(text, "XYX");
}
#[test]
fn test_strip_pdf_comments() {
// Basic comment stripping
+7 -21
View File
@@ -858,6 +858,13 @@ pub(crate) fn extract_text_from_operand(
// unmapped. Don't fall through to text-interpretation fallbacks
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
// 2-byte CIDs (Identity-H) are by far the common case; for
// an odd byte count we still emit at least one marker so
// detection downstream fires.
let cid_count = (bytes.len() / 2).max(1);
return Some("\u{FFFD}".repeat(cid_count));
}
// Try our custom encoding map from Differences arrays.
// The Differences array overrides specific codes in a base encoding (typically
@@ -966,27 +973,6 @@ pub(crate) fn extract_text_from_operand(
return Some(symbol_text);
}
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
// encodings — for these, an unmapped byte is a valid character
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
// production scrape_id 019de78c-... samples).
//
// For a CID font (has_cmap is set OR a /ToUnicode reference
// exists) with any non-ASCII bytes, emit a single U+FFFD per
// CID instead. This both replaces the mojibake with a proper
// "decode failed" marker AND keeps `detect_encoding_issues`
// tripping so the page is flagged for OCR — the existing
// garbage-detection path that the high-Latin-1 mojibake used
// to satisfy by accident.
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
// 2-byte CIDs (Identity-H) are by far the common case; for
// an odd byte count we still emit at least one marker so
// detection downstream fires.
let cid_count = (bytes.len() / 2).max(1);
return Some("\u{FFFD}".repeat(cid_count));
}
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
// use single-byte encodings where Latin-1 fallback is the
+80 -18
View File
@@ -1262,9 +1262,9 @@ pub(crate) fn group_into_lines_with_thresholds(
if is_newspaper {
// Newspaper: columns are independent text flows.
// 1. Split each column into its densest cluster (core) and stragglers
// 2. Use core columns to determine the above/below threshold
// 3. Emit: above items → core columns sequentially → below items
// 1. Split each column into its densest cluster (core) and stragglers.
// 2. Promote only true top matter above all column starts.
// 3. Emit: above items → each complete column flow → below spanning items.
let mut core_columns: Vec<Vec<TextLine>> = Vec::new();
let mut col_stragglers: Vec<Vec<TextLine>> = Vec::new();
for col in per_column_lines {
@@ -1273,12 +1273,15 @@ pub(crate) fn group_into_lines_with_thresholds(
col_stragglers.push(stragglers);
}
// col_top = min of max Y across core columns
let col_top = core_columns
// Use the highest core top as the top-matter cutoff. A column
// that starts lower because of a figure or large gap should not
// pull another column's opening prose above the abstract/title
// area; those fragments still belong to that column's flow.
let top_matter_cutoff = core_columns
.iter()
.filter(|c| !c.is_empty())
.map(|c| c.iter().map(|l| l.y).fold(f32::NEG_INFINITY, f32::max))
.fold(f32::INFINITY, f32::min);
.fold(f32::NEG_INFINITY, f32::max);
let margin = 5.0;
let mut above: Vec<TextLine> = Vec::new();
@@ -1286,37 +1289,37 @@ pub(crate) fn group_into_lines_with_thresholds(
// Spanning items: above or below the column region
for line in spanning_lines {
if line.y > col_top + margin {
if line.y > top_matter_cutoff + margin {
above.push(line);
} else {
below_spanning.push(line);
}
}
// Column stragglers above col_top go to "above";
// below col_top they stay with their column to avoid
// re-interleaving when sorted by Y.
let mut col_below: Vec<Vec<TextLine>> = vec![Vec::new(); core_columns.len()];
for (ci, stragglers) in col_stragglers.into_iter().enumerate() {
let mut column_flows: Vec<Vec<TextLine>> = Vec::with_capacity(core_columns.len());
for (ci, mut flow) in core_columns.into_iter().enumerate() {
let stragglers = col_stragglers
.get_mut(ci)
.map(std::mem::take)
.unwrap_or_default();
for line in stragglers {
if line.y > col_top + margin {
if line.y > top_matter_cutoff + margin {
above.push(line);
} else {
col_below[ci].push(line);
flow.push(line);
}
}
flow.sort_by(|a, b| b.y.total_cmp(&a.y));
column_flows.push(flow);
}
above.sort_by(|a, b| b.y.total_cmp(&a.y));
below_spanning.sort_by(|a, b| b.y.total_cmp(&a.y));
all_lines.extend(above);
for col in core_columns {
for col in column_flows {
all_lines.extend(col);
}
for cb in col_below {
all_lines.extend(cb);
}
all_lines.extend(below_spanning);
} else {
// Tabular: Y-interleaved merge — rows at the same Y from
@@ -1917,4 +1920,63 @@ mod tests {
let spanning_count = mask.iter().filter(|&&m| m).count();
assert_eq!(spanning_count, 0, "Narrow header should NOT be pre-masked");
}
#[test]
fn newspaper_keeps_staggered_right_column_fragment_in_column_flow() {
let mut items = Vec::new();
for i in 0..30 {
let text = if i == 0 {
"ABSTRACT".to_string()
} else {
format!("Left body line {i}")
};
let mut item = make_item(1, 54.0, 588.0 - i as f32 * 10.0, &text);
item.width = 220.0;
items.push(item);
}
for i in 0..4 {
let text = if i == 0 {
"options, people often need comments".to_string()
} else {
format!("Right opening fragment {i}")
};
let mut item = make_item(1, 318.0, 592.0 - i as f32 * 10.0, &text);
item.width = 220.0;
items.push(item);
}
for i in 0..20 {
let text = if i == 0 {
"Figure 1 caption starts here".to_string()
} else {
format!("Right lower body line {i}")
};
let mut item = make_item(1, 318.0, 330.0 - i as f32 * 10.0, &text);
item.width = 220.0;
items.push(item);
}
let lines = group_into_lines_with_thresholds(items, &HashMap::new(), &HashSet::new());
let texts: Vec<String> = lines.iter().map(TextLine::text).collect();
let abstract_pos = texts.iter().position(|text| text == "ABSTRACT").unwrap();
let options_pos = texts
.iter()
.position(|text| text.starts_with("options, people often"))
.unwrap();
let figure_pos = texts
.iter()
.position(|text| text.starts_with("Figure 1 caption"))
.unwrap();
assert!(
abstract_pos < options_pos,
"right-column opening fragment must not be promoted above the abstract: {texts:?}"
);
assert!(
options_pos < figure_pos,
"right-column opening fragment should stay before the lower right-column core"
);
}
}
+228 -26
View File
@@ -368,6 +368,7 @@ pub fn extract_pages_markdown_mem(
// Extract ALL pages to get accurate, document-wide font stats.
let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
let text_quality = analyze_text_quality(&all_items);
// Compute layout complexity from full document (near-zero cost).
let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
@@ -416,6 +417,7 @@ pub fn extract_pages_markdown_mem(
.collect();
let has_gid = gid_pages.contains(&page_1idx);
let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx);
// Build markdown with document-wide font stats
let options = MarkdownOptions {
@@ -425,17 +427,22 @@ pub fn extract_pages_markdown_mem(
..MarkdownOptions::default()
};
let md = markdown::to_markdown_from_items_with_rects_and_lines(
page_items,
options,
&page_rects,
&[],
&page_thresholds,
None,
&[],
);
let md = if has_text_quality_issue {
String::new()
} else {
markdown::to_markdown_from_items_with_rects_and_lines(
page_items,
options,
&page_rects,
&[],
&page_thresholds,
None,
&[],
)
};
let needs_ocr = md.trim().is_empty()
let needs_ocr = has_text_quality_issue
|| md.trim().is_empty()
|| has_gid
|| is_garbage_text(&md)
|| is_cid_garbage(&md)
@@ -592,24 +599,23 @@ pub fn extract_text_in_regions_mem(
for rect in regions {
let [rx1, ry1, rx2, ry2] = *rect;
let text = match items {
Some(items) => collect_text_in_region_with_options(
items,
rx1,
ry1,
rx2,
ry2,
page_h,
coords,
adaptive_threshold,
),
None => String::new(),
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
let matched: Vec<TextItem> = match items {
Some(items) => items
.iter()
.filter(|item| region_overlaps_item(item, bounds))
.cloned()
.collect(),
None => Vec::new(),
};
let has_text_quality_issue = region_items_have_decoding_issue(&matched);
let text = collect_text_from_matched_items(matched, adaptive_threshold);
// Check per-region text quality instead of blanket page-level
// GID rejection. A GID font in a logo elsewhere on the page
// shouldn't force GPU OCR for clean text regions.
let needs_ocr = text.trim().is_empty()
let needs_ocr = has_text_quality_issue
|| text.trim().is_empty()
|| is_garbage_text(&text)
|| is_cid_garbage(&text)
|| detect_encoding_issues(&text);
@@ -729,6 +735,14 @@ pub fn extract_tables_in_regions_mem(
continue;
}
if region_items_have_decoding_issue(&matched) {
page_results.push(RegionText {
text: String::new(),
needs_ocr: true,
});
continue;
}
// Compute base_font_size as most common font size in the region
let base_font_size = {
let mut freq: HashMap<i32, usize> = HashMap::new();
@@ -3401,7 +3415,7 @@ fn process_document(
})
.unwrap_or((None, Vec::new()));
let (markdown, layout, has_encoding_issues, gid_pages) = match extracted {
let (markdown, layout, has_encoding_issues, gid_pages, text_quality_pages) = match extracted {
Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => {
// For TextBased PDFs with pages flagged for OCR (Identity-H or
// Type3 fonts without ToUnicode), check whether the CID-as-Unicode
@@ -3451,6 +3465,7 @@ fn process_document(
}
};
let text_quality = analyze_text_quality(&items);
let layout = compute_layout_complexity(&items, &rects, &lines);
let md = if options.mode == ProcessMode::Analyze {
@@ -3467,14 +3482,22 @@ fn process_document(
))
};
let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m));
(md, layout, enc, gid_encoded_pages)
let enc = text_quality.has_encoding_issues
|| md.as_ref().is_some_and(|m| detect_encoding_issues(m));
(
md,
layout,
enc,
gid_encoded_pages,
text_quality.pages_needing_ocr,
)
}
None => (
None,
LayoutComplexity::default(),
false,
std::collections::HashSet::new(),
Vec::new(),
),
};
@@ -3516,6 +3539,18 @@ fn process_document(
}
pages_needing_ocr.sort_unstable();
}
if !text_quality_pages.is_empty() {
log::debug!(
"pages with suspicious text-layer decoding (need OCR): {:?}",
text_quality_pages
);
for page in text_quality_pages {
if !pages_needing_ocr.contains(&page) {
pages_needing_ocr.push(page);
}
}
pages_needing_ocr.sort_unstable();
}
// Detect sparse extraction: when a TEXT-BASED PDF produces very few
// characters per page, the text is likely embedded in images/forms
@@ -3601,6 +3636,104 @@ fn detect_encoding_issues(markdown: &str) -> bool {
false
}
#[derive(Debug, Default)]
struct TextQualityReport {
pages_needing_ocr: Vec<u32>,
has_encoding_issues: bool,
}
fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
let mut pages = HashSet::new();
for item in items {
if !matches!(item.item_type, crate::types::ItemType::Text) {
continue;
}
if text_span_has_decoding_issue(&item.text) {
pages.insert(item.page);
}
}
let mut pages_needing_ocr: Vec<u32> = pages.into_iter().collect();
pages_needing_ocr.sort_unstable();
TextQualityReport {
has_encoding_issues: !pages_needing_ocr.is_empty(),
pages_needing_ocr,
}
}
fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
items.iter().any(|item| {
matches!(item.item_type, crate::types::ItemType::Text)
&& text_span_has_decoding_issue(&item.text)
})
}
fn text_span_has_decoding_issue(text: &str) -> bool {
let text = text.trim();
if text.is_empty() {
return false;
}
detect_encoding_issues(text)
|| has_private_use_text_run(text)
|| is_cid_garbage(text)
|| has_cid_control_token(text)
}
fn has_private_use_text_run(text: &str) -> bool {
let mut total = 0usize;
let mut private_use = 0usize;
let mut current_run = 0usize;
let mut longest_run = 0usize;
for ch in text.chars() {
if ch.is_whitespace() {
current_run = 0;
continue;
}
total += 1;
if is_private_use_char(ch) {
private_use += 1;
current_run += 1;
longest_run = longest_run.max(current_run);
} else {
current_run = 0;
}
}
if private_use == 0 {
return false;
}
longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
}
fn has_cid_control_token(text: &str) -> bool {
text.split_whitespace().any(token_has_cid_control)
}
fn token_has_cid_control(token: &str) -> bool {
let mut total = 0usize;
let mut c1_control = 0usize;
for ch in token.chars() {
total += 1;
if ('\u{0080}'..='\u{009F}').contains(&ch) {
c1_control += 1;
}
}
total >= 5 && c1_control > 0 && c1_control * 20 >= total
}
fn is_private_use_char(ch: char) -> bool {
matches!(
ch as u32,
0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
)
}
/// Check if extracted text is predominantly garbage (non-alphanumeric).
///
/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---."
@@ -5543,6 +5676,13 @@ mod tests {
}
}
fn test_text_item_on_page(page: u32, text: &str) -> TextItem {
TextItem {
page,
..test_item(text, 10.0, 10.0, text.len() as f32 * 5.0, 12.0)
}
}
#[test]
fn test_detect_encoding_issues_fffd() {
assert!(detect_encoding_issues(
@@ -5578,6 +5718,68 @@ mod tests {
assert!(!detect_encoding_issues(text));
}
#[test]
fn test_text_quality_flags_localized_cid_mojibake_span() {
let items = vec![
test_text_item_on_page(
1,
"Waiting Period 等待期 Maternity and newborn infant care benefit",
),
test_text_item_on_page(
1,
"Inpatient and Day-care Benefits DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯",
),
test_text_item_on_page(1, "Covered up to annual maximum. 赔付至年度最高保额。"),
test_text_item_on_page(2, "A clean second page should not be routed to OCR."),
];
let quality = analyze_text_quality(&items);
assert!(quality.has_encoding_issues);
assert_eq!(quality.pages_needing_ocr, vec![1]);
}
#[test]
fn test_text_quality_flags_replacement_and_private_use_runs() {
let items = vec![
test_text_item_on_page(1, "broken \u{FFFD} text"),
test_text_item_on_page(3, "\u{E000}\u{E001}\u{E002}"),
];
let quality = analyze_text_quality(&items);
assert_eq!(quality.pages_needing_ocr, vec![1, 3]);
}
#[test]
fn test_text_quality_allows_clean_multilingual_and_latin1_text() {
let items = vec![
test_text_item_on_page(1, "你好世界,这是一段正常的中文文本。"),
test_text_item_on_page(1, "Résumé déjà vu: façade, São Paulo, año 2026."),
test_text_item_on_page(1, "A single icon \u{E000} should not force OCR."),
];
let quality = analyze_text_quality(&items);
assert!(!quality.has_encoding_issues);
assert!(quality.pages_needing_ocr.is_empty());
}
#[test]
fn test_region_text_quality_is_scoped_to_matched_items() {
let clean_region = vec![
test_text_item_on_page(1, "Clean native text"),
test_text_item_on_page(1, "Résumé déjà vu"),
];
let garbled_region = vec![
test_text_item_on_page(1, "Clean prefix"),
test_text_item_on_page(1, "DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯"),
];
assert!(!region_items_have_decoding_issue(&clean_region));
assert!(region_items_have_decoding_issue(&garbled_region));
}
#[test]
fn test_garbage_text_detection() {
// Simulates garbage output from Identity-H fonts without ToUnicode.
+232 -2
View File
@@ -149,6 +149,79 @@ fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32)
set
}
/// Pre-scan body-size all-bold runs that are too long to be headings.
///
/// Some academic PDFs use an all-bold abstract/summary paragraph immediately
/// after the author block. A line-local bold heading heuristic sees each
/// wrapped visual line as "standalone" once the first line is misclassified,
/// producing a stack of `##` headings. Multi-line body-size bold runs with a
/// paragraph-sized word count should stay paragraph text.
fn find_wrapped_bold_paragraph_lines(
lines: &[TextLine],
base_size: f32,
para_threshold: f32,
) -> HashSet<usize> {
let mut set = HashSet::new();
let mut i = 0usize;
while i < lines.len() {
if !is_body_size_all_bold_line(&lines[i], base_size) {
i += 1;
continue;
}
let start = i;
let mut end = i;
let mut word_count = lines[i].text().split_whitespace().count();
while end + 1 < lines.len()
&& is_body_size_all_bold_line(&lines[end + 1], base_size)
&& is_wrapped_same_style_line(&lines[end], &lines[end + 1], para_threshold)
{
end += 1;
word_count += lines[end].text().split_whitespace().count();
}
let line_count = end - start + 1;
if line_count >= 3 && word_count > 20 {
for idx in start..=end {
set.insert(idx);
}
}
i = end + 1;
}
set
}
fn is_body_size_all_bold_line(line: &TextLine, base_size: f32) -> bool {
let Some(first) = line.items.first() else {
return false;
};
first.font_size >= base_size * 0.95
&& first.font_size < base_size * 1.2
&& line
.items
.iter()
.all(|item| item.is_bold && (item.font_size - first.font_size).abs() < 0.5)
}
fn is_wrapped_same_style_line(prev: &TextLine, next: &TextLine, para_threshold: f32) -> bool {
if prev.page != next.page {
return false;
}
let y_gap = prev.y - next.y;
if !(y_gap > 0.0 && y_gap <= para_threshold) {
return false;
}
let prev_x = prev.items.first().map(|item| item.x).unwrap_or(0.0);
let next_x = next.items.first().map(|item| item.x).unwrap_or(0.0);
(prev_x - next_x).abs() <= 40.0
}
/// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
///
/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
@@ -397,6 +470,8 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
// between paragraphs at body font size. Inspired by opendataloader's
// lookahead in HeadingProcessor (prevNode/nextNode context).
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
let wrapped_bold_paragraph_lines =
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
// Detect struct heading levels that are overused (body text mistagged as headings)
let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
@@ -410,6 +485,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
let mut last_list_x: Option<f32> = None;
let mut in_code_block = false;
let mut prev_had_dot_leaders = false;
let mut paragraph_in_wrapped_bold_run = false;
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
@@ -475,6 +551,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
current_page = line.page;
prev_y = f32::MAX;
prev_x = 0.0;
paragraph_in_wrapped_bold_run = false;
if options.include_page_numbers {
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
@@ -489,6 +566,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push('\n');
output.push_str(table_md);
@@ -506,6 +584,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push('\n');
output.push_str(image_md);
@@ -527,9 +606,18 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
&& y_gap.abs() <= para_threshold
&& (prev_x - line_x).abs() > 50.0
&& prev_y < f32::MAX;
if (is_para_break || is_band_switch) && in_paragraph {
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
let is_bold_to_regular_break = in_paragraph
&& paragraph_in_wrapped_bold_run
&& !line_in_wrapped_bold_run
&& !line_all_bold
&& y_gap > base_size * 1.2
&& y_gap <= para_threshold;
if (is_para_break || is_band_switch || is_bold_to_regular_break) && in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
// Don't immediately end list on paragraph break
// Let the continuation check below decide if we're still in a list
@@ -572,6 +660,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push_str(trimmed);
output.push_str("\n\n");
@@ -625,6 +714,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if !(1..=15).contains(&word_count) {
return None;
}
if wrapped_bold_paragraph_lines.contains(&line_idx) {
return None;
}
let rarity = font_size_rarity(line_font_size, &font_stats);
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
let standalone = !in_paragraph;
@@ -656,6 +748,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
let prefix = "#".repeat(level);
// Use plain text for headers to avoid redundant formatting
@@ -678,6 +771,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push_str(&format!("- {}", trimmed));
output.push('\n');
@@ -691,6 +785,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
let formatted = format_list_item(trimmed);
output.push_str(&formatted);
@@ -737,6 +832,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push_str(&format!("> {}\n", trimmed));
continue;
@@ -747,6 +843,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
if !in_code_block {
output.push_str("```\n");
@@ -767,6 +864,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
}
}
output.push_str(trimmed);
paragraph_in_wrapped_bold_run = if in_paragraph {
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
} else {
line_in_wrapped_bold_run
};
in_paragraph = true;
prev_had_dot_leaders = cur_dot_leaders;
}
@@ -836,6 +938,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
let para_threshold = compute_paragraph_threshold(&lines, base_size);
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
let wrapped_bold_paragraph_lines =
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
let mut output = String::new();
let mut current_page = 0u32;
@@ -844,6 +948,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
let mut in_paragraph = false;
let mut last_list_x: Option<f32> = None;
let mut prev_had_dot_leaders = false;
let mut paragraph_in_wrapped_bold_run = false;
for (line_idx, line) in lines.iter().enumerate() {
// Page break
@@ -860,6 +965,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
in_list = false;
last_list_x = None;
prev_had_dot_leaders = false;
paragraph_in_wrapped_bold_run = false;
if options.include_page_numbers {
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
@@ -870,9 +976,18 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
// (newspaper columns emitted sequentially on the same page).
let y_gap = prev_y - line.y;
let is_para_break = y_gap.abs() > para_threshold;
if is_para_break && in_paragraph {
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
let is_bold_to_regular_break = in_paragraph
&& paragraph_in_wrapped_bold_run
&& !line_in_wrapped_bold_run
&& !line_all_bold
&& y_gap > base_size * 1.2
&& y_gap <= para_threshold;
if (is_para_break || is_bold_to_regular_break) && in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
// Don't immediately end list on paragraph break
// Let the continuation check below decide if we're still in a list
@@ -896,6 +1011,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
output.push_str(trimmed);
output.push_str("\n\n");
@@ -918,6 +1034,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if !(1..=15).contains(&word_count) {
return None;
}
if wrapped_bold_paragraph_lines.contains(&line_idx) {
return None;
}
let rarity = font_size_rarity(line_font_size, &font_stats);
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
let standalone = !in_paragraph;
@@ -935,6 +1054,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
let prefix = "#".repeat(header_level);
// Use plain text for headers to avoid redundant formatting
@@ -949,6 +1069,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
let formatted = format_list_item(trimmed);
output.push_str(&formatted);
@@ -993,6 +1114,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
// Use plain text for code blocks
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
@@ -1010,6 +1132,11 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
}
}
output.push_str(trimmed);
paragraph_in_wrapped_bold_run = if in_paragraph {
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
} else {
line_in_wrapped_bold_run
};
in_paragraph = true;
prev_had_dot_leaders = cur_dot_leaders;
}
@@ -1356,6 +1483,109 @@ mod tests {
);
}
#[test]
fn test_wrapped_bold_abstract_is_not_split_into_headings() {
// Regression for arXiv 1107.1353: the opening abstract paragraph is
// entirely bold at body size. The first wrapped lines used to become
// separate H2 headings, and the following body paragraph was joined to
// the bold abstract because the paragraph gap is modest.
let make = |text: &str, y: f32, font_size: f32, bold: bool| {
let mut item = make_item(text, 1, None);
item.y = y;
item.font_size = font_size;
item.height = font_size;
item.is_bold = bold;
item
};
let lines = vec![
make_line(vec![make(
"Quantum Nature of Light Measured With a Single Detector",
747.7,
25.0,
true,
)]),
make_line(vec![make(
"Gesine A. Steudle1*, Stefan Schietinger1, David Höckel1",
651.1,
11.0,
false,
)]),
make_line(vec![make(
"Zwiller2, and Oliver Benson1",
638.5,
11.0,
false,
)]),
make_line(vec![make(
"The introduction of light quanta by Einstein in 1905 triggered strong efforts to",
607.5,
11.0,
true,
)]),
make_line(vec![make(
"demonstrate the quantum properties of light directly, without involving matter",
594.8,
11.0,
true,
)]),
make_line(vec![make(
"quantization. It however took more than seven decades for the quantum granularity",
582.2,
11.0,
true,
)]),
make_line(vec![make(
"of light to be observed in the fluorescence of single atoms. Single atoms emit",
569.5,
11.0,
true,
)]),
make_line(vec![make(
"photons one at a time, this is typically demonstrated with a Hanbury-Brown-Twiss",
556.9,
11.0,
true,
)]),
make_line(vec![make(
"Our work significantly simplifies a widely used photon-correlation technique.",
544.2,
11.0,
true,
)]),
make_line(vec![make(
"A photon is a single excitation of a mode of the electromagnetic field.",
528.7,
11.0,
false,
)]),
];
let md = to_markdown_from_lines_with_tables_and_images(
lines,
MarkdownOptions::default(),
HashMap::new(),
HashMap::new(),
&std::collections::HashSet::new(),
None,
);
assert!(
md.contains("# Quantum Nature of Light Measured With a Single Detector"),
"title should remain a heading: {md}"
);
assert!(
!md.contains("## The introduction")
&& !md.contains("## demonstrate")
&& !md.contains("## quantization"),
"bold abstract lines should not become headings: {md}"
);
assert!(
md.contains("technique.**\n\nA photon is a single excitation"),
"body paragraph should be separated from bold abstract: {md}"
);
}
#[test]
fn test_struct_role_code_multiline_accumulation() {
let mut line1 = make_item("fn main() {", 1, Some(0));
+27
View File
@@ -29,6 +29,7 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
// text item, which combine with gap-based space insertion to produce
// double spaces ("Vice President" instead of "Vice President").
collapse_consecutive_spaces(&mut text);
remove_spaces_before_closing_brackets(&mut text);
// Remove excessive newlines (more than 2 in a row)
while text.contains("\n\n\n") {
@@ -71,6 +72,20 @@ fn collapse_consecutive_spaces(text: &mut String) {
*text = result;
}
/// Remove spaces before closing square brackets.
/// Unit markers and markdown links occasionally pick up a gap-inserted space
/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding.
fn remove_spaces_before_closing_brackets(text: &mut String) {
let mut result = String::with_capacity(text.len());
for ch in text.chars() {
if ch == ']' && result.ends_with(' ') {
result.pop();
}
result.push(ch);
}
*text = result;
}
/// Collapse dot leaders (runs of 4+ dots) into " ... "
/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
fn collapse_dot_leaders(text: &str) -> String {
@@ -342,6 +357,18 @@ mod tests {
assert!(result.contains("Chapter 2 ... 20"));
}
// --- remove_spaces_before_closing_brackets ---
#[test]
fn test_remove_spaces_before_closing_brackets() {
let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string();
remove_spaces_before_closing_brackets(&mut input);
assert_eq!(
input,
"Density [kg/m3] and [linked text](https://example.com)"
);
}
// --- fix_hyphenation ---
#[test]
+569 -51
View File
@@ -566,7 +566,7 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
})
.collect();
if page_items.len() < 4 {
if page_items.len() < 2 {
return None;
}
@@ -575,15 +575,12 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
.max(1.0);
let y_tol = (median_font_size * 0.75).clamp(4.0, 9.0);
let rows = group_key_value_visual_rows(page_items, y_tol);
if rows.len() < 2 || rows.len() > 80 {
if rows.is_empty() || rows.len() > 80 {
return None;
}
let split_x = infer_key_value_split_x(&rows, median_font_size)?;
let mut kv_rows: Vec<KeyValueRow> = Vec::new();
let mut paired_rows = 0usize;
let mut section_rows = 0usize;
let mut left_label_like = 0usize;
let mut left_starts = Vec::new();
let mut right_starts = Vec::new();
@@ -609,18 +606,12 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
item_indices.dedup();
if !left.is_empty() && !right.is_empty() {
paired_rows += 1;
if looks_like_key_value_label(&left) {
left_label_like += 1;
}
if let Some(x) = left_items.first().map(|ri| ri.item.x) {
left_starts.push(x);
}
if let Some(x) = right_items.first().map(|ri| ri.item.x) {
right_starts.push(x);
}
} else if !left.is_empty() {
section_rows += 1;
}
kv_rows.push(KeyValueRow {
@@ -631,11 +622,75 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
});
}
if kv_rows.len() < 2 || paired_rows < 2 {
if kv_rows.is_empty() {
return None;
}
let raw_left_only_rows = kv_rows
.iter()
.filter(|row| !row.left.is_empty() && row.right.is_empty())
.count();
let raw_right_only_rows = kv_rows
.iter()
.filter(|row| row.left.is_empty() && !row.right.is_empty())
.count();
let edgar_tag_rows = key_value_rows_look_like_edgar_tags(&kv_rows);
if edgar_tag_rows {
kv_rows.retain(|row| !row.right.is_empty() || !is_edgar_table_boundary_cell(&row.left));
}
let header_inferred = !edgar_tag_rows && key_value_first_pair_is_header(&kv_rows);
kv_rows = normalize_key_value_rows(kv_rows, header_inferred);
let paired_rows = kv_rows
.iter()
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
.count();
let section_rows = kv_rows
.iter()
.filter(|row| !row.left.is_empty() && row.right.is_empty())
.count();
let dangling_right_rows = kv_rows
.iter()
.filter(|row| row.left.is_empty() && !row.right.is_empty())
.count();
let left_label_like = kv_rows
.iter()
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
.filter(|row| looks_like_key_value_label(&row.left))
.count();
if paired_rows < 1 {
return None;
}
if dangling_right_rows > 0 {
return None;
}
let left_x = median_f32(left_starts).unwrap_or_else(|| {
rows.iter()
.flat_map(|row| row.items.iter().map(|ri| ri.item.x))
.fold(f32::INFINITY, f32::min)
});
let right_x = median_f32(right_starts).unwrap_or(split_x);
if !left_x.is_finite() || !right_x.is_finite() || right_x - left_x < 40.0 {
return None;
}
let single_pair_allowed = key_value_single_pair_allowed(
KeyValueSinglePairStats {
paired_rows,
section_rows,
raw_left_only_rows,
raw_right_only_rows,
},
&kv_rows,
header_inferred,
left_x,
right_x,
);
if (kv_rows.len() < 2 || paired_rows < 2) && !single_pair_allowed {
return None;
}
let header_inferred = key_value_first_pair_is_header(&kv_rows);
let data_pairs = if header_inferred {
paired_rows.saturating_sub(1)
} else {
@@ -645,7 +700,7 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
return None;
}
if section_rows > paired_rows * 2 + 2 {
if section_rows > paired_rows * 2 + 2 && !single_pair_allowed {
return None;
}
@@ -659,28 +714,25 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
} else {
left_label_like
};
if label_rows_for_score >= 2 && label_like_for_score * 2 < label_rows_for_score {
return None;
}
let left_x = median_f32(left_starts).unwrap_or_else(|| {
rows.iter()
.flat_map(|row| row.items.iter().map(|ri| ri.item.x))
.fold(f32::INFINITY, f32::min)
});
let right_x = median_f32(right_starts).unwrap_or(split_x);
if !left_x.is_finite() || !right_x.is_finite() || right_x - left_x < 40.0 {
return None;
}
let right_cluster_count = significant_side_x_clusters(&rows, split_x, false);
let marker_rows = marker_matrix_value_rows(&kv_rows);
if (right_cluster_count >= 5 && paired_rows >= 3)
|| (right_cluster_count >= 3 && marker_rows >= 3 && marker_rows * 2 >= paired_rows)
if !header_inferred
&& !edgar_tag_rows
&& label_rows_for_score >= 2
&& label_like_for_score * 2 < label_rows_for_score
{
return None;
}
if key_value_rows_look_like_prose(&kv_rows, header_inferred) {
let right_cluster_count = significant_side_x_clusters(&rows, split_x, false);
let marker_rows = marker_matrix_value_rows(&kv_rows);
if !single_pair_allowed
&& !edgar_tag_rows
&& ((right_cluster_count >= 5 && paired_rows >= 3)
|| (right_cluster_count >= 3 && marker_rows >= 3 && marker_rows * 2 >= paired_rows))
{
return None;
}
if !edgar_tag_rows && key_value_rows_look_like_prose(&kv_rows, header_inferred) {
return None;
}
@@ -765,6 +817,82 @@ struct KeyValueRow {
item_indices: Vec<usize>,
}
#[derive(Debug, Clone, Copy)]
struct KeyValueSinglePairStats {
paired_rows: usize,
section_rows: usize,
raw_left_only_rows: usize,
raw_right_only_rows: usize,
}
fn normalize_key_value_rows(rows: Vec<KeyValueRow>, header_inferred: bool) -> Vec<KeyValueRow> {
let mut normalized: Vec<KeyValueRow> = Vec::with_capacity(rows.len());
for row in rows {
if row.left.is_empty() && row.right.is_empty() {
continue;
}
if row.left.is_empty() && !row.right.is_empty() {
if let Some(last) = normalized.last_mut() {
if !last.right.is_empty() {
append_key_value_text(&mut last.right, &row.right);
last.item_indices.extend(row.item_indices);
continue;
}
}
normalized.push(row);
continue;
}
if !row.left.is_empty() && row.right.is_empty() {
let normalized_len = normalized.len();
if let Some(last) = normalized.last_mut() {
let last_is_header = header_inferred && normalized_len == 1;
if !last_is_header
&& !last.left.is_empty()
&& !last.right.is_empty()
&& key_value_left_continuation_allowed(&last.left, &row.left)
{
append_key_value_text(&mut last.left, &row.left);
last.item_indices.extend(row.item_indices);
continue;
}
}
}
normalized.push(row);
}
normalized
}
fn append_key_value_text(target: &mut String, addition: &str) {
let addition = addition.trim();
if addition.is_empty() {
return;
}
if !target.trim().is_empty() {
target.push(' ');
}
target.push_str(addition);
}
fn key_value_left_continuation_allowed(previous_left: &str, continuation: &str) -> bool {
let trimmed = continuation.trim();
if trimmed.is_empty() || looks_like_key_value_section_label(trimmed) {
return false;
}
let previous = previous_left.trim_end();
let continuation_chars = trimmed.chars().count();
let continuation_words = word_count_simple(trimmed);
previous.ends_with(['-', '/', ',', ';', ':'])
|| first_alpha_is_lowercase(trimmed)
|| continuation_chars > 28
|| continuation_words > 4
}
fn group_key_value_visual_rows(mut items: Vec<RowItem>, y_tol: f32) -> Vec<VisualRow> {
items.sort_by(|a, b| {
b.item
@@ -828,6 +956,13 @@ fn infer_key_value_split_x(rows: &[VisualRow], median_font_size: f32) -> Option<
}
if splits.len() < 2 {
let paired_visual_rows = rows.iter().filter(|row| row.items.len() >= 2).count();
if splits.len() == 1
&& paired_visual_rows == 1
&& (rows.len() == 1 || rows.iter().all(|row| row.items.len() <= 2))
{
return splits.into_iter().next();
}
return None;
}
@@ -902,16 +1037,169 @@ fn looks_like_key_value_label(cell: &str) -> bool {
trimmed.chars().any(|c| c.is_alphabetic())
}
fn key_value_rows_look_like_edgar_tags(rows: &[KeyValueRow]) -> bool {
let paired_rows = rows
.iter()
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
.count();
if paired_rows < 2 {
return false;
}
let tag_pairs = rows
.iter()
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
.filter(|row| is_edgar_tag_cell(&row.left))
.count();
let first_marker = rows.first().is_some_and(|row| {
row.left.eq_ignore_ascii_case("<S>") && row.right.eq_ignore_ascii_case("<C>")
});
tag_pairs >= 3 || (first_marker && tag_pairs >= 2)
}
fn is_edgar_tag_cell(cell: &str) -> bool {
let trimmed = cell.trim();
let Some(inner) = trimmed.strip_prefix('<').and_then(|s| s.strip_suffix('>')) else {
return false;
};
!inner.is_empty()
&& inner.len() <= 48
&& inner
.chars()
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_'))
}
fn is_edgar_table_boundary_cell(cell: &str) -> bool {
let trimmed = cell.trim();
trimmed.eq_ignore_ascii_case("<TABLE>") || trimmed.eq_ignore_ascii_case("</TABLE>")
}
fn key_value_single_pair_allowed(
stats: KeyValueSinglePairStats,
rows: &[KeyValueRow],
header_inferred: bool,
left_x: f32,
right_x: f32,
) -> bool {
if header_inferred || stats.paired_rows != 1 || stats.section_rows != 0 || rows.len() != 1 {
return false;
}
if right_x - left_x < 60.0 {
return false;
}
let Some(row) = rows
.iter()
.find(|row| !row.left.is_empty() && !row.right.is_empty())
else {
return false;
};
let left_chars = row.left.chars().count();
let right_chars = row.right.chars().count();
if !(2..=120).contains(&left_chars) || right_chars == 0 {
return false;
}
if key_value_cell_looks_like_sentence(&row.left) {
return false;
}
if stats.raw_left_only_rows == 0
&& stats.raw_right_only_rows >= 2
&& left_chars <= 70
&& right_chars <= 1_500
&& looks_like_key_value_label(&row.left)
{
return true;
}
if right_chars > 80 {
return false;
}
if key_value_cell_looks_like_sentence(&row.right) && !compact_key_value_scalar(&row.right) {
return false;
}
(looks_like_key_value_label(&row.left) || left_chars <= 90)
&& compact_key_value_scalar(&row.right)
}
fn compact_key_value_scalar(cell: &str) -> bool {
let trimmed = cell.trim();
let chars = trimmed.chars().count();
let words = word_count_simple(trimmed);
if trimmed.is_empty() || chars > 60 || words > 6 || trimmed.ends_with(['.', '!', '?']) {
return false;
}
let lower = trimmed.to_ascii_lowercase();
trimmed.chars().any(|ch| ch.is_ascii_digit())
|| matches!(
lower.as_str(),
"yes" | "no" | "true" | "false" | "none" | "n/a" | "na"
)
|| words <= 4
}
fn looks_like_key_value_section_label(cell: &str) -> bool {
let trimmed = cell.trim();
let chars = trimmed.chars().count();
let words = word_count_simple(trimmed);
if !(1..=5).contains(&words) || !(2..=48).contains(&chars) {
return false;
}
if trimmed.ends_with(['.', ',', ';', ':']) || first_alpha_is_lowercase(trimmed) {
return false;
}
if trimmed
.chars()
.any(|ch| matches!(ch, '.' | ',' | ';' | '(' | ')' | '[' | ']'))
{
return false;
}
trimmed.chars().any(|ch| ch.is_alphabetic())
}
fn first_alpha_is_lowercase(cell: &str) -> bool {
cell.chars()
.find(|ch| ch.is_alphabetic())
.is_some_and(|ch| ch.is_lowercase())
}
fn key_value_cell_looks_like_sentence(cell: &str) -> bool {
let trimmed = cell.trim();
let chars = trimmed.chars().count();
chars > 90
|| word_count_simple(trimmed) > 12
|| (chars > 42 && trimmed.ends_with(['.', '!', '?']))
}
fn key_value_rows_look_like_prose(rows: &[KeyValueRow], header_inferred: bool) -> bool {
let mut long_sentence_cells = 0usize;
let mut total_cells = 0usize;
let mut total_chars = 0usize;
let mut left_cells = 0usize;
let mut left_prose_cells = 0usize;
let mut left_label_like = 0usize;
let mut total_left_chars = 0usize;
let mut paired_rows = 0usize;
let mut paired_sentence_rows = 0usize;
let mut solo_prose_rows = 0usize;
for row in rows.iter().skip(usize::from(header_inferred)) {
if !row.left.is_empty() && !row.right.is_empty() {
paired_rows += 1;
let left = row.left.trim();
let right = row.right.trim();
let left_prose = key_value_cell_looks_like_sentence(left);
let right_prose = key_value_cell_looks_like_sentence(right);
left_cells += 1;
total_left_chars += left.chars().count();
if looks_like_key_value_label(left) {
left_label_like += 1;
}
if left_prose {
left_prose_cells += 1;
}
if left_prose && right_prose {
paired_sentence_rows += 1;
}
} else {
let solo = if row.left.is_empty() {
row.right.trim()
@@ -925,30 +1213,23 @@ fn key_value_rows_look_like_prose(rows: &[KeyValueRow], header_inferred: bool) -
solo_prose_rows += 1;
}
}
for cell in [&row.left, &row.right] {
let trimmed = cell.trim();
if trimmed.is_empty() {
continue;
}
total_cells += 1;
total_chars += trimmed.chars().count();
if trimmed.chars().count() > 100
|| (trimmed.chars().count() > 55 && trimmed.ends_with(['.', '!', '?']))
{
long_sentence_cells += 1;
}
}
}
if paired_rows < 1 || total_cells == 0 {
if paired_rows < 1 || left_cells == 0 {
return true;
}
if solo_prose_rows >= 3 {
return true;
}
if paired_rows >= 2 && paired_sentence_rows * 2 >= paired_rows {
return true;
}
if !header_inferred && left_prose_cells * 2 >= left_cells {
return true;
}
let avg_chars = total_chars as f32 / total_cells as f32;
avg_chars > 75.0 || long_sentence_cells * 2 >= total_cells
let avg_left_chars = total_left_chars as f32 / left_cells as f32;
!header_inferred && avg_left_chars > 70.0 && left_label_like * 2 < left_cells
}
fn marker_matrix_value_rows(rows: &[KeyValueRow]) -> usize {
@@ -1354,6 +1635,243 @@ mod tests {
assert!(md.contains("|Engine Code|1ZR-FAE|"), "{md}");
}
#[test]
fn test_key_value_builder_merges_wrapped_value_continuations() {
let items = vec![
make_char("Storage", 80.0, 700.0, 9.0, 42.0),
make_char(
"Store under normal conditions in dry rooms.",
250.0,
700.0,
9.0,
210.0,
),
make_char(
"Protect from heat and humidity in the original packaging material.",
250.0,
686.0,
9.0,
315.0,
),
make_char("Shelf Life", 80.0, 668.0, 9.0, 48.0),
make_char(
"To obtain best performance use within 24 months.",
250.0,
668.0,
9.0,
255.0,
),
make_char("Technical Information", 80.0, 650.0, 9.0, 104.0),
make_char(
"The product is designed for repeated industrial use and long service life.",
250.0,
650.0,
9.0,
340.0,
),
make_char(
"Additional details are provided for compatibility and installation planning.",
250.0,
636.0,
9.0,
350.0,
),
];
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
let md = table_to_markdown(&table);
assert!(md.contains("|Field|Value|"), "{md}");
assert!(
md.contains(
"|Storage|Store under normal conditions in dry rooms. Protect from heat and humidity in the original packaging material.|"
),
"{md}"
);
assert!(
md.contains(
"|Technical Information|The product is designed for repeated industrial use and long service life. Additional details are provided for compatibility and installation planning.|"
),
"{md}"
);
}
#[test]
fn test_key_value_builder_merges_wrapped_left_labels() {
let items = vec![
make_char("Title/Description", 76.0, 700.0, 9.0, 86.0),
make_char("Instances", 350.0, 700.0, 9.0, 48.0),
make_char("RE: Homes Gerald Ford lived in.", 76.0, 682.0, 9.0, 150.0),
make_char("Box 7", 350.0, 682.0, 9.0, 28.0),
make_char(
"Grand Rapids Remembers Gerald R. Ford issue. Grand",
76.0,
664.0,
9.0,
245.0,
),
make_char("Box 7", 350.0, 664.0, 9.0, 28.0),
make_char(
"Rapids Magazine, September 1987, p. 65.",
76.0,
650.0,
9.0,
196.0,
),
make_char(
"A Workhorse not a show horse: Gerald Ford remembered as humble.",
76.0,
632.0,
9.0,
290.0,
),
make_char("Box 7", 350.0, 632.0, 9.0, 28.0),
make_char(
"not flashy during his public life.",
76.0,
618.0,
9.0,
150.0,
),
];
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
let md = table_to_markdown(&table);
assert!(md.starts_with("|Title/Description|Instances|"), "{md}");
assert!(
md.contains(
"|Grand Rapids Remembers Gerald R. Ford issue. Grand Rapids Magazine, September 1987, p. 65.|Box 7|"
),
"{md}"
);
assert!(
md.contains(
"|A Workhorse not a show horse: Gerald Ford remembered as humble. not flashy during his public life.|Box 7|"
),
"{md}"
);
}
#[test]
fn test_key_value_builder_allows_tiny_two_cell_region() {
let items = vec![
make_char(
"3M E-A-R Classic Small Earplug Uncorded",
80.0,
700.0,
9.0,
210.0,
),
make_char("02/05/24", 360.0, 700.0, 9.0, 42.0),
];
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
let md = table_to_markdown(&table);
assert!(md.contains("|Field|Value|"), "{md}");
assert!(
md.contains("|3M E-A-R Classic Small Earplug Uncorded|02/05/24|"),
"{md}"
);
}
#[test]
fn test_key_value_builder_allows_single_wrapped_value_region() {
let items = vec![
make_char("Intrinsic Safety", 42.0, 174.0, 9.0, 60.0),
make_char(
"The powered air purifying respirator has been tested and classified",
311.0,
174.0,
9.0,
260.0,
),
make_char(
"for intrinsic safety in hazardous locations by Underwriters Laboratory",
311.0,
160.0,
9.0,
270.0,
),
make_char(
"for the following classes, divisions, groups, and temperature ratings.",
311.0,
146.0,
9.0,
275.0,
),
];
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
let md = table_to_markdown(&table);
assert!(md.contains("|Field|Value|"), "{md}");
assert!(
md.contains(
"|Intrinsic Safety|The powered air purifying respirator has been tested and classified for intrinsic safety in hazardous locations by Underwriters Laboratory for the following classes, divisions, groups, and temperature ratings.|"
),
"{md}"
);
}
#[test]
fn test_key_value_builder_rejects_leading_value_only_prose() {
let items = vec![
make_char(
"3rd Party Authorization documenting the reason for the hardship.",
260.0,
714.0,
9.0,
310.0,
),
make_char("Borrower", 80.0, 696.0, 9.0, 44.0),
make_char(
"Homeowner has adequate income to support modified payments.",
260.0,
696.0,
9.0,
300.0,
),
make_char("Servicer", 80.0, 678.0, 9.0, 42.0),
make_char(
"Collects documentation and reviews hardship status.",
260.0,
678.0,
9.0,
260.0,
),
];
assert!(try_build_key_value_table_from_rows(&items, 1).is_none());
}
#[test]
fn test_key_value_builder_recovers_edgar_tag_value_rows() {
let items = vec![
make_char("<S>", 70.0, 700.0, 9.0, 18.0),
make_char("<C>", 240.0, 700.0, 9.0, 18.0),
make_char("<PERIOD-TYPE>", 70.0, 684.0, 9.0, 78.0),
make_char("3-MOS", 240.0, 684.0, 9.0, 30.0),
make_char("<FISCAL-YEAR-END>", 70.0, 668.0, 9.0, 104.0),
make_char("DEC-31-2000", 240.0, 668.0, 9.0, 66.0),
make_char("<PERIOD-END>", 70.0, 652.0, 9.0, 76.0),
make_char("MAR-31-2000", 240.0, 652.0, 9.0, 66.0),
make_char("<CASH>", 70.0, 636.0, 9.0, 38.0),
make_char("214", 240.0, 636.0, 9.0, 18.0),
make_char("</TABLE>", 70.0, 620.0, 9.0, 46.0),
];
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
let md = table_to_markdown(&table);
assert!(md.starts_with("|Field|Value|"), "{md}");
assert!(md.contains("|<S>|<C>|"), "{md}");
assert!(md.contains("|<FISCAL-YEAR-END>|DEC-31-2000|"), "{md}");
assert!(md.contains("|<CASH>|214|"), "{md}");
assert!(!md.contains("</TABLE>"), "{md}");
}
#[test]
fn test_key_value_builder_rejects_split_prose() {
let items = vec![