Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
82c0a758cd | ||
|
Abimael Martell
|
1f6497197a |
@@ -1,87 +0,0 @@
|
||||
name: Publish Rust crate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['Cargo.toml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
|
||||
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
|
||||
"https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/crate-version.json
|
||||
echo "Unexpected crates.io response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish:
|
||||
name: Publish to crates.io
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
environment: crates-io
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Verify package
|
||||
run: cargo publish --dry-run
|
||||
|
||||
- name: Authenticate with crates.io
|
||||
id: auth
|
||||
uses: rust-lang/crates-io-auth-action@v1
|
||||
|
||||
- name: Publish crate
|
||||
run: cargo publish
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.2"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2.0"
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# pdf-inspector
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
@@ -74,17 +71,9 @@ console.log(result.markdown); // Markdown string or null
|
||||
|
||||
### Rust
|
||||
|
||||
Install from [crates.io](https://crates.io/crates/pdf-inspector):
|
||||
|
||||
```bash
|
||||
cargo add pdf-inspector
|
||||
```
|
||||
|
||||
Or add it manually:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-inspector = "0.1"
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
```
|
||||
|
||||
```rust
|
||||
@@ -102,34 +91,29 @@ if let Some(markdown) = &result.markdown {
|
||||
### CLI
|
||||
|
||||
```bash
|
||||
# Install the CLI tools
|
||||
cargo install pdf-inspector
|
||||
|
||||
# Convert PDF to Markdown
|
||||
pdf2md document.pdf
|
||||
cargo run --bin pdf2md -- document.pdf
|
||||
|
||||
# JSON output (for piping)
|
||||
pdf2md document.pdf --json
|
||||
cargo run --bin pdf2md -- document.pdf --json
|
||||
|
||||
# Raw markdown only (no headers)
|
||||
pdf2md document.pdf --raw
|
||||
cargo run --bin pdf2md -- document.pdf --raw
|
||||
|
||||
# Insert page break markers (<!-- Page N -->)
|
||||
pdf2md document.pdf --pages
|
||||
cargo run --bin pdf2md -- document.pdf --pages
|
||||
|
||||
# Process only specific pages
|
||||
pdf2md document.pdf --select-pages 1,3,5-10
|
||||
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
|
||||
|
||||
# Detection only (no extraction)
|
||||
detect-pdf document.pdf
|
||||
detect-pdf document.pdf --json
|
||||
cargo run --bin detect-pdf -- document.pdf
|
||||
cargo run --bin detect-pdf -- document.pdf --json
|
||||
|
||||
# Detection + layout analysis (tables, columns)
|
||||
detect-pdf document.pdf --analyze --json
|
||||
cargo run --bin detect-pdf -- document.pdf --analyze --json
|
||||
```
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
# Publishing
|
||||
|
||||
The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
|
||||
|
||||
## crates.io Trusted Publisher
|
||||
|
||||
Configure the trusted publisher for the `pdf-inspector` crate with:
|
||||
|
||||
- Repository: `firecrawl/pdf-inspector`
|
||||
- Workflow: `publish-crate.yml`
|
||||
- Environment: `crates-io`
|
||||
|
||||
The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
|
||||
|
||||
## Release Steps
|
||||
|
||||
1. Update `version` in `Cargo.toml`.
|
||||
2. Merge the version bump to `main`.
|
||||
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
|
||||
|
||||
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
|
||||
Generated
+4
-5
@@ -672,9 +672,8 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d"
|
||||
version = "0.40.0"
|
||||
source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"bitflags",
|
||||
@@ -830,7 +829,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.1"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"log",
|
||||
@@ -845,7 +844,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.1"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.1"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.7",
|
||||
"version": "1.9.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -188,17 +188,7 @@ pub(crate) fn extract_page_text_items(
|
||||
// Graphics state tracking
|
||||
let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
|
||||
let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
|
||||
#[derive(Clone)]
|
||||
struct SavedGraphicsState {
|
||||
ctm: [f32; 6],
|
||||
text_rendering_mode: i32,
|
||||
char_spacing: f32,
|
||||
word_spacing: f32,
|
||||
text_leading: f32,
|
||||
current_font: String,
|
||||
current_font_size: f32,
|
||||
}
|
||||
let mut gstate_stack: Vec<SavedGraphicsState> = Vec::new();
|
||||
let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();
|
||||
|
||||
// Text state tracking
|
||||
let mut current_font = String::new();
|
||||
@@ -237,26 +227,15 @@ pub(crate) fn extract_page_text_items(
|
||||
match op.operator.as_str() {
|
||||
"q" => {
|
||||
// Save graphics state
|
||||
gstate_stack.push(SavedGraphicsState {
|
||||
ctm,
|
||||
text_rendering_mode,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
text_leading,
|
||||
current_font: current_font.clone(),
|
||||
current_font_size,
|
||||
});
|
||||
gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
|
||||
}
|
||||
"Q" => {
|
||||
// Restore graphics state
|
||||
if let Some(saved) = gstate_stack.pop() {
|
||||
ctm = saved.ctm;
|
||||
text_rendering_mode = saved.text_rendering_mode;
|
||||
char_spacing = saved.char_spacing;
|
||||
word_spacing = saved.word_spacing;
|
||||
text_leading = saved.text_leading;
|
||||
current_font = saved.current_font;
|
||||
current_font_size = saved.current_font_size;
|
||||
if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
|
||||
ctm = saved_ctm;
|
||||
text_rendering_mode = saved_tr;
|
||||
char_spacing = saved_tc;
|
||||
word_spacing = saved_tw;
|
||||
}
|
||||
}
|
||||
"cm" => {
|
||||
@@ -1307,91 +1286,6 @@ mod tests {
|
||||
assert!(lines.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_q_restores_current_font_for_text_decoding() {
|
||||
use crate::tounicode::FontCMaps;
|
||||
use lopdf::{dictionary, Object, Stream};
|
||||
|
||||
fn cmap_stream(dst_hex: &str) -> Stream {
|
||||
let cmap = format!(
|
||||
r#"/CIDInit /ProcSet findresource begin
|
||||
12 dict begin
|
||||
begincmap
|
||||
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
|
||||
/CMapName /Test-UCS def
|
||||
/CMapType 2 def
|
||||
1 begincodespacerange
|
||||
<00> <FF>
|
||||
endcodespacerange
|
||||
1 beginbfchar
|
||||
<41> <{dst_hex}>
|
||||
endbfchar
|
||||
endcmap
|
||||
CMapName currentdict /CMap defineresource pop
|
||||
end
|
||||
end"#
|
||||
);
|
||||
Stream::new(dictionary! {}, cmap.into_bytes())
|
||||
}
|
||||
|
||||
let mut doc = lopdf::Document::new();
|
||||
let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X
|
||||
let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y
|
||||
let f1 = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
"ToUnicode" => Object::Reference(f1_cmap),
|
||||
});
|
||||
let f2 = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
"ToUnicode" => Object::Reference(f2_cmap),
|
||||
});
|
||||
|
||||
let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET
|
||||
q
|
||||
BT /F2 12 Tf 20 700 Tm <41> Tj ET
|
||||
Q
|
||||
BT 30 700 Tm <41> Tj ET";
|
||||
let content_id = doc.add_object(Object::Stream(Stream::new(
|
||||
dictionary! {},
|
||||
content.to_vec(),
|
||||
)));
|
||||
let page_id = doc.add_object(dictionary! {
|
||||
"Type" => "Page",
|
||||
"Contents" => Object::Reference(content_id),
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => Object::Reference(f1),
|
||||
"F2" => Object::Reference(f2),
|
||||
},
|
||||
},
|
||||
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
|
||||
});
|
||||
let pages_id = doc.add_object(dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Count" => Object::Integer(1),
|
||||
"Kids" => vec![Object::Reference(page_id)],
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => Object::Reference(pages_id),
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let text = items
|
||||
.iter()
|
||||
.map(|item| item.text.as_str())
|
||||
.collect::<String>();
|
||||
|
||||
assert_eq!(text, "XYX");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_strip_pdf_comments() {
|
||||
// Basic comment stripping
|
||||
|
||||
+21
-7
@@ -858,13 +858,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
// unmapped. Don't fall through to text-interpretation fallbacks
|
||||
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
|
||||
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
|
||||
// Try our custom encoding map from Differences arrays.
|
||||
// The Differences array overrides specific codes in a base encoding (typically
|
||||
@@ -973,6 +966,27 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
|
||||
+18
-80
@@ -1262,9 +1262,9 @@ pub(crate) fn group_into_lines_with_thresholds(
|
||||
|
||||
if is_newspaper {
|
||||
// Newspaper: columns are independent text flows.
|
||||
// 1. Split each column into its densest cluster (core) and stragglers.
|
||||
// 2. Promote only true top matter above all column starts.
|
||||
// 3. Emit: above items → each complete column flow → below spanning items.
|
||||
// 1. Split each column into its densest cluster (core) and stragglers
|
||||
// 2. Use core columns to determine the above/below threshold
|
||||
// 3. Emit: above items → core columns sequentially → below items
|
||||
let mut core_columns: Vec<Vec<TextLine>> = Vec::new();
|
||||
let mut col_stragglers: Vec<Vec<TextLine>> = Vec::new();
|
||||
for col in per_column_lines {
|
||||
@@ -1273,15 +1273,12 @@ pub(crate) fn group_into_lines_with_thresholds(
|
||||
col_stragglers.push(stragglers);
|
||||
}
|
||||
|
||||
// Use the highest core top as the top-matter cutoff. A column
|
||||
// that starts lower because of a figure or large gap should not
|
||||
// pull another column's opening prose above the abstract/title
|
||||
// area; those fragments still belong to that column's flow.
|
||||
let top_matter_cutoff = core_columns
|
||||
// col_top = min of max Y across core columns
|
||||
let col_top = core_columns
|
||||
.iter()
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(|c| c.iter().map(|l| l.y).fold(f32::NEG_INFINITY, f32::max))
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
.fold(f32::INFINITY, f32::min);
|
||||
let margin = 5.0;
|
||||
|
||||
let mut above: Vec<TextLine> = Vec::new();
|
||||
@@ -1289,37 +1286,37 @@ pub(crate) fn group_into_lines_with_thresholds(
|
||||
|
||||
// Spanning items: above or below the column region
|
||||
for line in spanning_lines {
|
||||
if line.y > top_matter_cutoff + margin {
|
||||
if line.y > col_top + margin {
|
||||
above.push(line);
|
||||
} else {
|
||||
below_spanning.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
let mut column_flows: Vec<Vec<TextLine>> = Vec::with_capacity(core_columns.len());
|
||||
for (ci, mut flow) in core_columns.into_iter().enumerate() {
|
||||
let stragglers = col_stragglers
|
||||
.get_mut(ci)
|
||||
.map(std::mem::take)
|
||||
.unwrap_or_default();
|
||||
// Column stragglers above col_top go to "above";
|
||||
// below col_top they stay with their column to avoid
|
||||
// re-interleaving when sorted by Y.
|
||||
let mut col_below: Vec<Vec<TextLine>> = vec![Vec::new(); core_columns.len()];
|
||||
for (ci, stragglers) in col_stragglers.into_iter().enumerate() {
|
||||
for line in stragglers {
|
||||
if line.y > top_matter_cutoff + margin {
|
||||
if line.y > col_top + margin {
|
||||
above.push(line);
|
||||
} else {
|
||||
flow.push(line);
|
||||
col_below[ci].push(line);
|
||||
}
|
||||
}
|
||||
flow.sort_by(|a, b| b.y.total_cmp(&a.y));
|
||||
column_flows.push(flow);
|
||||
}
|
||||
|
||||
above.sort_by(|a, b| b.y.total_cmp(&a.y));
|
||||
below_spanning.sort_by(|a, b| b.y.total_cmp(&a.y));
|
||||
|
||||
all_lines.extend(above);
|
||||
for col in column_flows {
|
||||
for col in core_columns {
|
||||
all_lines.extend(col);
|
||||
}
|
||||
for cb in col_below {
|
||||
all_lines.extend(cb);
|
||||
}
|
||||
all_lines.extend(below_spanning);
|
||||
} else {
|
||||
// Tabular: Y-interleaved merge — rows at the same Y from
|
||||
@@ -1920,63 +1917,4 @@ mod tests {
|
||||
let spanning_count = mask.iter().filter(|&&m| m).count();
|
||||
assert_eq!(spanning_count, 0, "Narrow header should NOT be pre-masked");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newspaper_keeps_staggered_right_column_fragment_in_column_flow() {
|
||||
let mut items = Vec::new();
|
||||
|
||||
for i in 0..30 {
|
||||
let text = if i == 0 {
|
||||
"ABSTRACT".to_string()
|
||||
} else {
|
||||
format!("Left body line {i}")
|
||||
};
|
||||
let mut item = make_item(1, 54.0, 588.0 - i as f32 * 10.0, &text);
|
||||
item.width = 220.0;
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
let text = if i == 0 {
|
||||
"options, people often need comments".to_string()
|
||||
} else {
|
||||
format!("Right opening fragment {i}")
|
||||
};
|
||||
let mut item = make_item(1, 318.0, 592.0 - i as f32 * 10.0, &text);
|
||||
item.width = 220.0;
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
for i in 0..20 {
|
||||
let text = if i == 0 {
|
||||
"Figure 1 caption starts here".to_string()
|
||||
} else {
|
||||
format!("Right lower body line {i}")
|
||||
};
|
||||
let mut item = make_item(1, 318.0, 330.0 - i as f32 * 10.0, &text);
|
||||
item.width = 220.0;
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
let lines = group_into_lines_with_thresholds(items, &HashMap::new(), &HashSet::new());
|
||||
let texts: Vec<String> = lines.iter().map(TextLine::text).collect();
|
||||
let abstract_pos = texts.iter().position(|text| text == "ABSTRACT").unwrap();
|
||||
let options_pos = texts
|
||||
.iter()
|
||||
.position(|text| text.starts_with("options, people often"))
|
||||
.unwrap();
|
||||
let figure_pos = texts
|
||||
.iter()
|
||||
.position(|text| text.starts_with("Figure 1 caption"))
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
abstract_pos < options_pos,
|
||||
"right-column opening fragment must not be promoted above the abstract: {texts:?}"
|
||||
);
|
||||
assert!(
|
||||
options_pos < figure_pos,
|
||||
"right-column opening fragment should stay before the lower right-column core"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+41
-468
@@ -368,7 +368,6 @@ pub fn extract_pages_markdown_mem(
|
||||
// Extract ALL pages to get accurate, document-wide font stats.
|
||||
let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
|
||||
extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
|
||||
let text_quality = analyze_text_quality(&all_items);
|
||||
|
||||
// Compute layout complexity from full document (near-zero cost).
|
||||
let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
|
||||
@@ -417,7 +416,6 @@ pub fn extract_pages_markdown_mem(
|
||||
.collect();
|
||||
|
||||
let has_gid = gid_pages.contains(&page_1idx);
|
||||
let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx);
|
||||
|
||||
// Build markdown with document-wide font stats
|
||||
let options = MarkdownOptions {
|
||||
@@ -427,22 +425,17 @@ pub fn extract_pages_markdown_mem(
|
||||
..MarkdownOptions::default()
|
||||
};
|
||||
|
||||
let md = if has_text_quality_issue {
|
||||
String::new()
|
||||
} else {
|
||||
markdown::to_markdown_from_items_with_rects_and_lines(
|
||||
page_items,
|
||||
options,
|
||||
&page_rects,
|
||||
&[],
|
||||
&page_thresholds,
|
||||
None,
|
||||
&[],
|
||||
)
|
||||
};
|
||||
let md = markdown::to_markdown_from_items_with_rects_and_lines(
|
||||
page_items,
|
||||
options,
|
||||
&page_rects,
|
||||
&[],
|
||||
&page_thresholds,
|
||||
None,
|
||||
&[],
|
||||
);
|
||||
|
||||
let needs_ocr = has_text_quality_issue
|
||||
|| md.trim().is_empty()
|
||||
let needs_ocr = md.trim().is_empty()
|
||||
|| has_gid
|
||||
|| is_garbage_text(&md)
|
||||
|| is_cid_garbage(&md)
|
||||
@@ -599,23 +592,24 @@ pub fn extract_text_in_regions_mem(
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
let matched: Vec<TextItem> = match items {
|
||||
Some(items) => items
|
||||
.iter()
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect(),
|
||||
None => Vec::new(),
|
||||
let text = match items {
|
||||
Some(items) => collect_text_in_region_with_options(
|
||||
items,
|
||||
rx1,
|
||||
ry1,
|
||||
rx2,
|
||||
ry2,
|
||||
page_h,
|
||||
coords,
|
||||
adaptive_threshold,
|
||||
),
|
||||
None => String::new(),
|
||||
};
|
||||
let has_text_quality_issue = region_items_have_decoding_issue(&matched);
|
||||
let text = collect_text_from_matched_items(matched, adaptive_threshold);
|
||||
|
||||
// Check per-region text quality instead of blanket page-level
|
||||
// GID rejection. A GID font in a logo elsewhere on the page
|
||||
// shouldn't force GPU OCR for clean text regions.
|
||||
let needs_ocr = has_text_quality_issue
|
||||
|| text.trim().is_empty()
|
||||
let needs_ocr = text.trim().is_empty()
|
||||
|| is_garbage_text(&text)
|
||||
|| is_cid_garbage(&text)
|
||||
|| detect_encoding_issues(&text);
|
||||
@@ -735,14 +729,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
continue;
|
||||
}
|
||||
|
||||
if region_items_have_decoding_issue(&matched) {
|
||||
page_results.push(RegionText {
|
||||
text: String::new(),
|
||||
needs_ocr: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compute base_font_size as most common font size in the region
|
||||
let base_font_size = {
|
||||
let mut freq: HashMap<i32, usize> = HashMap::new();
|
||||
@@ -837,10 +823,7 @@ pub fn extract_tables_in_regions_mem(
|
||||
// symmetrically low under font-decode failure — this
|
||||
// guard breaks that symmetry by comparing against
|
||||
// bbox area, which is independent of extraction.
|
||||
if source != TableCandidateSource::KeyValue
|
||||
&& region_text_density_too_low(region_text_chars, region_area)
|
||||
&& !markdown_table_body_is_dense(&md)
|
||||
{
|
||||
if region_text_density_too_low(region_text_chars, region_area) {
|
||||
return None;
|
||||
}
|
||||
let shape = markdown_table_shape(&md);
|
||||
@@ -851,11 +834,7 @@ pub fn extract_tables_in_regions_mem(
|
||||
Some(TableCandidateIssue::LineRowUndercount)
|
||||
} else if wide_table_sparse_prefix_undercount(&md) {
|
||||
Some(TableCandidateIssue::SparseWideUndercount)
|
||||
} else if !matches!(
|
||||
source,
|
||||
TableCandidateSource::Line | TableCandidateSource::KeyValue
|
||||
) && text_cluster_column_undercount(&matched, shape)
|
||||
{
|
||||
} else if text_cluster_column_undercount(&matched, shape) {
|
||||
Some(TableCandidateIssue::TextColumnUndercount)
|
||||
} else if prose_grid_fragment_needs_ocr(&md) {
|
||||
Some(TableCandidateIssue::ProseGridFragment)
|
||||
@@ -898,16 +877,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
{
|
||||
candidates.push(candidate);
|
||||
}
|
||||
if let Some(table) = tables::try_build_table_from_columns(&matched, page_1idx) {
|
||||
if let Some(candidate) = evaluate(TableCandidateSource::Column, &table) {
|
||||
candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
if let Some(table) = tables::try_build_key_value_table_from_rows(&matched, page_1idx) {
|
||||
if let Some(candidate) = evaluate(TableCandidateSource::KeyValue, &table) {
|
||||
candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
match select_table_candidate(&candidates) {
|
||||
Some(candidate) => page_results.push(RegionText {
|
||||
@@ -3415,7 +3384,7 @@ fn process_document(
|
||||
})
|
||||
.unwrap_or((None, Vec::new()));
|
||||
|
||||
let (markdown, layout, has_encoding_issues, gid_pages, text_quality_pages) = match extracted {
|
||||
let (markdown, layout, has_encoding_issues, gid_pages) = match extracted {
|
||||
Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => {
|
||||
// For TextBased PDFs with pages flagged for OCR (Identity-H or
|
||||
// Type3 fonts without ToUnicode), check whether the CID-as-Unicode
|
||||
@@ -3465,7 +3434,6 @@ fn process_document(
|
||||
}
|
||||
};
|
||||
|
||||
let text_quality = analyze_text_quality(&items);
|
||||
let layout = compute_layout_complexity(&items, &rects, &lines);
|
||||
|
||||
let md = if options.mode == ProcessMode::Analyze {
|
||||
@@ -3482,22 +3450,14 @@ fn process_document(
|
||||
))
|
||||
};
|
||||
|
||||
let enc = text_quality.has_encoding_issues
|
||||
|| md.as_ref().is_some_and(|m| detect_encoding_issues(m));
|
||||
(
|
||||
md,
|
||||
layout,
|
||||
enc,
|
||||
gid_encoded_pages,
|
||||
text_quality.pages_needing_ocr,
|
||||
)
|
||||
let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m));
|
||||
(md, layout, enc, gid_encoded_pages)
|
||||
}
|
||||
None => (
|
||||
None,
|
||||
LayoutComplexity::default(),
|
||||
false,
|
||||
std::collections::HashSet::new(),
|
||||
Vec::new(),
|
||||
),
|
||||
};
|
||||
|
||||
@@ -3539,18 +3499,6 @@ fn process_document(
|
||||
}
|
||||
pages_needing_ocr.sort_unstable();
|
||||
}
|
||||
if !text_quality_pages.is_empty() {
|
||||
log::debug!(
|
||||
"pages with suspicious text-layer decoding (need OCR): {:?}",
|
||||
text_quality_pages
|
||||
);
|
||||
for page in text_quality_pages {
|
||||
if !pages_needing_ocr.contains(&page) {
|
||||
pages_needing_ocr.push(page);
|
||||
}
|
||||
}
|
||||
pages_needing_ocr.sort_unstable();
|
||||
}
|
||||
|
||||
// Detect sparse extraction: when a TEXT-BASED PDF produces very few
|
||||
// characters per page, the text is likely embedded in images/forms
|
||||
@@ -3636,104 +3584,6 @@ fn detect_encoding_issues(markdown: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TextQualityReport {
|
||||
pages_needing_ocr: Vec<u32>,
|
||||
has_encoding_issues: bool,
|
||||
}
|
||||
|
||||
fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
|
||||
let mut pages = HashSet::new();
|
||||
|
||||
for item in items {
|
||||
if !matches!(item.item_type, crate::types::ItemType::Text) {
|
||||
continue;
|
||||
}
|
||||
if text_span_has_decoding_issue(&item.text) {
|
||||
pages.insert(item.page);
|
||||
}
|
||||
}
|
||||
|
||||
let mut pages_needing_ocr: Vec<u32> = pages.into_iter().collect();
|
||||
pages_needing_ocr.sort_unstable();
|
||||
TextQualityReport {
|
||||
has_encoding_issues: !pages_needing_ocr.is_empty(),
|
||||
pages_needing_ocr,
|
||||
}
|
||||
}
|
||||
|
||||
fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
|
||||
items.iter().any(|item| {
|
||||
matches!(item.item_type, crate::types::ItemType::Text)
|
||||
&& text_span_has_decoding_issue(&item.text)
|
||||
})
|
||||
}
|
||||
|
||||
fn text_span_has_decoding_issue(text: &str) -> bool {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
detect_encoding_issues(text)
|
||||
|| has_private_use_text_run(text)
|
||||
|| is_cid_garbage(text)
|
||||
|| has_cid_control_token(text)
|
||||
}
|
||||
|
||||
fn has_private_use_text_run(text: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut private_use = 0usize;
|
||||
let mut current_run = 0usize;
|
||||
let mut longest_run = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
current_run = 0;
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
if is_private_use_char(ch) {
|
||||
private_use += 1;
|
||||
current_run += 1;
|
||||
longest_run = longest_run.max(current_run);
|
||||
} else {
|
||||
current_run = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if private_use == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
|
||||
}
|
||||
|
||||
fn has_cid_control_token(text: &str) -> bool {
|
||||
text.split_whitespace().any(token_has_cid_control)
|
||||
}
|
||||
|
||||
fn token_has_cid_control(token: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
|
||||
for ch in token.chars() {
|
||||
total += 1;
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total >= 5 && c1_control > 0 && c1_control * 20 >= total
|
||||
}
|
||||
|
||||
fn is_private_use_char(ch: char) -> bool {
|
||||
matches!(
|
||||
ch as u32,
|
||||
0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if extracted text is predominantly garbage (non-alphanumeric).
|
||||
///
|
||||
/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---."
|
||||
@@ -3898,8 +3748,6 @@ enum TableCandidateSource {
|
||||
Rect,
|
||||
Line,
|
||||
Heuristic,
|
||||
Column,
|
||||
KeyValue,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -3934,12 +3782,8 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida
|
||||
// serving a tidy-looking fragment.
|
||||
if first.issue == Some(TableCandidateIssue::LineRowUndercount) {
|
||||
return candidates.iter().find(|candidate| {
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Heuristic
|
||||
| TableCandidateSource::Column
|
||||
| TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
candidate.source == TableCandidateSource::Heuristic
|
||||
&& candidate.issue.is_none()
|
||||
&& candidate.shape.cols * 10 >= first.shape.cols * 13
|
||||
});
|
||||
}
|
||||
@@ -3957,31 +3801,14 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida
|
||||
TableCandidateSource::Rect | TableCandidateSource::Line
|
||||
) {
|
||||
if let Some(heuristic) = candidates.iter().find(|candidate| {
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Heuristic
|
||||
| TableCandidateSource::Column
|
||||
| TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
candidate.source == TableCandidateSource::Heuristic
|
||||
&& candidate.issue.is_none()
|
||||
&& heuristic_substantially_better(candidate.shape, accepted.shape)
|
||||
}) {
|
||||
accepted = heuristic;
|
||||
}
|
||||
}
|
||||
|
||||
if accepted.source == TableCandidateSource::Heuristic {
|
||||
if let Some(layout_candidate) = candidates.iter().find(|candidate| {
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Column | TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
&& candidate.shape.cols >= accepted.shape.cols
|
||||
&& candidate.shape.rows > accepted.shape.rows
|
||||
}) {
|
||||
accepted = layout_candidate;
|
||||
}
|
||||
}
|
||||
|
||||
Some(accepted)
|
||||
}
|
||||
|
||||
@@ -4426,21 +4253,12 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
}
|
||||
|
||||
// Failure mode 2: header has empty cells in a multi-column table.
|
||||
// When layout-assisted, tolerate merged/spanning header gaps if the
|
||||
// body is dense. Region bboxes from a layout model often start at a
|
||||
// visual table whose header cannot be represented faithfully in a
|
||||
// flat pipe table, while the body rows are still complete enough to use.
|
||||
let header_empty_indices: Vec<usize> = header_cells
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(idx, cell)| cell.is_empty().then_some(idx))
|
||||
.collect();
|
||||
let empty_count = header_empty_indices.len();
|
||||
// When layout-assisted, allow up to 1 empty header cell (common in
|
||||
// tables with merged/spanning header cells that we can't represent).
|
||||
let empty_count = header_cells.iter().filter(|c| c.is_empty()).count();
|
||||
if layout_assisted {
|
||||
if n_cols >= 3
|
||||
&& empty_count >= 2
|
||||
&& !layout_assisted_empty_header_has_dense_body(markdown, n_cols)
|
||||
{
|
||||
// Reject only if >1 empty header cell (2+ means serious boundary issue)
|
||||
if n_cols >= 3 && empty_count >= 2 {
|
||||
return true;
|
||||
}
|
||||
} else if n_cols >= 3 && empty_count >= 1 {
|
||||
@@ -4478,16 +4296,7 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
// (totals, subtotals) are common.
|
||||
let threshold = if layout_assisted { 2 } else { 3 };
|
||||
if n_cols >= 3 && empty_data * threshold >= n_cols {
|
||||
let sparse_row_shares_header_spacer = layout_assisted
|
||||
&& data_inner.iter().enumerate().any(|(idx, cell)| {
|
||||
cell.trim().is_empty() && header_empty_indices.contains(&idx)
|
||||
})
|
||||
&& layout_assisted_empty_header_has_dense_body(markdown, n_cols);
|
||||
let sparse_row_is_section_label = layout_assisted
|
||||
&& layout_assisted_sparse_section_row_is_ok(data_inner, markdown, n_cols);
|
||||
if !sparse_row_shares_header_spacer && !sparse_row_is_section_label {
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4545,88 +4354,6 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn layout_assisted_empty_header_has_dense_body(markdown: &str, n_cols: usize) -> bool {
|
||||
let rows = markdown_pipe_rows(markdown);
|
||||
let data_rows: Vec<&Vec<&str>> = rows
|
||||
.iter()
|
||||
.skip(1)
|
||||
.filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
|
||||
.collect();
|
||||
if data_rows.len() < 2 || n_cols < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let total_cells = data_rows.len() * n_cols;
|
||||
let mut filled_cells = 0usize;
|
||||
let mut rows_with_multiple_cells = 0usize;
|
||||
let mut max_filled_in_row = 0usize;
|
||||
for row in &data_rows {
|
||||
let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count();
|
||||
filled_cells += filled;
|
||||
max_filled_in_row = max_filled_in_row.max(filled);
|
||||
if filled >= 2 {
|
||||
rows_with_multiple_cells += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Dense enough to be a useful extraction despite lossy merged headers.
|
||||
// The row-count gate avoids accepting a single tidy row under a broken
|
||||
// header, and the density gate keeps sparse fragments on the OCR path.
|
||||
rows_with_multiple_cells * 2 >= data_rows.len()
|
||||
&& max_filled_in_row >= n_cols.min(3)
|
||||
&& filled_cells * 100 >= total_cells * 45
|
||||
}
|
||||
|
||||
fn layout_assisted_sparse_section_row_is_ok(row: &[&str], markdown: &str, n_cols: usize) -> bool {
|
||||
let labels: Vec<&str> = row
|
||||
.iter()
|
||||
.map(|cell| cell.trim())
|
||||
.filter(|cell| !cell.is_empty())
|
||||
.collect();
|
||||
if labels.len() != 1 {
|
||||
return false;
|
||||
}
|
||||
let label = labels[0];
|
||||
if label.len() > 40 || !label.chars().any(|ch| ch.is_alphabetic()) {
|
||||
return false;
|
||||
}
|
||||
if label.ends_with('.') || label.ends_with('!') || label.ends_with('?') || label.ends_with(':')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
layout_assisted_empty_header_has_dense_body(markdown, n_cols)
|
||||
}
|
||||
|
||||
fn markdown_table_body_is_dense(markdown: &str) -> bool {
|
||||
let rows = markdown_pipe_rows(markdown);
|
||||
let data_rows: Vec<&Vec<&str>> = rows
|
||||
.iter()
|
||||
.skip(1)
|
||||
.filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
|
||||
.collect();
|
||||
if data_rows.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let cols = rows.iter().map(|row| row.len()).max().unwrap_or_default();
|
||||
if cols < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut filled_cells = 0usize;
|
||||
let mut rows_with_multiple_cells = 0usize;
|
||||
for row in &data_rows {
|
||||
let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count();
|
||||
filled_cells += filled;
|
||||
if filled >= cols.min(3) {
|
||||
rows_with_multiple_cells += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let total_cells = data_rows.len() * cols;
|
||||
rows_with_multiple_cells * 2 >= data_rows.len() && filled_cells * 100 >= total_cells * 45
|
||||
}
|
||||
|
||||
/// Original strict validation (no layout assistance). Used by tests and
|
||||
/// full-page extraction paths that don't have layout model assistance.
|
||||
#[cfg(test)]
|
||||
@@ -4978,16 +4705,6 @@ mod table_candidate_selection_tests {
|
||||
assert_eq!(selected.source, TableCandidateSource::Heuristic);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefers_clean_column_fallback_when_it_recovers_more_rows() {
|
||||
let candidates = vec![
|
||||
candidate(TableCandidateSource::Heuristic, 6, 5, None),
|
||||
candidate(TableCandidateSource::Column, 7, 5, None),
|
||||
];
|
||||
let selected = select_table_candidate(&candidates).unwrap();
|
||||
assert_eq!(selected.source, TableCandidateSource::Column);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_candidate_collapsing_captured_y_clusters_is_suspicious() {
|
||||
let long = "value value value value value value value value value value value value";
|
||||
@@ -5092,9 +4809,7 @@ mod table_candidate_selection_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod looks_like_partial_table_tests {
|
||||
use super::{
|
||||
looks_like_partial_table, looks_like_partial_table_ex, markdown_table_body_is_dense,
|
||||
};
|
||||
use super::{looks_like_partial_table, looks_like_partial_table_ex};
|
||||
|
||||
#[test]
|
||||
fn good_table_passes() {
|
||||
@@ -5229,29 +4944,11 @@ mod looks_like_partial_table_tests {
|
||||
|
||||
#[test]
|
||||
fn two_empty_headers_still_rejected_when_layout_assisted() {
|
||||
// A single tidy row is not enough evidence to trust a badly gapped header.
|
||||
// 2+ empty headers is still bad even with layout assistance.
|
||||
let md = "|A|||D|\n|---|---|---|---|\n|x|y|z|w|";
|
||||
assert!(
|
||||
looks_like_partial_table_ex(md, true),
|
||||
"2 empty headers with only one body row are rejected even layout-assisted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_body_with_empty_merged_header_passes_when_layout_assisted() {
|
||||
let md = "|Year||Unadjusted Basis|||\n\
|
||||
|---|---|---|---|---|\n\
|
||||
|1|.1667|$100,000|$16,670|$16,670|\n\
|
||||
|2|.3333|$100,000|$33,330|$50,000|\n\
|
||||
|3|.3333|$100,000|$33,330|$88,330|\n\
|
||||
|4|.1667|$100,000|$16,670|$100,000|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode still rejects merged-header gaps"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should trust a dense body under a merged header"
|
||||
"2 empty headers rejected even layout-assisted"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -5276,40 +4973,6 @@ mod looks_like_partial_table_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_first_row_with_header_spacer_passes_when_layout_assisted() {
|
||||
let md = "|Properties|Instruction||Training Datasets Alignment|\n\
|
||||
|---|---|---|---|\n\
|
||||
||Alpaca-GPT4 OpenOrca Synth. Math-Instruct||Orca DPO Pairs Ultrafeedback Cleaned|\n\
|
||||
|Total # Samples|52K 2.91M 126K||12.9K 60.8K 126K|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode rejects the sparse first row"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should allow sparse rows that share a header spacer column"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_section_row_passes_when_layout_assisted_body_is_dense() {
|
||||
let md = "|Properties|Conditions|Method|Typical values|Units|\n\
|
||||
|---|---|---|---|---|\n\
|
||||
|Rheology|||||\n\
|
||||
|Melt Flow Rate|230 C/2.16 kg|ASTM D1238|3.0|g/10 min|\n\
|
||||
|Tensile Stress at Yield|50 mm/min|ASTM D638|31|MPa|\n\
|
||||
|Elongation at Yield|50 mm/min|ASTM D638|8|%|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode rejects the sparse first row"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should allow a short section label above dense table rows"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paragraph_still_rejected_when_layout_assisted() {
|
||||
// Paragraph detection is not relaxed — it's a genuine extraction issue.
|
||||
@@ -5363,27 +5026,6 @@ mod looks_like_partial_table_tests {
|
||||
"duplicate headers rejected even layout-assisted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_numeric_table_body_is_structurally_trusted() {
|
||||
let md = "|Year|3-Year|5-Year|7-Year|\n\
|
||||
|---|---|---|---|\n\
|
||||
|1|33.0%|20.00%|14.29%|\n\
|
||||
|2|44.45%|32.00%|24.49%|\n\
|
||||
|3|14.81%|19.20%|17.49%|\n\
|
||||
|4|7.41%|11.52%|12.49%|";
|
||||
assert!(markdown_table_body_is_dense(md));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_markdown_fragment_is_not_structurally_trusted() {
|
||||
let md = "|A|B|C|D|\n\
|
||||
|---|---|---|---|\n\
|
||||
|x||||\n\
|
||||
|||y||\n\
|
||||
||||z|";
|
||||
assert!(!markdown_table_body_is_dense(md));
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyse extracted items and rects for layout complexity.
|
||||
@@ -5676,13 +5318,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn test_text_item_on_page(page: u32, text: &str) -> TextItem {
|
||||
TextItem {
|
||||
page,
|
||||
..test_item(text, 10.0, 10.0, text.len() as f32 * 5.0, 12.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_encoding_issues_fffd() {
|
||||
assert!(detect_encoding_issues(
|
||||
@@ -5718,68 +5353,6 @@ mod tests {
|
||||
assert!(!detect_encoding_issues(text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_quality_flags_localized_cid_mojibake_span() {
|
||||
let items = vec![
|
||||
test_text_item_on_page(
|
||||
1,
|
||||
"Waiting Period 等待期 Maternity and newborn infant care benefit",
|
||||
),
|
||||
test_text_item_on_page(
|
||||
1,
|
||||
"Inpatient and Day-care Benefits DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯",
|
||||
),
|
||||
test_text_item_on_page(1, "Covered up to annual maximum. 赔付至年度最高保额。"),
|
||||
test_text_item_on_page(2, "A clean second page should not be routed to OCR."),
|
||||
];
|
||||
|
||||
let quality = analyze_text_quality(&items);
|
||||
|
||||
assert!(quality.has_encoding_issues);
|
||||
assert_eq!(quality.pages_needing_ocr, vec![1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_quality_flags_replacement_and_private_use_runs() {
|
||||
let items = vec![
|
||||
test_text_item_on_page(1, "broken \u{FFFD} text"),
|
||||
test_text_item_on_page(3, "\u{E000}\u{E001}\u{E002}"),
|
||||
];
|
||||
|
||||
let quality = analyze_text_quality(&items);
|
||||
|
||||
assert_eq!(quality.pages_needing_ocr, vec![1, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_quality_allows_clean_multilingual_and_latin1_text() {
|
||||
let items = vec![
|
||||
test_text_item_on_page(1, "你好世界，这是一段正常的中文文本。"),
|
||||
test_text_item_on_page(1, "Résumé déjà vu: façade, São Paulo, año 2026."),
|
||||
test_text_item_on_page(1, "A single icon \u{E000} should not force OCR."),
|
||||
];
|
||||
|
||||
let quality = analyze_text_quality(&items);
|
||||
|
||||
assert!(!quality.has_encoding_issues);
|
||||
assert!(quality.pages_needing_ocr.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_region_text_quality_is_scoped_to_matched_items() {
|
||||
let clean_region = vec![
|
||||
test_text_item_on_page(1, "Clean native text"),
|
||||
test_text_item_on_page(1, "Résumé déjà vu"),
|
||||
];
|
||||
let garbled_region = vec![
|
||||
test_text_item_on_page(1, "Clean prefix"),
|
||||
test_text_item_on_page(1, "DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯"),
|
||||
];
|
||||
|
||||
assert!(!region_items_have_decoding_issue(&clean_region));
|
||||
assert!(region_items_have_decoding_issue(&garbled_region));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_garbage_text_detection() {
|
||||
// Simulates garbage output from Identity-H fonts without ToUnicode.
|
||||
|
||||
+2
-232
@@ -149,79 +149,6 @@ fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32)
|
||||
set
|
||||
}
|
||||
|
||||
/// Pre-scan body-size all-bold runs that are too long to be headings.
|
||||
///
|
||||
/// Some academic PDFs use an all-bold abstract/summary paragraph immediately
|
||||
/// after the author block. A line-local bold heading heuristic sees each
|
||||
/// wrapped visual line as "standalone" once the first line is misclassified,
|
||||
/// producing a stack of `##` headings. Multi-line body-size bold runs with a
|
||||
/// paragraph-sized word count should stay paragraph text.
|
||||
fn find_wrapped_bold_paragraph_lines(
|
||||
lines: &[TextLine],
|
||||
base_size: f32,
|
||||
para_threshold: f32,
|
||||
) -> HashSet<usize> {
|
||||
let mut set = HashSet::new();
|
||||
let mut i = 0usize;
|
||||
|
||||
while i < lines.len() {
|
||||
if !is_body_size_all_bold_line(&lines[i], base_size) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = i;
|
||||
let mut end = i;
|
||||
let mut word_count = lines[i].text().split_whitespace().count();
|
||||
|
||||
while end + 1 < lines.len()
|
||||
&& is_body_size_all_bold_line(&lines[end + 1], base_size)
|
||||
&& is_wrapped_same_style_line(&lines[end], &lines[end + 1], para_threshold)
|
||||
{
|
||||
end += 1;
|
||||
word_count += lines[end].text().split_whitespace().count();
|
||||
}
|
||||
|
||||
let line_count = end - start + 1;
|
||||
if line_count >= 3 && word_count > 20 {
|
||||
for idx in start..=end {
|
||||
set.insert(idx);
|
||||
}
|
||||
}
|
||||
|
||||
i = end + 1;
|
||||
}
|
||||
|
||||
set
|
||||
}
|
||||
|
||||
fn is_body_size_all_bold_line(line: &TextLine, base_size: f32) -> bool {
|
||||
let Some(first) = line.items.first() else {
|
||||
return false;
|
||||
};
|
||||
first.font_size >= base_size * 0.95
|
||||
&& first.font_size < base_size * 1.2
|
||||
&& line
|
||||
.items
|
||||
.iter()
|
||||
.all(|item| item.is_bold && (item.font_size - first.font_size).abs() < 0.5)
|
||||
}
|
||||
|
||||
fn is_wrapped_same_style_line(prev: &TextLine, next: &TextLine, para_threshold: f32) -> bool {
|
||||
if prev.page != next.page {
|
||||
return false;
|
||||
}
|
||||
|
||||
let y_gap = prev.y - next.y;
|
||||
if !(y_gap > 0.0 && y_gap <= para_threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let prev_x = prev.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
let next_x = next.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
(prev_x - next_x).abs() <= 40.0
|
||||
}
|
||||
|
||||
/// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
|
||||
///
|
||||
/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
|
||||
@@ -470,8 +397,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// between paragraphs at body font size. Inspired by opendataloader's
|
||||
// lookahead in HeadingProcessor (prevNode/nextNode context).
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
// Detect struct heading levels that are overused (body text mistagged as headings)
|
||||
let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
|
||||
@@ -485,7 +410,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
|
||||
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
|
||||
|
||||
@@ -551,7 +475,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
current_page = line.page;
|
||||
prev_y = f32::MAX;
|
||||
prev_x = 0.0;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -566,7 +489,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(table_md);
|
||||
@@ -584,7 +506,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(image_md);
|
||||
@@ -606,18 +527,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& y_gap.abs() <= para_threshold
|
||||
&& (prev_x - line_x).abs() > 50.0
|
||||
&& prev_y < f32::MAX;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_band_switch || is_bold_to_regular_break) && in_paragraph {
|
||||
if (is_para_break || is_band_switch) && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -660,7 +572,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -714,9 +625,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -748,7 +656,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -771,7 +678,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("- {}", trimmed));
|
||||
output.push('\n');
|
||||
@@ -785,7 +691,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -832,7 +737,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("> {}\n", trimmed));
|
||||
continue;
|
||||
@@ -843,7 +747,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
if !in_code_block {
|
||||
output.push_str("```\n");
|
||||
@@ -864,11 +767,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -938,8 +836,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
||||
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
let mut output = String::new();
|
||||
let mut current_page = 0u32;
|
||||
@@ -948,7 +844,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let mut in_paragraph = false;
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
for (line_idx, line) in lines.iter().enumerate() {
|
||||
// Page break
|
||||
@@ -965,7 +860,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
in_list = false;
|
||||
last_list_x = None;
|
||||
prev_had_dot_leaders = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -976,18 +870,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
// (newspaper columns emitted sequentially on the same page).
|
||||
let y_gap = prev_y - line.y;
|
||||
let is_para_break = y_gap.abs() > para_threshold;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_bold_to_regular_break) && in_paragraph {
|
||||
if is_para_break && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -1011,7 +896,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -1034,9 +918,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -1054,7 +935,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(header_level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -1069,7 +949,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -1114,7 +993,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Use plain text for code blocks
|
||||
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
||||
@@ -1132,11 +1010,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -1483,109 +1356,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrapped_bold_abstract_is_not_split_into_headings() {
|
||||
// Regression for arXiv 1107.1353: the opening abstract paragraph is
|
||||
// entirely bold at body size. The first wrapped lines used to become
|
||||
// separate H2 headings, and the following body paragraph was joined to
|
||||
// the bold abstract because the paragraph gap is modest.
|
||||
let make = |text: &str, y: f32, font_size: f32, bold: bool| {
|
||||
let mut item = make_item(text, 1, None);
|
||||
item.y = y;
|
||||
item.font_size = font_size;
|
||||
item.height = font_size;
|
||||
item.is_bold = bold;
|
||||
item
|
||||
};
|
||||
|
||||
let lines = vec![
|
||||
make_line(vec![make(
|
||||
"Quantum Nature of Light Measured With a Single Detector",
|
||||
747.7,
|
||||
25.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Gesine A. Steudle1*, Stefan Schietinger1, David Höckel1",
|
||||
651.1,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Zwiller2, and Oliver Benson1",
|
||||
638.5,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"The introduction of light quanta by Einstein in 1905 triggered strong efforts to",
|
||||
607.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"demonstrate the quantum properties of light directly, without involving matter",
|
||||
594.8,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"quantization. It however took more than seven decades for the quantum granularity",
|
||||
582.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"of light to be observed in the fluorescence of single atoms. Single atoms emit",
|
||||
569.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"photons one at a time, this is typically demonstrated with a Hanbury-Brown-Twiss",
|
||||
556.9,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Our work significantly simplifies a widely used photon-correlation technique.",
|
||||
544.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"A photon is a single excitation of a mode of the electromagnetic field.",
|
||||
528.7,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
];
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(
|
||||
md.contains("# Quantum Nature of Light Measured With a Single Detector"),
|
||||
"title should remain a heading: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("## The introduction")
|
||||
&& !md.contains("## demonstrate")
|
||||
&& !md.contains("## quantization"),
|
||||
"bold abstract lines should not become headings: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("technique.**\n\nA photon is a single excitation"),
|
||||
"body paragraph should be separated from bold abstract: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_code_multiline_accumulation() {
|
||||
let mut line1 = make_item("fn main() {", 1, Some(0));
|
||||
|
||||
@@ -29,7 +29,6 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
|
||||
// text item, which combine with gap-based space insertion to produce
|
||||
// double spaces ("Vice President" instead of "Vice President").
|
||||
collapse_consecutive_spaces(&mut text);
|
||||
remove_spaces_before_closing_brackets(&mut text);
|
||||
|
||||
// Remove excessive newlines (more than 2 in a row)
|
||||
while text.contains("\n\n\n") {
|
||||
@@ -72,20 +71,6 @@ fn collapse_consecutive_spaces(text: &mut String) {
|
||||
*text = result;
|
||||
}
|
||||
|
||||
/// Remove spaces before closing square brackets.
|
||||
/// Unit markers and markdown links occasionally pick up a gap-inserted space
|
||||
/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding.
|
||||
fn remove_spaces_before_closing_brackets(text: &mut String) {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
for ch in text.chars() {
|
||||
if ch == ']' && result.ends_with(' ') {
|
||||
result.pop();
|
||||
}
|
||||
result.push(ch);
|
||||
}
|
||||
*text = result;
|
||||
}
|
||||
|
||||
/// Collapse dot leaders (runs of 4+ dots) into " ... "
|
||||
/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
|
||||
fn collapse_dot_leaders(text: &str) -> String {
|
||||
@@ -357,18 +342,6 @@ mod tests {
|
||||
assert!(result.contains("Chapter 2 ... 20"));
|
||||
}
|
||||
|
||||
// --- remove_spaces_before_closing_brackets ---
|
||||
|
||||
#[test]
|
||||
fn test_remove_spaces_before_closing_brackets() {
|
||||
let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string();
|
||||
remove_spaces_before_closing_brackets(&mut input);
|
||||
assert_eq!(
|
||||
input,
|
||||
"Density [kg/m3] and [linked text](https://example.com)"
|
||||
);
|
||||
}
|
||||
|
||||
// --- fix_hyphenation ---
|
||||
|
||||
#[test]
|
||||
|
||||
+4
-65
@@ -208,24 +208,6 @@ fn looks_like_compact_entry_label(cell: &str) -> bool {
|
||||
(1..=6).contains(&words)
|
||||
}
|
||||
|
||||
fn looks_like_plain_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 4 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-'))
|
||||
&& starts_with_uppercase_alpha(trimmed)
|
||||
&& (1..=4).contains(&alpha_word_count(trimmed))
|
||||
}
|
||||
|
||||
fn ends_like_incomplete_phrase(cell: &str) -> bool {
|
||||
let lower = cell.trim_end().to_ascii_lowercase();
|
||||
lower.ends_with(" and")
|
||||
@@ -323,10 +305,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.and_then(|r| r.first())
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
@@ -350,10 +328,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
&& non_first_cells
|
||||
.iter()
|
||||
.any(|cell| looks_like_compact_entry_label(cell));
|
||||
let looks_like_section_label_row = !first_cell.is_empty()
|
||||
&& filled_cells == 1
|
||||
&& header_filled >= 3
|
||||
&& looks_like_plain_section_label(first_cell);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
@@ -370,6 +344,10 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.last()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(0);
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
// Merge when the row has significantly fewer filled cells than header.
|
||||
// For wide tables (5+ cols), require ≤50% of header cells.
|
||||
// For narrow tables (2-4 cols), require fewer than header cells.
|
||||
@@ -391,7 +369,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& !looks_like_new_first_column_entry
|
||||
&& !looks_like_section_label_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -537,44 +514,6 @@ mod tests {
|
||||
assert!(cleaned[1][1].contains("continued text here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_first_column_section_label_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Properties".into(),
|
||||
"Conditions".into(),
|
||||
"Method".into(),
|
||||
"Typical values".into(),
|
||||
"Units".into(),
|
||||
],
|
||||
vec![
|
||||
"Melt Flow Rate".into(),
|
||||
"230 C/2.16 kg".into(),
|
||||
"ASTM D1238".into(),
|
||||
"3.0".into(),
|
||||
"g/10 min".into(),
|
||||
],
|
||||
vec![
|
||||
"Mechanical".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"Tensile Stress at Yield".into(),
|
||||
"50 mm/min".into(),
|
||||
"ASTM D638".into(),
|
||||
"31".into(),
|
||||
"MPa".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 4);
|
||||
assert_eq!(cleaned[2][0], "Mechanical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_short_subheader_not_merged() {
|
||||
let cells = vec![
|
||||
|
||||
-1233
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user