Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65444584ad | ||
|
|
252d87ac58 | ||
|
|
85890648c9 | ||
|
|
6e55e38b55 | ||
|
|
42befcea57 | ||
|
|
e547f616f9 | ||
|
|
455dfe5a74 | ||
|
|
839317525b | ||
|
|
8b63ceb084 |
@@ -0,0 +1,87 @@
|
||||
name: Publish Rust crate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['Cargo.toml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
|
||||
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
|
||||
"https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/crate-version.json
|
||||
echo "Unexpected crates.io response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish:
|
||||
name: Publish to crates.io
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
environment: crates-io
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Verify package
|
||||
run: cargo publish --dry-run
|
||||
|
||||
- name: Authenticate with crates.io
|
||||
id: auth
|
||||
uses: rust-lang/crates-io-auth-action@v1
|
||||
|
||||
- name: Publish crate
|
||||
run: cargo publish
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
|
||||
+1
-1
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2.0"
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# pdf-inspector
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
@@ -71,9 +74,17 @@ console.log(result.markdown); // Markdown string or null
|
||||
|
||||
### Rust
|
||||
|
||||
Install from [crates.io](https://crates.io/crates/pdf-inspector):
|
||||
|
||||
```bash
|
||||
cargo add pdf-inspector
|
||||
```
|
||||
|
||||
Or add it manually:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
pdf-inspector = "0.1"
|
||||
```
|
||||
|
||||
```rust
|
||||
@@ -91,29 +102,34 @@ if let Some(markdown) = &result.markdown {
|
||||
### CLI
|
||||
|
||||
```bash
|
||||
# Install the CLI tools
|
||||
cargo install pdf-inspector
|
||||
|
||||
# Convert PDF to Markdown
|
||||
cargo run --bin pdf2md -- document.pdf
|
||||
pdf2md document.pdf
|
||||
|
||||
# JSON output (for piping)
|
||||
cargo run --bin pdf2md -- document.pdf --json
|
||||
pdf2md document.pdf --json
|
||||
|
||||
# Raw markdown only (no headers)
|
||||
cargo run --bin pdf2md -- document.pdf --raw
|
||||
pdf2md document.pdf --raw
|
||||
|
||||
# Insert page break markers (<!-- Page N -->)
|
||||
cargo run --bin pdf2md -- document.pdf --pages
|
||||
pdf2md document.pdf --pages
|
||||
|
||||
# Process only specific pages
|
||||
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
|
||||
pdf2md document.pdf --select-pages 1,3,5-10
|
||||
|
||||
# Detection only (no extraction)
|
||||
cargo run --bin detect-pdf -- document.pdf
|
||||
cargo run --bin detect-pdf -- document.pdf --json
|
||||
detect-pdf document.pdf
|
||||
detect-pdf document.pdf --json
|
||||
|
||||
# Detection + layout analysis (tables, columns)
|
||||
cargo run --bin detect-pdf -- document.pdf --analyze --json
|
||||
detect-pdf document.pdf --analyze --json
|
||||
```
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Publishing
|
||||
|
||||
The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
|
||||
|
||||
## crates.io Trusted Publisher
|
||||
|
||||
Configure the trusted publisher for the `pdf-inspector` crate with:
|
||||
|
||||
- Repository: `firecrawl/pdf-inspector`
|
||||
- Workflow: `publish-crate.yml`
|
||||
- Environment: `crates-io`
|
||||
|
||||
The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
|
||||
|
||||
## Release Steps
|
||||
|
||||
1. Update `version` in `Cargo.toml`.
|
||||
2. Merge the version bump to `main`.
|
||||
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
|
||||
|
||||
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.8.15",
|
||||
"version": "1.9.5",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::fonts::{
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
use super::{get_number, multiply_matrices};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
|
||||
/// Strip PDF comments (% to end of line) from content stream bytes.
|
||||
///
|
||||
@@ -664,7 +664,29 @@ pub(crate) fn extract_page_text_items(
|
||||
if let Some(xobj_type) = xobjects.get(&xobj_name) {
|
||||
match xobj_type {
|
||||
XObjectType::Image => {
|
||||
// Skip images — text extraction only
|
||||
// Emit a positional placeholder for the image
|
||||
// so downstream consumers (layout-aware
|
||||
// pipelines, figure-OCR routers) can locate
|
||||
// raster figures without parsing the PDF
|
||||
// again. The text field carries the
|
||||
// XObject resource name in the legacy
|
||||
// `[Image: Im0]` format that the markdown
|
||||
// emitter already recognizes.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
XObjectType::Form(form_id) => {
|
||||
// Extract text from Form XObject
|
||||
|
||||
+7
-21
@@ -858,6 +858,13 @@ pub(crate) fn extract_text_from_operand(
|
||||
// unmapped. Don't fall through to text-interpretation fallbacks
|
||||
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
|
||||
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
|
||||
// Try our custom encoding map from Differences arrays.
|
||||
// The Differences array overrides specific codes in a base encoding (typically
|
||||
@@ -966,27 +973,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
|
||||
@@ -29,8 +29,12 @@ pub(crate) fn detect_columns(
|
||||
const MIN_ITEMS_PER_COLUMN: usize = 10;
|
||||
const NOISE_FRACTION: f32 = 0.15;
|
||||
|
||||
// Get items for this page
|
||||
let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect();
|
||||
// Get items for this page. Strip Image placeholders — an image's left edge
|
||||
// would otherwise count toward the column projection profile.
|
||||
let page_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.page == page && crate::extractor::is_text_layout_item(i))
|
||||
.collect();
|
||||
|
||||
if page_items.is_empty() {
|
||||
return vec![];
|
||||
|
||||
@@ -227,6 +227,69 @@ fn extract_positioned_text_impl(
|
||||
// Shared helpers (used by submodules via `super::`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Return true when this item should participate in text-layout
|
||||
/// heuristics (column detection, table grid detection, line grouping).
|
||||
///
|
||||
/// Image XObjects emit a positional placeholder via
|
||||
/// `extract_text_with_positions` (so layout-aware callers can crop +
|
||||
/// caption figures), but their bboxes don't carry text glyphs and would
|
||||
/// skew column/row clustering if they reached the heuristics. Hyperlinks
|
||||
/// and form fields *do* participate — the existing logic treats them as
|
||||
/// text-like and we keep that.
|
||||
pub(crate) fn is_text_layout_item(item: &crate::types::TextItem) -> bool {
|
||||
!matches!(item.item_type, crate::types::ItemType::Image)
|
||||
}
|
||||
|
||||
/// Map a (u, v) point in unit-square coordinates through the 6-element CTM
|
||||
/// to page-space. CTM format is `[a, b, c, d, e, f]` per
|
||||
/// [`multiply_matrices`].
|
||||
fn apply_ctm_point(ctm: &[f32; 6], u: f32, v: f32) -> (f32, f32) {
|
||||
(
|
||||
u * ctm[0] + v * ctm[2] + ctm[4],
|
||||
u * ctm[1] + v * ctm[3] + ctm[5],
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute the page-space axis-aligned bounding box of an Image XObject
|
||||
/// invoked under the given CTM.
|
||||
///
|
||||
/// Per the PDF spec, an image XObject is always rendered into a unit
|
||||
/// square `(0,0)–(1,1)` in its local coordinate system, and the `Do`
|
||||
/// operator applies the current CTM to position/scale/rotate that square
|
||||
/// onto the page. For the common axis-aligned case (no rotation/shear),
|
||||
/// the CTM reduces to `[w, 0, 0, h, x, y]` and the bbox is just
|
||||
/// `(x, y, w, h)`. For rotated/sheared images we transform all four
|
||||
/// corners and return their axis-aligned bbox so the caller always gets
|
||||
/// an upright rectangle.
|
||||
///
|
||||
/// Coordinates are PDF user space (origin at bottom-left, y-up). Width
|
||||
/// and height are non-negative.
|
||||
pub(crate) fn image_bbox_from_ctm(ctm: &[f32; 6]) -> (f32, f32, f32, f32) {
|
||||
let corners = [
|
||||
apply_ctm_point(ctm, 0.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 1.0),
|
||||
apply_ctm_point(ctm, 0.0, 1.0),
|
||||
];
|
||||
let (mut x_min, mut x_max) = (corners[0].0, corners[0].0);
|
||||
let (mut y_min, mut y_max) = (corners[0].1, corners[0].1);
|
||||
for (cx, cy) in corners.iter().skip(1) {
|
||||
if *cx < x_min {
|
||||
x_min = *cx;
|
||||
}
|
||||
if *cx > x_max {
|
||||
x_max = *cx;
|
||||
}
|
||||
if *cy < y_min {
|
||||
y_min = *cy;
|
||||
}
|
||||
if *cy > y_max {
|
||||
y_max = *cy;
|
||||
}
|
||||
}
|
||||
(x_min, y_min, x_max - x_min, y_max - y_min)
|
||||
}
|
||||
|
||||
/// Multiply two 2D transformation matrices
|
||||
/// Matrix format: [a, b, c, d, e, f] representing:
|
||||
/// | a b 0 |
|
||||
|
||||
+37
-13
@@ -10,7 +10,7 @@ use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::{get_number, multiply_matrices};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
|
||||
const MAX_FORM_XOBJECT_DEPTH: u8 = 5;
|
||||
|
||||
@@ -262,19 +262,43 @@ fn extract_form_xobject_text_inner(
|
||||
if !op.operands.is_empty() {
|
||||
if let Ok(name) = op.operands[0].as_name() {
|
||||
let xobj_name = String::from_utf8_lossy(name).to_string();
|
||||
if let Some(XObjectType::Form(nested_id)) = form_xobjects.get(&xobj_name) {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
match form_xobjects.get(&xobj_name) {
|
||||
Some(XObjectType::Form(nested_id)) => {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
}
|
||||
Some(XObjectType::Image) => {
|
||||
// Mirror the top-level Image-XObject emission
|
||||
// in content_stream.rs so figures embedded
|
||||
// inside Form XObjects (common in print-to-PDF
|
||||
// workflows) aren't silently dropped.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+244
-16
@@ -47,7 +47,10 @@ pub use detector::{
|
||||
detect_pdf_type, detect_pdf_type_mem, detect_pdf_type_mem_with_config,
|
||||
detect_pdf_type_with_config, DetectionConfig, PdfType, PdfTypeResult, ScanStrategy,
|
||||
};
|
||||
pub use extractor::{extract_text, extract_text_with_positions, extract_text_with_positions_pages};
|
||||
pub use extractor::{
|
||||
extract_text, extract_text_with_positions, extract_text_with_positions_mem,
|
||||
extract_text_with_positions_pages,
|
||||
};
|
||||
pub use markdown::{
|
||||
to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions,
|
||||
};
|
||||
@@ -820,7 +823,10 @@ pub fn extract_tables_in_regions_mem(
|
||||
// symmetrically low under font-decode failure — this
|
||||
// guard breaks that symmetry by comparing against
|
||||
// bbox area, which is independent of extraction.
|
||||
if region_text_density_too_low(region_text_chars, region_area) {
|
||||
if source != TableCandidateSource::KeyValue
|
||||
&& region_text_density_too_low(region_text_chars, region_area)
|
||||
&& !markdown_table_body_is_dense(&md)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let shape = markdown_table_shape(&md);
|
||||
@@ -831,7 +837,11 @@ pub fn extract_tables_in_regions_mem(
|
||||
Some(TableCandidateIssue::LineRowUndercount)
|
||||
} else if wide_table_sparse_prefix_undercount(&md) {
|
||||
Some(TableCandidateIssue::SparseWideUndercount)
|
||||
} else if text_cluster_column_undercount(&matched, shape) {
|
||||
} else if !matches!(
|
||||
source,
|
||||
TableCandidateSource::Line | TableCandidateSource::KeyValue
|
||||
) && text_cluster_column_undercount(&matched, shape)
|
||||
{
|
||||
Some(TableCandidateIssue::TextColumnUndercount)
|
||||
} else if prose_grid_fragment_needs_ocr(&md) {
|
||||
Some(TableCandidateIssue::ProseGridFragment)
|
||||
@@ -874,6 +884,16 @@ pub fn extract_tables_in_regions_mem(
|
||||
{
|
||||
candidates.push(candidate);
|
||||
}
|
||||
if let Some(table) = tables::try_build_table_from_columns(&matched, page_1idx) {
|
||||
if let Some(candidate) = evaluate(TableCandidateSource::Column, &table) {
|
||||
candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
if let Some(table) = tables::try_build_key_value_table_from_rows(&matched, page_1idx) {
|
||||
if let Some(candidate) = evaluate(TableCandidateSource::KeyValue, &table) {
|
||||
candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
match select_table_candidate(&candidates) {
|
||||
Some(candidate) => page_results.push(RegionText {
|
||||
@@ -3745,6 +3765,8 @@ enum TableCandidateSource {
|
||||
Rect,
|
||||
Line,
|
||||
Heuristic,
|
||||
Column,
|
||||
KeyValue,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -3779,8 +3801,12 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida
|
||||
// serving a tidy-looking fragment.
|
||||
if first.issue == Some(TableCandidateIssue::LineRowUndercount) {
|
||||
return candidates.iter().find(|candidate| {
|
||||
candidate.source == TableCandidateSource::Heuristic
|
||||
&& candidate.issue.is_none()
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Heuristic
|
||||
| TableCandidateSource::Column
|
||||
| TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
&& candidate.shape.cols * 10 >= first.shape.cols * 13
|
||||
});
|
||||
}
|
||||
@@ -3798,14 +3824,31 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida
|
||||
TableCandidateSource::Rect | TableCandidateSource::Line
|
||||
) {
|
||||
if let Some(heuristic) = candidates.iter().find(|candidate| {
|
||||
candidate.source == TableCandidateSource::Heuristic
|
||||
&& candidate.issue.is_none()
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Heuristic
|
||||
| TableCandidateSource::Column
|
||||
| TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
&& heuristic_substantially_better(candidate.shape, accepted.shape)
|
||||
}) {
|
||||
accepted = heuristic;
|
||||
}
|
||||
}
|
||||
|
||||
if accepted.source == TableCandidateSource::Heuristic {
|
||||
if let Some(layout_candidate) = candidates.iter().find(|candidate| {
|
||||
matches!(
|
||||
candidate.source,
|
||||
TableCandidateSource::Column | TableCandidateSource::KeyValue
|
||||
) && candidate.issue.is_none()
|
||||
&& candidate.shape.cols >= accepted.shape.cols
|
||||
&& candidate.shape.rows > accepted.shape.rows
|
||||
}) {
|
||||
accepted = layout_candidate;
|
||||
}
|
||||
}
|
||||
|
||||
Some(accepted)
|
||||
}
|
||||
|
||||
@@ -4250,12 +4293,21 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
}
|
||||
|
||||
// Failure mode 2: header has empty cells in a multi-column table.
|
||||
// When layout-assisted, allow up to 1 empty header cell (common in
|
||||
// tables with merged/spanning header cells that we can't represent).
|
||||
let empty_count = header_cells.iter().filter(|c| c.is_empty()).count();
|
||||
// When layout-assisted, tolerate merged/spanning header gaps if the
|
||||
// body is dense. Region bboxes from a layout model often start at a
|
||||
// visual table whose header cannot be represented faithfully in a
|
||||
// flat pipe table, while the body rows are still complete enough to use.
|
||||
let header_empty_indices: Vec<usize> = header_cells
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(idx, cell)| cell.is_empty().then_some(idx))
|
||||
.collect();
|
||||
let empty_count = header_empty_indices.len();
|
||||
if layout_assisted {
|
||||
// Reject only if >1 empty header cell (2+ means serious boundary issue)
|
||||
if n_cols >= 3 && empty_count >= 2 {
|
||||
if n_cols >= 3
|
||||
&& empty_count >= 2
|
||||
&& !layout_assisted_empty_header_has_dense_body(markdown, n_cols)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} else if n_cols >= 3 && empty_count >= 1 {
|
||||
@@ -4293,7 +4345,16 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
// (totals, subtotals) are common.
|
||||
let threshold = if layout_assisted { 2 } else { 3 };
|
||||
if n_cols >= 3 && empty_data * threshold >= n_cols {
|
||||
return true;
|
||||
let sparse_row_shares_header_spacer = layout_assisted
|
||||
&& data_inner.iter().enumerate().any(|(idx, cell)| {
|
||||
cell.trim().is_empty() && header_empty_indices.contains(&idx)
|
||||
})
|
||||
&& layout_assisted_empty_header_has_dense_body(markdown, n_cols);
|
||||
let sparse_row_is_section_label = layout_assisted
|
||||
&& layout_assisted_sparse_section_row_is_ok(data_inner, markdown, n_cols);
|
||||
if !sparse_row_shares_header_spacer && !sparse_row_is_section_label {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4351,6 +4412,88 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn layout_assisted_empty_header_has_dense_body(markdown: &str, n_cols: usize) -> bool {
|
||||
let rows = markdown_pipe_rows(markdown);
|
||||
let data_rows: Vec<&Vec<&str>> = rows
|
||||
.iter()
|
||||
.skip(1)
|
||||
.filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
|
||||
.collect();
|
||||
if data_rows.len() < 2 || n_cols < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let total_cells = data_rows.len() * n_cols;
|
||||
let mut filled_cells = 0usize;
|
||||
let mut rows_with_multiple_cells = 0usize;
|
||||
let mut max_filled_in_row = 0usize;
|
||||
for row in &data_rows {
|
||||
let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count();
|
||||
filled_cells += filled;
|
||||
max_filled_in_row = max_filled_in_row.max(filled);
|
||||
if filled >= 2 {
|
||||
rows_with_multiple_cells += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Dense enough to be a useful extraction despite lossy merged headers.
|
||||
// The row-count gate avoids accepting a single tidy row under a broken
|
||||
// header, and the density gate keeps sparse fragments on the OCR path.
|
||||
rows_with_multiple_cells * 2 >= data_rows.len()
|
||||
&& max_filled_in_row >= n_cols.min(3)
|
||||
&& filled_cells * 100 >= total_cells * 45
|
||||
}
|
||||
|
||||
fn layout_assisted_sparse_section_row_is_ok(row: &[&str], markdown: &str, n_cols: usize) -> bool {
|
||||
let labels: Vec<&str> = row
|
||||
.iter()
|
||||
.map(|cell| cell.trim())
|
||||
.filter(|cell| !cell.is_empty())
|
||||
.collect();
|
||||
if labels.len() != 1 {
|
||||
return false;
|
||||
}
|
||||
let label = labels[0];
|
||||
if label.len() > 40 || !label.chars().any(|ch| ch.is_alphabetic()) {
|
||||
return false;
|
||||
}
|
||||
if label.ends_with('.') || label.ends_with('!') || label.ends_with('?') || label.ends_with(':')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
layout_assisted_empty_header_has_dense_body(markdown, n_cols)
|
||||
}
|
||||
|
||||
fn markdown_table_body_is_dense(markdown: &str) -> bool {
|
||||
let rows = markdown_pipe_rows(markdown);
|
||||
let data_rows: Vec<&Vec<&str>> = rows
|
||||
.iter()
|
||||
.skip(1)
|
||||
.filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
|
||||
.collect();
|
||||
if data_rows.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let cols = rows.iter().map(|row| row.len()).max().unwrap_or_default();
|
||||
if cols < 3 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut filled_cells = 0usize;
|
||||
let mut rows_with_multiple_cells = 0usize;
|
||||
for row in &data_rows {
|
||||
let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count();
|
||||
filled_cells += filled;
|
||||
if filled >= cols.min(3) {
|
||||
rows_with_multiple_cells += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let total_cells = data_rows.len() * cols;
|
||||
rows_with_multiple_cells * 2 >= data_rows.len() && filled_cells * 100 >= total_cells * 45
|
||||
}
|
||||
|
||||
/// Original strict validation (no layout assistance). Used by tests and
|
||||
/// full-page extraction paths that don't have layout model assistance.
|
||||
#[cfg(test)]
|
||||
@@ -4702,6 +4845,16 @@ mod table_candidate_selection_tests {
|
||||
assert_eq!(selected.source, TableCandidateSource::Heuristic);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefers_clean_column_fallback_when_it_recovers_more_rows() {
|
||||
let candidates = vec![
|
||||
candidate(TableCandidateSource::Heuristic, 6, 5, None),
|
||||
candidate(TableCandidateSource::Column, 7, 5, None),
|
||||
];
|
||||
let selected = select_table_candidate(&candidates).unwrap();
|
||||
assert_eq!(selected.source, TableCandidateSource::Column);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_candidate_collapsing_captured_y_clusters_is_suspicious() {
|
||||
let long = "value value value value value value value value value value value value";
|
||||
@@ -4806,7 +4959,9 @@ mod table_candidate_selection_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod looks_like_partial_table_tests {
|
||||
use super::{looks_like_partial_table, looks_like_partial_table_ex};
|
||||
use super::{
|
||||
looks_like_partial_table, looks_like_partial_table_ex, markdown_table_body_is_dense,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn good_table_passes() {
|
||||
@@ -4941,11 +5096,29 @@ mod looks_like_partial_table_tests {
|
||||
|
||||
#[test]
|
||||
fn two_empty_headers_still_rejected_when_layout_assisted() {
|
||||
// 2+ empty headers is still bad even with layout assistance.
|
||||
// A single tidy row is not enough evidence to trust a badly gapped header.
|
||||
let md = "|A|||D|\n|---|---|---|---|\n|x|y|z|w|";
|
||||
assert!(
|
||||
looks_like_partial_table_ex(md, true),
|
||||
"2 empty headers rejected even layout-assisted"
|
||||
"2 empty headers with only one body row are rejected even layout-assisted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_body_with_empty_merged_header_passes_when_layout_assisted() {
|
||||
let md = "|Year||Unadjusted Basis|||\n\
|
||||
|---|---|---|---|---|\n\
|
||||
|1|.1667|$100,000|$16,670|$16,670|\n\
|
||||
|2|.3333|$100,000|$33,330|$50,000|\n\
|
||||
|3|.3333|$100,000|$33,330|$88,330|\n\
|
||||
|4|.1667|$100,000|$16,670|$100,000|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode still rejects merged-header gaps"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should trust a dense body under a merged header"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4970,6 +5143,40 @@ mod looks_like_partial_table_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_first_row_with_header_spacer_passes_when_layout_assisted() {
|
||||
let md = "|Properties|Instruction||Training Datasets Alignment|\n\
|
||||
|---|---|---|---|\n\
|
||||
||Alpaca-GPT4 OpenOrca Synth. Math-Instruct||Orca DPO Pairs Ultrafeedback Cleaned|\n\
|
||||
|Total # Samples|52K 2.91M 126K||12.9K 60.8K 126K|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode rejects the sparse first row"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should allow sparse rows that share a header spacer column"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_section_row_passes_when_layout_assisted_body_is_dense() {
|
||||
let md = "|Properties|Conditions|Method|Typical values|Units|\n\
|
||||
|---|---|---|---|---|\n\
|
||||
|Rheology|||||\n\
|
||||
|Melt Flow Rate|230 C/2.16 kg|ASTM D1238|3.0|g/10 min|\n\
|
||||
|Tensile Stress at Yield|50 mm/min|ASTM D638|31|MPa|\n\
|
||||
|Elongation at Yield|50 mm/min|ASTM D638|8|%|";
|
||||
assert!(
|
||||
looks_like_partial_table(md),
|
||||
"strict mode rejects the sparse first row"
|
||||
);
|
||||
assert!(
|
||||
!looks_like_partial_table_ex(md, true),
|
||||
"layout-assisted should allow a short section label above dense table rows"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paragraph_still_rejected_when_layout_assisted() {
|
||||
// Paragraph detection is not relaxed — it's a genuine extraction issue.
|
||||
@@ -5023,6 +5230,27 @@ mod looks_like_partial_table_tests {
|
||||
"duplicate headers rejected even layout-assisted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_numeric_table_body_is_structurally_trusted() {
|
||||
let md = "|Year|3-Year|5-Year|7-Year|\n\
|
||||
|---|---|---|---|\n\
|
||||
|1|33.0%|20.00%|14.29%|\n\
|
||||
|2|44.45%|32.00%|24.49%|\n\
|
||||
|3|14.81%|19.20%|17.49%|\n\
|
||||
|4|7.41%|11.52%|12.49%|";
|
||||
assert!(markdown_table_body_is_dense(md));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_markdown_fragment_is_not_structurally_trusted() {
|
||||
let md = "|A|B|C|D|\n\
|
||||
|---|---|---|---|\n\
|
||||
|x||||\n\
|
||||
|||y||\n\
|
||||
||||z|";
|
||||
assert!(!markdown_table_body_is_dense(md));
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyse extracted items and rects for layout complexity.
|
||||
|
||||
+232
-2
@@ -149,6 +149,79 @@ fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32)
|
||||
set
|
||||
}
|
||||
|
||||
/// Pre-scan body-size all-bold runs that are too long to be headings.
|
||||
///
|
||||
/// Some academic PDFs use an all-bold abstract/summary paragraph immediately
|
||||
/// after the author block. A line-local bold heading heuristic sees each
|
||||
/// wrapped visual line as "standalone" once the first line is misclassified,
|
||||
/// producing a stack of `##` headings. Multi-line body-size bold runs with a
|
||||
/// paragraph-sized word count should stay paragraph text.
|
||||
fn find_wrapped_bold_paragraph_lines(
|
||||
lines: &[TextLine],
|
||||
base_size: f32,
|
||||
para_threshold: f32,
|
||||
) -> HashSet<usize> {
|
||||
let mut set = HashSet::new();
|
||||
let mut i = 0usize;
|
||||
|
||||
while i < lines.len() {
|
||||
if !is_body_size_all_bold_line(&lines[i], base_size) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = i;
|
||||
let mut end = i;
|
||||
let mut word_count = lines[i].text().split_whitespace().count();
|
||||
|
||||
while end + 1 < lines.len()
|
||||
&& is_body_size_all_bold_line(&lines[end + 1], base_size)
|
||||
&& is_wrapped_same_style_line(&lines[end], &lines[end + 1], para_threshold)
|
||||
{
|
||||
end += 1;
|
||||
word_count += lines[end].text().split_whitespace().count();
|
||||
}
|
||||
|
||||
let line_count = end - start + 1;
|
||||
if line_count >= 3 && word_count > 20 {
|
||||
for idx in start..=end {
|
||||
set.insert(idx);
|
||||
}
|
||||
}
|
||||
|
||||
i = end + 1;
|
||||
}
|
||||
|
||||
set
|
||||
}
|
||||
|
||||
fn is_body_size_all_bold_line(line: &TextLine, base_size: f32) -> bool {
|
||||
let Some(first) = line.items.first() else {
|
||||
return false;
|
||||
};
|
||||
first.font_size >= base_size * 0.95
|
||||
&& first.font_size < base_size * 1.2
|
||||
&& line
|
||||
.items
|
||||
.iter()
|
||||
.all(|item| item.is_bold && (item.font_size - first.font_size).abs() < 0.5)
|
||||
}
|
||||
|
||||
fn is_wrapped_same_style_line(prev: &TextLine, next: &TextLine, para_threshold: f32) -> bool {
|
||||
if prev.page != next.page {
|
||||
return false;
|
||||
}
|
||||
|
||||
let y_gap = prev.y - next.y;
|
||||
if !(y_gap > 0.0 && y_gap <= para_threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let prev_x = prev.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
let next_x = next.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
(prev_x - next_x).abs() <= 40.0
|
||||
}
|
||||
|
||||
/// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
|
||||
///
|
||||
/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
|
||||
@@ -397,6 +470,8 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// between paragraphs at body font size. Inspired by opendataloader's
|
||||
// lookahead in HeadingProcessor (prevNode/nextNode context).
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
// Detect struct heading levels that are overused (body text mistagged as headings)
|
||||
let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
|
||||
@@ -410,6 +485,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
|
||||
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
|
||||
|
||||
@@ -475,6 +551,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
current_page = line.page;
|
||||
prev_y = f32::MAX;
|
||||
prev_x = 0.0;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -489,6 +566,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(table_md);
|
||||
@@ -506,6 +584,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(image_md);
|
||||
@@ -527,9 +606,18 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& y_gap.abs() <= para_threshold
|
||||
&& (prev_x - line_x).abs() > 50.0
|
||||
&& prev_y < f32::MAX;
|
||||
if (is_para_break || is_band_switch) && in_paragraph {
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_band_switch || is_bold_to_regular_break) && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -572,6 +660,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -625,6 +714,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -656,6 +748,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -678,6 +771,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("- {}", trimmed));
|
||||
output.push('\n');
|
||||
@@ -691,6 +785,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -737,6 +832,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("> {}\n", trimmed));
|
||||
continue;
|
||||
@@ -747,6 +843,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
if !in_code_block {
|
||||
output.push_str("```\n");
|
||||
@@ -767,6 +864,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -836,6 +938,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
||||
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
let mut output = String::new();
|
||||
let mut current_page = 0u32;
|
||||
@@ -844,6 +948,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let mut in_paragraph = false;
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
for (line_idx, line) in lines.iter().enumerate() {
|
||||
// Page break
|
||||
@@ -860,6 +965,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
in_list = false;
|
||||
last_list_x = None;
|
||||
prev_had_dot_leaders = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -870,9 +976,18 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
// (newspaper columns emitted sequentially on the same page).
|
||||
let y_gap = prev_y - line.y;
|
||||
let is_para_break = y_gap.abs() > para_threshold;
|
||||
if is_para_break && in_paragraph {
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_bold_to_regular_break) && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -896,6 +1011,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -918,6 +1034,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -935,6 +1054,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(header_level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -949,6 +1069,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -993,6 +1114,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Use plain text for code blocks
|
||||
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
||||
@@ -1010,6 +1132,11 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -1356,6 +1483,109 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrapped_bold_abstract_is_not_split_into_headings() {
|
||||
// Regression for arXiv 1107.1353: the opening abstract paragraph is
|
||||
// entirely bold at body size. The first wrapped lines used to become
|
||||
// separate H2 headings, and the following body paragraph was joined to
|
||||
// the bold abstract because the paragraph gap is modest.
|
||||
let make = |text: &str, y: f32, font_size: f32, bold: bool| {
|
||||
let mut item = make_item(text, 1, None);
|
||||
item.y = y;
|
||||
item.font_size = font_size;
|
||||
item.height = font_size;
|
||||
item.is_bold = bold;
|
||||
item
|
||||
};
|
||||
|
||||
let lines = vec![
|
||||
make_line(vec![make(
|
||||
"Quantum Nature of Light Measured With a Single Detector",
|
||||
747.7,
|
||||
25.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Gesine A. Steudle1*, Stefan Schietinger1, David Höckel1",
|
||||
651.1,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Zwiller2, and Oliver Benson1",
|
||||
638.5,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"The introduction of light quanta by Einstein in 1905 triggered strong efforts to",
|
||||
607.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"demonstrate the quantum properties of light directly, without involving matter",
|
||||
594.8,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"quantization. It however took more than seven decades for the quantum granularity",
|
||||
582.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"of light to be observed in the fluorescence of single atoms. Single atoms emit",
|
||||
569.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"photons one at a time, this is typically demonstrated with a Hanbury-Brown-Twiss",
|
||||
556.9,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Our work significantly simplifies a widely used photon-correlation technique.",
|
||||
544.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"A photon is a single excitation of a mode of the electromagnetic field.",
|
||||
528.7,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
];
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(
|
||||
md.contains("# Quantum Nature of Light Measured With a Single Detector"),
|
||||
"title should remain a heading: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("## The introduction")
|
||||
&& !md.contains("## demonstrate")
|
||||
&& !md.contains("## quantization"),
|
||||
"bold abstract lines should not become headings: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("technique.**\n\nA photon is a single excitation"),
|
||||
"body paragraph should be separated from bold abstract: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_code_multiline_accumulation() {
|
||||
let mut line1 = make_item("fn main() {", 1, Some(0));
|
||||
|
||||
+10
-1
@@ -422,7 +422,16 @@ impl Default for MarkdownOptions {
|
||||
fix_hyphenation: true,
|
||||
detect_bold: true,
|
||||
detect_italic: true,
|
||||
include_images: true,
|
||||
// `include_images: false` is intentional. The content-stream walker
|
||||
// now emits `ItemType::Image` `TextItem`s for every Image XObject
|
||||
// it encounters (see `extractor/content_stream.rs`). If we rendered
|
||||
// those into markdown by default, every existing caller would
|
||||
// suddenly see `` placeholders inserted
|
||||
// throughout their output — a silent regression for anyone who
|
||||
// upgrades. Image bboxes are still available via
|
||||
// `extract_text_with_positions` for callers (e.g. layout-aware
|
||||
// pipelines) that want to crop + caption figures themselves.
|
||||
include_images: false,
|
||||
include_links: true,
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: true,
|
||||
|
||||
@@ -231,6 +231,15 @@ pub fn detect_tables_from_rects(
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
) -> (Vec<Table>, Vec<RectHintRegion>) {
|
||||
// Strip Image placeholders before column/row clustering — an image's bbox
|
||||
// would otherwise show up as a spurious column edge. See `is_text_layout_item`.
|
||||
let items_owned: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| crate::extractor::is_text_layout_item(i))
|
||||
.cloned()
|
||||
.collect();
|
||||
let items = items_owned.as_slice();
|
||||
|
||||
// Filter rects on this page; normalize negative widths/heights; skip tiny rects.
|
||||
let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized
|
||||
for r in rects {
|
||||
|
||||
+65
-4
@@ -208,6 +208,24 @@ fn looks_like_compact_entry_label(cell: &str) -> bool {
|
||||
(1..=6).contains(&words)
|
||||
}
|
||||
|
||||
fn looks_like_plain_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 4 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-'))
|
||||
&& starts_with_uppercase_alpha(trimmed)
|
||||
&& (1..=4).contains(&alpha_word_count(trimmed))
|
||||
}
|
||||
|
||||
fn ends_like_incomplete_phrase(cell: &str) -> bool {
|
||||
let lower = cell.trim_end().to_ascii_lowercase();
|
||||
lower.ends_with(" and")
|
||||
@@ -305,6 +323,10 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.and_then(|r| r.first())
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
@@ -328,6 +350,10 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
&& non_first_cells
|
||||
.iter()
|
||||
.any(|cell| looks_like_compact_entry_label(cell));
|
||||
let looks_like_section_label_row = !first_cell.is_empty()
|
||||
&& filled_cells == 1
|
||||
&& header_filled >= 3
|
||||
&& looks_like_plain_section_label(first_cell);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
@@ -344,10 +370,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.last()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(0);
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
// Merge when the row has significantly fewer filled cells than header.
|
||||
// For wide tables (5+ cols), require ≤50% of header cells.
|
||||
// For narrow tables (2-4 cols), require fewer than header cells.
|
||||
@@ -369,6 +391,7 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& !looks_like_new_first_column_entry
|
||||
&& !looks_like_section_label_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -514,6 +537,44 @@ mod tests {
|
||||
assert!(cleaned[1][1].contains("continued text here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_first_column_section_label_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Properties".into(),
|
||||
"Conditions".into(),
|
||||
"Method".into(),
|
||||
"Typical values".into(),
|
||||
"Units".into(),
|
||||
],
|
||||
vec![
|
||||
"Melt Flow Rate".into(),
|
||||
"230 C/2.16 kg".into(),
|
||||
"ASTM D1238".into(),
|
||||
"3.0".into(),
|
||||
"g/10 min".into(),
|
||||
],
|
||||
vec![
|
||||
"Mechanical".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"Tensile Stress at Yield".into(),
|
||||
"50 mm/min".into(),
|
||||
"ASTM D638".into(),
|
||||
"31".into(),
|
||||
"MPa".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 4);
|
||||
assert_eq!(cleaned[2][0], "Mechanical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_short_subheader_not_merged() {
|
||||
let cells = vec![
|
||||
|
||||
+1233
File diff suppressed because it is too large
Load Diff
+187
-3
@@ -2,13 +2,14 @@
|
||||
|
||||
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::ItemType;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
|
||||
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, extract_text_with_positions_mem,
|
||||
process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
|
||||
PdfType, TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -3391,3 +3392,186 @@ fn test_synthetic_type0_broken_tounicode_emits_fffd_not_latin1_mojibake() {
|
||||
result.pages_needing_ocr
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Image XObject emission
|
||||
// ============================================================================
|
||||
|
||||
/// Build a minimal PDF containing one Image XObject placed at a known CTM.
|
||||
/// `image_ctm` is the 6-element matrix applied to the unit square by the
|
||||
/// `Do` operator (per PDF spec section 8.9.5 "Image Coordinate System").
|
||||
/// For an axis-aligned image at `(x, y)` with size `w × h`, that's
|
||||
/// `[w, 0, 0, h, x, y]`.
|
||||
fn make_pdf_with_image(image_ctm: [f32; 6]) -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize];
|
||||
|
||||
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(body.as_bytes());
|
||||
pdf.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
fn add_stream_object(
|
||||
pdf: &mut Vec<u8>,
|
||||
offsets: &mut Vec<usize>,
|
||||
id: usize,
|
||||
dict: &str,
|
||||
stream_bytes: &[u8],
|
||||
) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(
|
||||
format!("<< {} /Length {} >>\nstream\n", dict, stream_bytes.len()).as_bytes(),
|
||||
);
|
||||
pdf.extend_from_slice(stream_bytes);
|
||||
pdf.extend_from_slice(b"\nendstream\nendobj\n");
|
||||
}
|
||||
|
||||
// 1: catalog → 2: pages → 3: page with XObject /Im0 → 4: content stream
|
||||
// 5: font → 6: image XObject (1×1 grayscale)
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
1,
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
2,
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
3,
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
|
||||
/Resources << /Font << /F1 5 0 R >> /XObject << /Im0 6 0 R >> >> \
|
||||
/Contents 4 0 R >>",
|
||||
);
|
||||
let [a, b, c, d, e, f] = image_ctm;
|
||||
// BT/ET around a small text item just so the page isn't classified as
|
||||
// image-only (which would route to a different code path). Then save
|
||||
// graphics state, apply the image CTM, invoke Im0, restore.
|
||||
let content = format!(
|
||||
"BT /F1 12 Tf 100 700 Td (Hi) Tj ET\nq {} {} {} {} {} {} cm /Im0 Do Q",
|
||||
a, b, c, d, e, f
|
||||
);
|
||||
add_stream_object(&mut pdf, &mut offsets, 4, "", content.as_bytes());
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
5,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
);
|
||||
// 1×1 grayscale image; the single byte is mid-gray. Contents don't
|
||||
// matter to the extractor — it only cares about the XObject's
|
||||
// /Subtype and the CTM at the `Do` operator.
|
||||
let image_pixel = [128u8];
|
||||
add_stream_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
6,
|
||||
"/Type /XObject /Subtype /Image /Width 1 /Height 1 \
|
||||
/ColorSpace /DeviceGray /BitsPerComponent 8",
|
||||
&image_pixel,
|
||||
);
|
||||
|
||||
let xref_start = pdf.len();
|
||||
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
|
||||
pdf.extend_from_slice(b"0000000000 65535 f \n");
|
||||
for offset in offsets.iter().skip(1) {
|
||||
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
pdf.extend_from_slice(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
|
||||
offsets.len(),
|
||||
xref_start
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
pdf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_text_with_positions_emits_image_bboxes() {
|
||||
// Place a 200×100 image at (50, 600) in PDF user space (origin
|
||||
// bottom-left). The Do operator applies the CTM to a unit square,
|
||||
// so for an axis-aligned image, CTM = [w, 0, 0, h, x, y].
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
|
||||
let images: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| matches!(i.item_type, ItemType::Image))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
images.len(),
|
||||
1,
|
||||
"expected exactly one Image item, got items: {:?}",
|
||||
items
|
||||
.iter()
|
||||
.map(|i| (&i.text, &i.item_type))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let img = images[0];
|
||||
assert!((img.x - 50.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 600.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 200.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
assert_eq!(img.page, 1);
|
||||
// text field carries the legacy `[Image: <resource-name>]` form that
|
||||
// the markdown emitter already knows how to parse.
|
||||
assert_eq!(img.text, "[Image: Im0]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_xobject_bbox_handles_rotated_ctm() {
|
||||
// 90° rotation CTM: a unit square at the origin maps to a square
|
||||
// rotated counter-clockwise about (0,0), then translated to (200, 300).
|
||||
// For a 100×100 image, that's CTM = [0, 100, -100, 0, 200, 300]
|
||||
// (apply the rotation: (1,0) → (0,100); (0,1) → (-100,0)).
|
||||
// The page-space corners are:
|
||||
// (0,0) → (200, 300)
|
||||
// (1,0) → (200, 400)
|
||||
// (1,1) → (100, 400)
|
||||
// (0,1) → (100, 300)
|
||||
// → AABB: x=100..200 (w=100), y=300..400 (h=100).
|
||||
let pdf = make_pdf_with_image([0.0, 100.0, -100.0, 0.0, 200.0, 300.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
let img = items
|
||||
.iter()
|
||||
.find(|i| matches!(i.item_type, ItemType::Image))
|
||||
.expect("image item");
|
||||
assert!((img.x - 100.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 300.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 100.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_emission_does_not_change_default_markdown() {
|
||||
// Default `MarkdownOptions::include_images = false` — adding image
|
||||
// emission MUST NOT make `extract_pages_markdown` start producing
|
||||
// `![Image: …]` placeholders for everyone. Existing callers that
|
||||
// upgrade should see no diff in their markdown.
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let result = extract_pages_markdown_mem(&pdf, None).expect("extract");
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(
|
||||
!result.pages[0].markdown.contains("Image:"),
|
||||
"default markdown leaked an image placeholder: {:?}",
|
||||
result.pages[0].markdown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_options_default_has_include_images_false() {
|
||||
// Explicit assertion so anyone flipping this back catches it in CI.
|
||||
// See `MarkdownOptions::default` in src/markdown/mod.rs for the
|
||||
// long-form rationale.
|
||||
let opts = MarkdownOptions::default();
|
||||
assert!(!opts.include_images);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user