Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
512c50c64d |
@@ -20,7 +20,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --verbose
|
||||
@@ -51,9 +61,17 @@ jobs:
|
||||
components: clippy
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: clippy
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-clippy-
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy -- -D warnings
|
||||
@@ -71,9 +89,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: build
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-build-
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --verbose
|
||||
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you believe you've found a security vulnerability in pdf-inspector, please
|
||||
report it privately so we can fix it before public disclosure.
|
||||
|
||||
**Preferred:** Email **help@firecrawl.dev** with:
|
||||
|
||||
- A description of the issue and its impact
|
||||
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
|
||||
- The version or commit hash of pdf-inspector you tested against
|
||||
|
||||
**Alternative:** Use GitHub's private vulnerability reporting under the
|
||||
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
|
||||
|
||||
We'll acknowledge your report in a timely manner and keep you updated on
|
||||
remediation progress. Please do not open a public GitHub issue for security
|
||||
bugs.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
- Memory-safety issues (panics, OOB reads, UB) reachable from a crafted PDF
|
||||
- Denial-of-service vectors (unbounded allocation, infinite loops) on
|
||||
reasonably-sized inputs
|
||||
- Bugs in the `pdf2md` / `detect-pdf` binaries or the `pdf-inspector` crate
|
||||
that affect downstream consumers
|
||||
|
||||
Out of scope:
|
||||
- Bugs in upstream dependencies (`lopdf`, etc.) — please report those upstream
|
||||
- Extraction quality issues (wrong text, missing tables) — open a regular
|
||||
GitHub issue instead
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.2",
|
||||
"version": "1.8.7",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env node
|
||||
// Generalized one-shot probe: run detectVectorGridInRegion against a single
|
||||
// PDF + region without spinning up the api or the bench.
|
||||
//
|
||||
// Usage:
|
||||
// node probe-doc.mjs <pdf-path> [page] [bbox] [dpi]
|
||||
//
|
||||
// pdf-path : path to a PDF on disk (required)
|
||||
// page : 0-indexed page number (default 0)
|
||||
// bbox : "x0,y0,x1,y1" in PDF points (default = full standard letter)
|
||||
// dpi : render dpi (default 200)
|
||||
//
|
||||
// Examples:
|
||||
// node probe-doc.mjs ~/Code/opendataloader-bench/pdfs/01030000000127.pdf 0
|
||||
// node probe-doc.mjs /tmp/foo.pdf 3 "50,100,560,700" 200
|
||||
// node probe-doc.mjs /tmp/foo.pdf # page 0, full page, dpi 200
|
||||
//
|
||||
// Prints: cells / rows / cols / null. Useful for fast pdf-inspector loops:
|
||||
// cargo build --release && bun run build:debug && node probe-doc.mjs ...
|
||||
|
||||
import { readFileSync, existsSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { detectVectorGridInRegion } = require("./index.js");
|
||||
|
||||
const [, , pdfPath, pageArg, bboxArg, dpiArg] = process.argv;
|
||||
|
||||
if (!pdfPath) {
|
||||
console.error("usage: node probe-doc.mjs <pdf-path> [page] [bbox] [dpi]");
|
||||
process.exit(2);
|
||||
}
|
||||
if (!existsSync(pdfPath)) {
|
||||
console.error(`pdf not found: ${pdfPath}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const page = pageArg !== undefined ? Number(pageArg) : 0;
|
||||
const bbox = bboxArg
|
||||
? bboxArg.split(",").map(Number)
|
||||
: [0, 0, 612, 792]; // standard US letter
|
||||
const dpi = dpiArg !== undefined ? Number(dpiArg) : 200;
|
||||
|
||||
if (bbox.length !== 4 || bbox.some(Number.isNaN)) {
|
||||
console.error(`invalid bbox "${bboxArg}", expected "x0,y0,x1,y1"`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const pdf = readFileSync(pdfPath);
|
||||
|
||||
const t0 = Date.now();
|
||||
const result = detectVectorGridInRegion(pdf, page, bbox, dpi);
|
||||
const ms = Date.now() - t0;
|
||||
|
||||
if (!result) {
|
||||
console.log(`null (${ms}ms) page=${page} bbox=${bbox.join(",")} dpi=${dpi}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const rows = result.structureTokens.filter((t) => t === "<tr>").length;
|
||||
const cols = rows > 0 ? result.cellBboxes.length / rows : 0;
|
||||
console.log(
|
||||
`cells=${result.cellBboxes.length} rows=${rows} cols=${cols} (${ms}ms) page=${page} bbox=${bbox.join(",")} dpi=${dpi}`,
|
||||
);
|
||||
@@ -18,7 +18,7 @@ use super::fonts::{
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
/// Strip PDF comments (% to end of line) from content stream bytes.
|
||||
///
|
||||
@@ -664,29 +664,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if let Some(xobj_type) = xobjects.get(&xobj_name) {
|
||||
match xobj_type {
|
||||
XObjectType::Image => {
|
||||
// Emit a positional placeholder for the image
|
||||
// so downstream consumers (layout-aware
|
||||
// pipelines, figure-OCR routers) can locate
|
||||
// raster figures without parsing the PDF
|
||||
// again. The text field carries the
|
||||
// XObject resource name in the legacy
|
||||
// `[Image: Im0]` format that the markdown
|
||||
// emitter already recognizes.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
// Skip images — text extraction only
|
||||
}
|
||||
XObjectType::Form(form_id) => {
|
||||
// Extract text from Form XObject
|
||||
|
||||
@@ -29,12 +29,8 @@ pub(crate) fn detect_columns(
|
||||
const MIN_ITEMS_PER_COLUMN: usize = 10;
|
||||
const NOISE_FRACTION: f32 = 0.15;
|
||||
|
||||
// Get items for this page. Strip Image placeholders — an image's left edge
|
||||
// would otherwise count toward the column projection profile.
|
||||
let page_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.page == page && crate::extractor::is_text_layout_item(i))
|
||||
.collect();
|
||||
// Get items for this page
|
||||
let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect();
|
||||
|
||||
if page_items.is_empty() {
|
||||
return vec![];
|
||||
|
||||
@@ -227,69 +227,6 @@ fn extract_positioned_text_impl(
|
||||
// Shared helpers (used by submodules via `super::`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Return true when this item should participate in text-layout
|
||||
/// heuristics (column detection, table grid detection, line grouping).
|
||||
///
|
||||
/// Image XObjects emit a positional placeholder via
|
||||
/// `extract_text_with_positions` (so layout-aware callers can crop +
|
||||
/// caption figures), but their bboxes don't carry text glyphs and would
|
||||
/// skew column/row clustering if they reached the heuristics. Hyperlinks
|
||||
/// and form fields *do* participate — the existing logic treats them as
|
||||
/// text-like and we keep that.
|
||||
pub(crate) fn is_text_layout_item(item: &crate::types::TextItem) -> bool {
|
||||
!matches!(item.item_type, crate::types::ItemType::Image)
|
||||
}
|
||||
|
||||
/// Map a (u, v) point in unit-square coordinates through the 6-element CTM
|
||||
/// to page-space. CTM format is `[a, b, c, d, e, f]` per
|
||||
/// [`multiply_matrices`].
|
||||
fn apply_ctm_point(ctm: &[f32; 6], u: f32, v: f32) -> (f32, f32) {
|
||||
(
|
||||
u * ctm[0] + v * ctm[2] + ctm[4],
|
||||
u * ctm[1] + v * ctm[3] + ctm[5],
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute the page-space axis-aligned bounding box of an Image XObject
|
||||
/// invoked under the given CTM.
|
||||
///
|
||||
/// Per the PDF spec, an image XObject is always rendered into a unit
|
||||
/// square `(0,0)–(1,1)` in its local coordinate system, and the `Do`
|
||||
/// operator applies the current CTM to position/scale/rotate that square
|
||||
/// onto the page. For the common axis-aligned case (no rotation/shear),
|
||||
/// the CTM reduces to `[w, 0, 0, h, x, y]` and the bbox is just
|
||||
/// `(x, y, w, h)`. For rotated/sheared images we transform all four
|
||||
/// corners and return their axis-aligned bbox so the caller always gets
|
||||
/// an upright rectangle.
|
||||
///
|
||||
/// Coordinates are PDF user space (origin at bottom-left, y-up). Width
|
||||
/// and height are non-negative.
|
||||
pub(crate) fn image_bbox_from_ctm(ctm: &[f32; 6]) -> (f32, f32, f32, f32) {
|
||||
let corners = [
|
||||
apply_ctm_point(ctm, 0.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 1.0),
|
||||
apply_ctm_point(ctm, 0.0, 1.0),
|
||||
];
|
||||
let (mut x_min, mut x_max) = (corners[0].0, corners[0].0);
|
||||
let (mut y_min, mut y_max) = (corners[0].1, corners[0].1);
|
||||
for (cx, cy) in corners.iter().skip(1) {
|
||||
if *cx < x_min {
|
||||
x_min = *cx;
|
||||
}
|
||||
if *cx > x_max {
|
||||
x_max = *cx;
|
||||
}
|
||||
if *cy < y_min {
|
||||
y_min = *cy;
|
||||
}
|
||||
if *cy > y_max {
|
||||
y_max = *cy;
|
||||
}
|
||||
}
|
||||
(x_min, y_min, x_max - x_min, y_max - y_min)
|
||||
}
|
||||
|
||||
/// Multiply two 2D transformation matrices
|
||||
/// Matrix format: [a, b, c, d, e, f] representing:
|
||||
/// | a b 0 |
|
||||
|
||||
+13
-37
@@ -10,7 +10,7 @@ use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
const MAX_FORM_XOBJECT_DEPTH: u8 = 5;
|
||||
|
||||
@@ -262,43 +262,19 @@ fn extract_form_xobject_text_inner(
|
||||
if !op.operands.is_empty() {
|
||||
if let Ok(name) = op.operands[0].as_name() {
|
||||
let xobj_name = String::from_utf8_lossy(name).to_string();
|
||||
match form_xobjects.get(&xobj_name) {
|
||||
Some(XObjectType::Form(nested_id)) => {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
if let Some(XObjectType::Form(nested_id)) = form_xobjects.get(&xobj_name) {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
Some(XObjectType::Image) => {
|
||||
// Mirror the top-level Image-XObject emission
|
||||
// in content_stream.rs so figures embedded
|
||||
// inside Form XObjects (common in print-to-PDF
|
||||
// workflows) aren't silently dropped.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+49
-1415
File diff suppressed because it is too large
Load Diff
+1
-10
@@ -422,16 +422,7 @@ impl Default for MarkdownOptions {
|
||||
fix_hyphenation: true,
|
||||
detect_bold: true,
|
||||
detect_italic: true,
|
||||
// `include_images: false` is intentional. The content-stream walker
|
||||
// now emits `ItemType::Image` `TextItem`s for every Image XObject
|
||||
// it encounters (see `extractor/content_stream.rs`). If we rendered
|
||||
// those into markdown by default, every existing caller would
|
||||
// suddenly see `` placeholders inserted
|
||||
// throughout their output — a silent regression for anyone who
|
||||
// upgrades. Image bboxes are still available via
|
||||
// `extract_text_with_positions` for callers (e.g. layout-aware
|
||||
// pipelines) that want to crop + caption figures themselves.
|
||||
include_images: false,
|
||||
include_images: true,
|
||||
include_links: true,
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: true,
|
||||
|
||||
+29
-261
@@ -4,74 +4,11 @@
|
||||
//! gridlines. Many IRS forms and government PDFs use these instead of
|
||||
//! `re` (rectangle) operators.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::tables::Table;
|
||||
use crate::types::{PdfLine, TextItem};
|
||||
|
||||
use super::detect_rects::{assign_items_to_grid, snap_edges};
|
||||
|
||||
/// Derive column edges from the x-endpoints of horizontal-rule
|
||||
/// segments when no vertical lines were drawn.
|
||||
///
|
||||
/// Catalog and archival-finding-aid tables are commonly drawn with
|
||||
/// per-row horizontal rules broken into N segments (one segment per
|
||||
/// cell), with no vertical dividers at all. The segment break points
|
||||
/// (e.g. `[50, 127], [127, 485], [485, 562]` per row) implicitly
|
||||
/// encode the column boundaries.
|
||||
///
|
||||
/// Returns column edges if ≥3 distinct x-positions each show up as a
|
||||
/// segment endpoint on ≥50% of the unique horizontal-line rows.
|
||||
/// Returns `None` otherwise — decorative rules with varying widths
|
||||
/// shouldn't be mistaken for a table.
|
||||
fn derive_columns_from_horizontal_segments(horizontals: &[(f32, f32, f32)]) -> Option<Vec<f32>> {
|
||||
if horizontals.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut endpoints: Vec<f32> = Vec::with_capacity(horizontals.len() * 2);
|
||||
for &(_, x_min, x_max) in horizontals {
|
||||
endpoints.push(x_min);
|
||||
endpoints.push(x_max);
|
||||
}
|
||||
let clusters = snap_edges(&endpoints, 5.0);
|
||||
if clusters.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Bucket y-values to count unique rows. Tolerance ~0.1pt (×10
|
||||
// rounding) tolerates the snap_edges 3pt clustering used later
|
||||
// for row edges.
|
||||
let unique_rows: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
if unique_rows.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let min_rows = (unique_rows.len() as f32 * 0.5).ceil() as usize;
|
||||
|
||||
let qualifying: Vec<f32> = clusters
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&cluster_x| {
|
||||
let rows_touched: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.filter(|&&(_, x_min, x_max)| {
|
||||
(x_min - cluster_x).abs() < 5.0 || (x_max - cluster_x).abs() < 5.0
|
||||
})
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
rows_touched.len() >= min_rows
|
||||
})
|
||||
.collect();
|
||||
|
||||
if qualifying.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
Some(qualifying)
|
||||
}
|
||||
|
||||
/// Detect tables from line segments on a given page.
|
||||
///
|
||||
/// Lines are classified as horizontal or vertical, snapped into grid edges,
|
||||
@@ -115,50 +52,25 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
// Diagonal lines are ignored
|
||||
}
|
||||
|
||||
if horizontals.len() < 3 {
|
||||
if horizontals.len() < 3 || verticals.len() < 2 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// If no/very-few vertical lines are drawn, try to derive column edges
|
||||
// from the x-endpoints of the horizontal-rule segments. Catalog and
|
||||
// archival-finding-aid layouts commonly draw each row's horizontal
|
||||
// rule as N segments (one per cell), with no vertical dividers at
|
||||
// all — the segment break points encode the column boundaries.
|
||||
let implicit_col_edges: Option<Vec<f32>> = if verticals.len() < 2 {
|
||||
derive_columns_from_horizontal_segments(&horizontals)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if verticals.len() < 2 && implicit_col_edges.is_none() {
|
||||
return Vec::new();
|
||||
}
|
||||
let cols_from_segments = implicit_col_edges.is_some();
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page){}",
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
|
||||
page,
|
||||
horizontals.len(),
|
||||
verticals.len(),
|
||||
page_lines.len(),
|
||||
if cols_from_segments {
|
||||
" — columns from horizontal segments"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
page_lines.len()
|
||||
);
|
||||
|
||||
// Snap Y-values of horizontal lines → row edges
|
||||
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
|
||||
let row_edges = snap_edges(&h_ys, 3.0);
|
||||
|
||||
// Column edges from drawn verticals when present, else from the
|
||||
// horizontal-segment endpoints derived above.
|
||||
let col_edges = if let Some(c) = implicit_col_edges {
|
||||
c
|
||||
} else {
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
snap_edges(&v_xs, 3.0)
|
||||
};
|
||||
// Snap X-values of vertical lines → column edges
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
let col_edges = snap_edges(&v_xs, 3.0);
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} row edges, {} col edges after snap",
|
||||
@@ -198,21 +110,15 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Reject page-spanning frames: a decorative outer border has just 4
|
||||
// edges (top/bottom/left/right). Real full-page tables — common in
|
||||
// governmental ledgers, financial reports, etc. — span the same A4 /
|
||||
// Letter dimensions but have many internal row/column rules. Only
|
||||
// reject when the line set looks like a bare frame, not a grid.
|
||||
// Reject page-spanning frames: if the grid covers >90% of a standard page
|
||||
// dimension in both axes, it's a border frame, not a table.
|
||||
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
||||
if table_width > 500.0 && table_height > 700.0 && horizontals.len() <= 4 && verticals.len() <= 4
|
||||
{
|
||||
if table_width > 500.0 && table_height > 700.0 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0}, {} h + {} v)",
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})",
|
||||
page,
|
||||
table_width,
|
||||
table_height,
|
||||
horizontals.len(),
|
||||
verticals.len()
|
||||
table_height
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -240,33 +146,24 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
|
||||
// Validate vertical lines: at least 2 should span a meaningful height.
|
||||
// Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
|
||||
// for tables with partial column separators. Skipped entirely when
|
||||
// columns came from horizontal-segment endpoints — there are no
|
||||
// vertical lines to validate against, and the segment-endpoint
|
||||
// consistency check in `derive_columns_from_horizontal_segments`
|
||||
// is the equivalent guard.
|
||||
let spanning_v = if cols_from_segments {
|
||||
0
|
||||
} else {
|
||||
let s = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let p = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if s < 2 && p < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
s,
|
||||
p
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
s
|
||||
};
|
||||
// for tables with partial column separators.
|
||||
let spanning_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let partial_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if spanning_v < 2 && partial_v < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
spanning_v,
|
||||
partial_v
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Row edges need to be in descending order (top of page = higher Y first)
|
||||
let mut row_edges_desc = row_edges;
|
||||
@@ -513,135 +410,6 @@ mod tests {
|
||||
assert!(tables.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_only_implicit_columns_accepted() {
|
||||
// Catalog/finding-aid pattern: each row's horizontal rule is
|
||||
// drawn as 3 segments at consistent x-endpoints (50, 127, 485,
|
||||
// 562), with no vertical lines anywhere. The segment break
|
||||
// points must be inferred as column edges.
|
||||
let mut lines = Vec::new();
|
||||
// Slightly uneven row spacing so the chart-gridline rejector
|
||||
// (CV < 0.02) doesn't fire.
|
||||
let row_ys = [80.0_f32, 145.0, 215.0, 280.0, 350.0, 415.0, 485.0];
|
||||
for &y in &row_ys {
|
||||
lines.push(make_hline(y, 50.0, 127.0, 1));
|
||||
lines.push(make_hline(y, 127.0, 485.0, 1));
|
||||
lines.push(make_hline(y, 485.0, 562.0, 1));
|
||||
}
|
||||
// Populate every cell so capture / density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for w in row_ys.windows(2) {
|
||||
let row_y = (w[0] + w[1]) / 2.0;
|
||||
items.push(make_item("id", 80.0, row_y, 1));
|
||||
items.push(make_item("description here", 200.0, row_y, 1));
|
||||
items.push(make_item("date", 510.0, row_y, 1));
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"horizontal-segment-only grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 3, "expected 3 columns");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_with_inconsistent_endpoints_rejected() {
|
||||
// Decorative rules of varying widths shouldn't be detected as a
|
||||
// table — each line has its own x-endpoints, no consistent
|
||||
// column boundary survives the 50%-of-rows threshold.
|
||||
let lines = vec![
|
||||
make_hline(100.0, 50.0, 150.0, 1),
|
||||
make_hline(200.0, 50.0, 220.0, 1),
|
||||
make_hline(300.0, 50.0, 310.0, 1),
|
||||
make_hline(400.0, 50.0, 470.0, 1),
|
||||
];
|
||||
let items = vec![
|
||||
make_item("decorative", 100.0, 150.0, 1),
|
||||
make_item("text", 100.0, 250.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"varying-width decorative rules should not be detected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_bare_frame_rejected() {
|
||||
// Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.
|
||||
// No internal structure → decorative border, not a table.
|
||||
let lines = vec![
|
||||
make_hline(20.0, 20.0, 575.0, 1), // top
|
||||
make_hline(820.0, 20.0, 575.0, 1), // bottom
|
||||
make_vline(20.0, 20.0, 820.0, 1), // left
|
||||
make_vline(575.0, 20.0, 820.0, 1), // right
|
||||
];
|
||||
let items = vec![
|
||||
make_item("title", 100.0, 100.0, 1),
|
||||
make_item("body", 100.0, 200.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"Page-sized 4-edge frame should be rejected as decoration"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_grid_with_internal_lines_accepted() {
|
||||
// Full-page table (governmental-ledger pattern): A4-sized grid
|
||||
// that previously hit the "page-spanning frame" early reject
|
||||
// before downstream validation could even look at it.
|
||||
// Verticals span the full table height so we isolate the
|
||||
// frame-vs-grid decision under test.
|
||||
let mut lines = Vec::new();
|
||||
// 13 horizontal rules: header + 12 row separators
|
||||
let h_ys = [
|
||||
22.5, 37.9, 95.5, 144.5, 184.9, 233.9, 291.7, 340.7, 415.8, 499.6, 574.7, 623.7, 698.8,
|
||||
];
|
||||
for &y in &h_ys {
|
||||
lines.push(make_hline(y, 22.6, 566.6, 1));
|
||||
}
|
||||
// 7 column dividers spanning full table height.
|
||||
let v_xs = [22.6, 66.3, 116.3, 186.6, 263.1, 493.5, 566.5];
|
||||
for &x in &v_xs {
|
||||
lines.push(make_vline(x, 22.5, 698.8, 1));
|
||||
}
|
||||
// Populate every cell so the capture-ratio + density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for r in 0..(h_ys.len() - 1) {
|
||||
let row_y = (h_ys[r] + h_ys[r + 1]) / 2.0;
|
||||
for c in 0..(v_xs.len() - 1) {
|
||||
let col_x = (v_xs[c] + v_xs[c + 1]) / 2.0;
|
||||
items.push(make_item("x", col_x, row_y, 1));
|
||||
}
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"Full-page table with internal grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 6,
|
||||
"expected ≥6 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert!(
|
||||
t.cells[0].len() >= 3,
|
||||
"expected ≥3 columns, got {}",
|
||||
t.cells[0].len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_rejected() {
|
||||
// Only 2 col edges (1 column) — not a table even with verticals
|
||||
|
||||
+14
-195
@@ -231,15 +231,6 @@ pub fn detect_tables_from_rects(
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
) -> (Vec<Table>, Vec<RectHintRegion>) {
|
||||
// Strip Image placeholders before column/row clustering — an image's bbox
|
||||
// would otherwise show up as a spurious column edge. See `is_text_layout_item`.
|
||||
let items_owned: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| crate::extractor::is_text_layout_item(i))
|
||||
.cloned()
|
||||
.collect();
|
||||
let items = items_owned.as_slice();
|
||||
|
||||
// Filter rects on this page; normalize negative widths/heights; skip tiny rects.
|
||||
let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized
|
||||
for r in rects {
|
||||
@@ -1403,15 +1394,13 @@ fn detect_row_stripe_table(
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
// Allow longer cells for multi-column tables (descriptions in one column
|
||||
// are common). Narrow grids with giant cells are usually layout
|
||||
// backgrounds — but only when the row count is also small. A 4+-row
|
||||
// key/value table with one descriptive column reads as a real table
|
||||
// on every other gate, so don't reject it on cell length alone.
|
||||
// are common). Single-column or 2-column "tables" with giant cells are
|
||||
// almost always layout backgrounds.
|
||||
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
||||
if max_cell_len > max_allowed && non_empty_rows < 4 {
|
||||
if max_cell_len > max_allowed {
|
||||
debug!(
|
||||
" row-stripe rejected: max cell length {} > {} (layout background, {} rows)",
|
||||
max_cell_len, max_allowed, non_empty_rows
|
||||
" row-stripe rejected: max cell length {} > {} (layout background)",
|
||||
max_cell_len, max_allowed
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1643,49 +1632,7 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
}
|
||||
};
|
||||
|
||||
// For wired-grid tables whose header text is centered/right-aligned but
|
||||
// whose data is left-aligned, cluster_x_positions can drop the header-only
|
||||
// x-cluster in its singleton-filter pass and merge adjacent data clusters
|
||||
// when the gap is below threshold, losing a column. Rect borders are
|
||||
// ground truth in that case — but only when each rect column actually
|
||||
// holds text. Decorative or background rects (prose laid out in a frame,
|
||||
// cell-fill rects with extra borders) can produce more rect-derived
|
||||
// columns than the text supports; preferring rects there would split a
|
||||
// logical column into spurious sub-columns.
|
||||
let rect_cols_match_text = match (&rect_col_edges, &text_col_edges) {
|
||||
(Some(rect_edges), _) if rect_edges.len() >= 4 => {
|
||||
let num_rect_cols = rect_edges.len() - 1;
|
||||
let mut col_item_counts = vec![0usize; num_rect_cols];
|
||||
for (_, item) in &page_items {
|
||||
let cx = item.x + item.width / 2.0;
|
||||
for c in 0..num_rect_cols {
|
||||
if cx >= rect_edges[c] - 2.0 && cx <= rect_edges[c + 1] + 2.0 {
|
||||
col_item_counts[c] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Require every rect column to hold multiple text items. A rect
|
||||
// column with no (or only one) item is decorative or the rect grid
|
||||
// is detecting a spurious column the data does not need; in those
|
||||
// cases the old text-cluster preference is the safer fallback.
|
||||
col_item_counts.iter().all(|&n| n >= 2)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||
(Some(rect_edges), text_edges_opt) if rect_cols_match_text => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns (text clusters: {}; rect cols well-distributed)",
|
||||
rect_edges.len() - 1,
|
||||
text_edges_opt
|
||||
.as_ref()
|
||||
.map(|e| (e.len() - 1) as i32)
|
||||
.unwrap_or(-1)
|
||||
);
|
||||
(rect_edges, false)
|
||||
}
|
||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||
@@ -1772,21 +1719,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject tables with paragraph-length cells — typically layout
|
||||
// backgrounds (sidebars, banners) where a single big rectangle
|
||||
// contains a wall of prose. Spare multi-row key/value tables where
|
||||
// the value column is a multi-bullet description: those pass every
|
||||
// other gate and shouldn't get killed on cell length alone.
|
||||
// Reject tables with paragraph-length cells (layout backgrounds, not tables)
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" cell-rect rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" cell-rect rejected: max cell length {} > 500",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2200,20 +2143,18 @@ fn detect_merged_cluster_table(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject if any cell has excessive text — layout background rects
|
||||
// produce "cells" containing paragraphs, not short data-table values.
|
||||
// Multi-row key/value tables can legitimately have one column of
|
||||
// long descriptive text, so only reject narrow-row layouts here.
|
||||
// Reject if any cell has excessive text — layout background rects produce
|
||||
// "cells" containing paragraphs, not short data-table values.
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" merged-cluster rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" merged-cluster rejected: max cell length {} > 500 (layout background)",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2643,46 +2584,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_row_stripe_accepts_multi_row_key_value_long_cells() {
|
||||
// Multi-row 2-column key/value table where one value cell holds
|
||||
// a paragraph (>500 chars). The old `max_cell_len > 500` check
|
||||
// rejected this shape as a "layout background"; with the
|
||||
// multi-row guard, it should be accepted.
|
||||
let mut rects = Vec::new();
|
||||
let row_h = 25.0_f32;
|
||||
let y_top = 700.0_f32;
|
||||
for i in 0..8 {
|
||||
let y = y_top - (i as f32) * row_h;
|
||||
rects.push((40.0, y, 510.0, row_h));
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
for i in 0..8 {
|
||||
let row_center_y = y_top - (i as f32) * row_h + row_h / 2.0;
|
||||
// Left column: short label
|
||||
items.push(make_item(&format!("Field {}", i), 45.0, row_center_y, 10.0));
|
||||
// Right column: short value, except the last row which is a paragraph
|
||||
let value = if i == 7 {
|
||||
"X".repeat(800)
|
||||
} else {
|
||||
"value".to_string()
|
||||
};
|
||||
items.push(make_item(&value, 300.0, row_center_y, 10.0));
|
||||
}
|
||||
let result = detect_row_stripe_table(&items, &rects, 1);
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"multi-row key/value table with one long cell should be accepted"
|
||||
);
|
||||
let t = result.unwrap();
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 2, "expected 2 columns");
|
||||
}
|
||||
|
||||
// --- propagate_merged_cells ---
|
||||
|
||||
#[test]
|
||||
@@ -3441,88 +3342,6 @@ mod tests {
|
||||
assert!(table.cells[2][1].contains("deny unauthorized"));
|
||||
}
|
||||
|
||||
/// Wire-bordered 4-column table whose header text is centered/right-aligned
|
||||
/// inside each cell while the data is left-aligned: cluster_x_positions
|
||||
/// merges adjacent columns (data Item→EAN gap is below threshold) and
|
||||
/// drops the header-only x-clusters in the filter pass, leaving only 3
|
||||
/// text-derived columns. Rect borders are 4 columns of ground truth.
|
||||
/// Before the fix the cell-rect path preferred text edges when they were
|
||||
/// the smaller set — losing a column. After the fix, 3+ rect columns
|
||||
/// always win.
|
||||
#[test]
|
||||
fn wired_header_data_misaligned_keeps_all_columns_from_rects() {
|
||||
let page = 1;
|
||||
// 4 cols: Item | EAN | Nombre | Cant
|
||||
let col_xs = [380.0_f32, 410.0, 470.0, 660.0, 700.0];
|
||||
// Header + 9 data rows at 15pt tall each (y descending).
|
||||
let row_ys: Vec<f32> = (0..=10).map(|r| 400.0 - 15.0 * r as f32).collect();
|
||||
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||
for r in 0..10 {
|
||||
let y_top = row_ys[r];
|
||||
let y_bot = row_ys[r + 1];
|
||||
for c in 0..4 {
|
||||
rects.push((col_xs[c], y_bot, col_xs[c + 1] - col_xs[c], y_top - y_bot));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items: Vec<TextItem> = Vec::new();
|
||||
// Header row (y ≈ 392.5): headers sit further to the right than data
|
||||
// because they are centered/right-aligned in the cells.
|
||||
items.push(make_item("Item", 389.0, 392.5, 9.0));
|
||||
items.push(make_item("EAN", 432.0, 392.5, 9.0));
|
||||
items.push(make_item("Nombre", 552.0, 392.5, 9.0));
|
||||
items.push(make_item("Cant", 672.0, 392.5, 9.0));
|
||||
|
||||
let names = [
|
||||
"Arnes Frontal",
|
||||
"Arnes Motor",
|
||||
"Arnes Piso",
|
||||
"Arnes Techo",
|
||||
"Arnes Puerta",
|
||||
"Arnes Tablero",
|
||||
"Arnes Trasero",
|
||||
"Arnes Lateral",
|
||||
"Arnes Sensor",
|
||||
];
|
||||
for r in 0..9 {
|
||||
let y = 377.5 - 15.0 * r as f32;
|
||||
items.push(make_item(&(r + 1).to_string(), 396.0, y, 9.0));
|
||||
items.push(make_item("7701023403016", 410.0, y, 9.0));
|
||||
items.push(make_item(names[r], 480.0, y, 9.0));
|
||||
items.push(make_item("1", 680.0, y, 9.0));
|
||||
}
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("wired 4-column table with header/data x-misalignment must detect");
|
||||
assert_eq!(
|
||||
table.columns.len(),
|
||||
4,
|
||||
"expected 4 columns from rect borders; cells: {:?}",
|
||||
table.cells
|
||||
);
|
||||
for c in 0..4 {
|
||||
let any_populated = table.cells.iter().any(|row| !row[c].trim().is_empty());
|
||||
assert!(
|
||||
any_populated,
|
||||
"column {} empty across all rows; cells: {:?}",
|
||||
c, table.cells
|
||||
);
|
||||
}
|
||||
// Header row populated in all 4 cells.
|
||||
let header = &table.cells[0];
|
||||
assert_eq!(header[0].trim(), "Item");
|
||||
assert_eq!(header[1].trim(), "EAN");
|
||||
assert_eq!(header[2].trim(), "Nombre");
|
||||
assert_eq!(header[3].trim(), "Cant");
|
||||
// First data row: Item="1", EAN, name, count="1" — no Item↔EAN merge.
|
||||
let data1 = &table.cells[1];
|
||||
assert_eq!(data1[0].trim(), "1");
|
||||
assert_eq!(data1[1].trim(), "7701023403016");
|
||||
assert!(data1[2].trim().contains("Arnes"));
|
||||
assert_eq!(data1[3].trim(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cluster_no_hint_without_items() {
|
||||
// Rects with no text items inside → no failed-cluster hint generated.
|
||||
|
||||
+7
-269
@@ -160,81 +160,6 @@ fn starts_with_uppercase_word(cell: &str) -> bool {
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_uppercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_lowercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_lowercase())
|
||||
}
|
||||
|
||||
fn starts_with_numbered_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim_start();
|
||||
let digit_count = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
|
||||
|
||||
digit_count > 0
|
||||
&& digit_count <= 3
|
||||
&& trimmed
|
||||
.chars()
|
||||
.nth(digit_count)
|
||||
.is_some_and(|c| matches!(c, '.' | ')' | '-' | ':'))
|
||||
}
|
||||
|
||||
fn alpha_word_count(cell: &str) -> usize {
|
||||
cell.split_whitespace()
|
||||
.filter(|word| word.chars().any(|c| c.is_alphabetic()))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn looks_like_compact_entry_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 3 || trimmed.len() > 80 {
|
||||
return false;
|
||||
}
|
||||
|
||||
if !starts_with_uppercase_alpha(trimmed) && !starts_with_numbered_label(trimmed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let words = alpha_word_count(trimmed);
|
||||
(1..=6).contains(&words)
|
||||
}
|
||||
|
||||
fn looks_like_plain_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 4 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-'))
|
||||
&& starts_with_uppercase_alpha(trimmed)
|
||||
&& (1..=4).contains(&alpha_word_count(trimmed))
|
||||
}
|
||||
|
||||
fn ends_like_incomplete_phrase(cell: &str) -> bool {
|
||||
let lower = cell.trim_end().to_ascii_lowercase();
|
||||
lower.ends_with(" and")
|
||||
|| lower.ends_with(" or")
|
||||
|| lower.ends_with(',')
|
||||
|| lower.ends_with('-')
|
||||
|| lower.ends_with('/')
|
||||
}
|
||||
|
||||
/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows
|
||||
fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
let mut cleaned: Vec<Vec<String>> = Vec::new();
|
||||
@@ -260,9 +185,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
|
||||
// Check if this is a continuation row (first column is empty but others have content).
|
||||
// A row with only 1 short non-empty cell (besides the first) is more likely a
|
||||
// section sub-header (e.g. "JAN", "FEB") than overflow text — don't merge it.
|
||||
@@ -300,76 +222,31 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.iter()
|
||||
.filter(|cell| starts_with_uppercase_word(cell))
|
||||
.count();
|
||||
let first_non_empty_col = row.iter().position(|c| !c.trim().is_empty());
|
||||
let first_non_empty_cell = first_non_empty_col
|
||||
.and_then(|idx| row.get(idx))
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let title_like_later_cells = first_non_empty_col
|
||||
.map(|idx| {
|
||||
row.iter()
|
||||
.skip(idx + 1)
|
||||
.map(|c| c.trim())
|
||||
.filter(|c| !c.is_empty() && starts_with_uppercase_alpha(c))
|
||||
.count()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let prev_first_cell_empty = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.is_some_and(|c| c.trim().is_empty());
|
||||
let prev_first_cell = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
&& uppercase_leading_cells >= non_first_cells.len().saturating_sub(1);
|
||||
// Hierarchical tables often use a row-spanned first column: sub-rows
|
||||
// leave column 0 blank, then start a compact title-like label in
|
||||
// column 1. Wrapped continuations in the existing fixtures start
|
||||
// mid-sentence/lowercase ("continued text here", "with 3.5%...") or
|
||||
// carry lowercase fragments in the later cells, so keep those mergeable.
|
||||
let looks_like_hierarchical_subrow = first_cell.is_empty()
|
||||
&& row.len() >= 3
|
||||
&& first_non_empty_col == Some(1)
|
||||
&& looks_like_compact_entry_label(first_non_empty_cell)
|
||||
&& ((non_first_cells.len() >= 2 && title_like_later_cells > 0)
|
||||
|| (non_first_cells.len() == 1
|
||||
&& prev_first_cell_empty
|
||||
&& alpha_word_count(first_non_empty_cell) >= 2));
|
||||
let looks_like_new_first_column_entry = !first_cell.is_empty()
|
||||
&& (starts_with_numbered_label(first_cell) || starts_with_uppercase_alpha(first_cell))
|
||||
&& filled_cells >= 2
|
||||
&& non_first_cells
|
||||
.iter()
|
||||
.any(|cell| looks_like_compact_entry_label(cell));
|
||||
let looks_like_section_label_row = !first_cell.is_empty()
|
||||
&& filled_cells == 1
|
||||
&& header_filled >= 3
|
||||
&& looks_like_plain_section_label(first_cell);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
&& !is_short_subheader
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& cleaned.len() > 1;
|
||||
|
||||
// Wrapped-cell continuation: row has fewer filled cells than the header
|
||||
// row, suggesting it's overflow text from the previous row's cells.
|
||||
// Only trigger when the previous row has significantly more filled cells.
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
let prev_filled = cleaned
|
||||
.last()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(0);
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
// Merge when the row has significantly fewer filled cells than header.
|
||||
// For wide tables (5+ cols), require ≤50% of header cells.
|
||||
// For narrow tables (2-4 cols), require fewer than header cells.
|
||||
@@ -380,18 +257,11 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
} else {
|
||||
header_filled.saturating_sub(1)
|
||||
};
|
||||
let continues_wrapped_first_column_label = !first_cell.is_empty()
|
||||
&& starts_with_lowercase_alpha(first_cell)
|
||||
&& ends_like_incomplete_phrase(prev_first_cell);
|
||||
let is_wrapped_continuation = cleaned.len() > 1
|
||||
&& filled_cells <= max_filled_for_merge
|
||||
&& (prev_filled > filled_cells
|
||||
|| (continues_wrapped_first_column_label && prev_filled >= filled_cells))
|
||||
&& prev_filled > filled_cells
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& !looks_like_new_first_column_entry
|
||||
&& !looks_like_section_label_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -537,44 +407,6 @@ mod tests {
|
||||
assert!(cleaned[1][1].contains("continued text here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_first_column_section_label_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Properties".into(),
|
||||
"Conditions".into(),
|
||||
"Method".into(),
|
||||
"Typical values".into(),
|
||||
"Units".into(),
|
||||
],
|
||||
vec![
|
||||
"Melt Flow Rate".into(),
|
||||
"230 C/2.16 kg".into(),
|
||||
"ASTM D1238".into(),
|
||||
"3.0".into(),
|
||||
"g/10 min".into(),
|
||||
],
|
||||
vec![
|
||||
"Mechanical".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"Tensile Stress at Yield".into(),
|
||||
"50 mm/min".into(),
|
||||
"ASTM D638".into(),
|
||||
"31".into(),
|
||||
"MPa".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 4);
|
||||
assert_eq!(cleaned[2][0], "Mechanical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_short_subheader_not_merged() {
|
||||
let cells = vec![
|
||||
@@ -627,100 +459,6 @@ mod tests {
|
||||
assert_eq!(cleaned[2][1], "Uncertainty around other copies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_numbered_hierarchy_rows_not_overmerged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"1. Group alpha".into(),
|
||||
"Task setup and".into(),
|
||||
"Begin setup".into(),
|
||||
"Faster start".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"management".into(),
|
||||
"recommended profile".into(),
|
||||
"with saved defaults".into(),
|
||||
],
|
||||
vec![
|
||||
"2. Group beta and".into(),
|
||||
"Storage setup".into(),
|
||||
"Provides upload tools".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"fine-tuning".into(),
|
||||
"".into(),
|
||||
"for filtered inputs".into(),
|
||||
"service".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Label workspace".into(),
|
||||
"Creates review sets".into(),
|
||||
"Lets teams review".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Model training".into(),
|
||||
"".into(),
|
||||
"Supports custom model".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 5);
|
||||
assert_eq!(cleaned[1][0], "1. Group alpha");
|
||||
assert_eq!(cleaned[1][1], "Task setup and management");
|
||||
assert_eq!(cleaned[2][0], "2. Group beta and fine-tuning");
|
||||
assert_eq!(cleaned[2][1], "Storage setup");
|
||||
assert_eq!(cleaned[3][1], "Label workspace");
|
||||
assert_eq!(cleaned[4][1], "Model training");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_partial_hierarchical_subrow_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"Group A".into(),
|
||||
"Alpha task".into(),
|
||||
"Initial detail".into(),
|
||||
"Initial benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Beta task".into(),
|
||||
"Parallel detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"second line".into(),
|
||||
"additional detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 3);
|
||||
assert_eq!(cleaned[1][1], "Alpha task");
|
||||
assert_eq!(cleaned[2][0], "");
|
||||
assert_eq!(cleaned[2][1], "Beta task second line");
|
||||
assert_eq!(cleaned[2][2], "Parallel detail additional detail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_full_width_continuation_row_still_merges_when_lowercase() {
|
||||
let cells = vec![
|
||||
|
||||
@@ -460,7 +460,6 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
item_indices.push(item_idx);
|
||||
}
|
||||
}
|
||||
merge_superscript_marker_rows(&mut row_ys, &mut cells);
|
||||
|
||||
// Validate: need reasonable fill rate
|
||||
let total_cells = row_ys.len() * columns.len();
|
||||
@@ -548,60 +547,6 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
Some(Table::new(col_xs, row_ys, cells, item_indices))
|
||||
}
|
||||
|
||||
fn merge_superscript_marker_rows(row_ys: &mut Vec<f32>, cells: &mut Vec<Vec<String>>) {
|
||||
let mut row_idx = 0;
|
||||
while row_idx < cells.len() {
|
||||
let non_empty: Vec<(usize, String)> = cells[row_idx]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(col_idx, cell)| {
|
||||
let trimmed = cell.trim();
|
||||
(!trimmed.is_empty()).then_some((col_idx, trimmed.to_string()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if non_empty.len() != 1 || !is_superscript_marker_cell(&non_empty[0].1) {
|
||||
row_idx += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let (marker_col, marker) = &non_empty[0];
|
||||
let prev =
|
||||
(row_idx > 0).then(|| (row_idx - 1, (row_ys[row_idx - 1] - row_ys[row_idx]).abs()));
|
||||
let next = (row_idx + 1 < cells.len())
|
||||
.then(|| (row_idx + 1, (row_ys[row_idx] - row_ys[row_idx + 1]).abs()));
|
||||
let target = [prev, next]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|(_, gap)| *gap <= 10.0)
|
||||
.min_by(|(_, gap_a), (_, gap_b)| gap_a.total_cmp(gap_b))
|
||||
.map(|(idx, _)| idx);
|
||||
|
||||
let Some(target_idx) = target else {
|
||||
row_idx += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
let target_cell = &mut cells[target_idx][*marker_col];
|
||||
if target_cell.trim().is_empty() {
|
||||
*target_cell = marker.to_string();
|
||||
} else {
|
||||
target_cell.push_str(marker);
|
||||
}
|
||||
cells.remove(row_idx);
|
||||
row_ys.remove(row_idx);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_superscript_marker_cell(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
!trimmed.is_empty()
|
||||
&& trimmed.chars().count() <= 2
|
||||
&& trimmed
|
||||
.chars()
|
||||
.all(|ch| matches!(ch, '*' | '#' | 'o' | 'O' | '°' | 'º' | '†' | '‡'))
|
||||
}
|
||||
|
||||
/// What kind of structure a detected `Table` represents. Classification is
|
||||
/// computed once at construction so consumers don't have to re-analyze the
|
||||
/// cells (and stay consistent across detection backends).
|
||||
@@ -744,69 +689,6 @@ mod tests {
|
||||
assert!(md.contains("|Cell 1|"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_superscript_marker_rows() {
|
||||
let mut rows = vec![506.0, 500.0, 480.0];
|
||||
let mut cells = vec![
|
||||
vec!["".into(), "".into(), "*".into()],
|
||||
vec!["Name".into(), "Method".into(), "Typical values".into()],
|
||||
vec!["Flow".into(), "ASTM D1238".into(), "3.0".into()],
|
||||
];
|
||||
|
||||
merge_superscript_marker_rows(&mut rows, &mut cells);
|
||||
|
||||
assert_eq!(rows, vec![500.0, 480.0]);
|
||||
assert_eq!(cells[0][2], "Typical values*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_builder_handles_borderless_specs_table() {
|
||||
let items = vec![
|
||||
make_char("*", 458.1, 544.2, 8.0, 4.4),
|
||||
make_char("Properties", 36.0, 538.6, 12.0, 53.1),
|
||||
make_char("Conditions", 195.8, 538.6, 12.0, 55.0),
|
||||
make_char("Method", 297.2, 538.6, 12.0, 39.4),
|
||||
make_char("Typical values", 384.1, 538.6, 12.0, 74.0),
|
||||
make_char("Units", 510.6, 538.6, 8.0, 17.9),
|
||||
make_char("Rheology", 36.0, 508.3, 10.0, 40.6),
|
||||
make_char("o", 209.8, 492.5, 6.5, 3.5),
|
||||
make_char("Melt Flow Rate", 36.0, 488.0, 10.0, 65.2),
|
||||
make_char("230 ", 190.4, 488.0, 10.0, 19.4),
|
||||
make_char("C/2.16 kg", 213.3, 488.0, 10.0, 42.8),
|
||||
make_char("ASTM D1238", 288.4, 488.0, 10.0, 56.8),
|
||||
make_char("3.0 ", 416.4, 488.0, 10.0, 16.9),
|
||||
make_char("g/10 min", 504.1, 488.0, 10.0, 39.5),
|
||||
make_char("Mechanical", 36.0, 451.5, 10.0, 48.3),
|
||||
make_char("Tensile Stress at Yield", 36.0, 431.3, 10.0, 96.7),
|
||||
make_char("50 mm/min", 197.9, 431.3, 10.0, 50.8),
|
||||
make_char("ASTM D638", 291.2, 431.3, 10.0, 51.3),
|
||||
make_char("31 ", 417.9, 431.3, 10.0, 13.9),
|
||||
make_char("MPa", 514.7, 431.3, 10.0, 18.4),
|
||||
make_char("Elongation at Yield", 36.0, 403.0, 10.0, 82.2),
|
||||
make_char("50 mm/min", 197.9, 403.0, 10.0, 50.8),
|
||||
make_char("ASTM D638", 291.2, 403.0, 10.0, 51.3),
|
||||
make_char("8 ", 420.6, 403.0, 10.0, 8.5),
|
||||
make_char("%", 519.1, 403.0, 10.0, 9.7),
|
||||
make_char("Flexural Modulus", 36.0, 374.6, 10.0, 74.0),
|
||||
make_char("ASTM D790", 291.2, 374.6, 10.0, 51.3),
|
||||
make_char("1400", 412.4, 374.6, 10.0, 21.8),
|
||||
make_char("MPa", 514.7, 374.6, 10.0, 18.4),
|
||||
];
|
||||
|
||||
let table = try_build_table_from_columns(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(
|
||||
md.contains("|Properties|Conditions|Method|Typical values*|Units|"),
|
||||
"{md}"
|
||||
);
|
||||
assert!(md.contains("|Mechanical|||||"), "{md}");
|
||||
assert!(
|
||||
md.contains("|Flexural Modulus||ASTM D790|1400|MPa|"),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_body_font_table_detected() {
|
||||
let items = vec![
|
||||
|
||||
BIN
Binary file not shown.
+3
-214
@@ -2,14 +2,13 @@
|
||||
|
||||
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::ItemType;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
|
||||
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, extract_text_with_positions_mem,
|
||||
process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
|
||||
PdfType, TextItem,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -1649,33 +1648,6 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_in_regions_uses_line_grid() {
|
||||
// Stroked-grid table (m/l/S path operators forming a 2x2 grid).
|
||||
// The heuristic text-only detector handles the same cells already,
|
||||
// so this guards that the line-backed path doesn't regress: the
|
||||
// markdown still contains all four data cells.
|
||||
let buf = synthetic_vector_grid_pdf(false);
|
||||
let results =
|
||||
extract_tables_in_regions_mem(&buf, &[(0, vec![[40.0, 50.0, 220.0, 760.0]])]).unwrap();
|
||||
let region = &results[0].regions[0];
|
||||
assert!(
|
||||
!region.needs_ocr,
|
||||
"stroked-grid table should be extracted, got needs_ocr=true"
|
||||
);
|
||||
for tok in ["A1", "B1", "A2", "B2"] {
|
||||
assert!(
|
||||
region.text.contains(tok),
|
||||
"expected '{tok}' in output, got: {}",
|
||||
region.text
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
region.text.contains('|'),
|
||||
"expected pipe-delimited markdown"
|
||||
);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_mem tests (TSR-aware path)
|
||||
// =========================================================================
|
||||
@@ -3392,186 +3364,3 @@ fn test_synthetic_type0_broken_tounicode_emits_fffd_not_latin1_mojibake() {
|
||||
result.pages_needing_ocr
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Image XObject emission
|
||||
// ============================================================================
|
||||
|
||||
/// Build a minimal PDF containing one Image XObject placed at a known CTM.
|
||||
/// `image_ctm` is the 6-element matrix applied to the unit square by the
|
||||
/// `Do` operator (per PDF spec section 8.9.5 "Image Coordinate System").
|
||||
/// For an axis-aligned image at `(x, y)` with size `w × h`, that's
|
||||
/// `[w, 0, 0, h, x, y]`.
|
||||
fn make_pdf_with_image(image_ctm: [f32; 6]) -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize];
|
||||
|
||||
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(body.as_bytes());
|
||||
pdf.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
fn add_stream_object(
|
||||
pdf: &mut Vec<u8>,
|
||||
offsets: &mut Vec<usize>,
|
||||
id: usize,
|
||||
dict: &str,
|
||||
stream_bytes: &[u8],
|
||||
) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(
|
||||
format!("<< {} /Length {} >>\nstream\n", dict, stream_bytes.len()).as_bytes(),
|
||||
);
|
||||
pdf.extend_from_slice(stream_bytes);
|
||||
pdf.extend_from_slice(b"\nendstream\nendobj\n");
|
||||
}
|
||||
|
||||
// 1: catalog → 2: pages → 3: page with XObject /Im0 → 4: content stream
|
||||
// 5: font → 6: image XObject (1×1 grayscale)
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
1,
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
2,
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
3,
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
|
||||
/Resources << /Font << /F1 5 0 R >> /XObject << /Im0 6 0 R >> >> \
|
||||
/Contents 4 0 R >>",
|
||||
);
|
||||
let [a, b, c, d, e, f] = image_ctm;
|
||||
// BT/ET around a small text item just so the page isn't classified as
|
||||
// image-only (which would route to a different code path). Then save
|
||||
// graphics state, apply the image CTM, invoke Im0, restore.
|
||||
let content = format!(
|
||||
"BT /F1 12 Tf 100 700 Td (Hi) Tj ET\nq {} {} {} {} {} {} cm /Im0 Do Q",
|
||||
a, b, c, d, e, f
|
||||
);
|
||||
add_stream_object(&mut pdf, &mut offsets, 4, "", content.as_bytes());
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
5,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
);
|
||||
// 1×1 grayscale image; the single byte is mid-gray. Contents don't
|
||||
// matter to the extractor — it only cares about the XObject's
|
||||
// /Subtype and the CTM at the `Do` operator.
|
||||
let image_pixel = [128u8];
|
||||
add_stream_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
6,
|
||||
"/Type /XObject /Subtype /Image /Width 1 /Height 1 \
|
||||
/ColorSpace /DeviceGray /BitsPerComponent 8",
|
||||
&image_pixel,
|
||||
);
|
||||
|
||||
let xref_start = pdf.len();
|
||||
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
|
||||
pdf.extend_from_slice(b"0000000000 65535 f \n");
|
||||
for offset in offsets.iter().skip(1) {
|
||||
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
pdf.extend_from_slice(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
|
||||
offsets.len(),
|
||||
xref_start
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
pdf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_text_with_positions_emits_image_bboxes() {
|
||||
// Place a 200×100 image at (50, 600) in PDF user space (origin
|
||||
// bottom-left). The Do operator applies the CTM to a unit square,
|
||||
// so for an axis-aligned image, CTM = [w, 0, 0, h, x, y].
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
|
||||
let images: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| matches!(i.item_type, ItemType::Image))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
images.len(),
|
||||
1,
|
||||
"expected exactly one Image item, got items: {:?}",
|
||||
items
|
||||
.iter()
|
||||
.map(|i| (&i.text, &i.item_type))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let img = images[0];
|
||||
assert!((img.x - 50.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 600.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 200.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
assert_eq!(img.page, 1);
|
||||
// text field carries the legacy `[Image: <resource-name>]` form that
|
||||
// the markdown emitter already knows how to parse.
|
||||
assert_eq!(img.text, "[Image: Im0]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_xobject_bbox_handles_rotated_ctm() {
|
||||
// 90° rotation CTM: a unit square at the origin maps to a square
|
||||
// rotated counter-clockwise about (0,0), then translated to (200, 300).
|
||||
// For a 100×100 image, that's CTM = [0, 100, -100, 0, 200, 300]
|
||||
// (apply the rotation: (1,0) → (0,100); (0,1) → (-100,0)).
|
||||
// The page-space corners are:
|
||||
// (0,0) → (200, 300)
|
||||
// (1,0) → (200, 400)
|
||||
// (1,1) → (100, 400)
|
||||
// (0,1) → (100, 300)
|
||||
// → AABB: x=100..200 (w=100), y=300..400 (h=100).
|
||||
let pdf = make_pdf_with_image([0.0, 100.0, -100.0, 0.0, 200.0, 300.0]);
|
||||
let items = extract_text_with_positions_mem(&pdf).expect("extract");
|
||||
let img = items
|
||||
.iter()
|
||||
.find(|i| matches!(i.item_type, ItemType::Image))
|
||||
.expect("image item");
|
||||
assert!((img.x - 100.0).abs() < 0.01, "x={}", img.x);
|
||||
assert!((img.y - 300.0).abs() < 0.01, "y={}", img.y);
|
||||
assert!((img.width - 100.0).abs() < 0.01, "width={}", img.width);
|
||||
assert!((img.height - 100.0).abs() < 0.01, "height={}", img.height);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_emission_does_not_change_default_markdown() {
|
||||
// Default `MarkdownOptions::include_images = false` — adding image
|
||||
// emission MUST NOT make `extract_pages_markdown` start producing
|
||||
// `![Image: …]` placeholders for everyone. Existing callers that
|
||||
// upgrade should see no diff in their markdown.
|
||||
let pdf = make_pdf_with_image([200.0, 0.0, 0.0, 100.0, 50.0, 600.0]);
|
||||
let result = extract_pages_markdown_mem(&pdf, None).expect("extract");
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(
|
||||
!result.pages[0].markdown.contains("Image:"),
|
||||
"default markdown leaked an image placeholder: {:?}",
|
||||
result.pages[0].markdown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_options_default_has_include_images_false() {
|
||||
// Explicit assertion so anyone flipping this back catches it in CI.
|
||||
// See `MarkdownOptions::default` in src/markdown/mod.rs for the
|
||||
// long-form rationale.
|
||||
let opts = MarkdownOptions::default();
|
||||
assert!(!opts.include_images);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user