Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab073a2fa9 | ||
|
|
7539868bf8 | ||
|
|
79d75dbdca | ||
|
|
b5b91470db | ||
|
|
f26efe6673 | ||
|
|
06ccaf5732 | ||
|
|
5a84c29ea4 | ||
|
|
96c0f2102a |
+33
@@ -0,0 +1,33 @@
|
|||||||
|
# Security Policy
|
||||||
|
|
||||||
|
## Reporting a Vulnerability
|
||||||
|
|
||||||
|
If you believe you've found a security vulnerability in pdf-inspector, please
|
||||||
|
report it privately so we can fix it before public disclosure.
|
||||||
|
|
||||||
|
**Preferred:** Email **help@firecrawl.dev** with:
|
||||||
|
|
||||||
|
- A description of the issue and its impact
|
||||||
|
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
|
||||||
|
- The version or commit hash of pdf-inspector you tested against
|
||||||
|
|
||||||
|
**Alternative:** Use GitHub's private vulnerability reporting under the
|
||||||
|
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
|
||||||
|
|
||||||
|
We'll acknowledge your report in a timely manner and keep you updated on
|
||||||
|
remediation progress. Please do not open a public GitHub issue for security
|
||||||
|
bugs.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
- Memory-safety issues (panics, OOB reads, UB) reachable from a crafted PDF
|
||||||
|
- Denial-of-service vectors (unbounded allocation, infinite loops) on
|
||||||
|
reasonably-sized inputs
|
||||||
|
- Bugs in the `pdf2md` / `detect-pdf` binaries or the `pdf-inspector` crate
|
||||||
|
that affect downstream consumers
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
- Bugs in upstream dependencies (`lopdf`, etc.) — please report those upstream
|
||||||
|
- Extraction quality issues (wrong text, missing tables) — open a regular
|
||||||
|
GitHub issue instead
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@firecrawl/pdf-inspector",
|
"name": "@firecrawl/pdf-inspector",
|
||||||
"version": "1.8.7",
|
"version": "1.8.12",
|
||||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"types": "index.d.ts",
|
"types": "index.d.ts",
|
||||||
|
|||||||
+204
-31
@@ -644,6 +644,8 @@ pub fn extract_tables_in_regions_mem(
|
|||||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||||
|
|
||||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||||
|
let mut rects_by_page: HashMap<u32, Vec<PdfRect>> = HashMap::new();
|
||||||
|
let mut lines_by_page: HashMap<u32, Vec<PdfLine>> = HashMap::new();
|
||||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||||
@@ -656,7 +658,7 @@ pub fn extract_tables_in_regions_mem(
|
|||||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||||
page_heights.insert(*page_num, height);
|
page_heights.insert(*page_num, height);
|
||||||
|
|
||||||
let ((mut items, _rects, _lines), has_gid, coords_rotated) =
|
let ((mut items, rects, lines), has_gid, coords_rotated) =
|
||||||
extractor::content_stream::extract_page_text_items(
|
extractor::content_stream::extract_page_text_items(
|
||||||
&doc,
|
&doc,
|
||||||
page_id,
|
page_id,
|
||||||
@@ -675,6 +677,8 @@ pub fn extract_tables_in_regions_mem(
|
|||||||
rotated_pages.insert(*page_num);
|
rotated_pages.insert(*page_num);
|
||||||
}
|
}
|
||||||
items_by_page.insert(*page_num, items);
|
items_by_page.insert(*page_num, items);
|
||||||
|
rects_by_page.insert(*page_num, rects);
|
||||||
|
lines_by_page.insert(*page_num, lines);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut results = Vec::with_capacity(page_regions.len());
|
let mut results = Vec::with_capacity(page_regions.len());
|
||||||
@@ -704,15 +708,13 @@ pub fn extract_tables_in_regions_mem(
|
|||||||
// content. This avoids rejecting clean tables just because an
|
// content. This avoids rejecting clean tables just because an
|
||||||
// unrelated decorative font on the same page is GID-encoded.
|
// unrelated decorative font on the same page is GID-encoded.
|
||||||
|
|
||||||
let matched: Vec<TextItem> = match items {
|
|
||||||
Some(items) => {
|
|
||||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||||
items
|
let matched: Vec<TextItem> = match items {
|
||||||
|
Some(items) => items
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|item| region_overlaps_item(item, bounds))
|
.filter(|item| region_overlaps_item(item, bounds))
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect()
|
.collect(),
|
||||||
}
|
|
||||||
None => Vec::new(),
|
None => Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -736,42 +738,100 @@ pub fn extract_tables_in_regions_mem(
|
|||||||
.unwrap_or(12.0)
|
.unwrap_or(12.0)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Run heuristic table detection; skip_body_font = false since
|
// Try rect-backed and line-backed vector-grid detectors first,
|
||||||
// the layout model already identified this region as a table.
|
// then fall back to the heuristic text-only detector. Each
|
||||||
let detected = tables::detect_tables(&matched, base_font_size, false);
|
// candidate's markdown is quality-gated by the same
|
||||||
|
// needs_ocr checks the heuristic-only path used: if a vector
|
||||||
if let Some(table) = detected.into_iter().next() {
|
// detector produces a partial/garbled table, we ignore it and
|
||||||
let md = tables::table_to_markdown(&table);
|
// try the next path rather than degrade the output.
|
||||||
if md.trim().is_empty() {
|
|
||||||
page_results.push(RegionText {
|
|
||||||
text: String::new(),
|
|
||||||
needs_ocr: true,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
// needs_ocr fires on any of:
|
// needs_ocr fires on any of:
|
||||||
// - garbage text (non-alphanumeric heavy)
|
// - garbage text (non-alphanumeric heavy)
|
||||||
// - CID/Latin-1 mojibake
|
// - CID/Latin-1 mojibake
|
||||||
// - encoding issues (U+FFFD, dollar-as-space)
|
// - encoding issues (U+FFFD, dollar-as-space)
|
||||||
// - structural giveaways that the table is partial /
|
// - structural giveaways that the table is partial /
|
||||||
// mis-detected (numeric "header", empty header cells,
|
// mis-detected (numeric "header", empty header cells,
|
||||||
// duplicate header cells). Caught GLM-OCR-as-baseline
|
// duplicate header cells).
|
||||||
// scoring 0 TEDS on real prod tables in eval.
|
// skip_body_font = false / layout_assisted = true because the
|
||||||
// Layout model already identified this region as a table,
|
// layout model already identified this region as a table.
|
||||||
// so use relaxed partial-table checks (layout_assisted=true).
|
let region_rects: Vec<PdfRect> = rects_by_page
|
||||||
let needs_ocr = is_garbage_text(&md)
|
.get(&page_1idx)
|
||||||
|
.map(|rs| {
|
||||||
|
rs.iter()
|
||||||
|
.filter(|r| region_overlaps_rect(r, bounds))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let region_lines: Vec<PdfLine> = lines_by_page
|
||||||
|
.get(&page_1idx)
|
||||||
|
.map(|ls| {
|
||||||
|
ls.iter()
|
||||||
|
.filter(|l| region_overlaps_line(l, bounds))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Total length of text the page extractor saw inside this
|
||||||
|
// region, used by the captured-fragment guard below.
|
||||||
|
let region_text_chars: usize = matched.iter().map(|i| i.text.chars().count()).sum();
|
||||||
|
|
||||||
|
let evaluate = |t: &tables::Table| -> Option<String> {
|
||||||
|
let md = tables::table_to_markdown(t);
|
||||||
|
let trimmed = md.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if is_garbage_text(&md)
|
||||||
|| is_cid_garbage(&md)
|
|| is_cid_garbage(&md)
|
||||||
|| detect_encoding_issues(&md)
|
|| detect_encoding_issues(&md)
|
||||||
|| looks_like_partial_table_ex(&md, true);
|
|| looks_like_partial_table_ex(&md, true)
|
||||||
page_results.push(RegionText {
|
{
|
||||||
text: if needs_ocr { String::new() } else { md },
|
return None;
|
||||||
needs_ocr,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
} else {
|
// Reject extractions that only captured a small fraction
|
||||||
page_results.push(RegionText {
|
// of the text actually in the region. Two recurring
|
||||||
|
// failure shapes this catches:
|
||||||
|
// - "header-only": detector found the column-header band
|
||||||
|
// cleanly but missed every data row below (financial
|
||||||
|
// statements with multi-line column headers + many
|
||||||
|
// data rows are the dominant case).
|
||||||
|
// - "sparse": detector returned a couple of fragmentary
|
||||||
|
// cells even though the region has many lines of text.
|
||||||
|
// The region floor (200 chars) keeps short legitimate
|
||||||
|
// tables (timestamps, units, axis labels) from being
|
||||||
|
// rejected as partial.
|
||||||
|
if captured_only_a_fragment(&md, region_text_chars) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(md)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut accepted_md: Option<String> = None;
|
||||||
|
if !region_rects.is_empty() {
|
||||||
|
let (rect_tables, _) =
|
||||||
|
tables::detect_tables_from_rects(&matched, ®ion_rects, page_1idx);
|
||||||
|
accepted_md = rect_tables.iter().find_map(&evaluate);
|
||||||
|
}
|
||||||
|
if accepted_md.is_none() && !region_lines.is_empty() {
|
||||||
|
let line_tables =
|
||||||
|
tables::detect_tables_from_lines(&matched, ®ion_lines, page_1idx);
|
||||||
|
accepted_md = line_tables.iter().find_map(&evaluate);
|
||||||
|
}
|
||||||
|
if accepted_md.is_none() {
|
||||||
|
let detected = tables::detect_tables(&matched, base_font_size, false);
|
||||||
|
accepted_md = detected.iter().find_map(&evaluate);
|
||||||
|
}
|
||||||
|
|
||||||
|
match accepted_md {
|
||||||
|
Some(md) => page_results.push(RegionText {
|
||||||
|
text: md,
|
||||||
|
needs_ocr: false,
|
||||||
|
}),
|
||||||
|
None => page_results.push(RegionText {
|
||||||
text: String::new(),
|
text: String::new(),
|
||||||
needs_ocr: true,
|
needs_ocr: true,
|
||||||
});
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1280,6 +1340,46 @@ mod vector_grid_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Regression for `wired_header_data_misalign.pdf` — a single page from a
|
||||||
|
/// parts catalog with a 4-column wire-bordered table (`Item | EAN | Nombre
|
||||||
|
/// | Cant`). Column headers are centered/right-aligned inside their cells
|
||||||
|
/// while data is left-aligned, so cluster_x_positions merges or drops
|
||||||
|
/// columns and the cell-rect fallback used to assign text to the wrong
|
||||||
|
/// columns (lost a column, fragmented neighbor cells). The fix prefers
|
||||||
|
/// rect-border-derived column edges when they're well-distributed across
|
||||||
|
/// the actual text items. This test asserts the detector keeps all 4
|
||||||
|
/// columns and every column ends up populated.
|
||||||
|
#[test]
|
||||||
|
fn wired_header_data_misalign_keeps_all_columns() {
|
||||||
|
let tables = detect_rect_tables_in_fixture("tests/fixtures/wired_header_data_misalign.pdf");
|
||||||
|
let table = tables
|
||||||
|
.iter()
|
||||||
|
.find(|t| t.columns.len() == 4 && t.rows.len() >= 5)
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
panic!(
|
||||||
|
"expected a 4-column ≥5-row table; got {:?}",
|
||||||
|
tables
|
||||||
|
.iter()
|
||||||
|
.map(|t| (t.rows.len(), t.columns.len()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
)
|
||||||
|
});
|
||||||
|
for c in 0..4 {
|
||||||
|
let populated_rows = table
|
||||||
|
.cells
|
||||||
|
.iter()
|
||||||
|
.filter(|row| !row[c].trim().is_empty())
|
||||||
|
.count();
|
||||||
|
assert!(
|
||||||
|
populated_rows >= 2,
|
||||||
|
"column {} only populated in {} rows; cells: {:?}",
|
||||||
|
c,
|
||||||
|
populated_rows,
|
||||||
|
table.cells
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_crop_px_bbox_is_plausible_bounds() {
|
fn test_crop_px_bbox_is_plausible_bounds() {
|
||||||
let crop = [10.0, 20.0, 110.0, 220.0];
|
let crop = [10.0, 20.0, 110.0, 220.0];
|
||||||
@@ -3526,6 +3626,28 @@ fn is_cid_garbage(text: &str) -> bool {
|
|||||||
/// anymore, only "can we extract it correctly?". Paragraph and duplicate-
|
/// anymore, only "can we extract it correctly?". Paragraph and duplicate-
|
||||||
/// header checks stay, since those indicate genuine extraction quality
|
/// header checks stay, since those indicate genuine extraction quality
|
||||||
/// issues regardless of how the region was identified.
|
/// issues regardless of how the region was identified.
|
||||||
|
/// Return true when the captured table markdown represents only a small
|
||||||
|
/// fraction of the text the page extractor actually saw inside the
|
||||||
|
/// region — typically a header-only band or a sparse fragment where
|
||||||
|
/// the detector found valid grid structure but missed most of the
|
||||||
|
/// data rows below.
|
||||||
|
///
|
||||||
|
/// Tuned at a 25% floor: tables that captured at least a quarter of
|
||||||
|
/// the region's text are treated as complete-enough. Below 25%, the
|
||||||
|
/// caller falls back to `needs_ocr = true` so GLM-OCR can take over.
|
||||||
|
/// The 200-char region floor keeps short legitimate tables (units,
|
||||||
|
/// axis labels, single-row stat blocks) from being mis-flagged.
|
||||||
|
fn captured_only_a_fragment(markdown: &str, region_text_chars: usize) -> bool {
|
||||||
|
if region_text_chars <= 200 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let captured_text_chars: usize = markdown
|
||||||
|
.chars()
|
||||||
|
.filter(|c| !matches!(c, '|' | '-' | '\n'))
|
||||||
|
.count();
|
||||||
|
captured_text_chars * 4 < region_text_chars
|
||||||
|
}
|
||||||
|
|
||||||
fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
|
||||||
let lines: Vec<&str> = markdown.lines().filter(|l| l.starts_with('|')).collect();
|
let lines: Vec<&str> = markdown.lines().filter(|l| l.starts_with('|')).collect();
|
||||||
if lines.len() < 2 {
|
if lines.len() < 2 {
|
||||||
@@ -3679,6 +3801,57 @@ fn looks_like_partial_table(markdown: &str) -> bool {
|
|||||||
looks_like_partial_table_ex(markdown, false)
|
looks_like_partial_table_ex(markdown, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod captured_only_a_fragment_tests {
|
||||||
|
use super::captured_only_a_fragment;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn small_region_skips_check() {
|
||||||
|
// Short legitimate tables (axis labels, unit blocks) shouldn't be
|
||||||
|
// flagged even when the captured markdown is tiny.
|
||||||
|
let md = "|Year|Value|\n|---|---|\n|2024|10|";
|
||||||
|
assert!(!captured_only_a_fragment(md, 50));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn full_table_passes() {
|
||||||
|
// Captured markdown matches the region text — full extraction.
|
||||||
|
let md =
|
||||||
|
"|Name|Year|Country|\n|---|---|---|\n|Alice|2020|US|\n|Bob|2021|UK|\n|Carol|2019|FR|";
|
||||||
|
// Region had ~50 chars of text (rough estimate of just the data words).
|
||||||
|
assert!(!captured_only_a_fragment(md, 50));
|
||||||
|
// Even a much larger region matched by the markdown content passes.
|
||||||
|
assert!(!captured_only_a_fragment(md, md.len()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn header_only_extraction_rejected() {
|
||||||
|
// Captured the column-header band (~30 chars) while the region
|
||||||
|
// actually has many rows of data (~1500 chars).
|
||||||
|
let md = "|Description|Year|Amount|\n|---|---|---|";
|
||||||
|
assert!(captured_only_a_fragment(md, 1500));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sparse_fragment_rejected() {
|
||||||
|
// A couple of fragment cells captured from a content-rich region.
|
||||||
|
let md = "|percent|for|\n|---|---|\n|sites|15|";
|
||||||
|
assert!(captured_only_a_fragment(md, 2000));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn boundary_at_25_percent_floor() {
|
||||||
|
// Right at the 25% line: 250 captured chars of 1000 region chars.
|
||||||
|
// The check rejects when captured*4 < region, so 250*4=1000 is NOT
|
||||||
|
// less than 1000 — boundary is treated as acceptable.
|
||||||
|
let md = "x".repeat(250);
|
||||||
|
assert!(!captured_only_a_fragment(&md, 1000));
|
||||||
|
// Just under 25%: 249*4=996 < 1000 — flagged.
|
||||||
|
let md_under = "x".repeat(249);
|
||||||
|
assert!(captured_only_a_fragment(&md_under, 1000));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod looks_like_partial_table_tests {
|
mod looks_like_partial_table_tests {
|
||||||
use super::{looks_like_partial_table, looks_like_partial_table_ex};
|
use super::{looks_like_partial_table, looks_like_partial_table_ex};
|
||||||
|
|||||||
+248
-16
@@ -4,11 +4,74 @@
|
|||||||
//! gridlines. Many IRS forms and government PDFs use these instead of
|
//! gridlines. Many IRS forms and government PDFs use these instead of
|
||||||
//! `re` (rectangle) operators.
|
//! `re` (rectangle) operators.
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
|
|
||||||
use crate::tables::Table;
|
use crate::tables::Table;
|
||||||
use crate::types::{PdfLine, TextItem};
|
use crate::types::{PdfLine, TextItem};
|
||||||
|
|
||||||
use super::detect_rects::{assign_items_to_grid, snap_edges};
|
use super::detect_rects::{assign_items_to_grid, snap_edges};
|
||||||
|
|
||||||
|
/// Derive column edges from the x-endpoints of horizontal-rule
|
||||||
|
/// segments when no vertical lines were drawn.
|
||||||
|
///
|
||||||
|
/// Catalog and archival-finding-aid tables are commonly drawn with
|
||||||
|
/// per-row horizontal rules broken into N segments (one segment per
|
||||||
|
/// cell), with no vertical dividers at all. The segment break points
|
||||||
|
/// (e.g. `[50, 127], [127, 485], [485, 562]` per row) implicitly
|
||||||
|
/// encode the column boundaries.
|
||||||
|
///
|
||||||
|
/// Returns column edges if ≥3 distinct x-positions each show up as a
|
||||||
|
/// segment endpoint on ≥50% of the unique horizontal-line rows.
|
||||||
|
/// Returns `None` otherwise — decorative rules with varying widths
|
||||||
|
/// shouldn't be mistaken for a table.
|
||||||
|
fn derive_columns_from_horizontal_segments(horizontals: &[(f32, f32, f32)]) -> Option<Vec<f32>> {
|
||||||
|
if horizontals.len() < 3 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut endpoints: Vec<f32> = Vec::with_capacity(horizontals.len() * 2);
|
||||||
|
for &(_, x_min, x_max) in horizontals {
|
||||||
|
endpoints.push(x_min);
|
||||||
|
endpoints.push(x_max);
|
||||||
|
}
|
||||||
|
let clusters = snap_edges(&endpoints, 5.0);
|
||||||
|
if clusters.len() < 3 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bucket y-values to count unique rows. Tolerance ~0.1pt (×10
|
||||||
|
// rounding) tolerates the snap_edges 3pt clustering used later
|
||||||
|
// for row edges.
|
||||||
|
let unique_rows: HashSet<i32> = horizontals
|
||||||
|
.iter()
|
||||||
|
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||||
|
.collect();
|
||||||
|
if unique_rows.len() < 2 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let min_rows = (unique_rows.len() as f32 * 0.5).ceil() as usize;
|
||||||
|
|
||||||
|
let qualifying: Vec<f32> = clusters
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|&cluster_x| {
|
||||||
|
let rows_touched: HashSet<i32> = horizontals
|
||||||
|
.iter()
|
||||||
|
.filter(|&&(_, x_min, x_max)| {
|
||||||
|
(x_min - cluster_x).abs() < 5.0 || (x_max - cluster_x).abs() < 5.0
|
||||||
|
})
|
||||||
|
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||||
|
.collect();
|
||||||
|
rows_touched.len() >= min_rows
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if qualifying.len() < 3 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(qualifying)
|
||||||
|
}
|
||||||
|
|
||||||
/// Detect tables from line segments on a given page.
|
/// Detect tables from line segments on a given page.
|
||||||
///
|
///
|
||||||
/// Lines are classified as horizontal or vertical, snapped into grid edges,
|
/// Lines are classified as horizontal or vertical, snapped into grid edges,
|
||||||
@@ -52,25 +115,50 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
|||||||
// Diagonal lines are ignored
|
// Diagonal lines are ignored
|
||||||
}
|
}
|
||||||
|
|
||||||
if horizontals.len() < 3 || verticals.len() < 2 {
|
if horizontals.len() < 3 {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If no/very-few vertical lines are drawn, try to derive column edges
|
||||||
|
// from the x-endpoints of the horizontal-rule segments. Catalog and
|
||||||
|
// archival-finding-aid layouts commonly draw each row's horizontal
|
||||||
|
// rule as N segments (one per cell), with no vertical dividers at
|
||||||
|
// all — the segment break points encode the column boundaries.
|
||||||
|
let implicit_col_edges: Option<Vec<f32>> = if verticals.len() < 2 {
|
||||||
|
derive_columns_from_horizontal_segments(&horizontals)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
if verticals.len() < 2 && implicit_col_edges.is_none() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
let cols_from_segments = implicit_col_edges.is_some();
|
||||||
|
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
|
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page){}",
|
||||||
page,
|
page,
|
||||||
horizontals.len(),
|
horizontals.len(),
|
||||||
verticals.len(),
|
verticals.len(),
|
||||||
page_lines.len()
|
page_lines.len(),
|
||||||
|
if cols_from_segments {
|
||||||
|
" — columns from horizontal segments"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
// Snap Y-values of horizontal lines → row edges
|
// Snap Y-values of horizontal lines → row edges
|
||||||
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
|
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
|
||||||
let row_edges = snap_edges(&h_ys, 3.0);
|
let row_edges = snap_edges(&h_ys, 3.0);
|
||||||
|
|
||||||
// Snap X-values of vertical lines → column edges
|
// Column edges from drawn verticals when present, else from the
|
||||||
|
// horizontal-segment endpoints derived above.
|
||||||
|
let col_edges = if let Some(c) = implicit_col_edges {
|
||||||
|
c
|
||||||
|
} else {
|
||||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||||
let col_edges = snap_edges(&v_xs, 3.0);
|
snap_edges(&v_xs, 3.0)
|
||||||
|
};
|
||||||
|
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"detect_lines p{}: {} row edges, {} col edges after snap",
|
"detect_lines p{}: {} row edges, {} col edges after snap",
|
||||||
@@ -110,15 +198,21 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
|||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reject page-spanning frames: if the grid covers >90% of a standard page
|
// Reject page-spanning frames: a decorative outer border has just 4
|
||||||
// dimension in both axes, it's a border frame, not a table.
|
// edges (top/bottom/left/right). Real full-page tables — common in
|
||||||
|
// governmental ledgers, financial reports, etc. — span the same A4 /
|
||||||
|
// Letter dimensions but have many internal row/column rules. Only
|
||||||
|
// reject when the line set looks like a bare frame, not a grid.
|
||||||
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
||||||
if table_width > 500.0 && table_height > 700.0 {
|
if table_width > 500.0 && table_height > 700.0 && horizontals.len() <= 4 && verticals.len() <= 4
|
||||||
|
{
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})",
|
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0}, {} h + {} v)",
|
||||||
page,
|
page,
|
||||||
table_width,
|
table_width,
|
||||||
table_height
|
table_height,
|
||||||
|
horizontals.len(),
|
||||||
|
verticals.len()
|
||||||
);
|
);
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
@@ -146,24 +240,33 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
|||||||
|
|
||||||
// Validate vertical lines: at least 2 should span a meaningful height.
|
// Validate vertical lines: at least 2 should span a meaningful height.
|
||||||
// Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
|
// Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
|
||||||
// for tables with partial column separators.
|
// for tables with partial column separators. Skipped entirely when
|
||||||
let spanning_v = verticals
|
// columns came from horizontal-segment endpoints — there are no
|
||||||
|
// vertical lines to validate against, and the segment-endpoint
|
||||||
|
// consistency check in `derive_columns_from_horizontal_segments`
|
||||||
|
// is the equivalent guard.
|
||||||
|
let spanning_v = if cols_from_segments {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
let s = verticals
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||||
.count();
|
.count();
|
||||||
let partial_v = verticals
|
let p = verticals
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||||
.count();
|
.count();
|
||||||
if spanning_v < 2 && partial_v < 4 {
|
if s < 2 && p < 4 {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||||
page,
|
page,
|
||||||
spanning_v,
|
s,
|
||||||
partial_v
|
p
|
||||||
);
|
);
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
s
|
||||||
|
};
|
||||||
|
|
||||||
// Row edges need to be in descending order (top of page = higher Y first)
|
// Row edges need to be in descending order (top of page = higher Y first)
|
||||||
let mut row_edges_desc = row_edges;
|
let mut row_edges_desc = row_edges;
|
||||||
@@ -410,6 +513,135 @@ mod tests {
|
|||||||
assert!(tables.is_empty());
|
assert!(tables.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_horizontal_segments_only_implicit_columns_accepted() {
|
||||||
|
// Catalog/finding-aid pattern: each row's horizontal rule is
|
||||||
|
// drawn as 3 segments at consistent x-endpoints (50, 127, 485,
|
||||||
|
// 562), with no vertical lines anywhere. The segment break
|
||||||
|
// points must be inferred as column edges.
|
||||||
|
let mut lines = Vec::new();
|
||||||
|
// Slightly uneven row spacing so the chart-gridline rejector
|
||||||
|
// (CV < 0.02) doesn't fire.
|
||||||
|
let row_ys = [80.0_f32, 145.0, 215.0, 280.0, 350.0, 415.0, 485.0];
|
||||||
|
for &y in &row_ys {
|
||||||
|
lines.push(make_hline(y, 50.0, 127.0, 1));
|
||||||
|
lines.push(make_hline(y, 127.0, 485.0, 1));
|
||||||
|
lines.push(make_hline(y, 485.0, 562.0, 1));
|
||||||
|
}
|
||||||
|
// Populate every cell so capture / density checks pass.
|
||||||
|
let mut items = Vec::new();
|
||||||
|
for w in row_ys.windows(2) {
|
||||||
|
let row_y = (w[0] + w[1]) / 2.0;
|
||||||
|
items.push(make_item("id", 80.0, row_y, 1));
|
||||||
|
items.push(make_item("description here", 200.0, row_y, 1));
|
||||||
|
items.push(make_item("date", 510.0, row_y, 1));
|
||||||
|
}
|
||||||
|
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||||
|
assert_eq!(
|
||||||
|
tables.len(),
|
||||||
|
1,
|
||||||
|
"horizontal-segment-only grid should be accepted"
|
||||||
|
);
|
||||||
|
let t = &tables[0];
|
||||||
|
assert!(
|
||||||
|
t.cells.len() >= 4,
|
||||||
|
"expected ≥4 rows, got {}",
|
||||||
|
t.cells.len()
|
||||||
|
);
|
||||||
|
assert_eq!(t.cells[0].len(), 3, "expected 3 columns");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_horizontal_segments_with_inconsistent_endpoints_rejected() {
|
||||||
|
// Decorative rules of varying widths shouldn't be detected as a
|
||||||
|
// table — each line has its own x-endpoints, no consistent
|
||||||
|
// column boundary survives the 50%-of-rows threshold.
|
||||||
|
let lines = vec![
|
||||||
|
make_hline(100.0, 50.0, 150.0, 1),
|
||||||
|
make_hline(200.0, 50.0, 220.0, 1),
|
||||||
|
make_hline(300.0, 50.0, 310.0, 1),
|
||||||
|
make_hline(400.0, 50.0, 470.0, 1),
|
||||||
|
];
|
||||||
|
let items = vec![
|
||||||
|
make_item("decorative", 100.0, 150.0, 1),
|
||||||
|
make_item("text", 100.0, 250.0, 1),
|
||||||
|
];
|
||||||
|
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||||
|
assert!(
|
||||||
|
tables.is_empty(),
|
||||||
|
"varying-width decorative rules should not be detected"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_page_spanning_bare_frame_rejected() {
|
||||||
|
// Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.
|
||||||
|
// No internal structure → decorative border, not a table.
|
||||||
|
let lines = vec![
|
||||||
|
make_hline(20.0, 20.0, 575.0, 1), // top
|
||||||
|
make_hline(820.0, 20.0, 575.0, 1), // bottom
|
||||||
|
make_vline(20.0, 20.0, 820.0, 1), // left
|
||||||
|
make_vline(575.0, 20.0, 820.0, 1), // right
|
||||||
|
];
|
||||||
|
let items = vec![
|
||||||
|
make_item("title", 100.0, 100.0, 1),
|
||||||
|
make_item("body", 100.0, 200.0, 1),
|
||||||
|
];
|
||||||
|
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||||
|
assert!(
|
||||||
|
tables.is_empty(),
|
||||||
|
"Page-sized 4-edge frame should be rejected as decoration"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_page_spanning_grid_with_internal_lines_accepted() {
|
||||||
|
// Full-page table (governmental-ledger pattern): A4-sized grid
|
||||||
|
// that previously hit the "page-spanning frame" early reject
|
||||||
|
// before downstream validation could even look at it.
|
||||||
|
// Verticals span the full table height so we isolate the
|
||||||
|
// frame-vs-grid decision under test.
|
||||||
|
let mut lines = Vec::new();
|
||||||
|
// 13 horizontal rules: header + 12 row separators
|
||||||
|
let h_ys = [
|
||||||
|
22.5, 37.9, 95.5, 144.5, 184.9, 233.9, 291.7, 340.7, 415.8, 499.6, 574.7, 623.7, 698.8,
|
||||||
|
];
|
||||||
|
for &y in &h_ys {
|
||||||
|
lines.push(make_hline(y, 22.6, 566.6, 1));
|
||||||
|
}
|
||||||
|
// 7 column dividers spanning full table height.
|
||||||
|
let v_xs = [22.6, 66.3, 116.3, 186.6, 263.1, 493.5, 566.5];
|
||||||
|
for &x in &v_xs {
|
||||||
|
lines.push(make_vline(x, 22.5, 698.8, 1));
|
||||||
|
}
|
||||||
|
// Populate every cell so the capture-ratio + density checks pass.
|
||||||
|
let mut items = Vec::new();
|
||||||
|
for r in 0..(h_ys.len() - 1) {
|
||||||
|
let row_y = (h_ys[r] + h_ys[r + 1]) / 2.0;
|
||||||
|
for c in 0..(v_xs.len() - 1) {
|
||||||
|
let col_x = (v_xs[c] + v_xs[c + 1]) / 2.0;
|
||||||
|
items.push(make_item("x", col_x, row_y, 1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||||
|
assert_eq!(
|
||||||
|
tables.len(),
|
||||||
|
1,
|
||||||
|
"Full-page table with internal grid should be accepted"
|
||||||
|
);
|
||||||
|
let t = &tables[0];
|
||||||
|
assert!(
|
||||||
|
t.cells.len() >= 6,
|
||||||
|
"expected ≥6 rows, got {}",
|
||||||
|
t.cells.len()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
t.cells[0].len() >= 3,
|
||||||
|
"expected ≥3 columns, got {}",
|
||||||
|
t.cells[0].len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_single_column_rejected() {
|
fn test_single_column_rejected() {
|
||||||
// Only 2 col edges (1 column) — not a table even with verticals
|
// Only 2 col edges (1 column) — not a table even with verticals
|
||||||
|
|||||||
+186
-14
@@ -1394,13 +1394,15 @@ fn detect_row_stripe_table(
|
|||||||
.max()
|
.max()
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
// Allow longer cells for multi-column tables (descriptions in one column
|
// Allow longer cells for multi-column tables (descriptions in one column
|
||||||
// are common). Single-column or 2-column "tables" with giant cells are
|
// are common). Narrow grids with giant cells are usually layout
|
||||||
// almost always layout backgrounds.
|
// backgrounds — but only when the row count is also small. A 4+-row
|
||||||
|
// key/value table with one descriptive column reads as a real table
|
||||||
|
// on every other gate, so don't reject it on cell length alone.
|
||||||
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
||||||
if max_cell_len > max_allowed {
|
if max_cell_len > max_allowed && non_empty_rows < 4 {
|
||||||
debug!(
|
debug!(
|
||||||
" row-stripe rejected: max cell length {} > {} (layout background)",
|
" row-stripe rejected: max cell length {} > {} (layout background, {} rows)",
|
||||||
max_cell_len, max_allowed
|
max_cell_len, max_allowed, non_empty_rows
|
||||||
);
|
);
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -1632,7 +1634,49 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// For wired-grid tables whose header text is centered/right-aligned but
|
||||||
|
// whose data is left-aligned, cluster_x_positions can drop the header-only
|
||||||
|
// x-cluster in its singleton-filter pass and merge adjacent data clusters
|
||||||
|
// when the gap is below threshold, losing a column. Rect borders are
|
||||||
|
// ground truth in that case — but only when each rect column actually
|
||||||
|
// holds text. Decorative or background rects (prose laid out in a frame,
|
||||||
|
// cell-fill rects with extra borders) can produce more rect-derived
|
||||||
|
// columns than the text supports; preferring rects there would split a
|
||||||
|
// logical column into spurious sub-columns.
|
||||||
|
let rect_cols_match_text = match (&rect_col_edges, &text_col_edges) {
|
||||||
|
(Some(rect_edges), _) if rect_edges.len() >= 4 => {
|
||||||
|
let num_rect_cols = rect_edges.len() - 1;
|
||||||
|
let mut col_item_counts = vec![0usize; num_rect_cols];
|
||||||
|
for (_, item) in &page_items {
|
||||||
|
let cx = item.x + item.width / 2.0;
|
||||||
|
for c in 0..num_rect_cols {
|
||||||
|
if cx >= rect_edges[c] - 2.0 && cx <= rect_edges[c + 1] + 2.0 {
|
||||||
|
col_item_counts[c] += 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Require every rect column to hold multiple text items. A rect
|
||||||
|
// column with no (or only one) item is decorative or the rect grid
|
||||||
|
// is detecting a spurious column the data does not need; in those
|
||||||
|
// cases the old text-cluster preference is the safer fallback.
|
||||||
|
col_item_counts.iter().all(|&n| n >= 2)
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
|
||||||
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||||
|
(Some(rect_edges), text_edges_opt) if rect_cols_match_text => {
|
||||||
|
debug!(
|
||||||
|
" cell-rect using {} rect-derived columns (text clusters: {}; rect cols well-distributed)",
|
||||||
|
rect_edges.len() - 1,
|
||||||
|
text_edges_opt
|
||||||
|
.as_ref()
|
||||||
|
.map(|e| (e.len() - 1) as i32)
|
||||||
|
.unwrap_or(-1)
|
||||||
|
);
|
||||||
|
(rect_edges, false)
|
||||||
|
}
|
||||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||||
debug!(
|
debug!(
|
||||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||||
@@ -1719,17 +1763,21 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reject tables with paragraph-length cells (layout backgrounds, not tables)
|
// Reject tables with paragraph-length cells — typically layout
|
||||||
|
// backgrounds (sidebars, banners) where a single big rectangle
|
||||||
|
// contains a wall of prose. Spare multi-row key/value tables where
|
||||||
|
// the value column is a multi-bullet description: those pass every
|
||||||
|
// other gate and shouldn't get killed on cell length alone.
|
||||||
let max_cell_len = cells
|
let max_cell_len = cells
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(|row| row.iter())
|
.flat_map(|row| row.iter())
|
||||||
.map(|c| c.len())
|
.map(|c| c.len())
|
||||||
.max()
|
.max()
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
if max_cell_len > 500 {
|
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||||
debug!(
|
debug!(
|
||||||
" cell-rect rejected: max cell length {} > 500",
|
" cell-rect rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||||
max_cell_len
|
max_cell_len, non_empty_rows
|
||||||
);
|
);
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -2143,18 +2191,20 @@ fn detect_merged_cluster_table(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reject if any cell has excessive text — layout background rects produce
|
// Reject if any cell has excessive text — layout background rects
|
||||||
// "cells" containing paragraphs, not short data-table values.
|
// produce "cells" containing paragraphs, not short data-table values.
|
||||||
|
// Multi-row key/value tables can legitimately have one column of
|
||||||
|
// long descriptive text, so only reject narrow-row layouts here.
|
||||||
let max_cell_len = cells
|
let max_cell_len = cells
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(|row| row.iter())
|
.flat_map(|row| row.iter())
|
||||||
.map(|c| c.len())
|
.map(|c| c.len())
|
||||||
.max()
|
.max()
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
if max_cell_len > 500 {
|
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||||
debug!(
|
debug!(
|
||||||
" merged-cluster rejected: max cell length {} > 500 (layout background)",
|
" merged-cluster rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||||
max_cell_len
|
max_cell_len, non_empty_rows
|
||||||
);
|
);
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -2584,6 +2634,46 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_row_stripe_accepts_multi_row_key_value_long_cells() {
|
||||||
|
// Multi-row 2-column key/value table where one value cell holds
|
||||||
|
// a paragraph (>500 chars). The old `max_cell_len > 500` check
|
||||||
|
// rejected this shape as a "layout background"; with the
|
||||||
|
// multi-row guard, it should be accepted.
|
||||||
|
let mut rects = Vec::new();
|
||||||
|
let row_h = 25.0_f32;
|
||||||
|
let y_top = 700.0_f32;
|
||||||
|
for i in 0..8 {
|
||||||
|
let y = y_top - (i as f32) * row_h;
|
||||||
|
rects.push((40.0, y, 510.0, row_h));
|
||||||
|
}
|
||||||
|
let mut items = Vec::new();
|
||||||
|
for i in 0..8 {
|
||||||
|
let row_center_y = y_top - (i as f32) * row_h + row_h / 2.0;
|
||||||
|
// Left column: short label
|
||||||
|
items.push(make_item(&format!("Field {}", i), 45.0, row_center_y, 10.0));
|
||||||
|
// Right column: short value, except the last row which is a paragraph
|
||||||
|
let value = if i == 7 {
|
||||||
|
"X".repeat(800)
|
||||||
|
} else {
|
||||||
|
"value".to_string()
|
||||||
|
};
|
||||||
|
items.push(make_item(&value, 300.0, row_center_y, 10.0));
|
||||||
|
}
|
||||||
|
let result = detect_row_stripe_table(&items, &rects, 1);
|
||||||
|
assert!(
|
||||||
|
result.is_some(),
|
||||||
|
"multi-row key/value table with one long cell should be accepted"
|
||||||
|
);
|
||||||
|
let t = result.unwrap();
|
||||||
|
assert!(
|
||||||
|
t.cells.len() >= 4,
|
||||||
|
"expected ≥4 rows, got {}",
|
||||||
|
t.cells.len()
|
||||||
|
);
|
||||||
|
assert_eq!(t.cells[0].len(), 2, "expected 2 columns");
|
||||||
|
}
|
||||||
|
|
||||||
// --- propagate_merged_cells ---
|
// --- propagate_merged_cells ---
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -3342,6 +3432,88 @@ mod tests {
|
|||||||
assert!(table.cells[2][1].contains("deny unauthorized"));
|
assert!(table.cells[2][1].contains("deny unauthorized"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wire-bordered 4-column table whose header text is centered/right-aligned
|
||||||
|
/// inside each cell while the data is left-aligned: cluster_x_positions
|
||||||
|
/// merges adjacent columns (data Item→EAN gap is below threshold) and
|
||||||
|
/// drops the header-only x-clusters in the filter pass, leaving only 3
|
||||||
|
/// text-derived columns. Rect borders are 4 columns of ground truth.
|
||||||
|
/// Before the fix the cell-rect path preferred text edges when they were
|
||||||
|
/// the smaller set — losing a column. After the fix, 3+ rect columns
|
||||||
|
/// always win.
|
||||||
|
#[test]
|
||||||
|
fn wired_header_data_misaligned_keeps_all_columns_from_rects() {
|
||||||
|
let page = 1;
|
||||||
|
// 4 cols: Item | EAN | Nombre | Cant
|
||||||
|
let col_xs = [380.0_f32, 410.0, 470.0, 660.0, 700.0];
|
||||||
|
// Header + 9 data rows at 15pt tall each (y descending).
|
||||||
|
let row_ys: Vec<f32> = (0..=10).map(|r| 400.0 - 15.0 * r as f32).collect();
|
||||||
|
|
||||||
|
let mut rects: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||||
|
for r in 0..10 {
|
||||||
|
let y_top = row_ys[r];
|
||||||
|
let y_bot = row_ys[r + 1];
|
||||||
|
for c in 0..4 {
|
||||||
|
rects.push((col_xs[c], y_bot, col_xs[c + 1] - col_xs[c], y_top - y_bot));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut items: Vec<TextItem> = Vec::new();
|
||||||
|
// Header row (y ≈ 392.5): headers sit further to the right than data
|
||||||
|
// because they are centered/right-aligned in the cells.
|
||||||
|
items.push(make_item("Item", 389.0, 392.5, 9.0));
|
||||||
|
items.push(make_item("EAN", 432.0, 392.5, 9.0));
|
||||||
|
items.push(make_item("Nombre", 552.0, 392.5, 9.0));
|
||||||
|
items.push(make_item("Cant", 672.0, 392.5, 9.0));
|
||||||
|
|
||||||
|
let names = [
|
||||||
|
"Arnes Frontal",
|
||||||
|
"Arnes Motor",
|
||||||
|
"Arnes Piso",
|
||||||
|
"Arnes Techo",
|
||||||
|
"Arnes Puerta",
|
||||||
|
"Arnes Tablero",
|
||||||
|
"Arnes Trasero",
|
||||||
|
"Arnes Lateral",
|
||||||
|
"Arnes Sensor",
|
||||||
|
];
|
||||||
|
for r in 0..9 {
|
||||||
|
let y = 377.5 - 15.0 * r as f32;
|
||||||
|
items.push(make_item(&(r + 1).to_string(), 396.0, y, 9.0));
|
||||||
|
items.push(make_item("7701023403016", 410.0, y, 9.0));
|
||||||
|
items.push(make_item(names[r], 480.0, y, 9.0));
|
||||||
|
items.push(make_item("1", 680.0, y, 9.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||||
|
.expect("wired 4-column table with header/data x-misalignment must detect");
|
||||||
|
assert_eq!(
|
||||||
|
table.columns.len(),
|
||||||
|
4,
|
||||||
|
"expected 4 columns from rect borders; cells: {:?}",
|
||||||
|
table.cells
|
||||||
|
);
|
||||||
|
for c in 0..4 {
|
||||||
|
let any_populated = table.cells.iter().any(|row| !row[c].trim().is_empty());
|
||||||
|
assert!(
|
||||||
|
any_populated,
|
||||||
|
"column {} empty across all rows; cells: {:?}",
|
||||||
|
c, table.cells
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Header row populated in all 4 cells.
|
||||||
|
let header = &table.cells[0];
|
||||||
|
assert_eq!(header[0].trim(), "Item");
|
||||||
|
assert_eq!(header[1].trim(), "EAN");
|
||||||
|
assert_eq!(header[2].trim(), "Nombre");
|
||||||
|
assert_eq!(header[3].trim(), "Cant");
|
||||||
|
// First data row: Item="1", EAN, name, count="1" — no Item↔EAN merge.
|
||||||
|
let data1 = &table.cells[1];
|
||||||
|
assert_eq!(data1[0].trim(), "1");
|
||||||
|
assert_eq!(data1[1].trim(), "7701023403016");
|
||||||
|
assert!(data1[2].trim().contains("Arnes"));
|
||||||
|
assert_eq!(data1[3].trim(), "1");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn failed_cluster_no_hint_without_items() {
|
fn failed_cluster_no_hint_without_items() {
|
||||||
// Rects with no text items inside → no failed-cluster hint generated.
|
// Rects with no text items inside → no failed-cluster hint generated.
|
||||||
|
|||||||
BIN
Binary file not shown.
@@ -1648,6 +1648,33 @@ fn test_bits_pilani_page8_table_detection() {
|
|||||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_extract_tables_in_regions_uses_line_grid() {
|
||||||
|
// Stroked-grid table (m/l/S path operators forming a 2x2 grid).
|
||||||
|
// The heuristic text-only detector handles the same cells already,
|
||||||
|
// so this guards that the line-backed path doesn't regress: the
|
||||||
|
// markdown still contains all four data cells.
|
||||||
|
let buf = synthetic_vector_grid_pdf(false);
|
||||||
|
let results =
|
||||||
|
extract_tables_in_regions_mem(&buf, &[(0, vec![[40.0, 50.0, 220.0, 760.0]])]).unwrap();
|
||||||
|
let region = &results[0].regions[0];
|
||||||
|
assert!(
|
||||||
|
!region.needs_ocr,
|
||||||
|
"stroked-grid table should be extracted, got needs_ocr=true"
|
||||||
|
);
|
||||||
|
for tok in ["A1", "B1", "A2", "B2"] {
|
||||||
|
assert!(
|
||||||
|
region.text.contains(tok),
|
||||||
|
"expected '{tok}' in output, got: {}",
|
||||||
|
region.text
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
region.text.contains('|'),
|
||||||
|
"expected pipe-delimited markdown"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
// extract_tables_with_structure_mem tests (TSR-aware path)
|
// extract_tables_with_structure_mem tests (TSR-aware path)
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user