Compare commits

...
Author SHA1 Message Date
Abimael Martell 512c50c64d napi: add probe-doc.mjs for single-region detect_vector_grid checks
Generalized one-shot probe — calls detectVectorGridInRegion against
a single PDF + region without spinning up the api or running the bench.
Useful for fast pdf-inspector dev loops:

  cargo build --release && cd napi && bun run build:debug
  node probe-doc.mjs <pdf-path> [page] [bbox] [dpi]

Output: cells/rows/cols and timing, or null when the region has no
vector grid. ~5s round-trip vs ~30s for the full bench harness.

Generalizes the existing probe-indent.mjs pattern so it works on
arbitrary PDFs, not just the multi-line indent fixture.
2026-05-07 23:05:42 -07:00
Abimael Martell 6086577d11 tables/detect_rects: emit grid for multiline indented cells (#79) 2026-05-07 11:44:53 -07:00
Abimael MartellandCursor f2186ec1aa tables/detect_rects: don't accept relaxed grid on wireless prose (#78)
Require rect-derived column evidence before relaxing prose checks for two-column cell-rect fallbacks, so text-position alignment alone cannot synthesize a vector grid on wireless content.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-07 09:30:29 -07:00
8 changed files with 525 additions and 15 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.8.5",
"version": "1.8.7",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env node
// Generalized one-shot probe: run detectVectorGridInRegion against a single
// PDF + region without spinning up the api or the bench.
//
// Usage:
// node probe-doc.mjs <pdf-path> [page] [bbox] [dpi]
//
// pdf-path : path to a PDF on disk (required)
// page : 0-indexed page number (default 0)
// bbox : "x0,y0,x1,y1" in PDF points (default = full standard letter)
// dpi : render dpi (default 200)
//
// Examples:
// node probe-doc.mjs ~/Code/opendataloader-bench/pdfs/01030000000127.pdf 0
// node probe-doc.mjs /tmp/foo.pdf 3 "50,100,560,700" 200
// node probe-doc.mjs /tmp/foo.pdf # page 0, full page, dpi 200
//
// Prints: cells / rows / cols / null. Useful for fast pdf-inspector loops:
// cargo build --release && bun run build:debug && node probe-doc.mjs ...
import { readFileSync, existsSync } from "node:fs";
import { createRequire } from "node:module";
const require = createRequire(import.meta.url);
const { detectVectorGridInRegion } = require("./index.js");
const [, , pdfPath, pageArg, bboxArg, dpiArg] = process.argv;
if (!pdfPath) {
console.error("usage: node probe-doc.mjs <pdf-path> [page] [bbox] [dpi]");
process.exit(2);
}
if (!existsSync(pdfPath)) {
console.error(`pdf not found: ${pdfPath}`);
process.exit(2);
}
const page = pageArg !== undefined ? Number(pageArg) : 0;
const bbox = bboxArg
? bboxArg.split(",").map(Number)
: [0, 0, 612, 792]; // standard US letter
const dpi = dpiArg !== undefined ? Number(dpiArg) : 200;
if (bbox.length !== 4 || bbox.some(Number.isNaN)) {
console.error(`invalid bbox "${bboxArg}", expected "x0,y0,x1,y1"`);
process.exit(2);
}
const pdf = readFileSync(pdfPath);
const t0 = Date.now();
const result = detectVectorGridInRegion(pdf, page, bbox, dpi);
const ms = Date.now() - t0;
if (!result) {
console.log(`null (${ms}ms) page=${page} bbox=${bbox.join(",")} dpi=${dpi}`);
process.exit(0);
}
const rows = result.structureTokens.filter((t) => t === "<tr>").length;
const cols = rows > 0 ? result.cellBboxes.length / rows : 0;
console.log(
`cells=${result.cellBboxes.length} rows=${rows} cols=${cols} (${ms}ms) page=${page} bbox=${bbox.join(",")} dpi=${dpi}`,
);
+29
View File
@@ -0,0 +1,29 @@
import { readFileSync } from "node:fs";
import { createRequire } from "node:module";
const require = createRequire(import.meta.url);
const { detectVectorGridInRegion } = require("./index.js");
const pdfPath =
process.argv[2] ?? "/tmp/pdf_inspector_indent_fixtures/cis_edge_benchmark.pdf";
const pdf = readFileSync(pdfPath);
const dpi = Number(process.argv[3] ?? 200);
const crops = [
{ pageIdx: 29, box: [0, 0, 612, 792], label: "page30-full" },
{ pageIdx: 16, box: [0, 0, 612, 792], label: "page17-full" },
{ pageIdx: 23, box: [0, 0, 612, 792], label: "page24-full" },
];
for (const { pageIdx, box, label } of crops) {
const result = detectVectorGridInRegion(pdf, pageIdx, box, dpi);
if (!result) {
console.log(`${label}: null`);
continue;
}
const rows = result.structureTokens.filter((token) => token === "<tr>").length;
const cols = rows > 0 ? result.cellBboxes.length / rows : 0;
console.log(
`${label}: cells=${result.cellBboxes.length} rows=${rows} cols=${cols}`,
);
}
+119 -5
View File
@@ -1036,7 +1036,7 @@ mod vector_grid_tests {
}
/// Helper: load a fixture PDF and run the rect-based table detector.
fn detect_rect_tables_in_fixture(path: &str) -> Vec<crate::tables::Table> {
fn detect_rect_tables_in_fixture_page(path: &str, page_num: u32) -> Vec<crate::tables::Table> {
use crate::extractor::content_stream::extract_page_text_items;
use crate::tables::detect_tables_from_rects;
use crate::tounicode::FontCMaps;
@@ -1047,16 +1047,20 @@ mod vector_grid_tests {
let buf = fs::read(path).unwrap();
let doc = Document::load_mem(&buf).unwrap();
let pages = doc.get_pages();
let &page_id = pages.get(&1).unwrap();
let needed: HashSet<u32> = HashSet::from([1]);
let &page_id = pages.get(&page_num).unwrap();
let needed: HashSet<u32> = HashSet::from([page_num]);
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
let ((items, rects, _lines), _has_gid, _rotated) =
extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap();
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num);
rect_tables
}
fn detect_rect_tables_in_fixture(path: &str) -> Vec<crate::tables::Table> {
detect_rect_tables_in_fixture_page(path, 1)
}
/// Regression for the prose-in-a-frame failure mode introduced by the
/// shaded-header detection lift (PR #76). The accessory_building permit
/// form has a paragraph of legal text laid out in a 2-column justified
@@ -1094,6 +1098,116 @@ mod vector_grid_tests {
);
}
/// Wireless table regression: decorative/text-region rects may provide row
/// bands, but without a real rect-derived column scaffold they must not be
/// accepted as a vector grid.
#[test]
fn wireless_two_col_rejects_rect_grid() {
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_two_col_no_rects.pdf");
assert!(
tables.is_empty(),
"expected no rect-detected tables for wireless content; got {:?}",
tables
.iter()
.map(|t| (t.rows.len(), t.columns.len()))
.collect::<Vec<_>>()
);
}
#[test]
fn wireless_two_col_region_rejects_vector_grid() {
let buf = std::fs::read("tests/fixtures/wireless_two_col_no_rects.pdf").unwrap();
let crops = [
[49.32_f32, 52.92, 558.72, 214.2],
[49.32_f32, 288.72, 556.56, 378.0],
[51.48_f32, 478.44, 558.36, 567.36],
];
for crop in crops {
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
assert!(
detected.is_none(),
"expected no vector grid for wireless crop {crop:?}; got {} cells",
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
);
}
}
/// Wireless dense table regression: text-position columns alone are not
/// enough evidence for a rect-derived grid.
#[test]
fn wireless_dense_rejects_rect_grid() {
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_dense_no_rects.pdf");
assert!(
tables.is_empty(),
"expected no rect-detected tables for wireless content; got {:?}",
tables
.iter()
.map(|t| (t.rows.len(), t.columns.len()))
.collect::<Vec<_>>()
);
}
#[test]
fn wireless_dense_region_rejects_vector_grid() {
let buf = std::fs::read("tests/fixtures/wireless_dense_no_rects.pdf").unwrap();
let crops = [
[72.36_f32, 177.48, 243.72, 333.36],
[72.0_f32, 390.24, 286.92, 417.6],
];
for crop in crops {
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
assert!(
detected.is_none(),
"expected no vector grid for wireless crop {crop:?}; got {} cells",
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
);
}
}
#[test]
fn multiline_indent_cell_rect_grid_fixture_detects_table() {
let tables = detect_rect_tables_in_fixture_page(
"tests/fixtures/multiline_indent_cell_rect_grid.pdf",
30,
);
let table = tables
.iter()
.max_by_key(|t| t.rows.len() * t.columns.len())
.expect("expected a rect-detected table");
assert_eq!(
table.columns.len(),
5,
"expected the Controls Version / Control / IG table shape; got {:?}",
tables
.iter()
.map(|t| (t.rows.len(), t.columns.len()))
.collect::<Vec<_>>()
);
assert!(
table.rows.len() >= 3,
"expected at least header plus data rows; got {}",
table.rows.len()
);
}
#[test]
fn multiline_indent_cell_rect_grid_region_detects_vector_grid() {
let buf = std::fs::read("tests/fixtures/multiline_indent_cell_rect_grid.pdf").unwrap();
let detected =
crate::detect_vector_grid_in_region_mem(&buf, 29, [0.0, 0.0, 612.0, 792.0], 200.0)
.unwrap()
.expect("expected vector grid for multiline indented description table");
let rows = detected
.structure_tokens
.iter()
.filter(|token| token.as_str() == "<tr>")
.count();
assert_eq!(detected.cell_bboxes.len() % rows, 0);
assert_eq!(detected.cell_bboxes.len() / rows, 5);
assert!(rows >= 3);
assert!(!detected.cell_bboxes.is_empty());
}
/// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
/// glossary with a green-shaded header row and plain (line-drawn) body cells.
/// Mirrors the production failure cohort #1 (Contractions glossary) and #6
+312 -9
View File
@@ -1632,17 +1632,17 @@ fn detect_row_stripe_table_from_cell_rects(
}
};
let col_edges = match (rect_col_edges, text_col_edges) {
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
debug!(
" cell-rect using {} rect-derived columns over {} text clusters",
rect_edges.len() - 1,
text_edges.len() - 1
);
rect_edges
(rect_edges, false)
}
(_, Some(text_edges)) => text_edges,
(Some(rect_edges), None) => rect_edges,
(_, Some(text_edges)) => (text_edges, true),
(Some(rect_edges), None) => (rect_edges, false),
(None, None) => {
debug!(
" cell-rect rejected: only {} columns from text clustering",
@@ -1667,12 +1667,25 @@ fn detect_row_stripe_table_from_cell_rects(
page_items.len()
);
let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
let (mut cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
if item_indices.is_empty() {
return None;
}
let mut row_edges = row_edges;
let (collapsed_cells, collapsed_row_edges, collapsed_rows) =
collapse_multiline_description_rows(cells, row_edges, &col_edges);
let has_wrapped_description_rows = collapsed_rows > 0;
cells = collapsed_cells;
row_edges = collapsed_row_edges;
if collapsed_rows > 0 {
debug!(
" cell-rect collapsed {} wrapped description rows",
collapsed_rows
);
}
// Validate: >=2 non-empty rows, >=25% density
let non_empty_rows = cells
.iter()
@@ -1686,6 +1699,7 @@ fn detect_row_stripe_table_from_cell_rects(
return None;
}
let num_rows = cells.len();
let total_cells = (num_cols * num_rows) as f32;
let non_empty_cells = cells
.iter()
@@ -1742,7 +1756,7 @@ fn detect_row_stripe_table_from_cell_rects(
// well-distributed-cols check (both cols populated), so we need a
// content-based signal to tell them apart.
//
// Two layered checks combine after the 20%-of-cells prose-word
// Layered checks combine after the 20%-of-cells prose-word
// trigger fires:
// (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
// per non-empty cell (sentence fragments); real data tables
@@ -1753,7 +1767,10 @@ fn detect_row_stripe_table_from_cell_rects(
// overrides the well-distributed relaxation — long cells
// are the strongest prose signal even when both cols are
// populated.
// (b) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
// (b) Two-column text-only scaffold: when both columns were inferred
// from text starts rather than rect edges, prose fragments can look
// perfectly balanced. Require rect evidence for this relaxed shape.
// (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
// cells. Catches the prose-paragraph-as-many-cols shape
// while admitting real "label / value / description /
// benefit"-style tables.
@@ -1793,15 +1810,31 @@ fn detect_row_stripe_table_from_cell_rects(
// discriminator.
const PROSE_MEAN_CHAR_THRESHOLD: usize = 65;
let mean_chars = total_chars / counted;
if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
if mean_chars > PROSE_MEAN_CHAR_THRESHOLD && !has_wrapped_description_rows {
debug!(
" cell-rect rejected: prose-in-frame, mean non-empty cell {} chars > {} (prose words {}/{})",
mean_chars, PROSE_MEAN_CHAR_THRESHOLD, prose_cells, counted
);
return None;
} else if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
debug!(
" cell-rect prose check relaxed: wrapped description rows, mean {} chars (prose words {}/{})",
mean_chars, prose_cells, counted
);
}
// (b) Well-distributed columns.
// (b) Two text-derived columns are not enough vector evidence once
// the content looks prose-like. Real 2-col rect tables still pass
// when the column scaffold comes from drawn cell geometry.
if columns_from_text && num_cols == 2 {
debug!(
" cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
mean_chars, prose_cells, counted
);
return None;
}
// (c) Well-distributed columns.
let filled_cols = (0..num_cols)
.filter(|&c| {
cells
@@ -1849,6 +1882,132 @@ fn detect_row_stripe_table_from_cell_rects(
Some(Table::new(column_centers, row_centers, cells, item_indices))
}
/// Merge wrapped description-line bands back into their visual data rows.
///
/// Some Word/PDF exports draw enough rectangle geometry to prove a table exists
/// but expose Y bands per wrapped text line instead of per cell row. In the
/// common mapping-table shape, a narrow row-label column precedes one wide
/// description column, and wrapped continuation bands have content only in that
/// wide column. Merge only that high-confidence shape so framed prose still
/// falls through the existing prose guards.
fn collapse_multiline_description_rows(
cells: Vec<Vec<String>>,
row_edges: Vec<f32>,
col_edges: &[f32],
) -> (Vec<Vec<String>>, Vec<f32>, usize) {
let num_rows = cells.len();
let num_cols = col_edges.len().saturating_sub(1);
if num_rows < 3 || num_cols < 3 || row_edges.len() != num_rows + 1 {
return (cells, row_edges, 0);
}
let table_width = col_edges[num_cols] - col_edges[0];
if table_width <= 0.0 {
return (cells, row_edges, 0);
}
let Some((description_col, description_width)) = (0..num_cols)
.map(|c| (c, col_edges[c + 1] - col_edges[c]))
.max_by(|a, b| a.1.total_cmp(&b.1))
else {
return (cells, row_edges, 0);
};
// Require a preceding row-label column. Without it (e.g. a prose frame
// split into text-start columns), "one populated wide column" is not enough
// evidence to find visual row starts safely.
if description_col == 0 || description_width < table_width * 0.35 {
return (cells, row_edges, 0);
}
let row_has_left_label = |row: &[String]| {
row.iter()
.take(description_col)
.any(|cell| !cell.trim().is_empty())
};
let labeled_rows = cells.iter().filter(|row| row_has_left_label(row)).count();
if labeled_rows < 2 {
return (cells, row_edges, 0);
}
let mut merged_rows = 0usize;
let mut wrapped_description_rows = 0usize;
let mut new_cells: Vec<Vec<String>> = Vec::with_capacity(num_rows);
let mut new_edges = Vec::with_capacity(row_edges.len());
new_edges.push(row_edges[0]);
for (row_idx, row) in cells.into_iter().enumerate() {
let desc_text = row
.get(description_col)
.map(String::as_str)
.unwrap_or("")
.trim();
let left_label = row_has_left_label(&row);
let non_desc_non_empty = row
.iter()
.enumerate()
.filter(|(col, cell)| *col != description_col && !cell.trim().is_empty())
.count();
// Wrapped continuation bands contain only description-column text.
// The preceding label/marker column is empty because the visual row's
// label cell spans the whole wrapped block.
let is_description_continuation = row_idx > 0
&& !desc_text.is_empty()
&& !left_label
&& non_desc_non_empty == 0
&& !new_cells.is_empty();
// Header cells are often split as "Controls" / "Version" in the first
// column while the other header labels sit on the first band.
let only_first_col = row
.iter()
.enumerate()
.all(|(col, cell)| col == 0 || cell.trim().is_empty());
let is_header_continuation = row_idx > 0
&& only_first_col
&& row
.first()
.is_some_and(|cell| !cell.trim().is_empty() && cell.chars().count() <= 24)
&& !new_cells.is_empty()
&& new_cells
.last()
.is_some_and(|prev| prev.iter().filter(|c| !c.trim().is_empty()).count() >= 2);
if is_description_continuation || is_header_continuation {
if let Some(prev) = new_cells.last_mut() {
for (col, cell) in row.iter().enumerate() {
let text = cell.trim();
if text.is_empty() {
continue;
}
if !prev[col].trim().is_empty() {
prev[col].push(' ');
}
prev[col].push_str(text);
}
}
merged_rows += 1;
if is_description_continuation {
wrapped_description_rows += 1;
}
} else {
if !new_cells.is_empty() {
new_edges.push(row_edges[row_idx]);
}
new_cells.push(row);
}
}
new_edges.push(*row_edges.last().unwrap());
if merged_rows == 0 || new_cells.len() < 2 || new_edges.len() != new_cells.len() + 1 {
return (new_cells, row_edges, 0);
}
(new_cells, new_edges, wrapped_description_rows)
}
/// Detect a table by merging all cluster rects into one group.
///
/// This handles clip-path PDFs where each column's cell rects form a separate
@@ -3039,6 +3198,150 @@ mod tests {
// If tables were detected, that's also acceptable
}
#[test]
fn text_derived_two_col_prose_is_not_cell_rect_table() {
let page = 1;
let mut rects = Vec::new();
for row in 0..8 {
rects.push(PdfRect {
x: 50.0,
y: 100.0 + row as f32 * 20.0,
width: 180.0,
height: 18.0,
page,
});
}
let mut items = Vec::new();
let left = [
"the annual plan was revised",
"and the team noted changes",
"this section explains limits",
"with additional notes below",
"the policy was reviewed",
"and results are summarized",
"this appendix describes scope",
"with examples for reference",
];
let right = [
"for each area in the review",
"as part of the assessment",
"that were applied in context",
"to support the conclusion",
"for use by the committee",
"as shown in the narrative",
"that remain under discussion",
"to clarify the method",
];
for row in 0..8 {
let y = 104.0 + row as f32 * 20.0;
let mut left_item = make_item(left[row], 60.0, y, 9.0);
left_item.width = 50.0;
items.push(left_item);
let mut right_item = make_item(right[row], 150.0, y, 9.0);
right_item.width = 50.0;
items.push(right_item);
}
let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
assert!(
tables.is_empty(),
"text-derived two-column prose must not be accepted as a rect table; got {:?}",
tables
.iter()
.map(|t| (t.rows.len(), t.columns.len()))
.collect::<Vec<_>>()
);
}
#[test]
fn multiline_indented_description_rows_collapse_to_visual_rows() {
let page = 1;
let col_edges = [0.0, 60.0, 420.0, 460.0, 500.0, 540.0];
let row_edges = [
340.0, 320.0, 300.0, 270.0, 250.0, 230.0, 200.0, 180.0, 160.0,
];
let mut rects = Vec::new();
for row in 0..row_edges.len() - 1 {
let y_top = row_edges[row];
let y_bot = row_edges[row + 1];
for col in 0..col_edges.len() - 1 {
rects.push((
col_edges[col],
y_bot,
col_edges[col + 1] - col_edges[col],
y_top - y_bot,
));
}
}
let mut items = vec![
make_item("Controls", 8.0, 330.0, 9.0),
make_item("Control", 70.0, 330.0, 9.0),
make_item("IG 1", 428.0, 330.0, 9.0),
make_item("IG 2", 468.0, 330.0, 9.0),
make_item("IG 3", 508.0, 330.0, 9.0),
make_item("Version", 8.0, 310.0, 9.0),
make_item("v8", 20.0, 285.0, 9.0),
make_item(
"4.5 Implement and Manage a Firewall on End-User Devices",
70.0,
285.0,
9.0,
),
make_item("*", 438.0, 285.0, 9.0),
make_item("*", 478.0, 285.0, 9.0),
make_item("*", 518.0, 285.0, 9.0),
make_item("v7", 20.0, 215.0, 9.0),
make_item(
"9.4 Apply Host-based Firewalls or Port-Filtering",
70.0,
215.0,
9.0,
),
make_item("*", 478.0, 215.0, 9.0),
make_item("*", 518.0, 215.0, 9.0),
];
items.push(make_item(
"Implement and manage a host-based firewall or port-filtering tool",
84.0,
260.0,
8.0,
));
items.push(make_item(
"on end-user devices with a default-deny rule",
84.0,
240.0,
8.0,
));
items.push(make_item(
"Apply host-based firewalls or port filtering tools on end systems",
84.0,
190.0,
8.0,
));
items.push(make_item(
"and deny unauthorized network communication",
84.0,
170.0,
8.0,
));
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
.expect("expected multiline description table");
assert_eq!(table.columns.len(), 5);
assert_eq!(
table.rows.len(),
3,
"wrapped lines should collapse to header plus two data rows"
);
assert_eq!(table.cells[0][0], "Controls Version");
assert!(table.cells[1][1].contains("host-based firewall"));
assert!(table.cells[1][1].contains("default-deny rule"));
assert!(table.cells[2][1].contains("deny unauthorized"));
}
#[test]
fn failed_cluster_no_hint_without_items() {
// Rects with no text items inside → no failed-cluster hint generated.
Binary file not shown.
Binary file not shown.
Binary file not shown.