Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c3949f355f |
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@firecrawl/pdf-inspector",
|
"name": "@firecrawl/pdf-inspector",
|
||||||
"version": "1.8.5",
|
"version": "1.8.6",
|
||||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"types": "index.d.ts",
|
"types": "index.d.ts",
|
||||||
|
|||||||
+66
@@ -1094,6 +1094,72 @@ mod vector_grid_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wireless table regression: decorative/text-region rects may provide row
|
||||||
|
/// bands, but without a real rect-derived column scaffold they must not be
|
||||||
|
/// accepted as a vector grid.
|
||||||
|
#[test]
|
||||||
|
fn wireless_two_col_rejects_rect_grid() {
|
||||||
|
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_two_col_no_rects.pdf");
|
||||||
|
assert!(
|
||||||
|
tables.is_empty(),
|
||||||
|
"expected no rect-detected tables for wireless content; got {:?}",
|
||||||
|
tables
|
||||||
|
.iter()
|
||||||
|
.map(|t| (t.rows.len(), t.columns.len()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wireless_two_col_region_rejects_vector_grid() {
|
||||||
|
let buf = std::fs::read("tests/fixtures/wireless_two_col_no_rects.pdf").unwrap();
|
||||||
|
let crops = [
|
||||||
|
[49.32_f32, 52.92, 558.72, 214.2],
|
||||||
|
[49.32_f32, 288.72, 556.56, 378.0],
|
||||||
|
[51.48_f32, 478.44, 558.36, 567.36],
|
||||||
|
];
|
||||||
|
for crop in crops {
|
||||||
|
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
|
||||||
|
assert!(
|
||||||
|
detected.is_none(),
|
||||||
|
"expected no vector grid for wireless crop {crop:?}; got {} cells",
|
||||||
|
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wireless dense table regression: text-position columns alone are not
|
||||||
|
/// enough evidence for a rect-derived grid.
|
||||||
|
#[test]
|
||||||
|
fn wireless_dense_rejects_rect_grid() {
|
||||||
|
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_dense_no_rects.pdf");
|
||||||
|
assert!(
|
||||||
|
tables.is_empty(),
|
||||||
|
"expected no rect-detected tables for wireless content; got {:?}",
|
||||||
|
tables
|
||||||
|
.iter()
|
||||||
|
.map(|t| (t.rows.len(), t.columns.len()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wireless_dense_region_rejects_vector_grid() {
|
||||||
|
let buf = std::fs::read("tests/fixtures/wireless_dense_no_rects.pdf").unwrap();
|
||||||
|
let crops = [
|
||||||
|
[72.36_f32, 177.48, 243.72, 333.36],
|
||||||
|
[72.0_f32, 390.24, 286.92, 417.6],
|
||||||
|
];
|
||||||
|
for crop in crops {
|
||||||
|
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
|
||||||
|
assert!(
|
||||||
|
detected.is_none(),
|
||||||
|
"expected no vector grid for wireless crop {crop:?}; got {} cells",
|
||||||
|
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
|
/// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
|
||||||
/// glossary with a green-shaded header row and plain (line-drawn) body cells.
|
/// glossary with a green-shaded header row and plain (line-drawn) body cells.
|
||||||
/// Mirrors the production failure cohort #1 (Contractions glossary) and #6
|
/// Mirrors the production failure cohort #1 (Contractions glossary) and #6
|
||||||
|
|||||||
@@ -1632,17 +1632,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let col_edges = match (rect_col_edges, text_col_edges) {
|
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||||
debug!(
|
debug!(
|
||||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||||
rect_edges.len() - 1,
|
rect_edges.len() - 1,
|
||||||
text_edges.len() - 1
|
text_edges.len() - 1
|
||||||
);
|
);
|
||||||
rect_edges
|
(rect_edges, false)
|
||||||
}
|
}
|
||||||
(_, Some(text_edges)) => text_edges,
|
(_, Some(text_edges)) => (text_edges, true),
|
||||||
(Some(rect_edges), None) => rect_edges,
|
(Some(rect_edges), None) => (rect_edges, false),
|
||||||
(None, None) => {
|
(None, None) => {
|
||||||
debug!(
|
debug!(
|
||||||
" cell-rect rejected: only {} columns from text clustering",
|
" cell-rect rejected: only {} columns from text clustering",
|
||||||
@@ -1742,7 +1742,7 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
// well-distributed-cols check (both cols populated), so we need a
|
// well-distributed-cols check (both cols populated), so we need a
|
||||||
// content-based signal to tell them apart.
|
// content-based signal to tell them apart.
|
||||||
//
|
//
|
||||||
// Two layered checks combine after the 20%-of-cells prose-word
|
// Layered checks combine after the 20%-of-cells prose-word
|
||||||
// trigger fires:
|
// trigger fires:
|
||||||
// (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
|
// (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
|
||||||
// per non-empty cell (sentence fragments); real data tables
|
// per non-empty cell (sentence fragments); real data tables
|
||||||
@@ -1753,7 +1753,10 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
// overrides the well-distributed relaxation — long cells
|
// overrides the well-distributed relaxation — long cells
|
||||||
// are the strongest prose signal even when both cols are
|
// are the strongest prose signal even when both cols are
|
||||||
// populated.
|
// populated.
|
||||||
// (b) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
|
// (b) Two-column text-only scaffold: when both columns were inferred
|
||||||
|
// from text starts rather than rect edges, prose fragments can look
|
||||||
|
// perfectly balanced. Require rect evidence for this relaxed shape.
|
||||||
|
// (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
|
||||||
// cells. Catches the prose-paragraph-as-many-cols shape
|
// cells. Catches the prose-paragraph-as-many-cols shape
|
||||||
// while admitting real "label / value / description /
|
// while admitting real "label / value / description /
|
||||||
// benefit"-style tables.
|
// benefit"-style tables.
|
||||||
@@ -1801,7 +1804,18 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// (b) Well-distributed columns.
|
// (b) Two text-derived columns are not enough vector evidence once
|
||||||
|
// the content looks prose-like. Real 2-col rect tables still pass
|
||||||
|
// when the column scaffold comes from drawn cell geometry.
|
||||||
|
if columns_from_text && num_cols == 2 {
|
||||||
|
debug!(
|
||||||
|
" cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
|
||||||
|
mean_chars, prose_cells, counted
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// (c) Well-distributed columns.
|
||||||
let filled_cols = (0..num_cols)
|
let filled_cols = (0..num_cols)
|
||||||
.filter(|&c| {
|
.filter(|&c| {
|
||||||
cells
|
cells
|
||||||
@@ -3039,6 +3053,62 @@ mod tests {
|
|||||||
// If tables were detected, that's also acceptable
|
// If tables were detected, that's also acceptable
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn text_derived_two_col_prose_is_not_cell_rect_table() {
|
||||||
|
let page = 1;
|
||||||
|
let mut rects = Vec::new();
|
||||||
|
for row in 0..8 {
|
||||||
|
rects.push(PdfRect {
|
||||||
|
x: 50.0,
|
||||||
|
y: 100.0 + row as f32 * 20.0,
|
||||||
|
width: 180.0,
|
||||||
|
height: 18.0,
|
||||||
|
page,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut items = Vec::new();
|
||||||
|
let left = [
|
||||||
|
"the annual plan was revised",
|
||||||
|
"and the team noted changes",
|
||||||
|
"this section explains limits",
|
||||||
|
"with additional notes below",
|
||||||
|
"the policy was reviewed",
|
||||||
|
"and results are summarized",
|
||||||
|
"this appendix describes scope",
|
||||||
|
"with examples for reference",
|
||||||
|
];
|
||||||
|
let right = [
|
||||||
|
"for each area in the review",
|
||||||
|
"as part of the assessment",
|
||||||
|
"that were applied in context",
|
||||||
|
"to support the conclusion",
|
||||||
|
"for use by the committee",
|
||||||
|
"as shown in the narrative",
|
||||||
|
"that remain under discussion",
|
||||||
|
"to clarify the method",
|
||||||
|
];
|
||||||
|
for row in 0..8 {
|
||||||
|
let y = 104.0 + row as f32 * 20.0;
|
||||||
|
let mut left_item = make_item(left[row], 60.0, y, 9.0);
|
||||||
|
left_item.width = 50.0;
|
||||||
|
items.push(left_item);
|
||||||
|
let mut right_item = make_item(right[row], 150.0, y, 9.0);
|
||||||
|
right_item.width = 50.0;
|
||||||
|
items.push(right_item);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
|
||||||
|
assert!(
|
||||||
|
tables.is_empty(),
|
||||||
|
"text-derived two-column prose must not be accepted as a rect table; got {:?}",
|
||||||
|
tables
|
||||||
|
.iter()
|
||||||
|
.map(|t| (t.rows.len(), t.columns.len()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn failed_cluster_no_hint_without_items() {
|
fn failed_cluster_no_hint_without_items() {
|
||||||
// Rects with no text items inside → no failed-cluster hint generated.
|
// Rects with no text items inside → no failed-cluster hint generated.
|
||||||
|
|||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Reference in New Issue
Block a user