Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b8249dfd61 | ||
|
|
c7612ceb16 | ||
|
|
63b5573133 | ||
|
|
c186a036fc | ||
|
|
d196d435d1 | ||
|
|
fbab84fc20 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.7.0",
|
||||
"version": "1.8.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+102
@@ -99,6 +99,13 @@ pub struct PageRegionTexts {
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// Vector-grid detection result compatible with `extractTablesWithStructure*`.
|
||||
#[napi(object)]
|
||||
pub struct VectorGridDetectionJs {
|
||||
pub structure_tokens: Vec<String>,
|
||||
pub cell_bboxes: Vec<Vec<f64>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -317,6 +324,53 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect a vector ruled-line / rectangle grid inside one page region.
|
||||
///
|
||||
/// Returns TSR-compatible structure tokens plus crop-pixel cell bboxes, or
|
||||
/// `null` when the region does not contain a valid vector grid.
|
||||
///
|
||||
/// `pageIdx` is 0-indexed. `regionPdfPtBbox` is `[x1,y1,x2,y2]` in PDF
|
||||
/// points with top-left origin. `renderDpi` is the DPI of the crop image that
|
||||
/// will consume the returned cell bboxes.
|
||||
#[napi]
|
||||
pub fn detect_vector_grid_in_region(
|
||||
buffer: Buffer,
|
||||
page_idx: u32,
|
||||
region_pdf_pt_bbox: Vec<f64>,
|
||||
render_dpi: f64,
|
||||
) -> Result<Option<VectorGridDetectionJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let region = if region_pdf_pt_bbox.len() == 4 {
|
||||
[
|
||||
region_pdf_pt_bbox[0] as f32,
|
||||
region_pdf_pt_bbox[1] as f32,
|
||||
region_pdf_pt_bbox[2] as f32,
|
||||
region_pdf_pt_bbox[3] as f32,
|
||||
]
|
||||
} else {
|
||||
[0.0, 0.0, 0.0, 0.0]
|
||||
};
|
||||
|
||||
catch_panic("detect_vector_grid_in_region", move || {
|
||||
let result = pdf_inspector::detect_vector_grid_in_region_mem(
|
||||
&bytes,
|
||||
page_idx,
|
||||
region,
|
||||
render_dpi as f32,
|
||||
)
|
||||
.map_err(|e| to_napi_err(e, "detect_vector_grid_in_region"))?;
|
||||
|
||||
Ok(result.map(|r| VectorGridDetectionJs {
|
||||
structure_tokens: r.structure_tokens,
|
||||
cell_bboxes: r
|
||||
.cell_bboxes
|
||||
.into_iter()
|
||||
.map(|bbox| bbox.into_iter().map(|v| v as f64).collect())
|
||||
.collect(),
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
/// One cropped table region plus its raw structure-recovery output, for
|
||||
/// `extractTablesWithStructure`.
|
||||
///
|
||||
@@ -422,6 +476,54 @@ pub fn extract_tables_with_structure_cells(
|
||||
})
|
||||
}
|
||||
|
||||
/// One result from `extractTablesWithStructureAuto` — markdown plus a
|
||||
/// diagnostic flag identifying which path produced it.
|
||||
///
|
||||
/// `fallbackReason` is `null` when the TSR-hybrid path produced the
|
||||
/// markdown directly. When stage 1's quality check fires (the cells
|
||||
/// look like a SLANet detection pathology — phantom rows or multi-row
|
||||
/// content in a single cell), the auto path may expand the TSR cells
|
||||
/// in-place or run the heuristic table extractor on the same region.
|
||||
/// `fallbackReason` carries the diagnostic label (for example
|
||||
/// `"multi_row_in_cell_expanded"` or `"phantom_empty_row"`).
|
||||
#[napi(object)]
|
||||
pub struct TableExtractionResultJs {
|
||||
pub markdown: String,
|
||||
pub fallback_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Auto-fallback variant of [`extractTablesWithStructure`].
|
||||
///
|
||||
/// Runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies, expands multi-row cells in-place when
|
||||
/// possible, and otherwise falls back to the heuristic
|
||||
/// `extractTablesInRegions` for inputs where the TSR path looks
|
||||
/// compromised.
|
||||
///
|
||||
/// On clean inputs this returns identical markdown to
|
||||
/// `extractTablesWithStructure`; on flagged inputs `fallbackReason` is
|
||||
/// set to the recovery path that produced the result.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure_auto(
|
||||
buffer: Buffer,
|
||||
inputs: Vec<TsrTableInputJs>,
|
||||
) -> Result<Vec<TableExtractionResultJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let parsed = parse_tsr_inputs(&inputs);
|
||||
|
||||
catch_panic("extract_tables_with_structure_auto", move || {
|
||||
let result = pdf_inspector::extract_tables_with_structure_auto_mem(&bytes, &parsed)
|
||||
.map_err(|e| to_napi_err(e, "extract_tables_with_structure_auto"))?;
|
||||
Ok(result
|
||||
.into_iter()
|
||||
.map(|r| TableExtractionResultJs {
|
||||
markdown: r.markdown,
|
||||
fallback_reason: r.fallback_reason,
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_tsr_inputs(inputs: &[TsrTableInputJs]) -> Vec<pdf_inspector::TsrTableInput> {
|
||||
inputs
|
||||
.iter()
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
detectVectorGridInRegion,
|
||||
extractPagesMarkdown,
|
||||
} from './index.js';
|
||||
|
||||
@@ -90,6 +91,17 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
|
||||
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
|
||||
console.log(' extractTextInRegions: OK');
|
||||
|
||||
// --- detectVectorGridInRegion ---
|
||||
console.log('Testing detectVectorGridInRegion...');
|
||||
const vectorGrid = detectVectorGridInRegion(fixture, 0, [0, 0, 600, 800], 72);
|
||||
assert.ok(vectorGrid === null || typeof vectorGrid === 'object');
|
||||
if (vectorGrid) {
|
||||
assert.ok(Array.isArray(vectorGrid.structureTokens));
|
||||
assert.ok(Array.isArray(vectorGrid.cellBboxes));
|
||||
assert.ok(vectorGrid.cellBboxes.every(bbox => Array.isArray(bbox) && bbox.length === 4));
|
||||
}
|
||||
console.log(' detectVectorGridInRegion: OK');
|
||||
|
||||
// --- extractPagesMarkdown ---
|
||||
console.log('Testing extractPagesMarkdown...');
|
||||
|
||||
|
||||
+1247
-1
File diff suppressed because it is too large
Load Diff
+703
-4
@@ -4,10 +4,11 @@ use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, S
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
|
||||
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
|
||||
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
|
||||
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
|
||||
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
|
||||
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -1704,6 +1705,292 @@ fn synthetic_dense_table_pdf() -> Vec<u8> {
|
||||
bytes
|
||||
}
|
||||
|
||||
fn synthetic_vector_grid_pdf(two_tables: bool) -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
fn push_grid(
|
||||
operations: &mut Vec<Operation>,
|
||||
x_left: i64,
|
||||
x_mid: i64,
|
||||
x_right: i64,
|
||||
y_top: i64,
|
||||
y_mid: i64,
|
||||
y_bottom: i64,
|
||||
) {
|
||||
for y in [y_top, y_mid, y_bottom] {
|
||||
operations.push(Operation::new("m", vec![x_left.into(), y.into()]));
|
||||
operations.push(Operation::new("l", vec![x_right.into(), y.into()]));
|
||||
}
|
||||
for x in [x_left, x_mid, x_right] {
|
||||
operations.push(Operation::new("m", vec![x.into(), y_bottom.into()]));
|
||||
operations.push(Operation::new("l", vec![x.into(), y_top.into()]));
|
||||
}
|
||||
operations.push(Operation::new("S", vec![]));
|
||||
}
|
||||
|
||||
fn push_text(operations: &mut Vec<Operation>, x: i64, y: i64, text: &str) {
|
||||
operations.push(Operation::new(
|
||||
"Tm",
|
||||
vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
|
||||
));
|
||||
operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
|
||||
}
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let mut operations = Vec::new();
|
||||
push_grid(&mut operations, 50, 130, 210, 740, 710, 670);
|
||||
if two_tables {
|
||||
push_grid(&mut operations, 50, 130, 210, 560, 530, 490);
|
||||
}
|
||||
|
||||
operations.push(Operation::new("BT", vec![]));
|
||||
operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
|
||||
push_text(&mut operations, 70, 724, "A1");
|
||||
push_text(&mut operations, 150, 724, "B1");
|
||||
push_text(&mut operations, 70, 688, "A2");
|
||||
push_text(&mut operations, 150, 688, "B2");
|
||||
if two_tables {
|
||||
push_text(&mut operations, 70, 544, "C1");
|
||||
push_text(&mut operations, 150, 544, "D1");
|
||||
push_text(&mut operations, 70, 508, "C2");
|
||||
push_text(&mut operations, 150, 508, "D2");
|
||||
}
|
||||
operations.push(Operation::new("ET", vec![]));
|
||||
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
fn synthetic_vector_grid_three_row_pdf() -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let mut operations = Vec::new();
|
||||
for y in [740, 710, 680, 650] {
|
||||
operations.push(Operation::new("m", vec![50.into(), y.into()]));
|
||||
operations.push(Operation::new("l", vec![210.into(), y.into()]));
|
||||
}
|
||||
for x in [50, 130, 210] {
|
||||
operations.push(Operation::new("m", vec![x.into(), 650.into()]));
|
||||
operations.push(Operation::new("l", vec![x.into(), 740.into()]));
|
||||
}
|
||||
operations.push(Operation::new("S", vec![]));
|
||||
|
||||
operations.push(Operation::new("BT", vec![]));
|
||||
operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
|
||||
for (x, y, text) in [
|
||||
(70, 724, "Branch"),
|
||||
(150, 724, "Deposits"),
|
||||
(70, 694, "Oak"),
|
||||
(150, 694, "100"),
|
||||
(70, 664, "Boardwalk"),
|
||||
(150, 664, "200"),
|
||||
] {
|
||||
operations.push(Operation::new(
|
||||
"Tm",
|
||||
vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
|
||||
));
|
||||
operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
|
||||
}
|
||||
operations.push(Operation::new("ET", vec![]));
|
||||
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
fn assert_close(actual: f32, expected: f32) {
|
||||
assert!(
|
||||
(actual - expected).abs() < 0.75,
|
||||
"expected {actual} to be close to {expected}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_line_pdf() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_vector_grid_pdf(false);
|
||||
let crop = [50.0_f32, 60.0, 210.0, 130.0];
|
||||
let detected = detect_vector_grid_in_region_mem(&buf, 0, crop, 72.0)
|
||||
.unwrap()
|
||||
.expect("ruled vector table should be detected");
|
||||
|
||||
assert_eq!(detected.cell_bboxes.len(), 4);
|
||||
assert_eq!(
|
||||
detected
|
||||
.structure_tokens
|
||||
.iter()
|
||||
.filter(|tok| tok.as_str() == "<td></td>")
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
assert_eq!(detected.structure_tokens.first().unwrap(), "<table>");
|
||||
assert_eq!(detected.structure_tokens.last().unwrap(), "</table>");
|
||||
|
||||
let first = &detected.cell_bboxes[0];
|
||||
assert_close(first[0], 0.0);
|
||||
assert_close(first[1], 0.0);
|
||||
assert_close(first[2], 80.0);
|
||||
assert_close(first[3], 30.0);
|
||||
|
||||
let markdown = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: detected.structure_tokens,
|
||||
cell_bboxes: detected.cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
|
||||
assert!(markdown.contains("A1"));
|
||||
assert!(markdown.contains("B1"));
|
||||
assert!(markdown.contains("A2"));
|
||||
assert!(markdown.contains("B2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_text_pdf_returns_none() {
|
||||
let buf = make_minimal_text_pdf();
|
||||
let detected =
|
||||
detect_vector_grid_in_region_mem(&buf, 0, [0.0, 0.0, 300.0, 800.0], 72.0).unwrap();
|
||||
assert!(detected.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_filters_to_requested_table() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_vector_grid_pdf(true);
|
||||
let second_table_crop = [50.0_f32, 240.0, 210.0, 310.0];
|
||||
let detected = detect_vector_grid_in_region_mem(&buf, 0, second_table_crop, 72.0)
|
||||
.unwrap()
|
||||
.expect("second ruled table should be detected");
|
||||
|
||||
assert_eq!(detected.cell_bboxes.len(), 4);
|
||||
let markdown = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: second_table_crop,
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: detected.structure_tokens,
|
||||
cell_bboxes: detected.cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
|
||||
assert!(markdown.contains("C1"));
|
||||
assert!(markdown.contains("D2"));
|
||||
assert!(!markdown.contains("A1"));
|
||||
assert!(!markdown.contains("B2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
@@ -2052,6 +2339,418 @@ fn test_extract_tables_with_structure_separator_after_thead() {
|
||||
assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_auto_mem tests (TSR + heuristic fallback)
|
||||
// =========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_auto_passes_through_clean_tsr_output() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cells fit each visible row cleanly. Same shape as the existing
|
||||
// dense-overlap regression test — TSR should produce clean output
|
||||
// and the auto wrapper should pass through with no fallback.
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"expected no fallback, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
assert!(results[0].markdown.contains("Boardwalk"));
|
||||
assert!(!results[0].markdown.contains("Oak Street Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_expands_multi_row_in_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// TSR returns only 2 rows for what's actually 3 visible PDF rows.
|
||||
// Row 1's cells are tall enough to encompass both Oak Street and
|
||||
// Boardwalk text — the FNBO row-undercount pattern.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row at top-left y=[88, 105] (covers "Branch Name"/"Deposits"
|
||||
// at native y=700, top-left y≈92-103). The "data" row at top-left
|
||||
// y=[105, 145] is intentionally tall — covers BOTH the Oak Street
|
||||
// line (top-left y≈108-119) AND the Boardwalk line (y≈124-135).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(
|
||||
results[0].fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell_expanded"),
|
||||
"expected multi_row_in_cell_expanded, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
// The in-place expansion should preserve all three PDF rows.
|
||||
let md = &results[0].markdown;
|
||||
assert!(md.contains("Oak Street"), "missing Oak Street: {md}");
|
||||
assert!(md.contains("Boardwalk"), "missing Boardwalk: {md}");
|
||||
assert!(md.contains("100"), "missing 100: {md}");
|
||||
assert!(md.contains("200"), "missing 200: {md}");
|
||||
assert!(
|
||||
!md.contains("Oak Street Boardwalk"),
|
||||
"rows should not remain compressed: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_expands_under_counted_vector_grid_rows() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_vector_grid_three_row_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
let crop = [50.0, 60.0, 210.0, 150.0];
|
||||
let cell_bboxes = vec![
|
||||
poly(0.0, 0.0, 80.0, 30.0),
|
||||
poly(80.0, 0.0, 160.0, 30.0),
|
||||
poly(0.0, 30.0, 80.0, 90.0),
|
||||
poly(80.0, 30.0, 160.0, 90.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(
|
||||
results[0].fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell_expanded")
|
||||
);
|
||||
let md = &results[0].markdown;
|
||||
assert!(md.contains("|Branch|Deposits|"), "missing header: {md}");
|
||||
assert!(md.contains("|Oak|100|"), "missing row 1: {md}");
|
||||
assert!(md.contains("|Boardwalk|200|"), "missing row 2: {md}");
|
||||
assert!(
|
||||
!md.contains("Oak Boardwalk"),
|
||||
"rows stayed compressed: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_returns_empty_inputs() {
|
||||
use pdf_inspector::extract_tables_with_structure_auto_mem;
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[]).unwrap();
|
||||
assert!(results.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_does_not_fire_on_legit_rowspan_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// 2 columns, 3 rows in the visible PDF. SLANet emits a 2-row table
|
||||
// where the LEFT cell of row 1 is a rowspan=2 cell that legitimately
|
||||
// covers Oak Street + Boardwalk on two visual lines. The right
|
||||
// column has two normal rows. multi_row_in_cell must NOT fire on
|
||||
// the rowspan=2 cell.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
// First data cell explicitly declares rowspan=2.
|
||||
"<td",
|
||||
" rowspan=\"2\"",
|
||||
">",
|
||||
"</td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row, then a tall left cell covering both data lines, plus
|
||||
// two narrow right cells (one per line).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0), // rowspan=2 — covers both lines
|
||||
poly(90.0, 105.0, 180.0, 122.0), // row 1 only
|
||||
poly(90.0, 122.0, 180.0, 145.0), // row 2 only
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"rowspan=2 cell containing 2 visual lines should not trip multi_row_in_cell, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_expands_when_heuristic_region_is_empty() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// Same shape as the multi_row_in_cell regression — a tall data cell
|
||||
// that catches Oak Street + Boardwalk. The crop bbox we pass points
|
||||
// at a strip of the page that has NO text items, so the old heuristic
|
||||
// fallback would be empty. Expansion uses the cell bboxes directly.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cell bboxes overlap the actual PDF text (so multi_row_in_cell
|
||||
// fires) — but the crop_pdf_pt_bbox we hand to the heuristic is a
|
||||
// wholly-empty region of the page. The heuristic should return "".
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
// Crop is at the BOTTOM of the page where there's no text.
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 50.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
let r = &results[0];
|
||||
assert_eq!(
|
||||
r.fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell_expanded"),
|
||||
"expected expansion despite empty heuristic region, got {:?}",
|
||||
r.fallback_reason,
|
||||
);
|
||||
assert!(
|
||||
r.markdown.contains("|Oak Street|100|"),
|
||||
"missing row 1: {}",
|
||||
r.markdown
|
||||
);
|
||||
assert!(
|
||||
r.markdown.contains("|Boardwalk|200|"),
|
||||
"missing row 2: {}",
|
||||
r.markdown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_isolates_per_input_failures() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let good_tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// A clean input that should pass through with no fallback.
|
||||
let good_input = TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: good_tokens,
|
||||
cell_bboxes: vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
],
|
||||
};
|
||||
// A bad input that targets a non-existent page. The detection
|
||||
// helper short-circuits on missing pages with Ok(None), so this
|
||||
// shouldn't itself crash, but pairing it with a flagged input
|
||||
// exercises the per-input control flow regardless. The point of
|
||||
// this test is that one input's outcome doesn't poison the other.
|
||||
let bad_input = TsrTableInput {
|
||||
page: 9999,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: vec![
|
||||
"<table>".into(),
|
||||
"<tr>".into(),
|
||||
"<td></td>".into(),
|
||||
"</tr>".into(),
|
||||
"</table>".into(),
|
||||
],
|
||||
cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
|
||||
};
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[good_input, bad_input]).unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
// Good input still produces non-empty TSR markdown with no fallback.
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"good input should pass through, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
// Bad input collapses to empty markdown but doesn't take the
|
||||
// batch down with it.
|
||||
assert_eq!(results[1].markdown, "");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_pages_markdown_mem tests
|
||||
// =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user