Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af60e378af | ||
|
|
f61d139710 | ||
|
|
3f8fb645c9 | ||
|
|
f6d5e214f1 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.5.0",
|
||||
"version": "1.6.3",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+137
-3
@@ -317,6 +317,141 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// One cropped table region plus its raw structure-recovery output, for
|
||||
/// `extractTablesWithStructure`.
|
||||
///
|
||||
/// `structureTokens` and `cellBboxes` are typically produced by an external
|
||||
/// table-structure recognition model (e.g. SLANet on PaddleOCR) running on
|
||||
/// a rendered crop of the page. pdf-inspector uses the structure to lay out
|
||||
/// the cells and pulls the cell text from the native PDF — no OCR involved.
|
||||
#[napi(object)]
|
||||
pub struct TsrTableInputJs {
|
||||
/// 0-indexed page number where the crop was taken from.
|
||||
pub page: u32,
|
||||
/// Crop bbox on the page, `[x1, y1, x2, y2]` in PDF points with
|
||||
/// top-left origin.
|
||||
pub crop_pdf_pt_bbox: Vec<f64>,
|
||||
/// DPI the crop image was rendered at (e.g. `200.0`).
|
||||
pub render_dpi: f64,
|
||||
/// Raw structure tokens emitted by the TSR model, in document order.
|
||||
pub structure_tokens: Vec<String>,
|
||||
/// One bbox per cell (in document order). May be 4-element
|
||||
/// `[x1,y1,x2,y2]` or 8-element 4-corner polygon, in crop image-pixel
|
||||
/// space.
|
||||
pub cell_bboxes: Vec<Vec<f64>>,
|
||||
}
|
||||
|
||||
/// Extract markdown tables using externally-supplied structure recovery.
|
||||
///
|
||||
/// For each input, pairs structure tokens with cell bboxes (rowspan/colspan
|
||||
/// aware), converts each cell bbox from crop image-pixels into page PDF
|
||||
/// points, pulls the cell's text from the native PDF, and emits a markdown
|
||||
/// pipe-table.
|
||||
///
|
||||
/// Returns one markdown string per input, in input order.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure(
|
||||
buffer: Buffer,
|
||||
inputs: Vec<TsrTableInputJs>,
|
||||
) -> Result<Vec<String>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let parsed = parse_tsr_inputs(&inputs);
|
||||
|
||||
catch_panic("extract_tables_with_structure", move || {
|
||||
pdf_inspector::extract_tables_with_structure_mem(&bytes, &parsed)
|
||||
.map_err(|e| to_napi_err(e, "extract_tables_with_structure"))
|
||||
})
|
||||
}
|
||||
|
||||
/// One resolved cell from `extractTablesWithStructureCells`.
|
||||
#[napi(object)]
|
||||
pub struct StructuredCellJs {
|
||||
/// 0-indexed grid row.
|
||||
pub row: u32,
|
||||
/// 0-indexed grid column.
|
||||
pub col: u32,
|
||||
/// 1 for a normal cell.
|
||||
pub rowspan: u32,
|
||||
/// 1 for a normal cell.
|
||||
pub colspan: u32,
|
||||
/// `true` when the cell is a `<th>` or sits inside `<thead>`.
|
||||
pub is_header: bool,
|
||||
/// Text extracted from the native PDF for this cell (may be empty).
|
||||
pub text: String,
|
||||
/// Axis-aligned bbox `[x1, y1, x2, y2]` in page PDF-points, top-left
|
||||
/// origin. Useful for debug overlays or per-cell post-processing.
|
||||
pub page_pt_bbox: Vec<f64>,
|
||||
}
|
||||
|
||||
/// Extract structured cells using externally-supplied structure recovery.
|
||||
///
|
||||
/// Lower-level sibling of [`extractTablesWithStructure`]: instead of
|
||||
/// rendering markdown, returns the resolved cells (row, col, rowspan,
|
||||
/// colspan, isHeader, text, pagePtBbox) so callers can drive their own
|
||||
/// rendering, debug overlays, or per-cell post-processing.
|
||||
///
|
||||
/// Returns one `Array<StructuredCellJs>` per input, in input order.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure_cells(
|
||||
buffer: Buffer,
|
||||
inputs: Vec<TsrTableInputJs>,
|
||||
) -> Result<Vec<Vec<StructuredCellJs>>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let parsed = parse_tsr_inputs(&inputs);
|
||||
|
||||
catch_panic("extract_tables_with_structure_cells", move || {
|
||||
let result = pdf_inspector::extract_tables_with_structure_cells_mem(&bytes, &parsed)
|
||||
.map_err(|e| to_napi_err(e, "extract_tables_with_structure_cells"))?;
|
||||
Ok(result
|
||||
.into_iter()
|
||||
.map(|cells| {
|
||||
cells
|
||||
.into_iter()
|
||||
.map(|c| StructuredCellJs {
|
||||
row: c.row as u32,
|
||||
col: c.col as u32,
|
||||
rowspan: c.rowspan as u32,
|
||||
colspan: c.colspan as u32,
|
||||
is_header: c.is_header,
|
||||
text: c.text,
|
||||
page_pt_bbox: c.page_pt_bbox.iter().map(|v| *v as f64).collect(),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_tsr_inputs(inputs: &[TsrTableInputJs]) -> Vec<pdf_inspector::TsrTableInput> {
|
||||
inputs
|
||||
.iter()
|
||||
.map(|i| {
|
||||
let crop = if i.crop_pdf_pt_bbox.len() == 4 {
|
||||
[
|
||||
i.crop_pdf_pt_bbox[0] as f32,
|
||||
i.crop_pdf_pt_bbox[1] as f32,
|
||||
i.crop_pdf_pt_bbox[2] as f32,
|
||||
i.crop_pdf_pt_bbox[3] as f32,
|
||||
]
|
||||
} else {
|
||||
[0.0, 0.0, 0.0, 0.0]
|
||||
};
|
||||
let cell_bboxes: Vec<Vec<f32>> = i
|
||||
.cell_bboxes
|
||||
.iter()
|
||||
.map(|bb| bb.iter().map(|v| *v as f32).collect())
|
||||
.collect();
|
||||
pdf_inspector::TsrTableInput {
|
||||
page: i.page,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: i.render_dpi as f32,
|
||||
structure_tokens: i.structure_tokens.clone(),
|
||||
cell_bboxes,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[napi(object)]
|
||||
pub struct PageMarkdownResult {
|
||||
@@ -360,9 +495,8 @@ pub fn extract_pages_markdown(
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result =
|
||||
pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
|
||||
+896
@@ -784,6 +784,358 @@ pub fn extract_tables_in_regions_mem(
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Region-based table extraction with external structure recovery (TSR)
|
||||
// =========================================================================
|
||||
|
||||
/// Input for [`extract_tables_with_structure_mem`]: one cropped table region
|
||||
/// plus the raw structure-recovery output for it.
|
||||
///
|
||||
/// The structure tokens and bboxes are typically produced by an external
|
||||
/// table-structure recognition model (e.g. SLANet on PaddleOCR) running on
|
||||
/// a rendered crop of the page. pdf-inspector uses the structure to lay out
|
||||
/// the cells and pulls the cell text from the native PDF — no OCR involved.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TsrTableInput {
|
||||
/// 0-indexed page number where the crop was taken from.
|
||||
pub page: u32,
|
||||
/// Crop bbox on the page, `[x1, y1, x2, y2]` in PDF points with
|
||||
/// **top-left origin** (matches the layout model's coordinate space).
|
||||
pub crop_pdf_pt_bbox: [f32; 4],
|
||||
/// DPI the crop image was rendered at (e.g. `200.0`). Used to convert
|
||||
/// cell bboxes from image-pixels back to PDF points.
|
||||
pub render_dpi: f32,
|
||||
/// Raw structure tokens emitted by the TSR model, in document order.
|
||||
/// See [`tables::structured::parse_structure`] for the accepted grammar.
|
||||
pub structure_tokens: Vec<String>,
|
||||
/// One bbox per cell (in document order, parallel to the cell open-tags
|
||||
/// in `structure_tokens`). May be 4-element `[x1,y1,x2,y2]` or
|
||||
/// 8-element 4-corner polygon, in **crop image-pixel space**.
|
||||
pub cell_bboxes: Vec<Vec<f32>>,
|
||||
}
|
||||
|
||||
/// Extract structured cells using externally-supplied structure recovery.
|
||||
///
|
||||
/// For each input, this:
|
||||
/// 1. Pairs each cell open-tag in `structure_tokens` with the next bbox in
|
||||
/// `cell_bboxes` (document order), tracking row/col with rowspan/colspan
|
||||
/// awareness.
|
||||
/// 2. Converts each cell bbox from crop image-pixels into page PDF-points.
|
||||
/// 3. Pulls the cell's text by overlap-testing PDF text items inside that
|
||||
/// bbox — same primitives used by [`extract_text_in_regions_mem`].
|
||||
///
|
||||
/// Returns one `Vec<StructuredCell>` per input, in input order. Each cell
|
||||
/// carries its (row, col, rowspan, colspan, is_header) metadata, the
|
||||
/// extracted text, and its page-PDF-pt bbox so callers can do their own
|
||||
/// rendering, debug overlays, or per-cell post-processing.
|
||||
///
|
||||
/// Inputs whose page is out of range or whose tokens parse to zero cells
|
||||
/// produce an empty `Vec`.
|
||||
///
|
||||
/// See [`extract_tables_with_structure_mem`] if you just want the rendered
|
||||
/// markdown.
|
||||
pub fn extract_tables_with_structure_cells_mem(
|
||||
buffer: &[u8],
|
||||
inputs: &[TsrTableInput],
|
||||
) -> Result<Vec<Vec<tables::StructuredCell>>, PdfError> {
|
||||
use tables::structured::{
|
||||
cell_px_to_page_pt, normalize_cell_bands, parse_structure, polygon_to_aabb, StructuredCell,
|
||||
};
|
||||
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let pages = doc.get_pages();
|
||||
|
||||
let needed_pages: HashSet<u32> = inputs.iter().map(|t| t.page + 1).collect();
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
let ((mut items, _rects, _lines), _has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
if coords_rotated {
|
||||
rotated_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
}
|
||||
|
||||
let mut results: Vec<Vec<StructuredCell>> = Vec::with_capacity(inputs.len());
|
||||
|
||||
for input in inputs {
|
||||
let page_1idx = input.page + 1;
|
||||
let Some(items) = items_by_page.get(&page_1idx) else {
|
||||
// Out-of-range page or page with no extractable text — emit empty.
|
||||
results.push(Vec::new());
|
||||
continue;
|
||||
};
|
||||
let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
|
||||
let adaptive_threshold = page_thresholds.get(&page_1idx).copied().unwrap_or(0.10);
|
||||
let coords = if rotated_pages.contains(&page_1idx) {
|
||||
RegionCoordSpace::Rotated90Ccw
|
||||
} else {
|
||||
RegionCoordSpace::Standard
|
||||
};
|
||||
|
||||
let crop_origin = [input.crop_pdf_pt_bbox[0], input.crop_pdf_pt_bbox[1]];
|
||||
|
||||
let slots = parse_structure(&input.structure_tokens);
|
||||
if slots.is_empty() {
|
||||
results.push(Vec::new());
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut cells: Vec<StructuredCell> = Vec::with_capacity(slots.len());
|
||||
for slot in &slots {
|
||||
let page_pt_bbox;
|
||||
|
||||
if let Some(coords_arr) = input.cell_bboxes.get(slot.bbox_idx) {
|
||||
if let Some(aabb_px) = polygon_to_aabb(coords_arr) {
|
||||
page_pt_bbox = cell_px_to_page_pt(aabb_px, input.render_dpi, crop_origin);
|
||||
} else {
|
||||
page_pt_bbox = [0.0, 0.0, 0.0, 0.0];
|
||||
}
|
||||
} else {
|
||||
page_pt_bbox = [0.0, 0.0, 0.0, 0.0];
|
||||
}
|
||||
|
||||
cells.push(StructuredCell {
|
||||
row: slot.row,
|
||||
col: slot.col,
|
||||
rowspan: slot.rowspan,
|
||||
colspan: slot.colspan,
|
||||
is_header: slot.is_header,
|
||||
text: String::new(),
|
||||
page_pt_bbox,
|
||||
});
|
||||
}
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
// Stage 1: strict text fill — each cell gets the items whose centers
|
||||
// fall inside its (normalized) bbox or whose >=60% overlap rule fires.
|
||||
// Track which item indices any cell claimed so the orphan pass below
|
||||
// doesn't double-assign.
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
for cell in &mut cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, coords);
|
||||
let mut matched: Vec<TextItem> = Vec::new();
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if tsr_region_contains_item(item, bounds) {
|
||||
claimed.insert(i);
|
||||
matched.push(item.clone());
|
||||
}
|
||||
}
|
||||
// Markdown cells must be one line — collapse line breaks produced
|
||||
// by the line-grouping pass.
|
||||
cell.text = collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
.replace(['\n', '\r'], " ");
|
||||
}
|
||||
|
||||
// Stage 2: orphan assignment — text items that didn't land in any
|
||||
// cell during stage 1 get assigned to their nearest *empty* cell,
|
||||
// clamped by a plausibility cap derived from cell geometry.
|
||||
//
|
||||
// This recovers two failure modes left by `normalize_cell_bands`:
|
||||
// (a) header text positioned to the LEFT of a column whose band
|
||||
// was derived from data cells centered farther right, so the
|
||||
// header text falls outside the clamped band; and
|
||||
// (b) local SLANet row drift where a cell's bbox sits slightly
|
||||
// above/below its target text item, so the strict rules miss.
|
||||
// Empty-cell-only is the safety net: a cell already filled by stage 1
|
||||
// is never overwritten or augmented, so the cell-bleed case PR #62
|
||||
// closed cannot regress.
|
||||
tsr_assign_orphan_items(items, &mut cells, &claimed, page_h, coords);
|
||||
|
||||
results.push(cells);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Compute plausibility caps for the orphan-assignment pass. Returns
|
||||
/// `(cap_x, cap_y)` — the maximum x/y distance from a text item's center
|
||||
/// to a candidate empty cell's bbox before the candidate is rejected.
|
||||
///
|
||||
/// Caps are derived from cell geometry so they scale with the table:
|
||||
/// dense small-row tables get a tight cap, looser tables get more slack.
|
||||
/// Floor values guard against degenerate single-cell tables collapsing
|
||||
/// the cap to zero.
|
||||
fn tsr_assignment_caps(cells: &[tables::StructuredCell]) -> (f32, f32) {
|
||||
let mut widths: Vec<f32> = Vec::with_capacity(cells.len());
|
||||
let mut heights: Vec<f32> = Vec::with_capacity(cells.len());
|
||||
for cell in cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
let w = (x2 - x1).abs();
|
||||
let h = (y2 - y1).abs();
|
||||
if w > 0.0 && h > 0.0 {
|
||||
widths.push(w);
|
||||
heights.push(h);
|
||||
}
|
||||
}
|
||||
if widths.is_empty() {
|
||||
return (0.0, 0.0);
|
||||
}
|
||||
widths.sort_by(|a, b| a.total_cmp(b));
|
||||
heights.sort_by(|a, b| a.total_cmp(b));
|
||||
let median_w = widths[widths.len() / 2];
|
||||
let median_h = heights[heights.len() / 2];
|
||||
// Floor values: even on a dense table, a 5pt floor handles small
|
||||
// pixel-level bbox jitter without being so loose that we'd cross
|
||||
// into a neighboring row/column. Symmetric in both axes.
|
||||
let cap_x = median_w.max(5.0);
|
||||
let cap_y = median_h.max(5.0);
|
||||
(cap_x, cap_y)
|
||||
}
|
||||
|
||||
/// For each text item that wasn't claimed by any cell during stage 1,
|
||||
/// find the nearest *empty* cell within `(cap_x, cap_y)` of the item's
|
||||
/// center and append the item's text to that cell. Cells that already
|
||||
/// have content are skipped — stage 2 only fills, never augments.
|
||||
///
|
||||
/// Distance is point-to-rect: 0 if the item center is inside the cell's
|
||||
/// bbox, else the axis-aligned gap to the nearest edge. Both x-gap and
|
||||
/// y-gap must be within their respective caps for a candidate to qualify;
|
||||
/// among qualifying candidates, the smallest combined euclidean distance
|
||||
/// wins.
|
||||
fn tsr_assign_orphan_items(
|
||||
items: &[TextItem],
|
||||
cells: &mut [tables::StructuredCell],
|
||||
claimed: &std::collections::HashSet<usize>,
|
||||
page_height: f32,
|
||||
coord_space: RegionCoordSpace,
|
||||
) {
|
||||
if cells.is_empty() {
|
||||
return;
|
||||
}
|
||||
let (cap_x, cap_y) = tsr_assignment_caps(cells);
|
||||
if cap_x <= 0.0 || cap_y <= 0.0 {
|
||||
return;
|
||||
}
|
||||
// Y-tolerance for "same line as a previous orphan" — multi-token branch
|
||||
// names like "Blue Valley Parkway" are 3 separate text items and should
|
||||
// all stack into the same cell. But two orphans on different rows of
|
||||
// the PDF (different y values) targeting the same empty cell should
|
||||
// NOT merge — that produces the "Mitchell Woonsocket" / "Shawnee Blue
|
||||
// Valley Parkway" run-on cells. Half a row of slack is conservative.
|
||||
let y_tolerance = (cap_y * 0.5).max(3.0);
|
||||
|
||||
// Pre-compute each empty cell's region bounds so we don't re-flip
|
||||
// page coordinates per orphan-candidate pair.
|
||||
let cell_bounds: Vec<Option<RegionBounds>> = cells
|
||||
.iter()
|
||||
.map(|cell| {
|
||||
if !cell.text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
if x1 >= x2 || y1 >= y2 {
|
||||
return None;
|
||||
}
|
||||
Some(region_bounds(x1, y1, x2, y2, page_height, coord_space))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Track the y-center of the FIRST orphan that landed in each cell so
|
||||
// subsequent orphans only stack if they're on the same line.
|
||||
let mut stage2_first_y: std::collections::HashMap<usize, f32> =
|
||||
std::collections::HashMap::new();
|
||||
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if claimed.contains(&i) {
|
||||
continue;
|
||||
}
|
||||
let item_w = text_utils::effective_width(item);
|
||||
if item.text.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let cx = item.x + item_w * 0.5;
|
||||
let cy = item.y + item.height * 0.5;
|
||||
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
for (ci, bounds_opt) in cell_bounds.iter().enumerate() {
|
||||
let Some(bounds) = bounds_opt else {
|
||||
continue;
|
||||
};
|
||||
// If a previous orphan already landed in this cell, only let a
|
||||
// new orphan join if it's on the same line. Cross-line orphans
|
||||
// need to look elsewhere (next-nearest empty cell).
|
||||
if let Some(&first_y) = stage2_first_y.get(&ci) {
|
||||
if (first_y - cy).abs() > y_tolerance {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let dx = (bounds.x_min - cx).max(0.0).max(cx - bounds.x_max);
|
||||
let dy = (bounds.y_min - cy).max(0.0).max(cy - bounds.y_max);
|
||||
if dx > cap_x || dy > cap_y {
|
||||
continue;
|
||||
}
|
||||
let dist_sq = dx * dx + dy * dy;
|
||||
if best.is_none_or(|(_, d)| dist_sq < d) {
|
||||
best = Some((ci, dist_sq));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((ci, _)) = best {
|
||||
// Append, preserving stage 1's content. Same-line orphans
|
||||
// stack to support multi-token text (e.g. "Blue Valley
|
||||
// Parkway"); cross-line orphans are filtered out above.
|
||||
let trimmed = item.text.trim();
|
||||
if cells[ci].text.is_empty() {
|
||||
cells[ci].text = trimmed.to_string();
|
||||
} else {
|
||||
cells[ci].text.push(' ');
|
||||
cells[ci].text.push_str(trimmed);
|
||||
}
|
||||
stage2_first_y.entry(ci).or_insert(cy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract markdown tables using externally-supplied structure recovery.
|
||||
///
|
||||
/// Convenience wrapper around [`extract_tables_with_structure_cells_mem`]
|
||||
/// that renders each cell list to markdown via
|
||||
/// [`tables::cells_to_markdown`]. Returns one markdown string per input,
|
||||
/// in input order. Inputs whose page is out of range or whose tokens parse
|
||||
/// to zero cells produce an empty string.
|
||||
pub fn extract_tables_with_structure_mem(
|
||||
buffer: &[u8],
|
||||
inputs: &[TsrTableInput],
|
||||
) -> Result<Vec<String>, PdfError> {
|
||||
let cells_lists = extract_tables_with_structure_cells_mem(buffer, inputs)?;
|
||||
Ok(cells_lists
|
||||
.into_iter()
|
||||
.map(|cells| {
|
||||
if cells.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
tables::cells_to_markdown(&cells)
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Get page height in points from MediaBox.
|
||||
fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option<f32> {
|
||||
let page_dict = doc.get_dictionary(page_id).ok()?;
|
||||
@@ -870,6 +1222,31 @@ fn collect_text_in_region_with_options(
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect();
|
||||
collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn collect_text_in_tsr_cell(
|
||||
items: &[TextItem],
|
||||
rx1: f32,
|
||||
ry1: f32,
|
||||
rx2: f32,
|
||||
ry2: f32,
|
||||
page_height: f32,
|
||||
coord_space: RegionCoordSpace,
|
||||
adaptive_threshold: f32,
|
||||
) -> String {
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_height, coord_space);
|
||||
let matched: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|item| tsr_region_contains_item(item, bounds))
|
||||
.cloned()
|
||||
.collect();
|
||||
collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
}
|
||||
|
||||
fn collect_text_from_matched_items(matched: Vec<TextItem>, adaptive_threshold: f32) -> String {
|
||||
if matched.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
@@ -971,6 +1348,30 @@ fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
x_overlap > 0.0 && y_overlap > 0.0
|
||||
}
|
||||
|
||||
fn tsr_region_contains_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
let item_x_min = item.x;
|
||||
let item_x_max = item.x + text_utils::effective_width(item);
|
||||
let item_y_min = item.y;
|
||||
let item_y_max = item.y + item.height;
|
||||
|
||||
let center_x = (item_x_min + item_x_max) * 0.5;
|
||||
let center_y = (item_y_min + item_y_max) * 0.5;
|
||||
if center_x >= bounds.x_min
|
||||
&& center_x <= bounds.x_max
|
||||
&& center_y >= bounds.y_min
|
||||
&& center_y <= bounds.y_max
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
let x_overlap = (item_x_max.min(bounds.x_max) - item_x_min.max(bounds.x_min)).max(0.0);
|
||||
let y_overlap = (item_y_max.min(bounds.y_max) - item_y_min.max(bounds.y_min)).max(0.0);
|
||||
let item_width = (item_x_max - item_x_min).max(0.1);
|
||||
let item_height = (item_y_max - item_y_min).max(0.1);
|
||||
|
||||
x_overlap / item_width >= 0.6 && y_overlap / item_height >= 0.6
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Internal: single-load document pipeline
|
||||
// =========================================================================
|
||||
@@ -2025,6 +2426,24 @@ pub(crate) fn validate_pdf_file<P: AsRef<Path>>(path: P) -> Result<(), PdfError>
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::ItemType;
|
||||
|
||||
fn test_item(text: &str, x: f32, y: f32, width: f32, height: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.to_string(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: "Helvetica".to_string(),
|
||||
font_size: height,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_encoding_issues_fffd() {
|
||||
@@ -2105,4 +2524,481 @@ mod tests {
|
||||
"Valid Japanese text should not be flagged as garbage"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsr_text_fill_does_not_pull_neighboring_overlapping_rows() {
|
||||
use crate::tables::structured::normalize_cell_bands;
|
||||
use crate::tables::StructuredCell;
|
||||
|
||||
let items = vec![
|
||||
test_item("Branch Name", 12.0, 88.0, 55.0, 8.0),
|
||||
test_item("Deposits", 112.0, 88.0, 36.0, 8.0),
|
||||
test_item("Oak Street", 12.0, 72.0, 48.0, 8.0),
|
||||
test_item("100", 112.0, 72.0, 18.0, 8.0),
|
||||
test_item("Boardwalk", 12.0, 55.2, 46.0, 8.0),
|
||||
test_item("200", 112.0, 55.2, 18.0, 8.0),
|
||||
];
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 100.0, 100.0, 125.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [100.0, 100.0, 170.0, 125.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 116.0, 100.0, 141.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [100.0, 116.0, 170.0, 141.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 2,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 132.8, 100.0, 157.8],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 2,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [100.0, 132.8, 170.0, 157.8],
|
||||
},
|
||||
];
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
for cell in &mut cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
cell.text = collect_text_in_tsr_cell(
|
||||
&items,
|
||||
x1,
|
||||
y1,
|
||||
x2,
|
||||
y2,
|
||||
200.0,
|
||||
RegionCoordSpace::Standard,
|
||||
0.10,
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(cells[0].text, "Branch Name");
|
||||
assert_eq!(cells[2].text, "Oak Street");
|
||||
assert_eq!(cells[4].text, "Boardwalk");
|
||||
assert!(!cells[0].text.contains("Oak Street"));
|
||||
assert!(!cells[2].text.contains("Branch Name"));
|
||||
assert!(!cells[2].text.contains("Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsr_assignment_caps_uses_median_geometry() {
|
||||
use crate::tables::StructuredCell;
|
||||
let cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [0.0, 0.0, 100.0, 20.0], // 100x20
|
||||
},
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [100.0, 0.0, 200.0, 20.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [0.0, 20.0, 100.0, 40.0],
|
||||
},
|
||||
];
|
||||
let (cap_x, cap_y) = tsr_assignment_caps(&cells);
|
||||
assert_eq!(cap_x, 100.0);
|
||||
assert_eq!(cap_y, 20.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsr_assignment_caps_floor_protects_degenerate_input() {
|
||||
use crate::tables::StructuredCell;
|
||||
let cells = vec![StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [0.0, 0.0, 1.0, 1.0],
|
||||
}];
|
||||
let (cap_x, cap_y) = tsr_assignment_caps(&cells);
|
||||
assert_eq!(cap_x, 5.0);
|
||||
assert_eq!(cap_y, 5.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage2_recovers_left_aligned_header_text_outside_data_band() {
|
||||
// Symptom A reproduction: the column band derived from data-cell
|
||||
// centers ends up too far right, so header text positioned at the
|
||||
// left of the column falls outside the band and stage 1's strict
|
||||
// membership rejects it. Stage 2 should re-attach by proximity.
|
||||
//
|
||||
// Item coords are bottom-left native; cell page_pt_bbox is top-left.
|
||||
// page_height=200 so a top-left bbox y=[88, 100] flips to native y
|
||||
// bounds [100, 112]; an item at native y=104 (center 108) lands in.
|
||||
use crate::tables::StructuredCell;
|
||||
let items = vec![
|
||||
// Header text — centered in row 0 (native y=104, center 108) but
|
||||
// at the LEFT of the column (x=175, far left of the [410, 700]
|
||||
// data-derived band).
|
||||
test_item("Address", 175.0, 104.0, 50.0, 8.0),
|
||||
// Data row 1 — fits its cell.
|
||||
test_item("205 W Oak St", 420.0, 84.0, 100.0, 8.0),
|
||||
// Data row 2 — fits its cell.
|
||||
test_item("155 E Boardwalk Dr", 420.0, 64.0, 100.0, 8.0),
|
||||
];
|
||||
// Cells AFTER normalize_cell_bands would have run — col 0 band
|
||||
// shifted right by data-cell centers, header cell now excludes
|
||||
// the "Address" text at center x=200.
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [410.0, 88.0, 700.0, 100.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [410.0, 108.0, 700.0, 116.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 2,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [410.0, 128.0, 700.0, 136.0],
|
||||
},
|
||||
];
|
||||
let page_h = 200.0;
|
||||
|
||||
// Stage 1 mimic — fill cells via the strict rule, track claimed.
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
for cell in &mut cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, RegionCoordSpace::Standard);
|
||||
let mut matched: Vec<TextItem> = Vec::new();
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if tsr_region_contains_item(item, bounds) {
|
||||
claimed.insert(i);
|
||||
matched.push(item.clone());
|
||||
}
|
||||
}
|
||||
cell.text = collect_text_from_matched_items(matched, 0.10).replace(['\n', '\r'], " ");
|
||||
}
|
||||
// Header is empty after stage 1 (Address fell outside col 0 band).
|
||||
assert_eq!(cells[0].text, "", "header should be empty after stage 1");
|
||||
// Data rows already populated.
|
||||
assert!(
|
||||
cells[1].text.contains("Oak"),
|
||||
"data row 1 should contain Oak: got {:?}",
|
||||
cells[1].text
|
||||
);
|
||||
assert!(
|
||||
cells[2].text.contains("Boardwalk"),
|
||||
"data row 2 should contain Boardwalk: got {:?}",
|
||||
cells[2].text
|
||||
);
|
||||
|
||||
// Stage 2 should fill the orphan "Address" into the empty header.
|
||||
tsr_assign_orphan_items(
|
||||
&items,
|
||||
&mut cells,
|
||||
&claimed,
|
||||
page_h,
|
||||
RegionCoordSpace::Standard,
|
||||
);
|
||||
assert_eq!(cells[0].text, "Address");
|
||||
// Data rows must NOT have been augmented (already filled by stage 1).
|
||||
assert!(!cells[1].text.contains("Address"));
|
||||
assert!(!cells[2].text.contains("Address"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage2_recovers_y_shifted_col0_in_consecutive_rows() {
|
||||
// Symptom B reproduction: a stretch of rows where col 0 cell bboxes
|
||||
// sit just above the actual branch-name text. After stage 1 those
|
||||
// cells are empty; stage 2 should pull the orphan items in by
|
||||
// y-proximity.
|
||||
//
|
||||
// page_height=800. Cells are 14pt tall in top-left; flipped native
|
||||
// bounds are [240,254], [220,234], [200,214]. Items sit ~1pt below
|
||||
// each cell's native y range (still within ~1pt of the edge), so
|
||||
// both center-containment and 60% overlap fail in stage 1.
|
||||
use crate::tables::StructuredCell;
|
||||
let items = vec![
|
||||
// Bellevue: native y=235, center 239 — just below row 0's
|
||||
// cell native bottom (240). Closer to row 0 than row 1.
|
||||
test_item("Bellevue", 30.0, 235.0, 45.0, 8.0),
|
||||
// Glenwood: native y=215, center 219 — just below row 1.
|
||||
test_item("Glenwood", 30.0, 215.0, 45.0, 8.0),
|
||||
// Metro Crossing: native y=195, center 199 — just below row 2.
|
||||
test_item("Metro Crossing", 30.0, 195.0, 70.0, 8.0),
|
||||
];
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 546.0, 200.0, 560.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 566.0, 200.0, 580.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 2,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 586.0, 200.0, 600.0],
|
||||
},
|
||||
];
|
||||
let page_h = 800.0;
|
||||
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
for cell in &mut cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, RegionCoordSpace::Standard);
|
||||
let mut matched: Vec<TextItem> = Vec::new();
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if tsr_region_contains_item(item, bounds) {
|
||||
claimed.insert(i);
|
||||
matched.push(item.clone());
|
||||
}
|
||||
}
|
||||
cell.text = collect_text_from_matched_items(matched, 0.10).replace(['\n', '\r'], " ");
|
||||
}
|
||||
// All three cells empty after stage 1 (text falls just below each).
|
||||
for c in &cells {
|
||||
assert!(
|
||||
c.text.is_empty(),
|
||||
"stage 1 should leave all cells empty: {:?}",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
tsr_assign_orphan_items(
|
||||
&items,
|
||||
&mut cells,
|
||||
&claimed,
|
||||
page_h,
|
||||
RegionCoordSpace::Standard,
|
||||
);
|
||||
assert_eq!(cells[0].text, "Bellevue");
|
||||
assert_eq!(cells[1].text, "Glenwood");
|
||||
assert_eq!(cells[2].text, "Metro Crossing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage2_rejects_cross_line_stacking_into_same_cell() {
|
||||
// Two orphans on different rows of the PDF, both equidistant from
|
||||
// the same empty cell. Without the same-line guard they'd both stack
|
||||
// into that cell ("Shawnee Blue Valley Parkway" run-on); the guard
|
||||
// keeps the first orphan and routes the second to the next-nearest
|
||||
// empty cell on its own line.
|
||||
use crate::tables::StructuredCell;
|
||||
// page_h=200. Two empty cells:
|
||||
// cell X (row 0): top-left y=[100, 110], native [90, 100]
|
||||
// cell Y (row 1): top-left y=[112, 122], native [78, 88]
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 100.0, 100.0, 110.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 112.0, 100.0, 122.0],
|
||||
},
|
||||
];
|
||||
// Two orphans, different rows of the PDF (y differs by 14pt = a
|
||||
// full row), both 2pt outside their target cell — both within
|
||||
// cap_y, both equidistant-ish to cell X. Without the same-line
|
||||
// guard they'd both land in X.
|
||||
// "Shawnee" should belong to cell X (row 0) — center y=98 is
|
||||
// 2pt below X's native min=100.
|
||||
// "BlueValley" should belong to cell Y (row 1) — center y=84
|
||||
// is 4pt above Y's native max=88.
|
||||
let items = vec![
|
||||
// Shawnee orphan — closer to X (dy=2) than Y (dy=6 from native min=78).
|
||||
test_item("Shawnee", 30.0, 94.0, 50.0, 8.0),
|
||||
// BlueValley orphan — closer to Y (dy=4) than X (dy=8 from native max=100).
|
||||
test_item("BlueValley", 30.0, 80.0, 60.0, 8.0),
|
||||
];
|
||||
let claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
|
||||
tsr_assign_orphan_items(
|
||||
&items,
|
||||
&mut cells,
|
||||
&claimed,
|
||||
200.0,
|
||||
RegionCoordSpace::Standard,
|
||||
);
|
||||
assert_eq!(cells[0].text, "Shawnee");
|
||||
assert_eq!(cells[1].text, "BlueValley");
|
||||
assert!(!cells[0].text.contains("BlueValley"));
|
||||
assert!(!cells[1].text.contains("Shawnee"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage2_allows_same_line_orphans_to_stack_into_one_cell() {
|
||||
// Multi-token branch names like "Blue Valley Parkway" are 3 PDF
|
||||
// text items at the SAME y-coordinate. They should all stack into
|
||||
// the cell their row's branch-name belongs to, not get split
|
||||
// across rows by the cross-line guard.
|
||||
use crate::tables::StructuredCell;
|
||||
let mut cells = vec![StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 100.0, 200.0, 110.0],
|
||||
}];
|
||||
// Three same-line items, all 2pt below the cell's native bottom.
|
||||
let items = vec![
|
||||
test_item("Blue", 30.0, 94.0, 25.0, 8.0),
|
||||
test_item("Valley", 60.0, 94.0, 35.0, 8.0),
|
||||
test_item("Parkway", 100.0, 94.0, 45.0, 8.0),
|
||||
];
|
||||
let claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
|
||||
tsr_assign_orphan_items(
|
||||
&items,
|
||||
&mut cells,
|
||||
&claimed,
|
||||
200.0,
|
||||
RegionCoordSpace::Standard,
|
||||
);
|
||||
// All three same-line orphans stacked into the single empty cell.
|
||||
assert_eq!(cells[0].text, "Blue Valley Parkway");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage2_does_not_overwrite_filled_cells_or_admit_far_orphans() {
|
||||
// Stage 2 must only fill EMPTY cells (preserves stage 1's strict
|
||||
// behavior on bleed cases) and must reject orphans that fall far
|
||||
// outside any cell (prevents pulling a figure title into a table).
|
||||
use crate::tables::StructuredCell;
|
||||
let items = vec![
|
||||
test_item("Real", 50.0, 100.0, 30.0, 8.0),
|
||||
// Far orphan — at native y=20 (page bottom edge) on a page where
|
||||
// the table sits around native y=92..104 (top-left y=96..108).
|
||||
// y-distance to nearest cell is ~70pt, far exceeding the ~12pt
|
||||
// cap from median row height.
|
||||
test_item("FigureTitle", 50.0, 20.0, 60.0, 8.0),
|
||||
];
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [40.0, 96.0, 100.0, 108.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "Pre-filled".to_string(),
|
||||
page_pt_bbox: [40.0, 116.0, 100.0, 128.0],
|
||||
},
|
||||
];
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
// Pretend "Real" got claimed by a different cell (won't be re-assigned).
|
||||
// Don't claim "FigureTitle" — it's the far orphan.
|
||||
claimed.insert(0);
|
||||
|
||||
tsr_assign_orphan_items(
|
||||
&items,
|
||||
&mut cells,
|
||||
&claimed,
|
||||
200.0,
|
||||
RegionCoordSpace::Standard,
|
||||
);
|
||||
// Empty cell stayed empty (orphan was too far).
|
||||
assert_eq!(cells[0].text, "");
|
||||
// Pre-filled cell was not touched.
|
||||
assert_eq!(cells[1].text, "Pre-filled");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ mod detect_struct;
|
||||
mod financial;
|
||||
mod format;
|
||||
mod grid;
|
||||
pub mod structured;
|
||||
|
||||
pub use detect_heuristic::detect_tables;
|
||||
pub(crate) use detect_heuristic::is_table_of_contents;
|
||||
@@ -17,6 +18,7 @@ pub(crate) use detect_rects::cluster_rects;
|
||||
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
||||
pub use detect_struct::detect_tables_from_struct_tree;
|
||||
pub use format::table_to_markdown;
|
||||
pub use structured::{cells_to_markdown, StructuredCell};
|
||||
|
||||
use crate::types::TextItem;
|
||||
|
||||
|
||||
@@ -0,0 +1,972 @@
|
||||
//! Structure-recovery-aware (TSR) table assembly.
|
||||
//!
|
||||
//! Consumes the raw output of an external table-structure recognition model
|
||||
//! (e.g. SLANet on PaddleOCR): a flat list of HTML structure tokens plus a
|
||||
//! parallel list of per-cell bboxes. Pairs each cell open-tag with its bbox
|
||||
//! in document order, tracks row/column position with rowspan/colspan
|
||||
//! awareness, and emits a markdown pipe-table.
|
||||
//!
|
||||
//! No real HTML parser is needed — the token grammar is restricted (see
|
||||
//! [`parse_structure`]), so a small state machine is enough.
|
||||
//!
|
||||
//! Cell text is supplied separately by the caller (typically by overlap-
|
||||
//! testing PDF text items against each cell's page-PDF-pt bbox).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
/// A single resolved cell, with both structural metadata and its bbox in
|
||||
/// page PDF-points (top-left origin).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StructuredCell {
|
||||
/// 0-indexed grid row.
|
||||
pub row: usize,
|
||||
/// 0-indexed grid column.
|
||||
pub col: usize,
|
||||
/// 1 for a normal cell.
|
||||
pub rowspan: usize,
|
||||
/// 1 for a normal cell.
|
||||
pub colspan: usize,
|
||||
/// `true` when the cell is a `<th>` or sits inside `<thead>`.
|
||||
pub is_header: bool,
|
||||
/// Cell text (filled in by the caller after overlap-testing PDF items).
|
||||
pub text: String,
|
||||
/// Axis-aligned bbox `[x1, y1, x2, y2]` in page PDF-points, top-left origin.
|
||||
pub page_pt_bbox: [f32; 4],
|
||||
}
|
||||
|
||||
/// Intermediate parse result before the caller fills in text + page coords.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct CellSlot {
|
||||
pub row: usize,
|
||||
pub col: usize,
|
||||
pub rowspan: usize,
|
||||
pub colspan: usize,
|
||||
pub is_header: bool,
|
||||
/// Index into the parallel `cell_bboxes` array.
|
||||
pub bbox_idx: usize,
|
||||
}
|
||||
|
||||
/// Parse a sequence of SLANet structure tokens into ordered cell slots.
|
||||
///
|
||||
/// Token grammar (no real HTML parsing required):
|
||||
/// - Section markers: `<thead>`, `</thead>`, `<tbody>`, `</tbody>` and
|
||||
/// wrapper tokens (`<html>`, `<body>`, `<table>`, plus closing variants)
|
||||
/// are tracked or skipped.
|
||||
/// - Row markers: `<tr>` opens a new row, `</tr>` is informational.
|
||||
/// - Empty cell, single token: `<td></td>` or `<th></th>`.
|
||||
/// - Cell with attributes, multi-token sequence: `<td` (or `<th`), then
|
||||
/// attribute fragments like ` colspan="4"`, then `>`, then later `</td>`
|
||||
/// (or `</th>`). Cells get paired with the next bbox in document order.
|
||||
///
|
||||
/// Cells inside `<thead>` and any `<th>` cells are flagged as headers.
|
||||
/// rowspan/colspan attributes are honoured and prior-row rowspans push
|
||||
/// later-row cells to the right.
|
||||
pub(crate) fn parse_structure(tokens: &[String]) -> Vec<CellSlot> {
|
||||
let mut slots: Vec<CellSlot> = Vec::new();
|
||||
let mut occupied: HashSet<(usize, usize)> = HashSet::new();
|
||||
let mut row: usize = 0;
|
||||
let mut col: usize = 0;
|
||||
let mut bbox_idx: usize = 0;
|
||||
let mut in_thead = false;
|
||||
let mut started_first_row = false;
|
||||
|
||||
let mut i = 0;
|
||||
while i < tokens.len() {
|
||||
let tok = tokens[i].trim();
|
||||
match tok {
|
||||
"<thead>" => {
|
||||
in_thead = true;
|
||||
}
|
||||
"</thead>" => {
|
||||
in_thead = false;
|
||||
}
|
||||
"<tr>" => {
|
||||
if started_first_row {
|
||||
row += 1;
|
||||
}
|
||||
col = 0;
|
||||
started_first_row = true;
|
||||
}
|
||||
"<td></td>" | "<th></th>" => {
|
||||
let is_th = tok == "<th></th>";
|
||||
while occupied.contains(&(row, col)) {
|
||||
col += 1;
|
||||
}
|
||||
slots.push(CellSlot {
|
||||
row,
|
||||
col,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: in_thead || is_th,
|
||||
bbox_idx,
|
||||
});
|
||||
bbox_idx += 1;
|
||||
col += 1;
|
||||
}
|
||||
"<td" | "<th" => {
|
||||
let is_th = tok == "<th";
|
||||
let mut rowspan: usize = 1;
|
||||
let mut colspan: usize = 1;
|
||||
// Consume attribute fragments until we hit ">".
|
||||
i += 1;
|
||||
while i < tokens.len() && tokens[i].trim() != ">" {
|
||||
let attr = tokens[i].as_str();
|
||||
if let Some(v) = parse_int_attr(attr, "rowspan") {
|
||||
rowspan = v.max(1);
|
||||
} else if let Some(v) = parse_int_attr(attr, "colspan") {
|
||||
colspan = v.max(1);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
// i now points at the `>` token (or off the end if malformed).
|
||||
while occupied.contains(&(row, col)) {
|
||||
col += 1;
|
||||
}
|
||||
slots.push(CellSlot {
|
||||
row,
|
||||
col,
|
||||
rowspan,
|
||||
colspan,
|
||||
is_header: in_thead || is_th,
|
||||
bbox_idx,
|
||||
});
|
||||
for r in row..row + rowspan {
|
||||
for c in col..col + colspan {
|
||||
occupied.insert((r, c));
|
||||
}
|
||||
}
|
||||
bbox_idx += 1;
|
||||
col += colspan;
|
||||
}
|
||||
// Wrapper / informational tokens — no-op.
|
||||
_ => {}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
slots
|
||||
}
|
||||
|
||||
/// Parse an attribute fragment like ` colspan="4"` or `rowspan='2'`.
|
||||
///
|
||||
/// Tolerates leading whitespace and either single or double quotes.
|
||||
fn parse_int_attr(s: &str, name: &str) -> Option<usize> {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.starts_with(name) {
|
||||
return None;
|
||||
}
|
||||
let rest = trimmed[name.len()..].trim_start();
|
||||
let rest = rest.strip_prefix('=')?.trim_start();
|
||||
let value = rest
|
||||
.trim_start_matches(['"', '\''])
|
||||
.trim_end_matches(['"', '\'']);
|
||||
value.parse().ok()
|
||||
}
|
||||
|
||||
/// Convert a SLANet polygon (4 or 8 elements) into an axis-aligned
|
||||
/// `[x1, y1, x2, y2]` rect.
|
||||
///
|
||||
/// 8-element form: `[x1,y1, x2,y1, x2,y2, x1,y2]` (4 corners). We ignore the
|
||||
/// implicit corner order and just take min/max so rotated polygons collapse
|
||||
/// to a sane bounding box.
|
||||
///
|
||||
/// 4-element form: `[x1, y1, x2, y2]` (axis-aligned, older SLANet variants).
|
||||
pub(crate) fn polygon_to_aabb(coords: &[f32]) -> Option<[f32; 4]> {
|
||||
match coords.len() {
|
||||
4 => {
|
||||
let x1 = coords[0].min(coords[2]);
|
||||
let y1 = coords[1].min(coords[3]);
|
||||
let x2 = coords[0].max(coords[2]);
|
||||
let y2 = coords[1].max(coords[3]);
|
||||
Some([x1, y1, x2, y2])
|
||||
}
|
||||
8 => {
|
||||
let xs = [coords[0], coords[2], coords[4], coords[6]];
|
||||
let ys = [coords[1], coords[3], coords[5], coords[7]];
|
||||
let x1 = xs.iter().copied().fold(f32::INFINITY, f32::min);
|
||||
let y1 = ys.iter().copied().fold(f32::INFINITY, f32::min);
|
||||
let x2 = xs.iter().copied().fold(f32::NEG_INFINITY, f32::max);
|
||||
let y2 = ys.iter().copied().fold(f32::NEG_INFINITY, f32::max);
|
||||
if x1.is_finite() && y1.is_finite() && x2.is_finite() && y2.is_finite() {
|
||||
Some([x1, y1, x2, y2])
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a cell rect from crop image-pixel space to page PDF-points
|
||||
/// (top-left origin), given the crop's PDF-point offset on the page and the
|
||||
/// DPI the crop image was rendered at.
|
||||
pub(crate) fn cell_px_to_page_pt(
|
||||
cell_px: [f32; 4],
|
||||
render_dpi: f32,
|
||||
crop_origin_pt: [f32; 2],
|
||||
) -> [f32; 4] {
|
||||
let pt_per_px = if render_dpi > 0.0 {
|
||||
72.0 / render_dpi
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
let [x_off, y_off] = crop_origin_pt;
|
||||
[
|
||||
cell_px[0] * pt_per_px + x_off,
|
||||
cell_px[1] * pt_per_px + y_off,
|
||||
cell_px[2] * pt_per_px + x_off,
|
||||
cell_px[3] * pt_per_px + y_off,
|
||||
]
|
||||
}
|
||||
|
||||
/// Refine TSR cell bboxes into non-overlapping row/column bands.
|
||||
///
|
||||
/// SLANet-style bboxes are often plausible but too tall on dense borderless
|
||||
/// tables. Native PDF text assignment is more reliable when each parsed row
|
||||
/// owns the band between neighboring row centers instead of the full model box.
|
||||
pub(crate) fn normalize_cell_bands(cells: &mut [StructuredCell]) {
|
||||
if cells.len() < 2 {
|
||||
return;
|
||||
}
|
||||
|
||||
let row_bands = derive_axis_bands(cells, Axis::Y);
|
||||
let col_bands = derive_axis_bands(cells, Axis::X);
|
||||
|
||||
for cell in cells {
|
||||
let row_end = cell.row + cell.rowspan.max(1).saturating_sub(1);
|
||||
if let (Some(&(y1, _)), Some(&(_, y2))) =
|
||||
(row_bands.get(&cell.row), row_bands.get(&row_end))
|
||||
{
|
||||
let clamped_y1 = cell.page_pt_bbox[1].max(y1);
|
||||
let clamped_y2 = cell.page_pt_bbox[3].min(y2);
|
||||
if clamped_y1 < clamped_y2 {
|
||||
cell.page_pt_bbox[1] = clamped_y1;
|
||||
cell.page_pt_bbox[3] = clamped_y2;
|
||||
}
|
||||
}
|
||||
|
||||
let col_end = cell.col + cell.colspan.max(1).saturating_sub(1);
|
||||
if let (Some(&(x1, _)), Some(&(_, x2))) =
|
||||
(col_bands.get(&cell.col), col_bands.get(&col_end))
|
||||
{
|
||||
let clamped_x1 = cell.page_pt_bbox[0].max(x1);
|
||||
let clamped_x2 = cell.page_pt_bbox[2].min(x2);
|
||||
if clamped_x1 < clamped_x2 {
|
||||
cell.page_pt_bbox[0] = clamped_x1;
|
||||
cell.page_pt_bbox[2] = clamped_x2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Axis {
|
||||
X,
|
||||
Y,
|
||||
}
|
||||
|
||||
fn derive_axis_bands(cells: &[StructuredCell], axis: Axis) -> HashMap<usize, (f32, f32)> {
|
||||
let mut by_index: HashMap<usize, Vec<(f32, f32)>> = HashMap::new();
|
||||
|
||||
// Prefer non-spanning cells so colspan/rowspan boxes do not skew a single
|
||||
// column/row center. If an axis has no non-spanning examples for an index,
|
||||
// fall back to anchored cells below.
|
||||
for cell in cells {
|
||||
let span = match axis {
|
||||
Axis::X => cell.colspan.max(1),
|
||||
Axis::Y => cell.rowspan.max(1),
|
||||
};
|
||||
if span == 1 {
|
||||
let idx = match axis {
|
||||
Axis::X => cell.col,
|
||||
Axis::Y => cell.row,
|
||||
};
|
||||
by_index
|
||||
.entry(idx)
|
||||
.or_default()
|
||||
.push(axis_bounds(cell.page_pt_bbox, axis));
|
||||
}
|
||||
}
|
||||
|
||||
for cell in cells {
|
||||
let idx = match axis {
|
||||
Axis::X => cell.col,
|
||||
Axis::Y => cell.row,
|
||||
};
|
||||
if !by_index.contains_key(&idx) {
|
||||
by_index
|
||||
.entry(idx)
|
||||
.or_default()
|
||||
.push(axis_bounds(cell.page_pt_bbox, axis));
|
||||
}
|
||||
}
|
||||
|
||||
let mut rows: Vec<(usize, f32, f32, f32)> = by_index
|
||||
.into_iter()
|
||||
.filter_map(|(idx, bounds)| {
|
||||
let mut min_edge = f32::INFINITY;
|
||||
let mut max_edge = f32::NEG_INFINITY;
|
||||
let mut center_sum = 0.0;
|
||||
let mut count = 0usize;
|
||||
for (lo, hi) in bounds {
|
||||
if lo.is_finite() && hi.is_finite() && lo < hi {
|
||||
min_edge = min_edge.min(lo);
|
||||
max_edge = max_edge.max(hi);
|
||||
center_sum += (lo + hi) * 0.5;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
(count > 0).then_some((idx, center_sum / count as f32, min_edge, max_edge))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if rows.len() < 2 {
|
||||
return rows
|
||||
.into_iter()
|
||||
.map(|(idx, _center, lo, hi)| (idx, (lo, hi)))
|
||||
.collect();
|
||||
}
|
||||
|
||||
rows.sort_by_key(|(idx, _, _, _)| *idx);
|
||||
|
||||
let mut bands = HashMap::new();
|
||||
for i in 0..rows.len() {
|
||||
let (idx, _center, min_edge, max_edge) = rows[i];
|
||||
let lo = if i == 0 {
|
||||
min_edge
|
||||
} else {
|
||||
(rows[i - 1].1 + rows[i].1) * 0.5
|
||||
};
|
||||
let hi = if i + 1 == rows.len() {
|
||||
max_edge
|
||||
} else {
|
||||
(rows[i].1 + rows[i + 1].1) * 0.5
|
||||
};
|
||||
if lo.is_finite() && hi.is_finite() && lo < hi {
|
||||
bands.insert(idx, (lo, hi));
|
||||
}
|
||||
}
|
||||
|
||||
bands
|
||||
}
|
||||
|
||||
fn axis_bounds(bbox: [f32; 4], axis: Axis) -> (f32, f32) {
|
||||
match axis {
|
||||
Axis::X => (bbox[0].min(bbox[2]), bbox[0].max(bbox[2])),
|
||||
Axis::Y => (bbox[1].min(bbox[3]), bbox[1].max(bbox[3])),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize cell text for inclusion in a markdown pipe-table cell:
|
||||
/// collapse whitespace runs, drop newlines/tabs (cells must be one line),
|
||||
/// and escape pipes that would otherwise break the table.
|
||||
fn sanitize_cell(text: &str) -> String {
|
||||
let mut s = String::with_capacity(text.len());
|
||||
let mut prev_space = false;
|
||||
for c in text.chars() {
|
||||
match c {
|
||||
'|' => {
|
||||
s.push_str("\\|");
|
||||
prev_space = false;
|
||||
}
|
||||
'\n' | '\r' | '\t' | ' ' => {
|
||||
if !prev_space {
|
||||
s.push(' ');
|
||||
}
|
||||
prev_space = true;
|
||||
}
|
||||
other => {
|
||||
s.push(other);
|
||||
prev_space = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
s.trim().to_string()
|
||||
}
|
||||
|
||||
/// Render a list of explicitly-positioned cells as a markdown pipe-table.
|
||||
///
|
||||
/// Grid dimensions are inferred from the cells' (row, col, rowspan, colspan)
|
||||
/// extents. A cell with colspan/rowspan > 1 is rendered in its top-left
|
||||
/// position; the absorbed grid positions are emitted as empty cells so the
|
||||
/// markdown stays a valid rectangular grid that downstream readers can
|
||||
/// column-count correctly.
|
||||
///
|
||||
/// The separator row (`|---|...|`) is emitted after the **last** row that
|
||||
/// contains a header cell (`is_header == true`). When no cells are flagged
|
||||
/// as headers — e.g. the upstream TSR model didn't emit `<thead>`/`<th>` —
|
||||
/// the separator falls back to "after row 0" so the output is still a
|
||||
/// valid pipe-table.
|
||||
pub fn cells_to_markdown(cells: &[StructuredCell]) -> String {
|
||||
if cells.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let num_rows = cells
|
||||
.iter()
|
||||
.map(|c| c.row + c.rowspan.max(1))
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let num_cols = cells
|
||||
.iter()
|
||||
.map(|c| c.col + c.colspan.max(1))
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if num_rows == 0 || num_cols == 0 {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Separator goes after the last header row, falling back to row 0 when
|
||||
// no header cells exist. Clamped into range so a malformed cell with
|
||||
// row >= num_rows can't push it past the table.
|
||||
let separator_after_row = cells
|
||||
.iter()
|
||||
.filter(|c| c.is_header)
|
||||
.map(|c| c.row)
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
.min(num_rows.saturating_sub(1));
|
||||
|
||||
let mut grid: Vec<Vec<String>> = vec![vec![String::new(); num_cols]; num_rows];
|
||||
for cell in cells {
|
||||
if cell.row < num_rows && cell.col < num_cols {
|
||||
grid[cell.row][cell.col] = sanitize_cell(&cell.text);
|
||||
}
|
||||
}
|
||||
|
||||
let mut output = String::new();
|
||||
for (row_idx, row) in grid.iter().enumerate() {
|
||||
output.push('|');
|
||||
for cell in row {
|
||||
output.push_str(cell);
|
||||
output.push('|');
|
||||
}
|
||||
output.push('\n');
|
||||
if row_idx == separator_after_row {
|
||||
output.push('|');
|
||||
for _ in 0..num_cols {
|
||||
output.push_str("---|");
|
||||
}
|
||||
output.push('\n');
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn t(s: &str) -> String {
|
||||
s.to_string()
|
||||
}
|
||||
|
||||
/// Tokens for the synthetic 3×3 grid example (one colspan-4 row + two
|
||||
/// data rows of 4 cells each = 9 cells total, 3 rows × 4 cols).
|
||||
fn synthetic_3x3_tokens() -> Vec<String> {
|
||||
vec![
|
||||
"<html>",
|
||||
"<body>",
|
||||
"<table>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td",
|
||||
" colspan=\"4\"",
|
||||
">",
|
||||
"</td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(t)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Bboxes for the synthetic 3×3 grid (8-element polygon form), all
|
||||
/// within a 400×120 px crop.
|
||||
fn synthetic_3x3_bboxes() -> Vec<Vec<f32>> {
|
||||
vec![
|
||||
vec![3.0, 2.0, 395.0, 2.0, 396.0, 59.0, 3.0, 59.0],
|
||||
vec![26.0, 62.0, 140.0, 62.0, 141.0, 120.0, 26.0, 120.0],
|
||||
vec![149.0, 64.0, 248.0, 64.0, 248.0, 119.0, 149.0, 119.0],
|
||||
vec![257.0, 64.0, 350.0, 64.0, 350.0, 119.0, 257.0, 119.0],
|
||||
vec![359.0, 64.0, 395.0, 64.0, 395.0, 119.0, 359.0, 119.0],
|
||||
vec![26.0, 122.0, 140.0, 122.0, 140.0, 178.0, 26.0, 178.0],
|
||||
vec![149.0, 124.0, 248.0, 124.0, 248.0, 179.0, 149.0, 179.0],
|
||||
vec![257.0, 124.0, 350.0, 124.0, 350.0, 179.0, 257.0, 179.0],
|
||||
vec![359.0, 124.0, 395.0, 124.0, 395.0, 179.0, 359.0, 179.0],
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_structure_synthetic_3x3() {
|
||||
let tokens = synthetic_3x3_tokens();
|
||||
let slots = parse_structure(&tokens);
|
||||
|
||||
assert_eq!(slots.len(), 9, "should parse 9 cells");
|
||||
|
||||
// Cell 0: row 0 col 0, colspan 4
|
||||
assert_eq!(slots[0].row, 0);
|
||||
assert_eq!(slots[0].col, 0);
|
||||
assert_eq!(slots[0].colspan, 4);
|
||||
assert_eq!(slots[0].rowspan, 1);
|
||||
|
||||
// Cells 1..5: row 1, cols 0..3
|
||||
for (i, slot) in slots.iter().enumerate().skip(1).take(4) {
|
||||
assert_eq!(slot.row, 1, "cell {i}: row should be 1");
|
||||
assert_eq!(slot.col, i - 1, "cell {i}: col should be {}", i - 1);
|
||||
assert_eq!(slot.colspan, 1);
|
||||
assert_eq!(slot.rowspan, 1);
|
||||
}
|
||||
|
||||
// Cells 5..9: row 2, cols 0..3
|
||||
for (i, slot) in slots.iter().enumerate().skip(5).take(4) {
|
||||
assert_eq!(slot.row, 2, "cell {i}: row should be 2");
|
||||
assert_eq!(slot.col, i - 5);
|
||||
assert_eq!(slot.colspan, 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn polygon_to_aabb_8elt() {
|
||||
// Synthetic cell bbox 0
|
||||
let coords = vec![3.0, 2.0, 395.0, 2.0, 396.0, 59.0, 3.0, 59.0];
|
||||
let aabb = polygon_to_aabb(&coords).unwrap();
|
||||
assert_eq!(aabb, [3.0, 2.0, 396.0, 59.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn polygon_to_aabb_4elt() {
|
||||
let coords = vec![5.0, 10.0, 50.0, 60.0];
|
||||
let aabb = polygon_to_aabb(&coords).unwrap();
|
||||
assert_eq!(aabb, [5.0, 10.0, 50.0, 60.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn polygon_to_aabb_4elt_unordered() {
|
||||
// Caller may pass corners in any order; min/max should normalise.
|
||||
let coords = vec![50.0, 60.0, 5.0, 10.0];
|
||||
let aabb = polygon_to_aabb(&coords).unwrap();
|
||||
assert_eq!(aabb, [5.0, 10.0, 50.0, 60.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn polygon_to_aabb_invalid_len() {
|
||||
assert!(polygon_to_aabb(&[1.0, 2.0, 3.0]).is_none());
|
||||
assert!(polygon_to_aabb(&[1.0; 6]).is_none());
|
||||
assert!(polygon_to_aabb(&[]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_3x3_aabbs_inside_crop() {
|
||||
// All 9 bboxes should produce valid (x1<x2, y1<y2) rects within the
|
||||
// crop bounds (400 wide, ~180 tall by inspection of the fixture).
|
||||
let bboxes = synthetic_3x3_bboxes();
|
||||
assert_eq!(bboxes.len(), 9);
|
||||
for (i, bb) in bboxes.iter().enumerate() {
|
||||
let aabb = polygon_to_aabb(bb).unwrap_or_else(|| panic!("bbox {i} invalid"));
|
||||
assert!(aabb[0] < aabb[2], "bbox {i}: x1 < x2");
|
||||
assert!(aabb[1] < aabb[3], "bbox {i}: y1 < y2");
|
||||
assert!(aabb[0] >= 0.0 && aabb[2] <= 500.0, "bbox {i}: within crop");
|
||||
assert!(aabb[1] >= 0.0 && aabb[3] <= 200.0, "bbox {i}: within crop");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_cell_bands_splits_overlapping_slanet_rows() {
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 100.0, 90.0, 120.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [90.0, 100.0, 170.0, 120.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 116.0, 90.0, 136.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [90.0, 116.0, 170.0, 136.0],
|
||||
},
|
||||
];
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
assert_eq!(cells[0].page_pt_bbox[3], cells[2].page_pt_bbox[1]);
|
||||
assert_eq!(cells[1].page_pt_bbox[3], cells[3].page_pt_bbox[1]);
|
||||
assert!(
|
||||
(cells[0].page_pt_bbox[3] - 118.0).abs() < 0.01,
|
||||
"row separator should be midpoint between row centers: {:?}",
|
||||
cells
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_cell_bands_preserves_colspan_extent() {
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 2,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [8.0, 80.0, 172.0, 98.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 96.0, 90.0, 114.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [88.0, 96.0, 170.0, 114.0],
|
||||
},
|
||||
];
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
assert!(
|
||||
cells[0].page_pt_bbox[0] <= cells[1].page_pt_bbox[0],
|
||||
"spanning cell should retain the first column's left edge"
|
||||
);
|
||||
assert!(
|
||||
cells[0].page_pt_bbox[2] >= cells[2].page_pt_bbox[2],
|
||||
"spanning cell should retain the last column's right edge"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_int_attr_basic() {
|
||||
assert_eq!(parse_int_attr(" colspan=\"4\"", "colspan"), Some(4));
|
||||
assert_eq!(parse_int_attr(" rowspan=\"2\"", "rowspan"), Some(2));
|
||||
assert_eq!(parse_int_attr("colspan='3'", "colspan"), Some(3));
|
||||
assert_eq!(parse_int_attr(" colspan=\"4\"", "rowspan"), None);
|
||||
assert_eq!(parse_int_attr(" class=\"foo\"", "colspan"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_structure_rowspan_pushes_next_row_right() {
|
||||
// <tr><td rowspan="2">A</td><td>B</td></tr><tr><td>C</td></tr>
|
||||
// Expected: A at (0,0), B at (0,1), C at (1,1) — col 0 of row 1
|
||||
// is occupied by A's rowspan.
|
||||
let tokens: Vec<String> = vec![
|
||||
"<table>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td",
|
||||
" rowspan=\"2\"",
|
||||
">",
|
||||
"</td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(t)
|
||||
.collect();
|
||||
|
||||
let slots = parse_structure(&tokens);
|
||||
assert_eq!(slots.len(), 3);
|
||||
assert_eq!((slots[0].row, slots[0].col), (0, 0));
|
||||
assert_eq!(slots[0].rowspan, 2);
|
||||
assert_eq!((slots[1].row, slots[1].col), (0, 1));
|
||||
// C should be at (1, 1) because (1, 0) is occupied by A's rowspan.
|
||||
assert_eq!((slots[2].row, slots[2].col), (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_structure_thead_marks_headers() {
|
||||
// <thead><tr><th>H1</th><th>H2</th></tr></thead>
|
||||
// <tbody><tr><td>D1</td><td>D2</td></tr></tbody>
|
||||
let tokens: Vec<String> = vec![
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(t)
|
||||
.collect();
|
||||
|
||||
let slots = parse_structure(&tokens);
|
||||
assert_eq!(slots.len(), 4);
|
||||
assert!(slots[0].is_header && slots[1].is_header);
|
||||
assert!(!slots[2].is_header && !slots[3].is_header);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_structure_th_outside_thead_still_header() {
|
||||
// A row-header style: leading <th> in tbody.
|
||||
let tokens: Vec<String> = vec![
|
||||
"<table>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(t)
|
||||
.collect();
|
||||
|
||||
let slots = parse_structure(&tokens);
|
||||
assert_eq!(slots.len(), 2);
|
||||
assert!(slots[0].is_header);
|
||||
assert!(!slots[1].is_header);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_structure_th_with_attrs() {
|
||||
let tokens: Vec<String> = vec![
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th",
|
||||
" colspan=\"2\"",
|
||||
">",
|
||||
"</th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(t)
|
||||
.collect();
|
||||
let slots = parse_structure(&tokens);
|
||||
assert_eq!(slots.len(), 1);
|
||||
assert_eq!(slots[0].colspan, 2);
|
||||
assert!(slots[0].is_header);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_synthetic_3x3() {
|
||||
// Build the cells the parser would produce for the synthetic grid,
|
||||
// and provide some sample text so we can sanity-check output.
|
||||
let cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 4,
|
||||
is_header: false,
|
||||
text: "Title".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "a".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "b".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 2,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "c".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 3,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "d".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
];
|
||||
|
||||
let md = cells_to_markdown(&cells);
|
||||
// Header row contains the spanning cell text in col 0 and pads to 4 cols.
|
||||
// Absorbed-by-colspan positions render as empty cells (no padding).
|
||||
assert!(md.starts_with("|Title||||\n"), "got: {md}");
|
||||
assert!(md.contains("|---|---|---|---|\n"));
|
||||
assert!(md.contains("|a|b|c|d|\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_escapes_pipes() {
|
||||
let cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "a|b".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "x".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
},
|
||||
];
|
||||
let md = cells_to_markdown(&cells);
|
||||
assert!(md.contains("|a\\|b|x|"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_collapses_whitespace_and_newlines() {
|
||||
let cells = vec![StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: "foo \n bar\tbaz".into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
}];
|
||||
let md = cells_to_markdown(&cells);
|
||||
assert!(md.contains("|foo bar baz|"));
|
||||
}
|
||||
|
||||
fn cell(row: usize, col: usize, is_header: bool, text: &str) -> StructuredCell {
|
||||
StructuredCell {
|
||||
row,
|
||||
col,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header,
|
||||
text: text.into(),
|
||||
page_pt_bbox: [0.0, 0.0, 0.0, 0.0],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_separator_after_last_header_row() {
|
||||
// Two-row header (a multi-row thead), then two body rows. Separator
|
||||
// should land after row 1 (the LAST header row), not after row 0.
|
||||
let cells = vec![
|
||||
cell(0, 0, true, "H0a"),
|
||||
cell(0, 1, true, "H0b"),
|
||||
cell(1, 0, true, "H1a"),
|
||||
cell(1, 1, true, "H1b"),
|
||||
cell(2, 0, false, "d0a"),
|
||||
cell(2, 1, false, "d0b"),
|
||||
cell(3, 0, false, "d1a"),
|
||||
cell(3, 1, false, "d1b"),
|
||||
];
|
||||
let md = cells_to_markdown(&cells);
|
||||
let expected = "|H0a|H0b|\n|H1a|H1b|\n|---|---|\n|d0a|d0b|\n|d1a|d1b|\n";
|
||||
assert_eq!(md, expected, "got: {md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_separator_when_row_0_not_header() {
|
||||
// Row 0 is not flagged as a header but row 1 is. Separator should
|
||||
// follow row 1 (the header), demonstrating that we don't blindly
|
||||
// emit after row 0.
|
||||
let cells = vec![
|
||||
cell(0, 0, false, "x0a"),
|
||||
cell(0, 1, false, "x0b"),
|
||||
cell(1, 0, true, "Hdr1"),
|
||||
cell(1, 1, true, "Hdr2"),
|
||||
cell(2, 0, false, "data1"),
|
||||
cell(2, 1, false, "data2"),
|
||||
];
|
||||
let md = cells_to_markdown(&cells);
|
||||
// Confirm the separator is NOT after row 0.
|
||||
assert!(!md.starts_with("|x0a|x0b|\n|---|"), "got: {md}");
|
||||
// Confirm it IS after row 1.
|
||||
assert!(
|
||||
md.contains("|Hdr1|Hdr2|\n|---|---|\n|data1|data2|"),
|
||||
"got: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_to_markdown_no_headers_falls_back_to_row_0() {
|
||||
// No header cells at all — fallback: separator after row 0 so the
|
||||
// output is still a valid markdown pipe-table.
|
||||
let cells = vec![
|
||||
cell(0, 0, false, "a"),
|
||||
cell(0, 1, false, "b"),
|
||||
cell(1, 0, false, "c"),
|
||||
cell(1, 1, false, "d"),
|
||||
];
|
||||
let md = cells_to_markdown(&cells);
|
||||
assert_eq!(md, "|a|b|\n|---|---|\n|c|d|\n");
|
||||
}
|
||||
}
|
||||
@@ -1474,6 +1474,440 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_mem tests (TSR-aware path)
|
||||
// =========================================================================
|
||||
|
||||
/// Build an 8-element 4-corner polygon `[x1,y1, x2,y1, x2,y2, x1,y2]` from
|
||||
/// an axis-aligned rect — matches the format SLANet emits for cell bboxes.
|
||||
fn poly(x1: f32, y1: f32, x2: f32, y2: f32) -> Vec<f32> {
|
||||
vec![x1, y1, x2, y1, x2, y2, x1, y2]
|
||||
}
|
||||
|
||||
fn synthetic_dense_table_pdf() -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let operations = vec![
|
||||
Operation::new("BT", vec![]),
|
||||
Operation::new("Tf", vec!["F1".into(), 10.into()]),
|
||||
Operation::new("Td", vec![20.into(), 700.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Branch Name")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Deposits")]),
|
||||
Operation::new("Td", vec![Object::Integer(-100), Object::Real(-16.8)]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Oak Street")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("100")]),
|
||||
Operation::new("Td", vec![Object::Integer(-100), Object::Real(-16.8)]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Boardwalk")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("200")]),
|
||||
Operation::new("ET", vec![]),
|
||||
];
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 200.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
// Hand-crafted TSR fixture targeting page 4 (0-indexed=3) of
|
||||
// bits_pilani_feedback.pdf, which contains a clean tabular layout.
|
||||
//
|
||||
// We construct a 2×2 table:
|
||||
// row 0 (header): "Department" "Core Courses"
|
||||
// row 1 (data): "BIO" "8.23"
|
||||
//
|
||||
// The PDF page is US Letter (792pt tall). We render at 72 dpi so
|
||||
// image-px maps 1:1 to PDF-pt — that lets us write cell bboxes in
|
||||
// the same units as our hand-measured page-pt coordinates.
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
|
||||
// The PDF page is A4 in points (≈595.44 × 841.68). The table sits in
|
||||
// the upper part of the page; we crop a window large enough to enclose
|
||||
// both rows we care about.
|
||||
//
|
||||
// Crop bounds in PDF points (top-left origin):
|
||||
// x: 80..280, y: 170..240
|
||||
let crop = [80.0_f32, 170.0, 280.0, 240.0];
|
||||
let dpi = 72.0_f32;
|
||||
|
||||
// Cell bboxes in CROP image-pixel space (= crop-relative PDF-pt at
|
||||
// 72 dpi). The y ranges are tightened against neighbouring rows
|
||||
// ("First Degree" above the header at native y=666.7, "Feedback Score"
|
||||
// between the header and data rows at native y=640.9, "CE" below the
|
||||
// BIO row at native y=591.1) so each cell only overlaps its target
|
||||
// text item.
|
||||
let cell_bboxes = vec![
|
||||
// Header row: y crop-relative (7, 18) → page-pt y (177, 188)
|
||||
poly(10.0, 7.0, 100.0, 18.0), // "Department" (item at page-pt x=107.1)
|
||||
poly(110.0, 7.0, 200.0, 18.0), // "Core Courses" (item at page-pt x=199.0)
|
||||
// Data row: y crop-relative (35, 60) → page-pt y (205, 230)
|
||||
poly(10.0, 35.0, 100.0, 60.0), // "BIO" (item at page-pt x=104.1)
|
||||
poly(110.0, 35.0, 200.0, 60.0), // "8.23" (item at page-pt x=221.2)
|
||||
];
|
||||
|
||||
// Minimal SLANet-style token stream: a 2-row table with a thead and tbody.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
let inputs = vec![TsrTableInput {
|
||||
page: 3,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: dpi,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}];
|
||||
|
||||
let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
|
||||
assert_eq!(mds.len(), 1);
|
||||
let md = &mds[0];
|
||||
|
||||
// Hand-written gold standard for the rendered markdown.
|
||||
let expected = "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n";
|
||||
assert_eq!(
|
||||
md, expected,
|
||||
"structured-table markdown should match the gold standard exactly\nactual: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_dense_overlapping_slanet_boxes() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
// Rows are spaced 16.8pt apart, while the SLANet-style boxes are 40pt
|
||||
// tall and overlap adjacent rows. Text must still land in only one row.
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
];
|
||||
|
||||
let mds = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let expected = "|Branch Name|Deposits|\n|---|---|\n|Oak Street|100|\n|Boardwalk|200|\n";
|
||||
assert_eq!(mds[0], expected);
|
||||
assert!(!mds[0].contains("Branch Name Oak Street"));
|
||||
assert!(!mds[0].contains("Oak Street Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_input_order_preserved() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
|
||||
// Two inputs; both target the same page but with different shapes.
|
||||
// We just need to confirm we get 2 outputs in the same order.
|
||||
let make_input = |toks: Vec<&str>, cells: Vec<Vec<f32>>| TsrTableInput {
|
||||
page: 3,
|
||||
crop_pdf_pt_bbox: [80.0, 170.0, 280.0, 240.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: toks.into_iter().map(String::from).collect(),
|
||||
cell_bboxes: cells,
|
||||
};
|
||||
|
||||
let inputs = vec![
|
||||
make_input(
|
||||
vec!["<table>", "<tr>", "<td></td>", "</tr>", "</table>"],
|
||||
vec![poly(10.0, 35.0, 100.0, 60.0)],
|
||||
),
|
||||
make_input(
|
||||
vec!["<table>", "<tr>", "<td></td>", "</tr>", "</table>"],
|
||||
vec![poly(110.0, 35.0, 200.0, 60.0)],
|
||||
),
|
||||
];
|
||||
|
||||
let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
|
||||
assert_eq!(mds.len(), 2);
|
||||
assert!(
|
||||
mds[0].contains("BIO"),
|
||||
"input 0 should pull 'BIO': {}",
|
||||
mds[0]
|
||||
);
|
||||
assert!(
|
||||
mds[1].contains("8.23"),
|
||||
"input 1 should pull '8.23': {}",
|
||||
mds[1]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_out_of_range_page() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
|
||||
let inputs = vec![TsrTableInput {
|
||||
page: 9999,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: vec![
|
||||
"<table>".into(),
|
||||
"<tr>".into(),
|
||||
"<td></td>".into(),
|
||||
"</tr>".into(),
|
||||
"</table>".into(),
|
||||
],
|
||||
cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
|
||||
}];
|
||||
|
||||
let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
|
||||
assert_eq!(mds.len(), 1);
|
||||
assert!(
|
||||
mds[0].is_empty(),
|
||||
"out-of-range page should yield empty string"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_not_a_pdf() {
|
||||
use pdf_inspector::extract_tables_with_structure_mem;
|
||||
let result = extract_tables_with_structure_mem(b"not a pdf", &[]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_empty_inputs() {
|
||||
use pdf_inspector::extract_tables_with_structure_mem;
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
let mds = extract_tables_with_structure_mem(&buf, &[]).unwrap();
|
||||
assert!(mds.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_cells_real_pdf_bits_pilani() {
|
||||
use pdf_inspector::{extract_tables_with_structure_cells_mem, TsrTableInput};
|
||||
// Same fixture as test_extract_tables_with_structure_real_pdf_bits_pilani
|
||||
// but exercising the cell-level API. Verifies that callers receive
|
||||
// structured per-cell metadata (row/col/spans/is_header/page_pt_bbox)
|
||||
// alongside the extracted text.
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
|
||||
let crop = [80.0_f32, 170.0, 280.0, 240.0];
|
||||
let dpi = 72.0_f32;
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 7.0, 100.0, 18.0),
|
||||
poly(110.0, 7.0, 200.0, 18.0),
|
||||
poly(10.0, 35.0, 100.0, 60.0),
|
||||
poly(110.0, 35.0, 200.0, 60.0),
|
||||
];
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
let inputs = vec![TsrTableInput {
|
||||
page: 3,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: dpi,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}];
|
||||
|
||||
let cells_lists = extract_tables_with_structure_cells_mem(&buf, &inputs).unwrap();
|
||||
assert_eq!(cells_lists.len(), 1);
|
||||
let cells = &cells_lists[0];
|
||||
assert_eq!(cells.len(), 4);
|
||||
|
||||
// Header row: both cells flagged as headers (they were in <thead>/<th>).
|
||||
assert!(cells[0].is_header);
|
||||
assert!(cells[1].is_header);
|
||||
assert_eq!((cells[0].row, cells[0].col), (0, 0));
|
||||
assert_eq!((cells[1].row, cells[1].col), (0, 1));
|
||||
assert_eq!(cells[0].text, "Department");
|
||||
assert_eq!(cells[1].text, "Core Courses");
|
||||
|
||||
// Data row: not flagged as header.
|
||||
assert!(!cells[2].is_header);
|
||||
assert!(!cells[3].is_header);
|
||||
assert_eq!((cells[2].row, cells[2].col), (1, 0));
|
||||
assert_eq!((cells[3].row, cells[3].col), (1, 1));
|
||||
assert_eq!(cells[2].text, "BIO");
|
||||
assert_eq!(cells[3].text, "8.23");
|
||||
|
||||
// Every cell carries a non-degenerate page-pt bbox.
|
||||
for c in cells {
|
||||
let [x1, y1, x2, y2] = c.page_pt_bbox;
|
||||
assert!(
|
||||
x1 < x2 && y1 < y2,
|
||||
"cell bbox should be non-empty: {:?}",
|
||||
c.page_pt_bbox
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_separator_after_thead() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
// Re-run the same 2x2 fixture but assert exact markdown output: with
|
||||
// <thead> + <th> headers, the separator should land after the header
|
||||
// row (which is also row 0 here, so the gold-standard hasn't changed).
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
|
||||
let crop = [80.0_f32, 170.0, 280.0, 240.0];
|
||||
let dpi = 72.0_f32;
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 7.0, 100.0, 18.0),
|
||||
poly(110.0, 7.0, 200.0, 18.0),
|
||||
poly(10.0, 35.0, 100.0, 60.0),
|
||||
poly(110.0, 35.0, 200.0, 60.0),
|
||||
];
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
let mds = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 3,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: dpi,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(mds.len(), 1);
|
||||
assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_pages_markdown_mem tests
|
||||
// =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user