From 839317525be368d55444e2d9eb7c662a2df4e738 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 27 May 2026 11:45:00 -0700 Subject: [PATCH] fix(pdf-inspector): relax vector table confidence gates (#96) --- napi/package.json | 2 +- src/lib.rs | 161 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 151 insertions(+), 12 deletions(-) diff --git a/napi/package.json b/napi/package.json index efa68e6..f28197f 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.9.0", + "version": "1.9.1", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/src/lib.rs b/src/lib.rs index 040942a..18f9cf1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -823,7 +823,9 @@ pub fn extract_tables_in_regions_mem( // symmetrically low under font-decode failure — this // guard breaks that symmetry by comparing against // bbox area, which is independent of extraction. - if region_text_density_too_low(region_text_chars, region_area) { + if region_text_density_too_low(region_text_chars, region_area) + && !markdown_table_body_is_dense(&md) + { return None; } let shape = markdown_table_shape(&md); @@ -834,7 +836,9 @@ pub fn extract_tables_in_regions_mem( Some(TableCandidateIssue::LineRowUndercount) } else if wide_table_sparse_prefix_undercount(&md) { Some(TableCandidateIssue::SparseWideUndercount) - } else if text_cluster_column_undercount(&matched, shape) { + } else if source != TableCandidateSource::Line + && text_cluster_column_undercount(&matched, shape) + { Some(TableCandidateIssue::TextColumnUndercount) } else if prose_grid_fragment_needs_ocr(&md) { Some(TableCandidateIssue::ProseGridFragment) @@ -4253,12 +4257,21 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool { } // Failure mode 2: header has empty cells in a multi-column table. - // When layout-assisted, allow up to 1 empty header cell (common in - // tables with merged/spanning header cells that we can't represent). - let empty_count = header_cells.iter().filter(|c| c.is_empty()).count(); + // When layout-assisted, tolerate merged/spanning header gaps if the + // body is dense. Region bboxes from a layout model often start at a + // visual table whose header cannot be represented faithfully in a + // flat pipe table, while the body rows are still complete enough to use. + let header_empty_indices: Vec = header_cells + .iter() + .enumerate() + .filter_map(|(idx, cell)| cell.is_empty().then_some(idx)) + .collect(); + let empty_count = header_empty_indices.len(); if layout_assisted { - // Reject only if >1 empty header cell (2+ means serious boundary issue) - if n_cols >= 3 && empty_count >= 2 { + if n_cols >= 3 + && empty_count >= 2 + && !layout_assisted_empty_header_has_dense_body(markdown, n_cols) + { return true; } } else if n_cols >= 3 && empty_count >= 1 { @@ -4296,7 +4309,14 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool { // (totals, subtotals) are common. let threshold = if layout_assisted { 2 } else { 3 }; if n_cols >= 3 && empty_data * threshold >= n_cols { - return true; + let sparse_row_shares_header_spacer = layout_assisted + && data_inner.iter().enumerate().any(|(idx, cell)| { + cell.trim().is_empty() && header_empty_indices.contains(&idx) + }) + && layout_assisted_empty_header_has_dense_body(markdown, n_cols); + if !sparse_row_shares_header_spacer { + return true; + } } } } @@ -4354,6 +4374,68 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool { false } +fn layout_assisted_empty_header_has_dense_body(markdown: &str, n_cols: usize) -> bool { + let rows = markdown_pipe_rows(markdown); + let data_rows: Vec<&Vec<&str>> = rows + .iter() + .skip(1) + .filter(|row| row.iter().any(|cell| !cell.trim().is_empty())) + .collect(); + if data_rows.len() < 2 || n_cols < 3 { + return false; + } + + let total_cells = data_rows.len() * n_cols; + let mut filled_cells = 0usize; + let mut rows_with_multiple_cells = 0usize; + let mut max_filled_in_row = 0usize; + for row in &data_rows { + let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count(); + filled_cells += filled; + max_filled_in_row = max_filled_in_row.max(filled); + if filled >= 2 { + rows_with_multiple_cells += 1; + } + } + + // Dense enough to be a useful extraction despite lossy merged headers. + // The row-count gate avoids accepting a single tidy row under a broken + // header, and the density gate keeps sparse fragments on the OCR path. + rows_with_multiple_cells * 2 >= data_rows.len() + && max_filled_in_row >= n_cols.min(3) + && filled_cells * 100 >= total_cells * 45 +} + +fn markdown_table_body_is_dense(markdown: &str) -> bool { + let rows = markdown_pipe_rows(markdown); + let data_rows: Vec<&Vec<&str>> = rows + .iter() + .skip(1) + .filter(|row| row.iter().any(|cell| !cell.trim().is_empty())) + .collect(); + if data_rows.len() < 3 { + return false; + } + + let cols = rows.iter().map(|row| row.len()).max().unwrap_or_default(); + if cols < 3 { + return false; + } + + let mut filled_cells = 0usize; + let mut rows_with_multiple_cells = 0usize; + for row in &data_rows { + let filled = row.iter().filter(|cell| !cell.trim().is_empty()).count(); + filled_cells += filled; + if filled >= cols.min(3) { + rows_with_multiple_cells += 1; + } + } + + let total_cells = data_rows.len() * cols; + rows_with_multiple_cells * 2 >= data_rows.len() && filled_cells * 100 >= total_cells * 45 +} + /// Original strict validation (no layout assistance). Used by tests and /// full-page extraction paths that don't have layout model assistance. #[cfg(test)] @@ -4809,7 +4891,9 @@ mod table_candidate_selection_tests { #[cfg(test)] mod looks_like_partial_table_tests { - use super::{looks_like_partial_table, looks_like_partial_table_ex}; + use super::{ + looks_like_partial_table, looks_like_partial_table_ex, markdown_table_body_is_dense, + }; #[test] fn good_table_passes() { @@ -4944,11 +5028,29 @@ mod looks_like_partial_table_tests { #[test] fn two_empty_headers_still_rejected_when_layout_assisted() { - // 2+ empty headers is still bad even with layout assistance. + // A single tidy row is not enough evidence to trust a badly gapped header. let md = "|A|||D|\n|---|---|---|---|\n|x|y|z|w|"; assert!( looks_like_partial_table_ex(md, true), - "2 empty headers rejected even layout-assisted" + "2 empty headers with only one body row are rejected even layout-assisted" + ); + } + + #[test] + fn dense_body_with_empty_merged_header_passes_when_layout_assisted() { + let md = "|Year||Unadjusted Basis|||\n\ + |---|---|---|---|---|\n\ + |1|.1667|$100,000|$16,670|$16,670|\n\ + |2|.3333|$100,000|$33,330|$50,000|\n\ + |3|.3333|$100,000|$33,330|$88,330|\n\ + |4|.1667|$100,000|$16,670|$100,000|"; + assert!( + looks_like_partial_table(md), + "strict mode still rejects merged-header gaps" + ); + assert!( + !looks_like_partial_table_ex(md, true), + "layout-assisted should trust a dense body under a merged header" ); } @@ -4973,6 +5075,22 @@ mod looks_like_partial_table_tests { ); } + #[test] + fn sparse_first_row_with_header_spacer_passes_when_layout_assisted() { + let md = "|Properties|Instruction||Training Datasets Alignment|\n\ + |---|---|---|---|\n\ + ||Alpaca-GPT4 OpenOrca Synth. Math-Instruct||Orca DPO Pairs Ultrafeedback Cleaned|\n\ + |Total # Samples|52K 2.91M 126K||12.9K 60.8K 126K|"; + assert!( + looks_like_partial_table(md), + "strict mode rejects the sparse first row" + ); + assert!( + !looks_like_partial_table_ex(md, true), + "layout-assisted should allow sparse rows that share a header spacer column" + ); + } + #[test] fn paragraph_still_rejected_when_layout_assisted() { // Paragraph detection is not relaxed — it's a genuine extraction issue. @@ -5026,6 +5144,27 @@ mod looks_like_partial_table_tests { "duplicate headers rejected even layout-assisted" ); } + + #[test] + fn dense_numeric_table_body_is_structurally_trusted() { + let md = "|Year|3-Year|5-Year|7-Year|\n\ + |---|---|---|---|\n\ + |1|33.0%|20.00%|14.29%|\n\ + |2|44.45%|32.00%|24.49%|\n\ + |3|14.81%|19.20%|17.49%|\n\ + |4|7.41%|11.52%|12.49%|"; + assert!(markdown_table_body_is_dense(md)); + } + + #[test] + fn sparse_markdown_fragment_is_not_structurally_trusted() { + let md = "|A|B|C|D|\n\ + |---|---|---|---|\n\ + |x||||\n\ + |||y||\n\ + ||||z|"; + assert!(!markdown_table_body_is_dense(md)); + } } /// Analyse extracted items and rects for layout complexity.