From 455dfe5a74504e589ef7b41e39c24d3b5c48fb98 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 28 May 2026 10:20:06 -0700 Subject: [PATCH] fix(pdf-inspector): recover borderless region tables (#97) --- napi/package.json | 2 +- src/lib.rs | 81 +++++++++++++++++++++++++++-- src/tables/format.rs | 69 +++++++++++++++++++++++-- src/tables/mod.rs | 118 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 10 deletions(-) diff --git a/napi/package.json b/napi/package.json index f28197f..15dd76e 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.9.1", + "version": "1.9.2", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/src/lib.rs b/src/lib.rs index 18f9cf1..a422065 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -881,6 +881,11 @@ pub fn extract_tables_in_regions_mem( { candidates.push(candidate); } + if let Some(table) = tables::try_build_table_from_columns(&matched, page_1idx) { + if let Some(candidate) = evaluate(TableCandidateSource::Column, &table) { + candidates.push(candidate); + } + } match select_table_candidate(&candidates) { Some(candidate) => page_results.push(RegionText { @@ -3752,6 +3757,7 @@ enum TableCandidateSource { Rect, Line, Heuristic, + Column, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -3786,8 +3792,10 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida // serving a tidy-looking fragment. if first.issue == Some(TableCandidateIssue::LineRowUndercount) { return candidates.iter().find(|candidate| { - candidate.source == TableCandidateSource::Heuristic - && candidate.issue.is_none() + matches!( + candidate.source, + TableCandidateSource::Heuristic | TableCandidateSource::Column + ) && candidate.issue.is_none() && candidate.shape.cols * 10 >= first.shape.cols * 13 }); } @@ -3805,14 +3813,27 @@ fn select_table_candidate(candidates: &[TableCandidate]) -> Option<&TableCandida TableCandidateSource::Rect | TableCandidateSource::Line ) { if let Some(heuristic) = candidates.iter().find(|candidate| { - candidate.source == TableCandidateSource::Heuristic - && candidate.issue.is_none() + matches!( + candidate.source, + TableCandidateSource::Heuristic | TableCandidateSource::Column + ) && candidate.issue.is_none() && heuristic_substantially_better(candidate.shape, accepted.shape) }) { accepted = heuristic; } } + if accepted.source == TableCandidateSource::Heuristic { + if let Some(column) = candidates.iter().find(|candidate| { + candidate.source == TableCandidateSource::Column + && candidate.issue.is_none() + && candidate.shape.cols >= accepted.shape.cols + && candidate.shape.rows > accepted.shape.rows + }) { + accepted = column; + } + } + Some(accepted) } @@ -4314,7 +4335,9 @@ fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool { cell.trim().is_empty() && header_empty_indices.contains(&idx) }) && layout_assisted_empty_header_has_dense_body(markdown, n_cols); - if !sparse_row_shares_header_spacer { + let sparse_row_is_section_label = layout_assisted + && layout_assisted_sparse_section_row_is_ok(data_inner, markdown, n_cols); + if !sparse_row_shares_header_spacer && !sparse_row_is_section_label { return true; } } @@ -4406,6 +4429,26 @@ fn layout_assisted_empty_header_has_dense_body(markdown: &str, n_cols: usize) -> && filled_cells * 100 >= total_cells * 45 } +fn layout_assisted_sparse_section_row_is_ok(row: &[&str], markdown: &str, n_cols: usize) -> bool { + let labels: Vec<&str> = row + .iter() + .map(|cell| cell.trim()) + .filter(|cell| !cell.is_empty()) + .collect(); + if labels.len() != 1 { + return false; + } + let label = labels[0]; + if label.len() > 40 || !label.chars().any(|ch| ch.is_alphabetic()) { + return false; + } + if label.ends_with('.') || label.ends_with('!') || label.ends_with('?') || label.ends_with(':') + { + return false; + } + layout_assisted_empty_header_has_dense_body(markdown, n_cols) +} + fn markdown_table_body_is_dense(markdown: &str) -> bool { let rows = markdown_pipe_rows(markdown); let data_rows: Vec<&Vec<&str>> = rows @@ -4787,6 +4830,16 @@ mod table_candidate_selection_tests { assert_eq!(selected.source, TableCandidateSource::Heuristic); } + #[test] + fn prefers_clean_column_fallback_when_it_recovers_more_rows() { + let candidates = vec![ + candidate(TableCandidateSource::Heuristic, 6, 5, None), + candidate(TableCandidateSource::Column, 7, 5, None), + ]; + let selected = select_table_candidate(&candidates).unwrap(); + assert_eq!(selected.source, TableCandidateSource::Column); + } + #[test] fn line_candidate_collapsing_captured_y_clusters_is_suspicious() { let long = "value value value value value value value value value value value value"; @@ -5091,6 +5144,24 @@ mod looks_like_partial_table_tests { ); } + #[test] + fn sparse_section_row_passes_when_layout_assisted_body_is_dense() { + let md = "|Properties|Conditions|Method|Typical values|Units|\n\ + |---|---|---|---|---|\n\ + |Rheology|||||\n\ + |Melt Flow Rate|230 C/2.16 kg|ASTM D1238|3.0|g/10 min|\n\ + |Tensile Stress at Yield|50 mm/min|ASTM D638|31|MPa|\n\ + |Elongation at Yield|50 mm/min|ASTM D638|8|%|"; + assert!( + looks_like_partial_table(md), + "strict mode rejects the sparse first row" + ); + assert!( + !looks_like_partial_table_ex(md, true), + "layout-assisted should allow a short section label above dense table rows" + ); + } + #[test] fn paragraph_still_rejected_when_layout_assisted() { // Paragraph detection is not relaxed — it's a genuine extraction issue. diff --git a/src/tables/format.rs b/src/tables/format.rs index 88003c0..a2e60a5 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -208,6 +208,24 @@ fn looks_like_compact_entry_label(cell: &str) -> bool { (1..=6).contains(&words) } +fn looks_like_plain_section_label(cell: &str) -> bool { + let trimmed = cell.trim(); + if trimmed.len() < 4 || trimmed.len() > 40 { + return false; + } + if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) { + return false; + } + if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) { + return false; + } + trimmed + .chars() + .all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-')) + && starts_with_uppercase_alpha(trimmed) + && (1..=4).contains(&alpha_word_count(trimmed)) +} + fn ends_like_incomplete_phrase(cell: &str) -> bool { let lower = cell.trim_end().to_ascii_lowercase(); lower.ends_with(" and") @@ -305,6 +323,10 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { .and_then(|r| r.first()) .map(|c| c.trim()) .unwrap_or(""); + let header_filled = cleaned + .first() + .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) + .unwrap_or(num_cols); let looks_like_spanning_first_column_row = first_cell.is_empty() && row.len() >= 4 && non_first_cells.len() == row.len().saturating_sub(1) @@ -328,6 +350,10 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { && non_first_cells .iter() .any(|cell| looks_like_compact_entry_label(cell)); + let looks_like_section_label_row = !first_cell.is_empty() + && filled_cells == 1 + && header_filled >= 3 + && looks_like_plain_section_label(first_cell); // Classic continuation: first cell empty, content in other cells let is_classic_continuation = first_cell.is_empty() && !non_first_cells.is_empty() @@ -344,10 +370,6 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { .last() .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) .unwrap_or(0); - let header_filled = cleaned - .first() - .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) - .unwrap_or(num_cols); // Merge when the row has significantly fewer filled cells than header. // For wide tables (5+ cols), require ≤50% of header cells. // For narrow tables (2-4 cols), require fewer than header cells. @@ -369,6 +391,7 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { && !looks_like_spanning_first_column_row && !looks_like_hierarchical_subrow && !looks_like_new_first_column_entry + && !looks_like_section_label_row && !is_short_subheader; let is_continuation = is_classic_continuation || is_wrapped_continuation; @@ -514,6 +537,44 @@ mod tests { assert!(cleaned[1][1].contains("continued text here")); } + #[test] + fn test_clean_table_cells_first_column_section_label_not_merged() { + let cells = vec![ + vec![ + "Properties".into(), + "Conditions".into(), + "Method".into(), + "Typical values".into(), + "Units".into(), + ], + vec![ + "Melt Flow Rate".into(), + "230 C/2.16 kg".into(), + "ASTM D1238".into(), + "3.0".into(), + "g/10 min".into(), + ], + vec![ + "Mechanical".into(), + "".into(), + "".into(), + "".into(), + "".into(), + ], + vec![ + "Tensile Stress at Yield".into(), + "50 mm/min".into(), + "ASTM D638".into(), + "31".into(), + "MPa".into(), + ], + ]; + let (cleaned, _) = clean_table_cells(&cells); + + assert_eq!(cleaned.len(), 4); + assert_eq!(cleaned[2][0], "Mechanical"); + } + #[test] fn test_clean_table_cells_short_subheader_not_merged() { let cells = vec![ diff --git a/src/tables/mod.rs b/src/tables/mod.rs index b69d2be..53b03cf 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -460,6 +460,7 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt item_indices.push(item_idx); } } + merge_superscript_marker_rows(&mut row_ys, &mut cells); // Validate: need reasonable fill rate let total_cells = row_ys.len() * columns.len(); @@ -547,6 +548,60 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt Some(Table::new(col_xs, row_ys, cells, item_indices)) } +fn merge_superscript_marker_rows(row_ys: &mut Vec, cells: &mut Vec>) { + let mut row_idx = 0; + while row_idx < cells.len() { + let non_empty: Vec<(usize, String)> = cells[row_idx] + .iter() + .enumerate() + .filter_map(|(col_idx, cell)| { + let trimmed = cell.trim(); + (!trimmed.is_empty()).then_some((col_idx, trimmed.to_string())) + }) + .collect(); + + if non_empty.len() != 1 || !is_superscript_marker_cell(&non_empty[0].1) { + row_idx += 1; + continue; + } + + let (marker_col, marker) = &non_empty[0]; + let prev = + (row_idx > 0).then(|| (row_idx - 1, (row_ys[row_idx - 1] - row_ys[row_idx]).abs())); + let next = (row_idx + 1 < cells.len()) + .then(|| (row_idx + 1, (row_ys[row_idx] - row_ys[row_idx + 1]).abs())); + let target = [prev, next] + .into_iter() + .flatten() + .filter(|(_, gap)| *gap <= 10.0) + .min_by(|(_, gap_a), (_, gap_b)| gap_a.total_cmp(gap_b)) + .map(|(idx, _)| idx); + + let Some(target_idx) = target else { + row_idx += 1; + continue; + }; + + let target_cell = &mut cells[target_idx][*marker_col]; + if target_cell.trim().is_empty() { + *target_cell = marker.to_string(); + } else { + target_cell.push_str(marker); + } + cells.remove(row_idx); + row_ys.remove(row_idx); + } +} + +fn is_superscript_marker_cell(value: &str) -> bool { + let trimmed = value.trim(); + !trimmed.is_empty() + && trimmed.chars().count() <= 2 + && trimmed + .chars() + .all(|ch| matches!(ch, '*' | '#' | 'o' | 'O' | '°' | 'º' | '†' | '‡')) +} + /// What kind of structure a detected `Table` represents. Classification is /// computed once at construction so consumers don't have to re-analyze the /// cells (and stay consistent across detection backends). @@ -689,6 +744,69 @@ mod tests { assert!(md.contains("|Cell 1|")); } + #[test] + fn test_merge_superscript_marker_rows() { + let mut rows = vec![506.0, 500.0, 480.0]; + let mut cells = vec![ + vec!["".into(), "".into(), "*".into()], + vec!["Name".into(), "Method".into(), "Typical values".into()], + vec!["Flow".into(), "ASTM D1238".into(), "3.0".into()], + ]; + + merge_superscript_marker_rows(&mut rows, &mut cells); + + assert_eq!(rows, vec![500.0, 480.0]); + assert_eq!(cells[0][2], "Typical values*"); + } + + #[test] + fn test_column_builder_handles_borderless_specs_table() { + let items = vec![ + make_char("*", 458.1, 544.2, 8.0, 4.4), + make_char("Properties", 36.0, 538.6, 12.0, 53.1), + make_char("Conditions", 195.8, 538.6, 12.0, 55.0), + make_char("Method", 297.2, 538.6, 12.0, 39.4), + make_char("Typical values", 384.1, 538.6, 12.0, 74.0), + make_char("Units", 510.6, 538.6, 8.0, 17.9), + make_char("Rheology", 36.0, 508.3, 10.0, 40.6), + make_char("o", 209.8, 492.5, 6.5, 3.5), + make_char("Melt Flow Rate", 36.0, 488.0, 10.0, 65.2), + make_char("230 ", 190.4, 488.0, 10.0, 19.4), + make_char("C/2.16 kg", 213.3, 488.0, 10.0, 42.8), + make_char("ASTM D1238", 288.4, 488.0, 10.0, 56.8), + make_char("3.0 ", 416.4, 488.0, 10.0, 16.9), + make_char("g/10 min", 504.1, 488.0, 10.0, 39.5), + make_char("Mechanical", 36.0, 451.5, 10.0, 48.3), + make_char("Tensile Stress at Yield", 36.0, 431.3, 10.0, 96.7), + make_char("50 mm/min", 197.9, 431.3, 10.0, 50.8), + make_char("ASTM D638", 291.2, 431.3, 10.0, 51.3), + make_char("31 ", 417.9, 431.3, 10.0, 13.9), + make_char("MPa", 514.7, 431.3, 10.0, 18.4), + make_char("Elongation at Yield", 36.0, 403.0, 10.0, 82.2), + make_char("50 mm/min", 197.9, 403.0, 10.0, 50.8), + make_char("ASTM D638", 291.2, 403.0, 10.0, 51.3), + make_char("8 ", 420.6, 403.0, 10.0, 8.5), + make_char("%", 519.1, 403.0, 10.0, 9.7), + make_char("Flexural Modulus", 36.0, 374.6, 10.0, 74.0), + make_char("ASTM D790", 291.2, 374.6, 10.0, 51.3), + make_char("1400", 412.4, 374.6, 10.0, 21.8), + make_char("MPa", 514.7, 374.6, 10.0, 18.4), + ]; + + let table = try_build_table_from_columns(&items, 1).unwrap(); + let md = table_to_markdown(&table); + + assert!( + md.contains("|Properties|Conditions|Method|Typical values*|Units|"), + "{md}" + ); + assert!(md.contains("|Mechanical|||||"), "{md}"); + assert!( + md.contains("|Flexural Modulus||ASTM D790|1400|MPa|"), + "{md}" + ); + } + #[test] fn test_body_font_table_detected() { let items = vec![