mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tables): Prevent paragraph text from being falsely detected as tables

Browse Source 
Add pairwise cross-row column alignment check to find_table_regions_strict().
Real tables have fixed column X positions that repeat across rows (score ~1.0),
while paragraph text has varying word positions (score ~0.3-0.4). Regions with
average pairwise alignment score < 0.5 are now rejected.

Also remove the num_cols >= 5 content-check bypass for BodyFont mode, requiring
body-font tables to always have ≥30% data-like cell content.

Fixes false table detection across 9+ PDFs including AI_Cultural_Collections
(-730 false lines), 131212888 (-740), Data-Processing-Agreement (-108),
CEP_DN_Interest_Rates (-93). No regressions on legitimate tables.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-12 15:10:10 -08:00
co-authored by Claude Opus 4.6
parent d0e14fe4cd
commit 56cfd895b5
2 changed files with 119 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+4
View File
@@ -41,3 +41,7 @@ path = "src/bin/pdf2md.rs"
name = "detect-pdf"
path = "src/bin/detect_pdf.rs"
[[bin]]
name = "debug_ygaps"
path = "src/bin/debug_ygaps.rs"
src/tables.rs
+115 -25
View File
@@ -149,7 +149,9 @@ fn find_table_regions(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> {
}
/// Find Y-regions for body-font table candidates using strict structural criteria.
/// Requires rows with 3+ distinct X-position clusters to qualify.
/// Requires rows with 3+ distinct X-position clusters to qualify, and verifies
/// that column positions are consistent across rows (tables have fixed columns,
/// paragraph text has varying word positions).
fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> {
if items.is_empty() {
return vec![];
@@ -172,7 +174,8 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> {
}
// Step 2: Filter to rows with 3+ distinct X-position clusters (20pt tolerance)
let mut qualifying_ys: Vec<f32> = Vec::new();
// Collect cluster start positions for cross-row alignment analysis
let mut qualifying_rows: Vec<(f32, Vec<f32>)> = Vec::new(); // (y, cluster_starts)
for (y, x_positions) in &row_groups {
let mut sorted_xs = x_positions.clone();
sorted_xs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
@@ -181,48 +184,87 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> {
continue;
}
let mut clusters = 1;
let mut cluster_starts: Vec<f32> = vec![sorted_xs[0]];
let mut last_x = sorted_xs[0];
for &x in &sorted_xs[1..] {
if x - last_x > 20.0 {
clusters += 1;
cluster_starts.push(x);
last_x = x;
}
}
if clusters >= 3 {
qualifying_ys.push(*y);
if cluster_starts.len() >= 3 {
qualifying_rows.push((*y, cluster_starts));
}
}
if qualifying_ys.len() < 3 {
if qualifying_rows.len() < 3 {
return vec![];
}
// Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap)
qualifying_ys.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));
let mut regions = Vec::new();
let mut region_start = qualifying_ys[0];
let mut region_end = qualifying_ys[0];
let mut region_count = 1;
let mut candidate_regions: Vec<Vec<&(f32, Vec<f32>)>> = Vec::new();
let mut current_region: Vec<&(f32, Vec<f32>)> = vec![&qualifying_rows[0]];
for &y in &qualifying_ys[1..] {
if y - region_end > 25.0 {
if region_count >= 3 {
regions.push((region_start - 5.0, region_end + 5.0));
for row in qualifying_rows.iter().skip(1) {
let prev_y = current_region.last().unwrap().0;
if row.0 - prev_y > 25.0 {
if current_region.len() >= 3 {
candidate_regions.push(current_region);
}
region_start = y;
region_end = y;
region_count = 1;
current_region = vec![row];
} else {
region_end = y;
region_count += 1;
current_region.push(row);
}
}
if current_region.len() >= 3 {
candidate_regions.push(current_region);
}
// Don't forget last region
if region_count >= 3 {
regions.push((region_start - 5.0, region_end + 5.0));
// Step 4: Cross-row column alignment check per region
// Real tables have consistent column X positions across rows (high pairwise score).
// Paragraph text has varying word positions line-to-line (low pairwise score).
let mut regions = Vec::new();
for region_rows in &candidate_regions {
let num_rows = region_rows.len();
let mut total_score = 0.0f32;
let mut pair_count = 0u32;
let tolerance = 10.0f32;
for i in 0..num_rows {
for j in (i + 1)..num_rows {
let centers_a = &region_rows[i].1;
let centers_b = &region_rows[j].1;
let matches_a = centers_a
.iter()
.filter(|&&a| centers_b.iter().any(|&b| (a - b).abs() < tolerance))
.count();
let matches_b = centers_b
.iter()
.filter(|&&b| centers_a.iter().any(|&a| (a - b).abs() < tolerance))
.count();
let max_len = centers_a.len().max(centers_b.len());
if max_len > 0 {
total_score += (matches_a + matches_b) as f32 / (2 * max_len) as f32;
pair_count += 1;
}
}
}
let avg_score = if pair_count > 0 {
total_score / pair_count as f32
} else {
0.0
};
if avg_score >= 0.5 {
let y_min = region_rows.first().unwrap().0;
let y_max = region_rows.last().unwrap().0;
regions.push((y_min - 5.0, y_max + 5.0));
}
}
regions
@@ -490,7 +532,9 @@ fn has_table_like_content(cells: &[Vec<String>], mode: TableDetectionMode) -> bo
TableDetectionMode::BodyFont => 0.3,
};
pct_data > min_pct || num_cols >= 5
// For SmallFont, bypass content check for wide tables (5+ columns may have text headers).
// For BodyFont, always require data-like content to prevent paragraph false positives.
pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5)
}
/// Check if a cell value looks like table data
@@ -1269,6 +1313,52 @@ mod tests {
);
}
#[test]
fn test_word_level_paragraph_not_detected_as_table() {
// Paragraph text with per-word TextItems (as produced by some PDFs).
// Word X positions vary from line to line — NOT a table.
let items = vec![
// Line 1
make_item("We", 72.0, 500.0, 10.0),
make_item("would", 95.0, 500.0, 10.0),
make_item("like", 145.0, 500.0, 10.0),
make_item("to", 180.0, 500.0, 10.0),
make_item("thank", 200.0, 500.0, 10.0),
make_item("all", 250.0, 500.0, 10.0),
make_item("the", 278.0, 500.0, 10.0),
make_item("practitioners", 305.0, 500.0, 10.0),
// Line 2
make_item("and", 72.0, 485.0, 10.0),
make_item("researchers", 105.0, 485.0, 10.0),
make_item("across", 185.0, 485.0, 10.0),
make_item("the", 232.0, 485.0, 10.0),
make_item("University", 260.0, 485.0, 10.0),
make_item("of", 335.0, 485.0, 10.0),
make_item("Leeds", 355.0, 485.0, 10.0),
// Line 3
make_item("Libraries", 72.0, 470.0, 10.0),
make_item("whose", 142.0, 470.0, 10.0),
make_item("contributions", 190.0, 470.0, 10.0),
make_item("made", 290.0, 470.0, 10.0),
make_item("this", 328.0, 470.0, 10.0),
make_item("report", 360.0, 470.0, 10.0),
// Line 4
make_item("possible", 72.0, 455.0, 10.0),
make_item("Both", 140.0, 455.0, 10.0),
make_item("constituent", 178.0, 455.0, 10.0),
make_item("studies", 262.0, 455.0, 10.0),
make_item("were", 315.0, 455.0, 10.0),
make_item("approved", 350.0, 455.0, 10.0),
];
let tables = detect_tables(&items, 10.0);
assert_eq!(
tables.len(),
0,
"Word-level paragraph text must not be detected as table"
);
}
#[test]
fn test_large_data_table_not_rejected() {
// 50-row table at small font — must not be rejected by row limit