From 56cfd895b5f82a739eab25560df14386b8a63d3a Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 12 Feb 2026 15:10:10 -0800 Subject: [PATCH] fix(tables): Prevent paragraph text from being falsely detected as tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pairwise cross-row column alignment check to find_table_regions_strict(). Real tables have fixed column X positions that repeat across rows (score ~1.0), while paragraph text has varying word positions (score ~0.3-0.4). Regions with average pairwise alignment score < 0.5 are now rejected. Also remove the num_cols >= 5 content-check bypass for BodyFont mode, requiring body-font tables to always have ≥30% data-like cell content. Fixes false table detection across 9+ PDFs including AI_Cultural_Collections (-730 false lines), 131212888 (-740), Data-Processing-Agreement (-108), CEP_DN_Interest_Rates (-93). No regressions on legitimate tables. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 4 ++ src/tables.rs | 140 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 119 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cfc8ed8..4e0c4c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,3 +41,7 @@ path = "src/bin/pdf2md.rs" name = "detect-pdf" path = "src/bin/detect_pdf.rs" + +[[bin]] +name = "debug_ygaps" +path = "src/bin/debug_ygaps.rs" diff --git a/src/tables.rs b/src/tables.rs index b620ba1..2056be7 100644 --- a/src/tables.rs +++ b/src/tables.rs @@ -149,7 +149,9 @@ fn find_table_regions(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { } /// Find Y-regions for body-font table candidates using strict structural criteria. -/// Requires rows with 3+ distinct X-position clusters to qualify. +/// Requires rows with 3+ distinct X-position clusters to qualify, and verifies +/// that column positions are consistent across rows (tables have fixed columns, +/// paragraph text has varying word positions). fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { if items.is_empty() { return vec![]; @@ -172,7 +174,8 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { } // Step 2: Filter to rows with 3+ distinct X-position clusters (20pt tolerance) - let mut qualifying_ys: Vec = Vec::new(); + // Collect cluster start positions for cross-row alignment analysis + let mut qualifying_rows: Vec<(f32, Vec)> = Vec::new(); // (y, cluster_starts) for (y, x_positions) in &row_groups { let mut sorted_xs = x_positions.clone(); sorted_xs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); @@ -181,48 +184,87 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32)> { continue; } - let mut clusters = 1; + let mut cluster_starts: Vec = vec![sorted_xs[0]]; let mut last_x = sorted_xs[0]; for &x in &sorted_xs[1..] { if x - last_x > 20.0 { - clusters += 1; + cluster_starts.push(x); last_x = x; } } - if clusters >= 3 { - qualifying_ys.push(*y); + + if cluster_starts.len() >= 3 { + qualifying_rows.push((*y, cluster_starts)); } } - if qualifying_ys.len() < 3 { + if qualifying_rows.len() < 3 { return vec![]; } // Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap) - qualifying_ys.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); - let mut regions = Vec::new(); - let mut region_start = qualifying_ys[0]; - let mut region_end = qualifying_ys[0]; - let mut region_count = 1; + let mut candidate_regions: Vec)>> = Vec::new(); + let mut current_region: Vec<&(f32, Vec)> = vec![&qualifying_rows[0]]; - for &y in &qualifying_ys[1..] { - if y - region_end > 25.0 { - if region_count >= 3 { - regions.push((region_start - 5.0, region_end + 5.0)); + for row in qualifying_rows.iter().skip(1) { + let prev_y = current_region.last().unwrap().0; + if row.0 - prev_y > 25.0 { + if current_region.len() >= 3 { + candidate_regions.push(current_region); } - region_start = y; - region_end = y; - region_count = 1; + current_region = vec![row]; } else { - region_end = y; - region_count += 1; + current_region.push(row); } } + if current_region.len() >= 3 { + candidate_regions.push(current_region); + } - // Don't forget last region - if region_count >= 3 { - regions.push((region_start - 5.0, region_end + 5.0)); + // Step 4: Cross-row column alignment check per region + // Real tables have consistent column X positions across rows (high pairwise score). + // Paragraph text has varying word positions line-to-line (low pairwise score). + let mut regions = Vec::new(); + for region_rows in &candidate_regions { + let num_rows = region_rows.len(); + let mut total_score = 0.0f32; + let mut pair_count = 0u32; + let tolerance = 10.0f32; + + for i in 0..num_rows { + for j in (i + 1)..num_rows { + let centers_a = ®ion_rows[i].1; + let centers_b = ®ion_rows[j].1; + + let matches_a = centers_a + .iter() + .filter(|&&a| centers_b.iter().any(|&b| (a - b).abs() < tolerance)) + .count(); + let matches_b = centers_b + .iter() + .filter(|&&b| centers_a.iter().any(|&a| (a - b).abs() < tolerance)) + .count(); + + let max_len = centers_a.len().max(centers_b.len()); + if max_len > 0 { + total_score += (matches_a + matches_b) as f32 / (2 * max_len) as f32; + pair_count += 1; + } + } + } + + let avg_score = if pair_count > 0 { + total_score / pair_count as f32 + } else { + 0.0 + }; + if avg_score >= 0.5 { + let y_min = region_rows.first().unwrap().0; + let y_max = region_rows.last().unwrap().0; + regions.push((y_min - 5.0, y_max + 5.0)); + } } regions @@ -490,7 +532,9 @@ fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bo TableDetectionMode::BodyFont => 0.3, }; - pct_data > min_pct || num_cols >= 5 + // For SmallFont, bypass content check for wide tables (5+ columns may have text headers). + // For BodyFont, always require data-like content to prevent paragraph false positives. + pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5) } /// Check if a cell value looks like table data @@ -1269,6 +1313,52 @@ mod tests { ); } + #[test] + fn test_word_level_paragraph_not_detected_as_table() { + // Paragraph text with per-word TextItems (as produced by some PDFs). + // Word X positions vary from line to line — NOT a table. + let items = vec![ + // Line 1 + make_item("We", 72.0, 500.0, 10.0), + make_item("would", 95.0, 500.0, 10.0), + make_item("like", 145.0, 500.0, 10.0), + make_item("to", 180.0, 500.0, 10.0), + make_item("thank", 200.0, 500.0, 10.0), + make_item("all", 250.0, 500.0, 10.0), + make_item("the", 278.0, 500.0, 10.0), + make_item("practitioners", 305.0, 500.0, 10.0), + // Line 2 + make_item("and", 72.0, 485.0, 10.0), + make_item("researchers", 105.0, 485.0, 10.0), + make_item("across", 185.0, 485.0, 10.0), + make_item("the", 232.0, 485.0, 10.0), + make_item("University", 260.0, 485.0, 10.0), + make_item("of", 335.0, 485.0, 10.0), + make_item("Leeds", 355.0, 485.0, 10.0), + // Line 3 + make_item("Libraries", 72.0, 470.0, 10.0), + make_item("whose", 142.0, 470.0, 10.0), + make_item("contributions", 190.0, 470.0, 10.0), + make_item("made", 290.0, 470.0, 10.0), + make_item("this", 328.0, 470.0, 10.0), + make_item("report", 360.0, 470.0, 10.0), + // Line 4 + make_item("possible", 72.0, 455.0, 10.0), + make_item("Both", 140.0, 455.0, 10.0), + make_item("constituent", 178.0, 455.0, 10.0), + make_item("studies", 262.0, 455.0, 10.0), + make_item("were", 315.0, 455.0, 10.0), + make_item("approved", 350.0, 455.0, 10.0), + ]; + + let tables = detect_tables(&items, 10.0); + assert_eq!( + tables.len(), + 0, + "Word-level paragraph text must not be detected as table" + ); + } + #[test] fn test_large_data_table_not_rejected() { // 50-row table at small font — must not be rejected by row limit