diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 7810219..5009b00 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -149,6 +149,62 @@ pub fn detect_tables_from_rects( page_rects.push((x, y, w, h)); } + // Remove rects that are much wider than typical cell rects — these are + // page-spanning clipping paths or row-spanning background fills that + // would add spurious X-edges and corrupt the grid. We use the median + // WIDTH (not area) because row-stripe tables have ALL rects at the same + // full width, so their median width equals the full table width and none + // get filtered. Cell-grid tables have narrow cell rects, so full-width + // background fills stand out clearly. + if page_rects.len() >= 6 { + let mut widths: Vec = page_rects.iter().map(|&(_, _, w, _)| w).collect(); + widths.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_width = widths[widths.len() / 2]; + let width_threshold = median_width * 10.0; + let before = page_rects.len(); + page_rects.retain(|&(_, _, w, _)| w <= width_threshold); + if page_rects.len() < before { + debug!( + "page {}: removed {} oversized rects (median_w={:.0}, threshold={:.0})", + page, + before - page_rects.len(), + median_width, + width_threshold, + ); + } + + // Deduplicate sub-rects: when a rect is fully contained within a + // slightly larger rect (same column, interior Y range), the smaller + // one is a cell-internal decoration (e.g. content-area shading + // inside the full cell background). Keeping both creates spurious + // Y-edges that split visual rows into thin sub-rows. + // + // Only remove when the container is a similarly-sized cell (height + // ratio < 4×), NOT when the container is a table-wide background + // that dwarfs the sub-rect. + let before = page_rects.len(); + let snapshot = page_rects.clone(); + page_rects.retain(|&(ax, ay, aw, ah)| { + let tol = 2.0; + !snapshot.iter().any(|&(bx, by, bw, bh)| { + // b must strictly contain a (b is larger in area) + bw * bh > aw * ah * 1.2 + && bh < ah * 4.0 // container must be similarly sized, not a table background + && bx <= ax + tol + && (bx + bw) >= (ax + aw) - tol + && by <= ay + tol + && (by + bh) >= (ay + ah) - tol + }) + }); + if page_rects.len() < before { + debug!( + "page {}: removed {} contained sub-rects", + page, + before - page_rects.len(), + ); + } + } + debug!( "page {}: {} rects after size filter (from {} raw)", page, @@ -266,6 +322,14 @@ pub(crate) fn detect_table_from_rect_group( let x_edges = snap_edges(&x_edges, 6.0); let y_edges = snap_edges(&y_edges, 6.0); + debug!( + " edges: {} x, {} y — grid {}x{}", + x_edges.len(), + y_edges.len(), + y_edges.len().saturating_sub(1), + x_edges.len().saturating_sub(1), + ); + if x_edges.len() < 3 || y_edges.len() < 4 { debug!( " rejected: {} x-edges, {} y-edges (need >=3, >=4)", @@ -288,9 +352,11 @@ pub(crate) fn detect_table_from_rect_group( return None; } - // Reject grids that are too large — real tables rarely exceed 12 columns. - // Form-style PDFs with scattered field boxes produce huge sparse grids. - if num_cols > 12 { + // Reject grids that are too large — form-style PDFs with scattered field + // boxes produce huge sparse grids. Statistical lookup tables (e.g. MWU, + // chi-square) can legitimately have 20+ columns, so allow up to 25. + if num_cols > 25 { + debug!(" rejected: {} columns > 25", num_cols); return None; } @@ -336,7 +402,12 @@ pub(crate) fn detect_table_from_rect_group( // Consolidate vertically-merged cells: rects spanning multiple grid rows // should have their text collected into the first sub-row. - propagate_merged_cells(&mut cells, &col_edges, &row_edges, group_rects); + // Skip for wide tables (>10 columns) where spanning rects are typically + // background fills rather than true merged cells (e.g. statistical lookup + // tables with row-grouping shading). + if num_cols <= 10 { + propagate_merged_cells(&mut cells, &col_edges, &row_edges, group_rects); + } // Compute column centers and row centers for the Table struct let columns: Vec = (0..num_cols) @@ -708,9 +779,9 @@ fn detect_row_stripe_table( .filter(|c| !c.trim().is_empty()) .count(); let content_ratio = non_empty_cells as f32 / total_cells; - if content_ratio < 0.25 { + if content_ratio < 0.40 { debug!( - " row-stripe rejected: content ratio {:.2} < 0.25", + " row-stripe rejected: content ratio {:.2} < 0.40", content_ratio ); return None; diff --git a/src/tables/format.rs b/src/tables/format.rs index 2c8e305..b6a9e52 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -88,6 +88,8 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { // Check if this is a continuation row (first column is empty but others have content). // A row with only 1 short non-empty cell (besides the first) is more likely a // section sub-header (e.g. "JAN", "FEB") than overflow text — don't merge it. + // A row with content in many columns is a real data row with a merged/spanning + // first-column cell (e.g. n₂ in a statistical table), not text overflow. let non_first_cells: Vec<&str> = row .iter() .skip(1) @@ -95,9 +97,31 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { .filter(|c| !c.is_empty()) .collect(); let is_short_subheader = non_first_cells.len() == 1 && non_first_cells[0].len() <= 5; + // Rows with multiple short-valued cells (e.g. numeric data in a lookup + // table) are data rows with a merged/spanning first column, not text + // overflow from the previous row. Continuation rows typically have + // longer descriptive text; data rows have short numeric values. + let avg_cell_len = if non_first_cells.is_empty() { + 0.0 + } else { + non_first_cells.iter().map(|c| c.len()).sum::() as f32 + / non_first_cells.len() as f32 + }; + let numeric_cells = non_first_cells + .iter() + .filter(|c| { + c.chars().all(|ch| { + ch.is_ascii_digit() || ch == '.' || ch == '-' || ch == ',' || ch == ' ' + }) + }) + .count(); + let looks_like_data_row = non_first_cells.len() >= 2 + && avg_cell_len <= 10.0 + && numeric_cells > non_first_cells.len() / 2; let is_continuation = first_cell.is_empty() && !non_first_cells.is_empty() && !is_short_subheader + && !looks_like_data_row && cleaned.len() > 1; // Don't merge into the first row (header) if is_continuation {