feat(tables): compete normalized table and chart evidence (#172)

* feat(tables): compete normalized table and chart evidence

* fix(tables): preserve normalized hypotheses

src/lib.rs

@@ -1527,16 +1527,10 @@ mod vector_grid_tests {
     /// strip while body rows are drawn with `m`/`l` operators, so the rect
     /// cluster has only 2 Y-edges and `try_build_grid` rejects.
     ///
-    /// IGNORED: lifting this shape required the exact-duplicate early-dedup
-    /// (PR #76 first iteration), which had broad collateral damage on
-    /// SEC 10-K TOCs and similar docs that draw rule-rects above + below
-    /// section dividers (production diff: 0001104659-25-093871 lost its
-    /// TOC structure, perf-graph data table, and qualifications matrix).
-    /// Re-enable once a more surgical lift exists in `try_build_grid` or
-    /// `snap_edges` that handles cell-border + inner-fill + text-bg rect
-    /// triplets without page-wide dedup.
+    /// The page repeats a full-page background many times. Those fills must be
+    /// removed from clustering before chart/table evidence is evaluated, or
+    /// they swamp the real cell rectangles and make this table look chart-like.
     #[test]
-    #[ignore]
     fn greencomp_competence_two_cols() {
         let tables = detect_rect_tables_in_fixture("tests/fixtures/greencomp_competence.pdf");
         assert!(

src/tables/detect_rects.rs

@@ -8,6 +8,9 @@ use crate::types::{PdfRect, TextItem};
 
 use super::Table;
 
+const DOMINANT_PAGE_BACKGROUND_MIN_REPETITIONS: usize = 8;
+const COMPETING_TABLE_MIN_ROWS: usize = 8;
+
 /// Disjoint-set (union-find) with component sizes for clustering indices.
 struct UnionFind {
     parent: Vec<usize>,
@@ -287,6 +290,16 @@ pub fn detect_chart_regions(
     regions
 }
 
+fn detect_direct_rect_table(
+    items: &[TextItem],
+    rects: &[(f32, f32, f32, f32)],
+    page: u32,
+) -> Option<Table> {
+    detect_table_from_rect_group(items, rects, page)
+        .or_else(|| detect_row_stripe_table(items, rects, page))
+        .or_else(|| detect_stacked_box_table(items, rects, page))
+}
+
 pub fn detect_tables_from_rects(
     items: &[TextItem],
     rects: &[PdfRect],
@@ -440,7 +453,7 @@ pub fn detect_tables_from_rects(
             .collect();
 
         debug!("page {}: {} clusters with >= 6 rects", page, clusters.len());
-        let mut chart_cluster_ids: Vec<usize> = Vec::new();
+        let mut merge_excluded_cluster_ids: Vec<usize> = Vec::new();
         for (cluster_id, cluster_indices) in clusters.iter().enumerate() {
             let group_rects: Vec<(f32, f32, f32, f32)> =
                 cluster_indices.iter().map(|&i| page_rects[i]).collect();
@@ -449,19 +462,68 @@ pub fn detect_tables_from_rects(
             // entirely so it can't reach any detector, the merged fallback,
             // or the hint fallback.
             if is_chart_bar_cluster(items, &group_rects, page) {
-                debug!(
-                    "page {}: skipping chart-bar cluster ({} rects)",
-                    page,
-                    group_rects.len()
-                );
-                chart_cluster_ids.push(cluster_id);
-                continue;
+                // Repeated page fills can dominate the geometry and make a
+                // real shaded-cell table look like a chart. Remove those fills,
+                // re-cluster the remaining geometry, and evaluate valid table
+                // candidates as a competing hypothesis before the chart
+                // rejection wins.
+                let normalized = without_dominant_page_backgrounds(&group_rects);
+                let normalized_table = (normalized.len() < group_rects.len())
+                    .then(|| {
+                        cluster_rects(&normalized, 3.0, 6)
+                            .iter()
+                            .filter_map(|indices| {
+                                let candidate: Vec<(f32, f32, f32, f32)> =
+                                    indices.iter().map(|&i| normalized[i]).collect();
+                                if is_chart_bar_cluster(items, &candidate, page) {
+                                    None
+                                } else {
+                                    detect_table_from_rect_group(items, &candidate, page)
+                                        .or_else(|| {
+                                            detect_row_stripe_table_from_cell_rects(
+                                                items, &candidate, page,
+                                            )
+                                        })
+                                        // Small chart panels can still form
+                                        // plausible grids from their labels.
+                                        // Require sustained row evidence; the
+                                        // motivating table has 17 rows.
+                                        .filter(|table| {
+                                            table.rows.len() >= COMPETING_TABLE_MIN_ROWS
+                                        })
+                                }
+                            })
+                            .max_by_key(|table| table.rows.len() * table.columns.len())
+                    })
+                    .flatten();
+                if let Some(table) = normalized_table {
+                    debug!(
+                        "page {}: chart-like cluster normalized from {} to {} rects; accepted {}x{} table hypothesis",
+                        page,
+                        group_rects.len(),
+                        normalized.len(),
+                        table.rows.len(),
+                        table.columns.len()
+                    );
+                    // The accepted hypothesis is based on normalized
+                    // geometry. Keep the original chart-like cluster out of
+                    // the merged fallback: reintroducing its repeated page
+                    // fills can manufacture a wider candidate that replaces
+                    // this valid narrow table below.
+                    merge_excluded_cluster_ids.push(cluster_id);
+                    tables.push(table);
+                    continue;
+                } else {
+                    debug!(
+                        "page {}: skipping chart-bar cluster ({} rects)",
+                        page,
+                        group_rects.len()
+                    );
+                    merge_excluded_cluster_ids.push(cluster_id);
+                    continue;
+                }
             }
-            if let Some(table) = detect_table_from_rect_group(items, &group_rects, page) {
-                tables.push(table);
-            } else if let Some(table) = detect_row_stripe_table(items, &group_rects, page) {
-                tables.push(table);
-            } else if let Some(table) = detect_stacked_box_table(items, &group_rects, page) {
+            if let Some(table) = detect_direct_rect_table(items, &group_rects, page) {
                 tables.push(table);
             } else if let Some((left, right)) = split_wide_cluster(&group_rects, 15.0, 6) {
                 // Cluster was too wide — retry each half independently
@@ -500,7 +562,7 @@ pub fn detect_tables_from_rects(
             let table_clusters: Vec<&Vec<usize>> = clusters
                 .iter()
                 .enumerate()
-                .filter(|(id, _)| !chart_cluster_ids.contains(id))
+                .filter(|(id, _)| !merge_excluded_cluster_ids.contains(id))
                 .map(|(_, c)| c)
                 .collect();
             let total_clustered: usize = table_clusters.iter().map(|c| c.len()).sum();
@@ -1904,6 +1966,36 @@ fn row_stripe_is_sparse_prose_outline(cells: &[Vec<String>]) -> bool {
     long_dense_cells * 2 >= dense_count
 }
 
+/// Remove repeated page-scale fills from a chart-like cluster so the actual
+/// cell/bar geometry can be evaluated independently. A small number of
+/// coincident origin frames may be meaningful table structure, so repetition
+/// only becomes normalization evidence when it dominates the cluster.
+fn without_dominant_page_backgrounds(rects: &[(f32, f32, f32, f32)]) -> Vec<(f32, f32, f32, f32)> {
+    let x_max = rects
+        .iter()
+        .map(|&(x, _, width, _)| x + width)
+        .fold(0.0_f32, f32::max);
+    let y_max = rects
+        .iter()
+        .map(|&(_, y, _, height)| y + height)
+        .fold(0.0_f32, f32::max);
+    let is_page_scale = |&(x, y, width, height): &(f32, f32, f32, f32)| {
+        x < 5.0 && y < 5.0 && width >= x_max * 0.9 && height >= y_max * 0.9
+    };
+
+    if rects.iter().filter(|rect| is_page_scale(rect)).count()
+        < DOMINANT_PAGE_BACKGROUND_MIN_REPETITIONS
+    {
+        return rects.to_vec();
+    }
+
+    rects
+        .iter()
+        .filter(|rect| !is_page_scale(rect))
+        .copied()
+        .collect()
+}
+
 /// Detect a table from cell-background rects that failed grid detection.
 ///
 /// Uses rect Y-edges for row boundaries and text X-position clustering for
@@ -2952,6 +3044,20 @@ mod tests {
         assert!(hints.is_empty(), "chart bars must not become a hint region");
     }
 
+    #[test]
+    fn dominant_page_backgrounds_are_normalized_only_after_repetition() {
+        let page_fill = (0.0, 0.0, 600.0, 800.0);
+        let cell = (100.0, 500.0, 120.0, 20.0);
+
+        let mut dominant = vec![page_fill; DOMINANT_PAGE_BACKGROUND_MIN_REPETITIONS];
+        dominant.push(cell);
+        assert_eq!(without_dominant_page_backgrounds(&dominant), vec![cell]);
+
+        let mut incidental = vec![page_fill; DOMINANT_PAGE_BACKGROUND_MIN_REPETITIONS - 1];
+        incidental.push(cell);
+        assert_eq!(without_dominant_page_backgrounds(&incidental), incidental);
+    }
+
     #[test]
     fn uniform_cell_grid_is_not_a_chart() {
         // Touching, uniform-height cell rects (a real table) must not match:

src/tables/format.rs

@@ -188,6 +188,21 @@ fn starts_with_numbered_label(cell: &str) -> bool {
             .is_some_and(|c| matches!(c, '.' | ')' | '-' | ':'))
 }
 
+fn starts_with_hierarchical_numbered_label(cell: &str) -> bool {
+    let token = cell
+        .split_whitespace()
+        .next()
+        .unwrap_or("")
+        .trim_end_matches(['.', ')', ':', '-']);
+    let levels: Vec<&str> = token.split('.').collect();
+    (2..=4).contains(&levels.len())
+        && levels.iter().all(|level| {
+            !level.is_empty()
+                && level.len() <= 3
+                && level.chars().all(|character| character.is_ascii_digit())
+        })
+}
+
 fn alpha_word_count(cell: &str) -> usize {
     cell.split_whitespace()
         .filter(|word| word.chars().any(|c| c.is_alphabetic()))
@@ -341,11 +356,12 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
         // mid-sentence/lowercase ("continued text here", "with 3.5%...") or
         // carry lowercase fragments in the later cells, so keep those mergeable.
         let looks_like_hierarchical_subrow = first_cell.is_empty()
-            && row.len() >= 3
             && first_non_empty_col == Some(1)
             && looks_like_compact_entry_label(first_non_empty_cell)
-            && ((non_first_cells.len() >= 2 && title_like_later_cells > 0)
+            && ((row.len() == 2 && starts_with_hierarchical_numbered_label(first_non_empty_cell))
+                || (row.len() >= 3 && non_first_cells.len() >= 2 && title_like_later_cells > 0)
                 || (non_first_cells.len() == 1
+                    && row.len() >= 3
                     && prev_first_cell_empty
                     && alpha_word_count(first_non_empty_cell) >= 2));
         let looks_like_new_first_column_entry = !first_cell.is_empty()
@@ -688,6 +704,46 @@ mod tests {
         assert_eq!(cleaned[4][1], "Model training");
     }
 
+    #[test]
+    fn test_clean_table_cells_two_column_numbered_subrows_not_merged() {
+        let cells = vec![
+            vec!["Area".into(), "Competence".into()],
+            vec![
+                "1. Embodying sustainability values".into(),
+                "1.1 Valuing sustainability".into(),
+            ],
+            vec!["".into(), "1.2 Supporting fairness".into()],
+            vec!["".into(), "1.3 Promoting nature".into()],
+            vec![
+                "2. Embracing complexity".into(),
+                "2.1 Systems thinking".into(),
+            ],
+            vec!["".into(), "2.2 Critical thinking".into()],
+        ];
+        let (cleaned, _) = clean_table_cells(&cells);
+
+        assert_eq!(cleaned.len(), 6);
+        assert_eq!(cleaned[2], vec!["", "1.2 Supporting fairness"]);
+        assert_eq!(cleaned[3], vec!["", "1.3 Promoting nature"]);
+        assert_eq!(cleaned[5], vec!["", "2.2 Critical thinking"]);
+    }
+
+    #[test]
+    fn test_clean_table_cells_two_column_numbered_continuation_merges() {
+        let cells = vec![
+            vec!["Area".into(), "Requirement".into()],
+            vec!["Safety".into(), "The program includes".into()],
+            vec!["".into(), "1. First requirement for every operator".into()],
+        ];
+        let (cleaned, _) = clean_table_cells(&cells);
+
+        assert_eq!(cleaned.len(), 2);
+        assert_eq!(
+            cleaned[1][1],
+            "The program includes 1. First requirement for every operator"
+        );
+    }
+
     #[test]
     fn test_clean_table_cells_partial_hierarchical_subrow_not_merged() {
         let cells = vec![