feat(tables): detect tables from clip-path rects via merged-cluster fallback

Extract axis-aligned rectangles from W/W* clip operators in content
streams — many PDFs define table cells as clipping paths instead of
stroked rects. Add merged-cluster fallback in detect_tables_from_rects()
that merges all cluster rects when per-cluster detection fails or only
produces narrow false-positives (≤3 columns). Uses rect Y-edges for
rows and text X-clustering for columns.

Also updates lopdf to firecrawl fork (fix-leading-whitespace branch).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Cargo.toml

@@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector"
 
 [dependencies]
 # PDF parsing
-lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "dc2887a", features = ["rayon"] }
+lopdf = { git = "https://github.com/firecrawl/lopdf", branch = "firecrawl/fix-leading-whitespace", features = ["rayon"] }
 
 # Error handling
 thiserror = "2.0"

src/extractor/content_stream.rs

@@ -30,6 +30,7 @@ pub(crate) fn extract_page_text_items(
 
     let mut items = Vec::new();
     let mut rects: Vec<PdfRect> = Vec::new();
+    let mut clip_rects: Vec<PdfRect> = Vec::new();
     let mut lines: Vec<PdfLine> = Vec::new();
 
     // Path construction state for m/l/h → S/s line extraction
@@ -698,6 +699,60 @@ pub(crate) fn extract_page_text_items(
                 path_subpath_start = None;
                 path_current = None;
             }
+            "W" | "W*" => {
+                // Clip operator: check if pending path forms an axis-aligned rectangle.
+                // Many PDFs define table cells as clipping paths instead of stroked rects.
+                let mut segs: Vec<(f32, f32, f32, f32)> = pending_lines.clone();
+                // If only 3 segments, synthesize closing segment back to subpath start
+                if segs.len() == 3 {
+                    if let Some((sx, sy)) = path_subpath_start {
+                        let (_, _, ex, ey) = segs[2];
+                        if (ex - sx).abs() > 0.01 || (ey - sy).abs() > 0.01 {
+                            segs.push((ex, ey, sx, sy));
+                        }
+                    }
+                }
+                if segs.len() == 4 {
+                    // Collect all endpoints and compute bounding box
+                    let mut xs = Vec::with_capacity(8);
+                    let mut ys = Vec::with_capacity(8);
+                    for &(x1, y1, x2, y2) in &segs {
+                        xs.push(x1);
+                        xs.push(x2);
+                        ys.push(y1);
+                        ys.push(y2);
+                    }
+                    let min_x = xs.iter().copied().fold(f32::INFINITY, f32::min);
+                    let max_x = xs.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+                    let min_y = ys.iter().copied().fold(f32::INFINITY, f32::min);
+                    let max_y = ys.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+                    let w = max_x - min_x;
+                    let h = max_y - min_y;
+                    // Verify all points lie on bounding box edges (axis-aligned rectangle)
+                    let eps: f32 = 0.5;
+                    let axis_aligned = xs
+                        .iter()
+                        .all(|&x| (x - min_x).abs() < eps || (x - max_x).abs() < eps)
+                        && ys
+                            .iter()
+                            .all(|&y| (y - min_y).abs() < eps || (y - max_y).abs() < eps);
+                    if axis_aligned && w > 1.0 && h > 1.0 {
+                        // Transform to device space using CTM (same as `re` handler)
+                        let x_dev = min_x * ctm[0] + min_y * ctm[2] + ctm[4];
+                        let y_dev = min_x * ctm[1] + min_y * ctm[3] + ctm[5];
+                        let w_dev = w * ctm[0];
+                        let h_dev = h * ctm[3];
+                        clip_rects.push(PdfRect {
+                            x: x_dev,
+                            y: y_dev,
+                            width: w_dev,
+                            height: h_dev,
+                            page: page_num,
+                        });
+                    }
+                }
+                // Do NOT clear pending_lines — the following `n` does that
+            }
             "n" => {
                 // end path (no-op): discard
                 pending_lines.clear();
@@ -708,6 +763,12 @@ pub(crate) fn extract_page_text_items(
         }
     }
 
+    // Only use clipping-path rects when no `re` rects exist on this page,
+    // to avoid diluting real table rects with decorative clip regions.
+    if rects.is_empty() && !clip_rects.is_empty() {
+        rects = clip_rects;
+    }
+
     let items = super::merge_text_items(items);
     Ok((items, rects, lines))
 }

src/tables/detect_rects.rs

@@ -229,6 +229,38 @@ pub fn detect_tables_from_rects(
                 tables.push(table);
             }
         }
+
+        // Merged-cluster fallback: when per-cluster attempts produce no tables
+        // or only narrow false-positives (≤3 columns from individual column
+        // clusters), merge all cluster rects and try row-stripe strategy with
+        // text-based column detection.
+        let only_narrow = !tables.is_empty() && tables.iter().all(|t| t.columns.len() <= 3);
+        if tables.is_empty() || only_narrow {
+            let total_clustered: usize = clusters.iter().map(|c| c.len()).sum();
+            if clusters.len() >= 3 && total_clustered >= 50 {
+                debug!(
+                    "page {}: trying merged-cluster fallback ({} clusters, {} rects{})",
+                    page,
+                    clusters.len(),
+                    total_clustered,
+                    if only_narrow {
+                        ", replacing narrow tables"
+                    } else {
+                        ""
+                    }
+                );
+                let all_cluster_rects: Vec<(f32, f32, f32, f32)> = clusters
+                    .iter()
+                    .flat_map(|idxs| idxs.iter().map(|&i| page_rects[i]))
+                    .collect();
+                if let Some(table) = detect_merged_cluster_table(items, &all_cluster_rects, page) {
+                    if only_narrow {
+                        tables.clear();
+                    }
+                    tables.push(table);
+                }
+            }
+        }
     }
 
     // On rect-sparse pages (≤ 6 rects), a few cell-border rects may define the
@@ -820,6 +852,174 @@ fn detect_row_stripe_table(
     })
 }
 
+/// Detect a table by merging all cluster rects into one group.
+///
+/// This handles clip-path PDFs where each column's cell rects form a separate
+/// cluster (no spatial overlap between columns). Uses rect Y-edges for rows
+/// and text X-position clustering for columns, similar to `detect_row_stripe_table`
+/// but without the width-uniformity check.
+fn detect_merged_cluster_table(
+    items: &[TextItem],
+    all_rects: &[(f32, f32, f32, f32)],
+    page: u32,
+) -> Option<Table> {
+    // Extract Y-edges from all rects
+    let mut y_vals: Vec<f32> = Vec::new();
+    for &(_, y, _, h) in all_rects {
+        y_vals.push(y);
+        y_vals.push(y + h);
+    }
+    let y_edges = snap_edges(&y_vals, 6.0);
+
+    if y_edges.len() < 4 {
+        debug!("  merged-cluster rejected: only {} y-edges", y_edges.len());
+        return None;
+    }
+
+    let mut row_edges = y_edges;
+    row_edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
+
+    // Bounding box of all rects
+    let y_top = row_edges[0];
+    let y_bottom = *row_edges.last().unwrap();
+    let x_left = all_rects
+        .iter()
+        .map(|&(x, _, _, _)| x)
+        .reduce(f32::min)
+        .unwrap();
+    let x_right = all_rects
+        .iter()
+        .map(|&(x, _, w, _)| x + w)
+        .reduce(f32::max)
+        .unwrap();
+
+    // Gather page items within the bounding box
+    let page_items: Vec<(usize, &TextItem)> = items
+        .iter()
+        .enumerate()
+        .filter(|(_, item)| {
+            item.page == page
+                && item.y >= y_bottom - 2.0
+                && item.y <= y_top + 2.0
+                && item.x >= x_left - 5.0
+                && item.x + item.width <= x_right + 5.0
+        })
+        .collect();
+
+    if page_items.is_empty() {
+        return None;
+    }
+
+    // Derive columns from text X-position clustering
+    let columns = cluster_x_positions(&page_items, 15.0);
+
+    if columns.len() < 2 {
+        debug!(
+            "  merged-cluster rejected: only {} columns from text clustering",
+            columns.len()
+        );
+        return None;
+    }
+
+    // Convert column centers to edges
+    let mut col_edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
+    let min_x = page_items
+        .iter()
+        .map(|(_, i)| i.x)
+        .reduce(f32::min)
+        .unwrap();
+    col_edges.push(min_x - 5.0);
+    for pair in columns.windows(2) {
+        col_edges.push((pair[0] + pair[1]) / 2.0);
+    }
+    let max_x_right = page_items
+        .iter()
+        .map(|(_, i)| i.x + i.width)
+        .reduce(f32::max)
+        .unwrap();
+    col_edges.push(max_x_right + 5.0);
+
+    let num_cols = col_edges.len() - 1;
+    let num_rows = row_edges.len() - 1;
+
+    debug!(
+        "  merged-cluster grid: {}x{} ({} col edges, {} row edges)",
+        num_rows,
+        num_cols,
+        col_edges.len(),
+        row_edges.len()
+    );
+
+    // Assign items to grid
+    let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
+
+    if item_indices.is_empty() {
+        debug!("  merged-cluster rejected: no items assigned");
+        return None;
+    }
+
+    // Validate: >=2 non-empty rows
+    let non_empty_rows = cells
+        .iter()
+        .filter(|row| row.iter().any(|c| !c.trim().is_empty()))
+        .count();
+    if non_empty_rows < 2 {
+        debug!(
+            "  merged-cluster rejected: only {} non-empty rows",
+            non_empty_rows
+        );
+        return None;
+    }
+
+    // Content density: >=40%
+    let total_cells = (num_cols * num_rows) as f32;
+    let non_empty_cells = cells
+        .iter()
+        .flat_map(|row| row.iter())
+        .filter(|c| !c.trim().is_empty())
+        .count();
+    let content_ratio = non_empty_cells as f32 / total_cells;
+    if content_ratio < 0.40 {
+        debug!(
+            "  merged-cluster rejected: content ratio {:.2} < 0.40",
+            content_ratio
+        );
+        return None;
+    }
+
+    // No empty columns
+    for col in 0..num_cols {
+        let col_has_content = cells
+            .iter()
+            .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty()));
+        if !col_has_content {
+            debug!("  merged-cluster rejected: column {} is empty", col);
+            return None;
+        }
+    }
+
+    let column_centers: Vec<f32> = (0..num_cols)
+        .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0)
+        .collect();
+    let row_centers: Vec<f32> = (0..num_rows)
+        .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0)
+        .collect();
+
+    debug!(
+        "  merged-cluster table accepted: {}x{}, {:.0}% density",
+        num_rows,
+        num_cols,
+        content_ratio * 100.0
+    );
+
+    Some(Table {
+        columns: column_centers,
+        rows: row_centers,
+        cells,
+        item_indices,
+    })
+}
+
 /// Cluster text item X positions into column centers with a given minimum threshold.
 ///
 /// Similar to `find_column_boundaries` in grid.rs but with a lower minimum threshold