diff --git a/Cargo.toml b/Cargo.toml index 8bb2f25..5abf91f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector" [dependencies] # PDF parsing -lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "dc2887a", features = ["rayon"] } +lopdf = { git = "https://github.com/firecrawl/lopdf", branch = "firecrawl/fix-leading-whitespace", features = ["rayon"] } # Error handling thiserror = "2.0" diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index f680663..c596754 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -30,6 +30,7 @@ pub(crate) fn extract_page_text_items( let mut items = Vec::new(); let mut rects: Vec = Vec::new(); + let mut clip_rects: Vec = Vec::new(); let mut lines: Vec = Vec::new(); // Path construction state for m/l/h → S/s line extraction @@ -698,6 +699,60 @@ pub(crate) fn extract_page_text_items( path_subpath_start = None; path_current = None; } + "W" | "W*" => { + // Clip operator: check if pending path forms an axis-aligned rectangle. + // Many PDFs define table cells as clipping paths instead of stroked rects. + let mut segs: Vec<(f32, f32, f32, f32)> = pending_lines.clone(); + // If only 3 segments, synthesize closing segment back to subpath start + if segs.len() == 3 { + if let Some((sx, sy)) = path_subpath_start { + let (_, _, ex, ey) = segs[2]; + if (ex - sx).abs() > 0.01 || (ey - sy).abs() > 0.01 { + segs.push((ex, ey, sx, sy)); + } + } + } + if segs.len() == 4 { + // Collect all endpoints and compute bounding box + let mut xs = Vec::with_capacity(8); + let mut ys = Vec::with_capacity(8); + for &(x1, y1, x2, y2) in &segs { + xs.push(x1); + xs.push(x2); + ys.push(y1); + ys.push(y2); + } + let min_x = xs.iter().copied().fold(f32::INFINITY, f32::min); + let max_x = xs.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let min_y = ys.iter().copied().fold(f32::INFINITY, f32::min); + let max_y = ys.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let w = max_x - min_x; + let h = max_y - min_y; + // Verify all points lie on bounding box edges (axis-aligned rectangle) + let eps: f32 = 0.5; + let axis_aligned = xs + .iter() + .all(|&x| (x - min_x).abs() < eps || (x - max_x).abs() < eps) + && ys + .iter() + .all(|&y| (y - min_y).abs() < eps || (y - max_y).abs() < eps); + if axis_aligned && w > 1.0 && h > 1.0 { + // Transform to device space using CTM (same as `re` handler) + let x_dev = min_x * ctm[0] + min_y * ctm[2] + ctm[4]; + let y_dev = min_x * ctm[1] + min_y * ctm[3] + ctm[5]; + let w_dev = w * ctm[0]; + let h_dev = h * ctm[3]; + clip_rects.push(PdfRect { + x: x_dev, + y: y_dev, + width: w_dev, + height: h_dev, + page: page_num, + }); + } + } + // Do NOT clear pending_lines — the following `n` does that + } "n" => { // end path (no-op): discard pending_lines.clear(); @@ -708,6 +763,12 @@ pub(crate) fn extract_page_text_items( } } + // Only use clipping-path rects when no `re` rects exist on this page, + // to avoid diluting real table rects with decorative clip regions. + if rects.is_empty() && !clip_rects.is_empty() { + rects = clip_rects; + } + let items = super::merge_text_items(items); Ok((items, rects, lines)) } diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 5009b00..0e24a56 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -229,6 +229,38 @@ pub fn detect_tables_from_rects( tables.push(table); } } + + // Merged-cluster fallback: when per-cluster attempts produce no tables + // or only narrow false-positives (≤3 columns from individual column + // clusters), merge all cluster rects and try row-stripe strategy with + // text-based column detection. + let only_narrow = !tables.is_empty() && tables.iter().all(|t| t.columns.len() <= 3); + if tables.is_empty() || only_narrow { + let total_clustered: usize = clusters.iter().map(|c| c.len()).sum(); + if clusters.len() >= 3 && total_clustered >= 50 { + debug!( + "page {}: trying merged-cluster fallback ({} clusters, {} rects{})", + page, + clusters.len(), + total_clustered, + if only_narrow { + ", replacing narrow tables" + } else { + "" + } + ); + let all_cluster_rects: Vec<(f32, f32, f32, f32)> = clusters + .iter() + .flat_map(|idxs| idxs.iter().map(|&i| page_rects[i])) + .collect(); + if let Some(table) = detect_merged_cluster_table(items, &all_cluster_rects, page) { + if only_narrow { + tables.clear(); + } + tables.push(table); + } + } + } } // On rect-sparse pages (≤ 6 rects), a few cell-border rects may define the @@ -820,6 +852,174 @@ fn detect_row_stripe_table( }) } +/// Detect a table by merging all cluster rects into one group. +/// +/// This handles clip-path PDFs where each column's cell rects form a separate +/// cluster (no spatial overlap between columns). Uses rect Y-edges for rows +/// and text X-position clustering for columns, similar to `detect_row_stripe_table` +/// but without the width-uniformity check. +fn detect_merged_cluster_table( + items: &[TextItem], + all_rects: &[(f32, f32, f32, f32)], + page: u32, +) -> Option { + // Extract Y-edges from all rects + let mut y_vals: Vec = Vec::new(); + for &(_, y, _, h) in all_rects { + y_vals.push(y); + y_vals.push(y + h); + } + let y_edges = snap_edges(&y_vals, 6.0); + + if y_edges.len() < 4 { + debug!(" merged-cluster rejected: only {} y-edges", y_edges.len()); + return None; + } + + let mut row_edges = y_edges; + row_edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + // Bounding box of all rects + let y_top = row_edges[0]; + let y_bottom = *row_edges.last().unwrap(); + let x_left = all_rects + .iter() + .map(|&(x, _, _, _)| x) + .reduce(f32::min) + .unwrap(); + let x_right = all_rects + .iter() + .map(|&(x, _, w, _)| x + w) + .reduce(f32::max) + .unwrap(); + + // Gather page items within the bounding box + let page_items: Vec<(usize, &TextItem)> = items + .iter() + .enumerate() + .filter(|(_, item)| { + item.page == page + && item.y >= y_bottom - 2.0 + && item.y <= y_top + 2.0 + && item.x >= x_left - 5.0 + && item.x + item.width <= x_right + 5.0 + }) + .collect(); + + if page_items.is_empty() { + return None; + } + + // Derive columns from text X-position clustering + let columns = cluster_x_positions(&page_items, 15.0); + + if columns.len() < 2 { + debug!( + " merged-cluster rejected: only {} columns from text clustering", + columns.len() + ); + return None; + } + + // Convert column centers to edges + let mut col_edges: Vec = Vec::with_capacity(columns.len() + 1); + let min_x = page_items + .iter() + .map(|(_, i)| i.x) + .reduce(f32::min) + .unwrap(); + col_edges.push(min_x - 5.0); + for pair in columns.windows(2) { + col_edges.push((pair[0] + pair[1]) / 2.0); + } + let max_x_right = page_items + .iter() + .map(|(_, i)| i.x + i.width) + .reduce(f32::max) + .unwrap(); + col_edges.push(max_x_right + 5.0); + + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + debug!( + " merged-cluster grid: {}x{} ({} col edges, {} row edges)", + num_rows, + num_cols, + col_edges.len(), + row_edges.len() + ); + + // Assign items to grid + let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + + if item_indices.is_empty() { + debug!(" merged-cluster rejected: no items assigned"); + return None; + } + + // Validate: >=2 non-empty rows + let non_empty_rows = cells + .iter() + .filter(|row| row.iter().any(|c| !c.trim().is_empty())) + .count(); + if non_empty_rows < 2 { + debug!( + " merged-cluster rejected: only {} non-empty rows", + non_empty_rows + ); + return None; + } + + // Content density: >=40% + let total_cells = (num_cols * num_rows) as f32; + let non_empty_cells = cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let content_ratio = non_empty_cells as f32 / total_cells; + if content_ratio < 0.40 { + debug!( + " merged-cluster rejected: content ratio {:.2} < 0.40", + content_ratio + ); + return None; + } + + // No empty columns + for col in 0..num_cols { + let col_has_content = cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); + if !col_has_content { + debug!(" merged-cluster rejected: column {} is empty", col); + return None; + } + } + + let column_centers: Vec = (0..num_cols) + .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) + .collect(); + let row_centers: Vec = (0..num_rows) + .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) + .collect(); + + debug!( + " merged-cluster table accepted: {}x{}, {:.0}% density", + num_rows, + num_cols, + content_ratio * 100.0 + ); + + Some(Table { + columns: column_centers, + rows: row_centers, + cells, + item_indices, + }) +} + /// Cluster text item X positions into column centers with a given minimum threshold. /// /// Similar to `find_column_boundaries` in grid.rs but with a lower minimum threshold