tables: lift detection on shaded-header + alt-row tables (#wired-grids)

Production telemetry on `wired_high_confidence`-classified table regions
showed `detect_vector_grid_in_region_mem` returning a usable grid only
~27% of the time, with the rest falling through to GLM-OCR. Three
surgical fixes target the dominant production shapes:

* Path-fill cell backgrounds: when the page has no `re` rects but draws
  cell backgrounds via `m`/`l`/`h`/`f*` sequences, prefer the fill-derived
  rects over the few section-level `W*` clip paths that previously won
  the priority gate. Activated when fill rects outnumber clip rects ≥3×.

* Dedup-induced cluster splits: page-background rects could pose as
  containers in the sub-rect dedup and evict a slightly smaller
  table-frame rect, breaking adjacency between column-cell groups so each
  column became its own cluster. Origin-anchored containers are now
  disqualified from sub-rect dedup. A separate exact-duplicate pass
  collapses the cell-padding/text-bg/cell-border triple emissions some
  PDFs produce, preserving original order to avoid reshuffling table
  output on multi-table pages.

* Prose-words rejection: the `cell-rect` fallback's whole-grid prose
  threshold also rejected real tables that include a description column.
  Now relaxed when content is well-distributed (≥75% of cols filled),
  while keeping the original strictness for prose-in-a-frame layouts.

Two regression fixtures from the opendataloader-bench corpus, covering
the dominant production failure categories:

* `greencomp_competence.pdf` — 2-col shaded-header + plain-body glossary.
  Mirrors production crops #1 (Contractions glossary) and #6 (BIO 350
  course header).
* `upstage_key_functions.pdf` — 4-col shaded-header + alt-row backgrounds
  + merged left column. Mirrors production crops #2 (Parameter/Value
  alt-row), #7 (Spanish XML schema), and #8 (Córdoba multi-row header).

Existing fixtures stay green (doc 51 wrapped-label, doc 128 forecast
six-cols, td9264 snapshot). 133 unit + integration tests pass; clippy
clean.

Bumps napi/package.json 1.8.4 → 1.8.5.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.8.4",
+  "version": "1.8.5",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/extractor/content_stream.rs

@@ -1015,9 +1015,17 @@ pub(crate) fn extract_page_text_items(
     // producing thousands of identical rects that yield a degenerate grid.
     // After dedup, if too few unique clip rects remain we fall through to
     // fill rects (explicitly drawn visible rectangles).
+    //
+    // When fill rects substantially outnumber clip rects, the clips are
+    // typically section-level wrappers and the fills are the actual table
+    // cell backgrounds (e.g. shaded-header tables drawn with `m`/`l`/`h`/`f*`
+    // sequences). In that case, prefer fills.
     if rects.is_empty() {
         dedup_rects(&mut clip_rects);
-        if clip_rects.len() >= 4 {
+        let prefer_fills = !fill_rects.is_empty() && fill_rects.len() >= clip_rects.len() * 3;
+        if prefer_fills {
+            rects = fill_rects;
+        } else if clip_rects.len() >= 4 {
             rects = clip_rects;
         } else if !fill_rects.is_empty() {
             rects = fill_rects;

src/lib.rs

@@ -1035,6 +1035,90 @@ mod vector_grid_tests {
         );
     }
 
+    /// Helper: load a fixture PDF and run the rect-based table detector.
+    fn detect_rect_tables_in_fixture(path: &str) -> Vec<crate::tables::Table> {
+        use crate::extractor::content_stream::extract_page_text_items;
+        use crate::tables::detect_tables_from_rects;
+        use crate::tounicode::FontCMaps;
+        use lopdf::Document;
+        use std::collections::HashSet;
+        use std::fs;
+
+        let buf = fs::read(path).unwrap();
+        let doc = Document::load_mem(&buf).unwrap();
+        let pages = doc.get_pages();
+        let &page_id = pages.get(&1).unwrap();
+        let needed: HashSet<u32> = HashSet::from([1]);
+        let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
+        let ((items, rects, _lines), _has_gid, _rotated) =
+            extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
+
+        let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
+        rect_tables
+    }
+
+    /// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
+    /// glossary with a green-shaded header row and plain (line-drawn) body cells.
+    /// Mirrors the production failure cohort #1 (Contractions glossary) and #6
+    /// (BIO 350 course header): a few colored header rects sit in a horizontal
+    /// strip while body rows are drawn with `m`/`l` operators, so the rect
+    /// cluster has only 2 Y-edges and `try_build_grid` rejects.
+    #[test]
+    fn greencomp_competence_two_cols() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/greencomp_competence.pdf");
+        assert!(
+            !tables.is_empty(),
+            "expected at least one rect-detected table for shaded-header + plain-body shape"
+        );
+        let t = tables
+            .iter()
+            .max_by_key(|t| t.rows.len() * t.columns.len())
+            .unwrap();
+        assert_eq!(
+            t.columns.len(),
+            2,
+            "GreenComp competence is a 2-column table; got {}: {:?}",
+            t.columns.len(),
+            t.columns
+        );
+        assert!(
+            t.rows.len() >= 6,
+            "expected at least 6 rows of competences; got {}",
+            t.rows.len()
+        );
+    }
+
+    /// Regression for `upstage_key_functions.pdf` — a 4-column "Service Stage /
+    /// Function Name / Explanation / Expected Benefit" table with a blue-shaded
+    /// header band plus alternating row backgrounds. Mirrors production crops
+    /// #2 (Parameter / Value with alternating blue rows) and #7 (Spanish XML
+    /// schema with shaded header). Currently `pdf2md` returns zero markdown
+    /// table rows.
+    #[test]
+    fn upstage_key_functions_four_cols() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/upstage_key_functions.pdf");
+        assert!(
+            !tables.is_empty(),
+            "expected at least one rect-detected table for shaded-header + alt-row shape"
+        );
+        let t = tables
+            .iter()
+            .max_by_key(|t| t.rows.len() * t.columns.len())
+            .unwrap();
+        assert_eq!(
+            t.columns.len(),
+            4,
+            "Service Flow is a 4-column table; got {}: {:?}",
+            t.columns.len(),
+            t.columns
+        );
+        assert!(
+            t.rows.len() >= 8,
+            "expected at least 8 visible body rows; got {}",
+            t.rows.len()
+        );
+    }
+
     #[test]
     fn test_crop_px_bbox_is_plausible_bounds() {
         let crop = [10.0, 20.0, 110.0, 220.0];

src/tables/detect_rects.rs

@@ -277,6 +277,39 @@ pub fn detect_tables_from_rects(
             );
         }
 
+        // Drop exact / near-exact duplicates first.  Many PDFs draw the
+        // same cell rectangle multiple times — once for the cell border,
+        // again for an inner padding fill, plus a per-text-run background
+        // wrapper.  Without this dedup, the contained-sub-rect pass below
+        // can't help (it requires container area to strictly exceed the
+        // sub-rect by 20%), and the duplicated edges over-segment the grid
+        // into spurious thin rows / columns that collapse content density.
+        //
+        // Preserve original order (no sort) — cluster output is keyed by
+        // first-seen index, and a sort here would shuffle the table-emission
+        // order on multi-table pages.
+        if page_rects.len() < MAX_CLUSTER_RECTS {
+            let before = page_rects.len();
+            let mut seen: std::collections::HashSet<(i32, i32, i32, i32)> =
+                std::collections::HashSet::new();
+            page_rects.retain(|&(x, y, w, h)| {
+                let key = (
+                    x.round() as i32,
+                    y.round() as i32,
+                    w.round() as i32,
+                    h.round() as i32,
+                );
+                seen.insert(key)
+            });
+            if page_rects.len() < before {
+                debug!(
+                    "page {}: removed {} duplicate rects",
+                    page,
+                    before - page_rects.len(),
+                );
+            }
+        }
+
         // Deduplicate sub-rects: when a rect is fully contained within a
         // slightly larger rect (same column, interior Y range), the smaller
         // one is a cell-internal decoration (e.g. content-area shading
@@ -285,7 +318,11 @@ pub fn detect_tables_from_rects(
         //
         // Only remove when the container is a similarly-sized cell (height
         // ratio < 4×), NOT when the container is a table-wide background
-        // that dwarfs the sub-rect.
+        // that dwarfs the sub-rect.  Origin-anchored page-background rects
+        // also disqualify as containers — they normally exceed the 4× ratio,
+        // but when the sub-rect is itself a tall table-frame the ratio can
+        // fall under the gate, and dropping the frame collapses cluster
+        // adjacency between adjacent column-cell groups.
         //
         // Skip this O(n²) dedup when there are too many rects — pages with
         // thousands of vector-drawing rects won't benefit from cell dedup.
@@ -295,9 +332,11 @@ pub fn detect_tables_from_rects(
             page_rects.retain(|&(ax, ay, aw, ah)| {
                 let tol = 2.0;
                 !snapshot.iter().any(|&(bx, by, bw, bh)| {
+                    let container_is_page_bg = bx < 5.0 && by < 5.0;
                     // b must strictly contain a (b is larger in area)
                     bw * bh > aw * ah * 1.2
                         && bh < ah * 4.0 // container must be similarly sized, not a table background
+                        && !container_is_page_bg
                         && bx <= ax + tol
                         && (bx + bw) >= (ax + aw) - tol
                         && by <= ay + tol
@@ -1728,9 +1767,17 @@ fn detect_row_stripe_table_from_cell_rects(
     // inside a bounding-box rect (e.g. chat-transcript figures) the
     // word-boundary gaps cluster into many spurious columns, and the
     // resulting cells hold sentence fragments riddled with common English
-    // function words. Count cells with any such word and reject when
-    // 20%+ of non-empty cells match — real tabular data (labels, units,
-    // numbers) rarely contains these words.
+    // function words.
+    //
+    // The 20%-of-cells threshold catches both shapes — a prose paragraph
+    // chunked across cols where every cell carries prose, and a single
+    // prose column flanked by empty cols where the prose dominates the
+    // small population of non-empty cells. To avoid rejecting real data
+    // tables that happen to include one description column, relax only
+    // when content is well-distributed: at least 75% of columns must hold
+    // ≥2 non-empty cells. That excludes the prose-in-a-frame case (one
+    // filled col, the rest empty) while admitting "label / value /
+    // explanation / benefit"-style tables.
     if num_cols >= 4 {
         const PROSE_WORDS: &[&str] = &[
             "a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on",
@@ -1758,11 +1805,33 @@ fn detect_row_stripe_table_from_cell_rects(
             }
         }
         if counted > 0 && prose_cells * 5 >= counted {
+            let filled_cols = (0..num_cols)
+                .filter(|&c| {
+                    cells
+                        .iter()
+                        .filter(|row| {
+                            !row.get(c)
+                                .map(String::as_str)
+                                .unwrap_or("")
+                                .trim()
+                                .is_empty()
+                        })
+                        .count()
+                        >= 2
+                })
+                .count();
+            let well_distributed = filled_cols * 4 >= num_cols * 3;
+            if !well_distributed {
+                debug!(
+                    "  cell-rect rejected: {}/{} cells contain prose function words — likely prose ({}/{} cols filled)",
+                    prose_cells, counted, filled_cols, num_cols
+                );
+                return None;
+            }
             debug!(
-                "  cell-rect rejected: {}/{} cells contain prose function words — likely prose",
-                prose_cells, counted
+                "  cell-rect prose check relaxed: {}/{} cols filled — table-with-description-col",
+                filled_cols, num_cols
             );
-            return None;
         }
     }