tables/detect_rects: don't accept relaxed grid on wireless prose

Require rect-derived column evidence before relaxing prose checks for two-column cell-rect fallbacks, so text-position alignment alone cannot synthesize a vector grid on wireless content.

Co-authored-by: Cursor <cursoragent@cursor.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.8.4",
+  "version": "1.8.6",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/extractor/content_stream.rs

@@ -1015,9 +1015,17 @@ pub(crate) fn extract_page_text_items(
     // producing thousands of identical rects that yield a degenerate grid.
     // After dedup, if too few unique clip rects remain we fall through to
     // fill rects (explicitly drawn visible rectangles).
+    //
+    // When fill rects substantially outnumber clip rects, the clips are
+    // typically section-level wrappers and the fills are the actual table
+    // cell backgrounds (e.g. shaded-header tables drawn with `m`/`l`/`h`/`f*`
+    // sequences). In that case, prefer fills.
     if rects.is_empty() {
         dedup_rects(&mut clip_rects);
-        if clip_rects.len() >= 4 {
+        let prefer_fills = !fill_rects.is_empty() && fill_rects.len() >= clip_rects.len() * 3;
+        if prefer_fills {
+            rects = fill_rects;
+        } else if clip_rects.len() >= 4 {
             rects = clip_rects;
         } else if !fill_rects.is_empty() {
             rects = fill_rects;

src/lib.rs

@@ -1035,6 +1035,203 @@ mod vector_grid_tests {
         );
     }
 
+    /// Helper: load a fixture PDF and run the rect-based table detector.
+    fn detect_rect_tables_in_fixture(path: &str) -> Vec<crate::tables::Table> {
+        use crate::extractor::content_stream::extract_page_text_items;
+        use crate::tables::detect_tables_from_rects;
+        use crate::tounicode::FontCMaps;
+        use lopdf::Document;
+        use std::collections::HashSet;
+        use std::fs;
+
+        let buf = fs::read(path).unwrap();
+        let doc = Document::load_mem(&buf).unwrap();
+        let pages = doc.get_pages();
+        let &page_id = pages.get(&1).unwrap();
+        let needed: HashSet<u32> = HashSet::from([1]);
+        let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
+        let ((items, rects, _lines), _has_gid, _rotated) =
+            extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
+
+        let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
+        rect_tables
+    }
+
+    /// Regression for the prose-in-a-frame failure mode introduced by the
+    /// shaded-header detection lift (PR #76). The accessory_building permit
+    /// form has a paragraph of legal text laid out in a 2-column justified
+    /// block; the new fill-priority + dedup changes start producing rects
+    /// for it, and the rect detector then admits a 10×2 fake table where
+    /// every cell holds a sentence fragment ("I agree to comply...", "I",
+    /// "It is the property owner's responsibility..."). This test asserts
+    /// the detector REJECTS that fake table — only the real 5×3 form data
+    /// table (TYPE / SIZE / SETBACKS) should survive. See pdf-evals PR #30
+    /// for the original score regression that surfaced this.
+    #[test]
+    fn accessory_building_rejects_prose_in_frame() {
+        let tables = detect_rect_tables_in_fixture(
+            "tests/fixtures/accessory_building_permit_prose_frame.pdf",
+        );
+        // Real form data table (TYPE / SIZE / SETBACKS) must still be detected.
+        let data_table = tables.iter().find(|t| t.columns.len() == 3);
+        assert!(
+            data_table.is_some(),
+            "expected to keep the 5×3 TYPE/SIZE/SETBACKS data table; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len()))
+                .collect::<Vec<_>>()
+        );
+        // Prose paragraph laid out in 2 cols must NOT be detected as a table.
+        // If the rejection regresses, the 10×2 fake table reappears and
+        // produces fragmented markdown like "I agree to comply..." | "I"
+        // that fragments mid-sentence.
+        let prose_table = tables.iter().find(|t| t.columns.len() == 2);
+        assert!(
+            prose_table.is_none(),
+            "expected the 10×2 prose-in-frame block to be rejected; got rows×cols = {:?}",
+            prose_table.map(|t| (t.rows.len(), t.columns.len()))
+        );
+    }
+
+    /// Wireless table regression: decorative/text-region rects may provide row
+    /// bands, but without a real rect-derived column scaffold they must not be
+    /// accepted as a vector grid.
+    #[test]
+    fn wireless_two_col_rejects_rect_grid() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_two_col_no_rects.pdf");
+        assert!(
+            tables.is_empty(),
+            "expected no rect-detected tables for wireless content; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len()))
+                .collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn wireless_two_col_region_rejects_vector_grid() {
+        let buf = std::fs::read("tests/fixtures/wireless_two_col_no_rects.pdf").unwrap();
+        let crops = [
+            [49.32_f32, 52.92, 558.72, 214.2],
+            [49.32_f32, 288.72, 556.56, 378.0],
+            [51.48_f32, 478.44, 558.36, 567.36],
+        ];
+        for crop in crops {
+            let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
+            assert!(
+                detected.is_none(),
+                "expected no vector grid for wireless crop {crop:?}; got {} cells",
+                detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
+            );
+        }
+    }
+
+    /// Wireless dense table regression: text-position columns alone are not
+    /// enough evidence for a rect-derived grid.
+    #[test]
+    fn wireless_dense_rejects_rect_grid() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_dense_no_rects.pdf");
+        assert!(
+            tables.is_empty(),
+            "expected no rect-detected tables for wireless content; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len()))
+                .collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn wireless_dense_region_rejects_vector_grid() {
+        let buf = std::fs::read("tests/fixtures/wireless_dense_no_rects.pdf").unwrap();
+        let crops = [
+            [72.36_f32, 177.48, 243.72, 333.36],
+            [72.0_f32, 390.24, 286.92, 417.6],
+        ];
+        for crop in crops {
+            let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
+            assert!(
+                detected.is_none(),
+                "expected no vector grid for wireless crop {crop:?}; got {} cells",
+                detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
+            );
+        }
+    }
+
+    /// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
+    /// glossary with a green-shaded header row and plain (line-drawn) body cells.
+    /// Mirrors the production failure cohort #1 (Contractions glossary) and #6
+    /// (BIO 350 course header): a few colored header rects sit in a horizontal
+    /// strip while body rows are drawn with `m`/`l` operators, so the rect
+    /// cluster has only 2 Y-edges and `try_build_grid` rejects.
+    ///
+    /// IGNORED: lifting this shape required the exact-duplicate early-dedup
+    /// (PR #76 first iteration), which had broad collateral damage on
+    /// SEC 10-K TOCs and similar docs that draw rule-rects above + below
+    /// section dividers (production diff: 0001104659-25-093871 lost its
+    /// TOC structure, perf-graph data table, and qualifications matrix).
+    /// Re-enable once a more surgical lift exists in `try_build_grid` or
+    /// `snap_edges` that handles cell-border + inner-fill + text-bg rect
+    /// triplets without page-wide dedup.
+    #[test]
+    #[ignore]
+    fn greencomp_competence_two_cols() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/greencomp_competence.pdf");
+        assert!(
+            !tables.is_empty(),
+            "expected at least one rect-detected table for shaded-header + plain-body shape"
+        );
+        let t = tables
+            .iter()
+            .max_by_key(|t| t.rows.len() * t.columns.len())
+            .unwrap();
+        assert_eq!(
+            t.columns.len(),
+            2,
+            "GreenComp competence is a 2-column table; got {}: {:?}",
+            t.columns.len(),
+            t.columns
+        );
+        assert!(
+            t.rows.len() >= 6,
+            "expected at least 6 rows of competences; got {}",
+            t.rows.len()
+        );
+    }
+
+    /// Regression for `upstage_key_functions.pdf` — a 4-column "Service Stage /
+    /// Function Name / Explanation / Expected Benefit" table with a blue-shaded
+    /// header band plus alternating row backgrounds. Mirrors production crops
+    /// #2 (Parameter / Value with alternating blue rows) and #7 (Spanish XML
+    /// schema with shaded header). Currently `pdf2md` returns zero markdown
+    /// table rows.
+    #[test]
+    fn upstage_key_functions_four_cols() {
+        let tables = detect_rect_tables_in_fixture("tests/fixtures/upstage_key_functions.pdf");
+        assert!(
+            !tables.is_empty(),
+            "expected at least one rect-detected table for shaded-header + alt-row shape"
+        );
+        let t = tables
+            .iter()
+            .max_by_key(|t| t.rows.len() * t.columns.len())
+            .unwrap();
+        assert_eq!(
+            t.columns.len(),
+            4,
+            "Service Flow is a 4-column table; got {}: {:?}",
+            t.columns.len(),
+            t.columns
+        );
+        assert!(
+            t.rows.len() >= 8,
+            "expected at least 8 visible body rows; got {}",
+            t.rows.len()
+        );
+    }
+
     #[test]
     fn test_crop_px_bbox_is_plausible_bounds() {
         let crop = [10.0, 20.0, 110.0, 220.0];

src/tables/detect_rects.rs

@@ -285,7 +285,11 @@ pub fn detect_tables_from_rects(
         //
         // Only remove when the container is a similarly-sized cell (height
         // ratio < 4×), NOT when the container is a table-wide background
-        // that dwarfs the sub-rect.
+        // that dwarfs the sub-rect.  Origin-anchored page-background rects
+        // also disqualify as containers — they normally exceed the 4× ratio,
+        // but when the sub-rect is itself a tall table-frame the ratio can
+        // fall under the gate, and dropping the frame collapses cluster
+        // adjacency between adjacent column-cell groups.
         //
         // Skip this O(n²) dedup when there are too many rects — pages with
         // thousands of vector-drawing rects won't benefit from cell dedup.
@@ -295,9 +299,11 @@ pub fn detect_tables_from_rects(
             page_rects.retain(|&(ax, ay, aw, ah)| {
                 let tol = 2.0;
                 !snapshot.iter().any(|&(bx, by, bw, bh)| {
+                    let container_is_page_bg = bx < 5.0 && by < 5.0;
                     // b must strictly contain a (b is larger in area)
                     bw * bh > aw * ah * 1.2
                         && bh < ah * 4.0 // container must be similarly sized, not a table background
+                        && !container_is_page_bg
                         && bx <= ax + tol
                         && (bx + bw) >= (ax + aw) - tol
                         && by <= ay + tol
@@ -1626,17 +1632,17 @@ fn detect_row_stripe_table_from_cell_rects(
         }
     };
 
-    let col_edges = match (rect_col_edges, text_col_edges) {
+    let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
         (Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
             debug!(
                 "  cell-rect using {} rect-derived columns over {} text clusters",
                 rect_edges.len() - 1,
                 text_edges.len() - 1
             );
-            rect_edges
+            (rect_edges, false)
         }
-        (_, Some(text_edges)) => text_edges,
-        (Some(rect_edges), None) => rect_edges,
+        (_, Some(text_edges)) => (text_edges, true),
+        (Some(rect_edges), None) => (rect_edges, false),
         (None, None) => {
             debug!(
                 "  cell-rect rejected: only {} columns from text clustering",
@@ -1725,13 +1731,36 @@ fn detect_row_stripe_table_from_cell_rects(
 
     // Reject "tables" that are actually prose in a framed region.
     // Columns here come from text X-position clustering; when prose wraps
-    // inside a bounding-box rect (e.g. chat-transcript figures) the
-    // word-boundary gaps cluster into many spurious columns, and the
-    // resulting cells hold sentence fragments riddled with common English
-    // function words. Count cells with any such word and reject when
-    // 20%+ of non-empty cells match — real tabular data (labels, units,
-    // numbers) rarely contains these words.
-    if num_cols >= 4 {
+    // inside a bounding-box rect (e.g. chat-transcript figures, two-column
+    // legal-text blocks in forms) the word-boundary gaps cluster into
+    // spurious columns, and the resulting cells hold sentence fragments
+    // riddled with common English function words.
+    //
+    // Apply at any column count >= 2. The 2-col case is the bite — a
+    // paragraph wrapped into 2 justified columns produces the same
+    // surface signal as a real "label / value" table in the
+    // well-distributed-cols check (both cols populated), so we need a
+    // content-based signal to tell them apart.
+    //
+    // Layered checks combine after the 20%-of-cells prose-word
+    // trigger fires:
+    //   (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
+    //       per non-empty cell (sentence fragments); real data tables
+    //       are typically <30 chars, occasionally up to ~55 for
+    //       descriptive 4-col tables. The 65-char threshold cleanly
+    //       separates them on observed fixtures (accessory_building
+    //       prose=74 chars, upstage data=53, greencomp=20). This
+    //       overrides the well-distributed relaxation — long cells
+    //       are the strongest prose signal even when both cols are
+    //       populated.
+    //   (b) Two-column text-only scaffold: when both columns were inferred
+    //       from text starts rather than rect edges, prose fragments can look
+    //       perfectly balanced. Require rect evidence for this relaxed shape.
+    //   (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
+    //       cells. Catches the prose-paragraph-as-many-cols shape
+    //       while admitting real "label / value / description /
+    //       benefit"-style tables.
+    if num_cols >= 2 {
         const PROSE_WORDS: &[&str] = &[
             "a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on",
             "at", "with", "for", "by", "as", "and", "or", "but", "this", "that", "these", "those",
@@ -1741,6 +1770,7 @@ fn detect_row_stripe_table_from_cell_rects(
         ];
         let mut prose_cells = 0usize;
         let mut counted = 0usize;
+        let mut total_chars = 0usize;
         for row in &cells {
             for cell in row {
                 let t = cell.trim();
@@ -1748,6 +1778,7 @@ fn detect_row_stripe_table_from_cell_rects(
                     continue;
                 }
                 counted += 1;
+                total_chars += t.chars().count();
                 let lower = t.to_ascii_lowercase();
                 let has_prose_word = lower
                     .split(|c: char| !c.is_ascii_alphabetic() && c != '\'')
@@ -1758,11 +1789,60 @@ fn detect_row_stripe_table_from_cell_rects(
             }
         }
         if counted > 0 && prose_cells * 5 >= counted {
+            // (a) Long-cell content: overrides the well-distributed
+            // relaxation. The 2-col prose-in-a-frame case populates
+            // both cols (passes well-distributed) but every cell
+            // holds a sentence fragment, so mean cell length is the
+            // discriminator.
+            const PROSE_MEAN_CHAR_THRESHOLD: usize = 65;
+            let mean_chars = total_chars / counted;
+            if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
+                debug!(
+                    "  cell-rect rejected: prose-in-frame, mean non-empty cell {} chars > {} (prose words {}/{})",
+                    mean_chars, PROSE_MEAN_CHAR_THRESHOLD, prose_cells, counted
+                );
+                return None;
+            }
+
+            // (b) Two text-derived columns are not enough vector evidence once
+            // the content looks prose-like. Real 2-col rect tables still pass
+            // when the column scaffold comes from drawn cell geometry.
+            if columns_from_text && num_cols == 2 {
+                debug!(
+                    "  cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
+                    mean_chars, prose_cells, counted
+                );
+                return None;
+            }
+
+            // (c) Well-distributed columns.
+            let filled_cols = (0..num_cols)
+                .filter(|&c| {
+                    cells
+                        .iter()
+                        .filter(|row| {
+                            !row.get(c)
+                                .map(String::as_str)
+                                .unwrap_or("")
+                                .trim()
+                                .is_empty()
+                        })
+                        .count()
+                        >= 2
+                })
+                .count();
+            let well_distributed = filled_cols * 4 >= num_cols * 3;
+            if !well_distributed {
+                debug!(
+                    "  cell-rect rejected: {}/{} cells contain prose function words — likely prose ({}/{} cols filled, mean {} chars)",
+                    prose_cells, counted, filled_cols, num_cols, mean_chars
+                );
+                return None;
+            }
             debug!(
-                "  cell-rect rejected: {}/{} cells contain prose function words — likely prose",
-                prose_cells, counted
+                "  cell-rect prose check relaxed: {}/{} cols filled, mean {} chars — table-with-description-col",
+                filled_cols, num_cols, mean_chars
             );
-            return None;
         }
     }
 
@@ -2973,6 +3053,62 @@ mod tests {
         // If tables were detected, that's also acceptable
     }
 
+    #[test]
+    fn text_derived_two_col_prose_is_not_cell_rect_table() {
+        let page = 1;
+        let mut rects = Vec::new();
+        for row in 0..8 {
+            rects.push(PdfRect {
+                x: 50.0,
+                y: 100.0 + row as f32 * 20.0,
+                width: 180.0,
+                height: 18.0,
+                page,
+            });
+        }
+
+        let mut items = Vec::new();
+        let left = [
+            "the annual plan was revised",
+            "and the team noted changes",
+            "this section explains limits",
+            "with additional notes below",
+            "the policy was reviewed",
+            "and results are summarized",
+            "this appendix describes scope",
+            "with examples for reference",
+        ];
+        let right = [
+            "for each area in the review",
+            "as part of the assessment",
+            "that were applied in context",
+            "to support the conclusion",
+            "for use by the committee",
+            "as shown in the narrative",
+            "that remain under discussion",
+            "to clarify the method",
+        ];
+        for row in 0..8 {
+            let y = 104.0 + row as f32 * 20.0;
+            let mut left_item = make_item(left[row], 60.0, y, 9.0);
+            left_item.width = 50.0;
+            items.push(left_item);
+            let mut right_item = make_item(right[row], 150.0, y, 9.0);
+            right_item.width = 50.0;
+            items.push(right_item);
+        }
+
+        let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
+        assert!(
+            tables.is_empty(),
+            "text-derived two-column prose must not be accepted as a rect table; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len()))
+                .collect::<Vec<_>>()
+        );
+    }
+
     #[test]
     fn failed_cluster_no_hint_without_items() {
         // Rects with no text items inside → no failed-cluster hint generated.

tests/snapshots/td9264.md

@@ -54,9 +54,14 @@ company other than a life insurance company shall make a return on Form 1120PC.
 
 annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return.
 
-||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and|
-|---|---|
-||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of|
+(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and
+(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company.
+(4) Exception for insurance companies filing their Federal income tax returns
+electronically. If an insurance company described in paragraph (c)(1), (c)(2), or
+
+(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e).
+(5) Definition. For purposes of this section, the term annual statement means
+the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of
 
 Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement.
 
@@ -201,3 +206,4 @@ CFR part or section where Current OMB identified or described control No.
 Deputy Commissioner for Services and Enforcement.
 
 Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy).
+