tables/detect_rects: don't accept relaxed grid on wireless prose

Require rect-derived column evidence before relaxing prose checks for two-column cell-rect fallbacks, so text-position alignment alone cannot synthesize a vector grid on wireless content.

Co-authored-by: Cursor <cursoragent@cursor.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.7.2",
+  "version": "1.8.6",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -99,6 +99,13 @@ pub struct PageRegionTexts {
     pub regions: Vec<RegionText>,
 }
 
+/// Vector-grid detection result compatible with `extractTablesWithStructure*`.
+#[napi(object)]
+pub struct VectorGridDetectionJs {
+    pub structure_tokens: Vec<String>,
+    pub cell_bboxes: Vec<Vec<f64>>,
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -317,6 +324,53 @@ pub fn extract_tables_in_regions(
     })
 }
 
+/// Detect a vector ruled-line / rectangle grid inside one page region.
+///
+/// Returns TSR-compatible structure tokens plus crop-pixel cell bboxes, or
+/// `null` when the region does not contain a valid vector grid.
+///
+/// `pageIdx` is 0-indexed. `regionPdfPtBbox` is `[x1,y1,x2,y2]` in PDF
+/// points with top-left origin. `renderDpi` is the DPI of the crop image that
+/// will consume the returned cell bboxes.
+#[napi]
+pub fn detect_vector_grid_in_region(
+    buffer: Buffer,
+    page_idx: u32,
+    region_pdf_pt_bbox: Vec<f64>,
+    render_dpi: f64,
+) -> Result<Option<VectorGridDetectionJs>> {
+    let bytes: Vec<u8> = buffer.to_vec();
+    let region = if region_pdf_pt_bbox.len() == 4 {
+        [
+            region_pdf_pt_bbox[0] as f32,
+            region_pdf_pt_bbox[1] as f32,
+            region_pdf_pt_bbox[2] as f32,
+            region_pdf_pt_bbox[3] as f32,
+        ]
+    } else {
+        [0.0, 0.0, 0.0, 0.0]
+    };
+
+    catch_panic("detect_vector_grid_in_region", move || {
+        let result = pdf_inspector::detect_vector_grid_in_region_mem(
+            &bytes,
+            page_idx,
+            region,
+            render_dpi as f32,
+        )
+        .map_err(|e| to_napi_err(e, "detect_vector_grid_in_region"))?;
+
+        Ok(result.map(|r| VectorGridDetectionJs {
+            structure_tokens: r.structure_tokens,
+            cell_bboxes: r
+                .cell_bboxes
+                .into_iter()
+                .map(|bbox| bbox.into_iter().map(|v| v as f64).collect())
+                .collect(),
+        }))
+    })
+}
+
 /// One cropped table region plus its raw structure-recovery output, for
 /// `extractTablesWithStructure`.
 ///
@@ -428,9 +482,10 @@ pub fn extract_tables_with_structure_cells(
 /// `fallbackReason` is `null` when the TSR-hybrid path produced the
 /// markdown directly. When stage 1's quality check fires (the cells
 /// look like a SLANet detection pathology — phantom rows or multi-row
-/// content in a single cell), the heuristic table extractor is run on
-/// the same region instead, and `fallbackReason` carries the diagnostic
-/// label (`"phantom_empty_row"`, `"multi_row_in_cell"`).
+/// content in a single cell), the auto path may expand the TSR cells
+/// in-place or run the heuristic table extractor on the same region.
+/// `fallbackReason` carries the diagnostic label (for example
+/// `"multi_row_in_cell_expanded"` or `"phantom_empty_row"`).
 #[napi(object)]
 pub struct TableExtractionResultJs {
     pub markdown: String,
@@ -440,13 +495,14 @@ pub struct TableExtractionResultJs {
 /// Auto-fallback variant of [`extractTablesWithStructure`].
 ///
 /// Runs the TSR-hybrid path, checks the resulting cells for known
-/// SLANet detection pathologies, and falls back to the heuristic
-/// `extractTablesInRegions` for any input where the TSR path looks
+/// SLANet detection pathologies, expands multi-row cells in-place when
+/// possible, and otherwise falls back to the heuristic
+/// `extractTablesInRegions` for inputs where the TSR path looks
 /// compromised.
 ///
 /// On clean inputs this returns identical markdown to
-/// `extractTablesWithStructure`; on flagged inputs the heuristic
-/// markdown replaces the TSR markdown and `fallbackReason` is set.
+/// `extractTablesWithStructure`; on flagged inputs `fallbackReason` is
+/// set to the recovery path that produced the result.
 #[napi]
 pub fn extract_tables_with_structure_auto(
     buffer: Buffer,

napi/test.mjs

@@ -7,6 +7,7 @@ import {
   extractText,
   extractTextWithPositions,
   extractTextInRegions,
+  detectVectorGridInRegion,
   extractPagesMarkdown,
 } from './index.js';
 
@@ -90,6 +91,17 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
 assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
 console.log('  extractTextInRegions: OK');
 
+// --- detectVectorGridInRegion ---
+console.log('Testing detectVectorGridInRegion...');
+const vectorGrid = detectVectorGridInRegion(fixture, 0, [0, 0, 600, 800], 72);
+assert.ok(vectorGrid === null || typeof vectorGrid === 'object');
+if (vectorGrid) {
+  assert.ok(Array.isArray(vectorGrid.structureTokens));
+  assert.ok(Array.isArray(vectorGrid.cellBboxes));
+  assert.ok(vectorGrid.cellBboxes.every(bbox => Array.isArray(bbox) && bbox.length === 4));
+}
+console.log('  detectVectorGridInRegion: OK');
+
 // --- extractPagesMarkdown ---
 console.log('Testing extractPagesMarkdown...');
 

src/extractor/content_stream.rs

@@ -385,6 +385,7 @@ pub(crate) fn extract_page_text_items(
                         &font_encodings,
                         &encoding_cache,
                         &mut cmap_decisions,
+                        &font_widths,
                     ) {
                         let combined = multiply_matrices(&text_matrix, &ctm);
                         let rendered_size = effective_font_size(current_font_size, &combined);
@@ -533,6 +534,7 @@ pub(crate) fn extract_page_text_items(
                                     &font_encodings,
                                     &encoding_cache,
                                     &mut cmap_decisions,
+                                    &font_widths,
                                 ) {
                                     current_text.push_str(&text);
                                 }
@@ -620,6 +622,7 @@ pub(crate) fn extract_page_text_items(
                         &font_encodings,
                         &encoding_cache,
                         &mut cmap_decisions,
+                        &font_widths,
                     ) {
                         if !text.trim().is_empty() {
                             let combined = multiply_matrices(&text_matrix, &ctm);
@@ -1012,9 +1015,17 @@ pub(crate) fn extract_page_text_items(
     // producing thousands of identical rects that yield a degenerate grid.
     // After dedup, if too few unique clip rects remain we fall through to
     // fill rects (explicitly drawn visible rectangles).
+    //
+    // When fill rects substantially outnumber clip rects, the clips are
+    // typically section-level wrappers and the fills are the actual table
+    // cell backgrounds (e.g. shaded-header tables drawn with `m`/`l`/`h`/`f*`
+    // sequences). In that case, prefer fills.
     if rects.is_empty() {
         dedup_rects(&mut clip_rects);
-        if clip_rects.len() >= 4 {
+        let prefer_fills = !fill_rects.is_empty() && fill_rects.len() >= clip_rects.len() * 3;
+        if prefer_fills {
+            rects = fill_rects;
+        } else if clip_rects.len() >= 4 {
             rects = clip_rects;
         } else if !fill_rects.is_empty() {
             rects = fill_rects;

src/extractor/fonts.rs

@@ -720,7 +720,11 @@ pub(crate) fn extract_text_from_operand(
     font_encodings: &PageFontEncodings,
     encoding_cache: &HashMap<String, Encoding<'_>>,
     cmap_decisions: &mut CMapDecisionCache,
+    font_widths: &PageFontWidths,
 ) -> Option<String> {
+    let is_type0_cid_font = font_widths
+        .get(current_font)
+        .is_some_and(|info| info.is_cid);
     let result = (|| -> Option<String> {
         if let Object::String(bytes, _) = obj {
             let mut decode_with_entry = |entry: &crate::tounicode::CMapEntry| -> Option<String> {
@@ -962,7 +966,31 @@ pub(crate) fn extract_text_from_operand(
                 return Some(symbol_text);
             }
 
-            // Latin-1 fallback
+            // Latin-1 fallback. Safe ONLY for fonts that use single-byte
+            // encodings — for these, an unmapped byte is a valid character
+            // code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
+            // emit multi-byte CIDs that aren't characters; per-byte Latin-1
+            // produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
+            // production scrape_id 019de78c-... samples).
+            //
+            // For a CID font (has_cmap is set OR a /ToUnicode reference
+            // exists) with any non-ASCII bytes, emit a single U+FFFD per
+            // CID instead. This both replaces the mojibake with a proper
+            // "decode failed" marker AND keeps `detect_encoding_issues`
+            // tripping so the page is flagged for OCR — the existing
+            // garbage-detection path that the high-Latin-1 mojibake used
+            // to satisfy by accident.
+            if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
+                // 2-byte CIDs (Identity-H) are by far the common case; for
+                // an odd byte count we still emit at least one marker so
+                // detection downstream fires.
+                let cid_count = (bytes.len() / 2).max(1);
+                return Some("\u{FFFD}".repeat(cid_count));
+            }
+            // Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
+            // 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
+            // use single-byte encodings where Latin-1 fallback is the
+            // canonical interpretation.
             Some(bytes.iter().map(|&b| b as char).collect())
         } else {
             None
@@ -1213,4 +1241,97 @@ mod tests {
         let bad = "###!!!@@@$$$";
         assert!(score_text(good) > score_text(bad));
     }
+
+    #[test]
+    fn cid_font_with_unparseable_cmap_does_not_emit_latin1_mojibake() {
+        // Type0/CID font (font_widths reports `is_cid=true`) where the
+        // ToUnicode CMap couldn't be parsed (FontCMaps doesn't have the
+        // obj_num). Bytes are a 2-byte CID stream containing high bytes
+        // that aren't valid UTF-8 — exactly the case in the production
+        // samples (Identity-H text where the ToUnicode CMap was missing
+        // or malformed, scrape_id 019de78c-..., e.g. "Í Ù Z)¿").
+        //
+        // Without the guard, the function falls through to the byte-by-byte
+        // Latin-1 fallback and produces "ÍÙ" (U+00CD U+00D9). The correct
+        // behavior is to emit U+FFFD per CID so downstream
+        // `detect_encoding_issues` flags the page for OCR.
+        let bytes = vec![0xCD_u8, 0xD9, 0xCD, 0xD9];
+        let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
+
+        let font_cmaps = FontCMaps::default();
+        let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
+        font_tounicode_refs.insert("F0".to_string(), 999);
+        let inline_cmaps = HashMap::new();
+        let font_encodings: PageFontEncodings = HashMap::new();
+        let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
+        let mut decisions = CMapDecisionCache::new();
+        let mut font_widths: PageFontWidths = HashMap::new();
+        font_widths.insert("F0".to_string(), make_font_info(&[], 1000, true));
+
+        let result = extract_text_from_operand(
+            &obj,
+            "F0",
+            None,
+            &font_cmaps,
+            &font_tounicode_refs,
+            &inline_cmaps,
+            &font_encodings,
+            &encoding_cache,
+            &mut decisions,
+            &font_widths,
+        );
+
+        let text = result.expect("CID font fallback should still emit a marker");
+        assert!(
+            !text.contains('\u{00CD}') && !text.contains('\u{00D9}'),
+            "CID font with unparseable CMap leaked Latin-1 mojibake: {text:?}"
+        );
+        assert!(
+            text.contains('\u{FFFD}'),
+            "CID font with unparseable CMap should emit U+FFFD so detect_encoding_issues fires: {text:?}"
+        );
+    }
+
+    #[test]
+    fn simple_font_latin1_fallback_passes_high_bytes_through() {
+        // A Type1/TrueType simple font (is_cid=false) with a `/ToUnicode`
+        // reference but no usable CMap and no `/Differences` map.
+        // Per-byte Latin-1 IS the canonical interpretation here — these
+        // bytes are character codes, not CIDs. The CID guard must NOT
+        // strip them. Reproduces the false positive that an earlier
+        // version of the guard introduced for fonts in PDFs like
+        // pdf-evals/Navigating-Artificial-Intelligence-..., where bytes
+        // like 0xB6 are legitimate Latin-1 character codes.
+        let bytes = vec![0x24_u8, 0x47, 0xB6, 0x56]; // "$G¶V"
+        let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
+
+        let font_cmaps = FontCMaps::default();
+        let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
+        font_tounicode_refs.insert("F1".to_string(), 999);
+        let inline_cmaps = HashMap::new();
+        let font_encodings: PageFontEncodings = HashMap::new();
+        let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
+        let mut decisions = CMapDecisionCache::new();
+        let mut font_widths: PageFontWidths = HashMap::new();
+        font_widths.insert("F1".to_string(), make_font_info(&[], 1000, false));
+
+        let text = extract_text_from_operand(
+            &obj,
+            "F1",
+            None,
+            &font_cmaps,
+            &font_tounicode_refs,
+            &inline_cmaps,
+            &font_encodings,
+            &encoding_cache,
+            &mut decisions,
+            &font_widths,
+        )
+        .expect("simple font should round-trip Latin-1 bytes");
+        assert_eq!(text, "$G\u{00B6}V");
+        assert!(
+            !text.contains('\u{FFFD}'),
+            "simple font fallback must not stamp FFFD over legitimate bytes: {text:?}"
+        );
+    }
 }

src/extractor/xobjects.rs

@@ -373,6 +373,7 @@ fn extract_form_xobject_text_inner(
                         &font_encodings,
                         &encoding_cache,
                         cmap_decisions,
+                        &font_widths,
                     ) {
                         let combined = multiply_matrices(&text_matrix, &ctm);
                         let rendered_size = effective_font_size(current_font_size, &combined);
@@ -517,6 +518,7 @@ fn extract_form_xobject_text_inner(
                                     &font_encodings,
                                     &encoding_cache,
                                     cmap_decisions,
+                                    &font_widths,
                                 ) {
                                     current_text.push_str(&text);
                                 }

src/tables/detect_rects.rs

@@ -285,7 +285,11 @@ pub fn detect_tables_from_rects(
         //
         // Only remove when the container is a similarly-sized cell (height
         // ratio < 4×), NOT when the container is a table-wide background
-        // that dwarfs the sub-rect.
+        // that dwarfs the sub-rect.  Origin-anchored page-background rects
+        // also disqualify as containers — they normally exceed the 4× ratio,
+        // but when the sub-rect is itself a tall table-frame the ratio can
+        // fall under the gate, and dropping the frame collapses cluster
+        // adjacency between adjacent column-cell groups.
         //
         // Skip this O(n²) dedup when there are too many rects — pages with
         // thousands of vector-drawing rects won't benefit from cell dedup.
@@ -295,9 +299,11 @@ pub fn detect_tables_from_rects(
             page_rects.retain(|&(ax, ay, aw, ah)| {
                 let tol = 2.0;
                 !snapshot.iter().any(|&(bx, by, bw, bh)| {
+                    let container_is_page_bg = bx < 5.0 && by < 5.0;
                     // b must strictly contain a (b is larger in area)
                     bw * bh > aw * ah * 1.2
                         && bh < ah * 4.0 // container must be similarly sized, not a table background
+                        && !container_is_page_bg
                         && bx <= ax + tol
                         && (bx + bw) >= (ax + aw) - tol
                         && by <= ay + tol
@@ -1587,25 +1593,69 @@ fn detect_row_stripe_table_from_cell_rects(
         return None;
     }
 
-    // Derive columns from text X-position clustering
+    // Derive columns from text X-position clustering, but prefer rect
+    // X-edges when they already provide a tighter scaffold.  Some PDFs draw
+    // only the row-index cells in the body plus a full header row; that is
+    // not dense enough for `try_build_grid`, but the header rects still define
+    // the real columns.  Text starts inside wide cells can otherwise split the
+    // table into spurious sub-columns.
     let columns = cluster_x_positions(&page_items, 15.0);
-    if columns.len() < 2 {
+    let text_col_edges = if columns.len() >= 2 {
+        let mut edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
+        let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
+        edges.push(min_x - 5.0);
+        for pair in columns.windows(2) {
+            edges.push((pair[0] + pair[1]) / 2.0);
+        }
+        let max_x_right = page_items
+            .iter()
+            .map(|(_, i)| i.x + i.width)
+            .reduce(f32::max)?;
+        edges.push(max_x_right + 5.0);
+        Some(edges)
+    } else {
+        None
+    };
+
+    let rect_col_edges = {
+        let mut x_vals = Vec::with_capacity(content_rects.len() * 2);
+        for &&(x, _, w, _) in &content_rects {
+            x_vals.push(x);
+            x_vals.push(x + w);
+        }
+        let mut edges = snap_edges(&x_vals, 6.0);
+        edges.sort_by(|a, b| a.total_cmp(b));
+        if (3..=26).contains(&edges.len()) {
+            Some(edges)
+        } else {
+            None
+        }
+    };
+
+    let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
+        (Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
+            debug!(
+                "  cell-rect using {} rect-derived columns over {} text clusters",
+                rect_edges.len() - 1,
+                text_edges.len() - 1
+            );
+            (rect_edges, false)
+        }
+        (_, Some(text_edges)) => (text_edges, true),
+        (Some(rect_edges), None) => (rect_edges, false),
+        (None, None) => {
+            debug!(
+                "  cell-rect rejected: only {} columns from text clustering",
+                columns.len()
+            );
+            return None;
+        }
+    };
+
+    if col_edges.len() < 3 {
         return None;
     }
 
-    // Build column edges
-    let mut col_edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
-    let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
-    col_edges.push(min_x - 5.0);
-    for pair in columns.windows(2) {
-        col_edges.push((pair[0] + pair[1]) / 2.0);
-    }
-    let max_x_right = page_items
-        .iter()
-        .map(|(_, i)| i.x + i.width)
-        .reduce(f32::max)?;
-    col_edges.push(max_x_right + 5.0);
-
     let num_cols = col_edges.len() - 1;
     let num_rows = row_edges.len() - 1;
 
@@ -1681,13 +1731,36 @@ fn detect_row_stripe_table_from_cell_rects(
 
     // Reject "tables" that are actually prose in a framed region.
     // Columns here come from text X-position clustering; when prose wraps
-    // inside a bounding-box rect (e.g. chat-transcript figures) the
-    // word-boundary gaps cluster into many spurious columns, and the
-    // resulting cells hold sentence fragments riddled with common English
-    // function words. Count cells with any such word and reject when
-    // 20%+ of non-empty cells match — real tabular data (labels, units,
-    // numbers) rarely contains these words.
-    if num_cols >= 4 {
+    // inside a bounding-box rect (e.g. chat-transcript figures, two-column
+    // legal-text blocks in forms) the word-boundary gaps cluster into
+    // spurious columns, and the resulting cells hold sentence fragments
+    // riddled with common English function words.
+    //
+    // Apply at any column count >= 2. The 2-col case is the bite — a
+    // paragraph wrapped into 2 justified columns produces the same
+    // surface signal as a real "label / value" table in the
+    // well-distributed-cols check (both cols populated), so we need a
+    // content-based signal to tell them apart.
+    //
+    // Layered checks combine after the 20%-of-cells prose-word
+    // trigger fires:
+    //   (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
+    //       per non-empty cell (sentence fragments); real data tables
+    //       are typically <30 chars, occasionally up to ~55 for
+    //       descriptive 4-col tables. The 65-char threshold cleanly
+    //       separates them on observed fixtures (accessory_building
+    //       prose=74 chars, upstage data=53, greencomp=20). This
+    //       overrides the well-distributed relaxation — long cells
+    //       are the strongest prose signal even when both cols are
+    //       populated.
+    //   (b) Two-column text-only scaffold: when both columns were inferred
+    //       from text starts rather than rect edges, prose fragments can look
+    //       perfectly balanced. Require rect evidence for this relaxed shape.
+    //   (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
+    //       cells. Catches the prose-paragraph-as-many-cols shape
+    //       while admitting real "label / value / description /
+    //       benefit"-style tables.
+    if num_cols >= 2 {
         const PROSE_WORDS: &[&str] = &[
             "a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on",
             "at", "with", "for", "by", "as", "and", "or", "but", "this", "that", "these", "those",
@@ -1697,6 +1770,7 @@ fn detect_row_stripe_table_from_cell_rects(
         ];
         let mut prose_cells = 0usize;
         let mut counted = 0usize;
+        let mut total_chars = 0usize;
         for row in &cells {
             for cell in row {
                 let t = cell.trim();
@@ -1704,6 +1778,7 @@ fn detect_row_stripe_table_from_cell_rects(
                     continue;
                 }
                 counted += 1;
+                total_chars += t.chars().count();
                 let lower = t.to_ascii_lowercase();
                 let has_prose_word = lower
                     .split(|c: char| !c.is_ascii_alphabetic() && c != '\'')
@@ -1714,11 +1789,60 @@ fn detect_row_stripe_table_from_cell_rects(
             }
         }
         if counted > 0 && prose_cells * 5 >= counted {
+            // (a) Long-cell content: overrides the well-distributed
+            // relaxation. The 2-col prose-in-a-frame case populates
+            // both cols (passes well-distributed) but every cell
+            // holds a sentence fragment, so mean cell length is the
+            // discriminator.
+            const PROSE_MEAN_CHAR_THRESHOLD: usize = 65;
+            let mean_chars = total_chars / counted;
+            if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
+                debug!(
+                    "  cell-rect rejected: prose-in-frame, mean non-empty cell {} chars > {} (prose words {}/{})",
+                    mean_chars, PROSE_MEAN_CHAR_THRESHOLD, prose_cells, counted
+                );
+                return None;
+            }
+
+            // (b) Two text-derived columns are not enough vector evidence once
+            // the content looks prose-like. Real 2-col rect tables still pass
+            // when the column scaffold comes from drawn cell geometry.
+            if columns_from_text && num_cols == 2 {
+                debug!(
+                    "  cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
+                    mean_chars, prose_cells, counted
+                );
+                return None;
+            }
+
+            // (c) Well-distributed columns.
+            let filled_cols = (0..num_cols)
+                .filter(|&c| {
+                    cells
+                        .iter()
+                        .filter(|row| {
+                            !row.get(c)
+                                .map(String::as_str)
+                                .unwrap_or("")
+                                .trim()
+                                .is_empty()
+                        })
+                        .count()
+                        >= 2
+                })
+                .count();
+            let well_distributed = filled_cols * 4 >= num_cols * 3;
+            if !well_distributed {
+                debug!(
+                    "  cell-rect rejected: {}/{} cells contain prose function words — likely prose ({}/{} cols filled, mean {} chars)",
+                    prose_cells, counted, filled_cols, num_cols, mean_chars
+                );
+                return None;
+            }
             debug!(
-                "  cell-rect rejected: {}/{} cells contain prose function words — likely prose",
-                prose_cells, counted
+                "  cell-rect prose check relaxed: {}/{} cols filled, mean {} chars — table-with-description-col",
+                filled_cols, num_cols, mean_chars
             );
-            return None;
         }
     }
 
@@ -2929,6 +3053,62 @@ mod tests {
         // If tables were detected, that's also acceptable
     }
 
+    #[test]
+    fn text_derived_two_col_prose_is_not_cell_rect_table() {
+        let page = 1;
+        let mut rects = Vec::new();
+        for row in 0..8 {
+            rects.push(PdfRect {
+                x: 50.0,
+                y: 100.0 + row as f32 * 20.0,
+                width: 180.0,
+                height: 18.0,
+                page,
+            });
+        }
+
+        let mut items = Vec::new();
+        let left = [
+            "the annual plan was revised",
+            "and the team noted changes",
+            "this section explains limits",
+            "with additional notes below",
+            "the policy was reviewed",
+            "and results are summarized",
+            "this appendix describes scope",
+            "with examples for reference",
+        ];
+        let right = [
+            "for each area in the review",
+            "as part of the assessment",
+            "that were applied in context",
+            "to support the conclusion",
+            "for use by the committee",
+            "as shown in the narrative",
+            "that remain under discussion",
+            "to clarify the method",
+        ];
+        for row in 0..8 {
+            let y = 104.0 + row as f32 * 20.0;
+            let mut left_item = make_item(left[row], 60.0, y, 9.0);
+            left_item.width = 50.0;
+            items.push(left_item);
+            let mut right_item = make_item(right[row], 150.0, y, 9.0);
+            right_item.width = 50.0;
+            items.push(right_item);
+        }
+
+        let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
+        assert!(
+            tables.is_empty(),
+            "text-derived two-column prose must not be accepted as a rect table; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len()))
+                .collect::<Vec<_>>()
+        );
+    }
+
     #[test]
     fn failed_cluster_no_hint_without_items() {
         // Rects with no text items inside → no failed-cluster hint generated.

tests/integration_tests.rs

@@ -4,10 +4,11 @@ use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, S
 use pdf_inspector::extractor::group_into_lines;
 use pdf_inspector::types::TextLine;
 use pdf_inspector::{
-    detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
-    extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
-    extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
-    MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
+    detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
+    extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
+    extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
+    process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
+    TextItem,
 };
 use std::collections::HashSet;
 
@@ -1188,9 +1189,38 @@ fn test_firecrawl_tagged_pdf_struct_tree() {
 #[test]
 fn test_identity_h_no_tounicode_suppresses_garbage() {
     // shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no
-    // ToUnicode CMap.  The raw CID values look like random Latin characters.
-    // We should suppress the garbage and flag the page for OCR.
+    // usable ToUnicode CMap. The raw CID bytes (e.g. 0x08 0x37, 0x0E 0x0F)
+    // contain non-ASCII high bytes and previously fell through to the
+    // per-byte Latin-1 fallback, producing high-Latin-1 mojibake that
+    // `is_cid_garbage` flagged. The Type0/CID guard in
+    // `extract_text_from_operand` now emits one U+FFFD per CID instead of
+    // mojibake; `detect_encoding_issues` trips on that and suppresses the
+    // markdown / flags the page for OCR — so we still pass this test, but
+    // via the deliberate marker path rather than by accident.
     let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
+
+    // Pre-suppression check: the raw text items must contain the U+FFFD
+    // markers that prove the Type0/CID fallback fired. This pins the
+    // mechanism so a future regression that re-enables Latin-1 mojibake
+    // would fail loudly here, not just silently change the suppression
+    // chain to one that depends on `is_cid_garbage` + high-Latin-1 chars.
+    let items = pdf_inspector::extractor::extract_text_with_positions_mem(&buf).unwrap();
+    let combined: String = items.iter().map(|i| i.text.as_str()).collect();
+    assert!(
+        combined.contains('\u{FFFD}'),
+        "Type0/CID font with unparseable ToUnicode CMap should emit U+FFFD per CID; \
+         got {} chars: {:?}",
+        combined.len(),
+        &combined[..combined.len().min(100)]
+    );
+    assert!(
+        !combined
+            .chars()
+            .any(|c| ('\u{0080}'..='\u{00FF}').contains(&c)),
+        "Latin-1 mojibake (high bytes) must not leak from Type0/CID fallback; got: {:?}",
+        &combined[..combined.len().min(100)]
+    );
+
     let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
 
     // Page 1 should be flagged for OCR
@@ -1704,6 +1734,292 @@ fn synthetic_dense_table_pdf() -> Vec<u8> {
     bytes
 }
 
+fn synthetic_vector_grid_pdf(two_tables: bool) -> Vec<u8> {
+    use lopdf::content::{Content, Operation};
+    use lopdf::{dictionary, Document, Object, Stream};
+
+    fn push_grid(
+        operations: &mut Vec<Operation>,
+        x_left: i64,
+        x_mid: i64,
+        x_right: i64,
+        y_top: i64,
+        y_mid: i64,
+        y_bottom: i64,
+    ) {
+        for y in [y_top, y_mid, y_bottom] {
+            operations.push(Operation::new("m", vec![x_left.into(), y.into()]));
+            operations.push(Operation::new("l", vec![x_right.into(), y.into()]));
+        }
+        for x in [x_left, x_mid, x_right] {
+            operations.push(Operation::new("m", vec![x.into(), y_bottom.into()]));
+            operations.push(Operation::new("l", vec![x.into(), y_top.into()]));
+        }
+        operations.push(Operation::new("S", vec![]));
+    }
+
+    fn push_text(operations: &mut Vec<Operation>, x: i64, y: i64, text: &str) {
+        operations.push(Operation::new(
+            "Tm",
+            vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
+        ));
+        operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
+    }
+
+    let mut doc = Document::with_version("1.5");
+    let pages_id = doc.new_object_id();
+    let page_id = doc.new_object_id();
+    let font_id = doc.new_object_id();
+    let content_id = doc.new_object_id();
+
+    doc.objects.insert(
+        font_id,
+        dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+        }
+        .into(),
+    );
+
+    let mut operations = Vec::new();
+    push_grid(&mut operations, 50, 130, 210, 740, 710, 670);
+    if two_tables {
+        push_grid(&mut operations, 50, 130, 210, 560, 530, 490);
+    }
+
+    operations.push(Operation::new("BT", vec![]));
+    operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
+    push_text(&mut operations, 70, 724, "A1");
+    push_text(&mut operations, 150, 724, "B1");
+    push_text(&mut operations, 70, 688, "A2");
+    push_text(&mut operations, 150, 688, "B2");
+    if two_tables {
+        push_text(&mut operations, 70, 544, "C1");
+        push_text(&mut operations, 150, 544, "D1");
+        push_text(&mut operations, 70, 508, "C2");
+        push_text(&mut operations, 150, 508, "D2");
+    }
+    operations.push(Operation::new("ET", vec![]));
+
+    let content = Content { operations }.encode().unwrap();
+    doc.objects
+        .insert(content_id, Stream::new(dictionary! {}, content).into());
+
+    doc.objects.insert(
+        page_id,
+        dictionary! {
+            "Type" => "Page",
+            "Parent" => pages_id,
+            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => font_id,
+                },
+            },
+            "Contents" => content_id,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        pages_id,
+        dictionary! {
+            "Type" => "Pages",
+            "Kids" => vec![page_id.into()],
+            "Count" => 1,
+        }
+        .into(),
+    );
+    let catalog_id = doc.add_object(dictionary! {
+        "Type" => "Catalog",
+        "Pages" => pages_id,
+    });
+    doc.trailer.set("Root", catalog_id);
+
+    let mut bytes = Vec::new();
+    doc.save_to(&mut bytes).unwrap();
+    bytes
+}
+
+fn synthetic_vector_grid_three_row_pdf() -> Vec<u8> {
+    use lopdf::content::{Content, Operation};
+    use lopdf::{dictionary, Document, Object, Stream};
+
+    let mut doc = Document::with_version("1.5");
+    let pages_id = doc.new_object_id();
+    let page_id = doc.new_object_id();
+    let font_id = doc.new_object_id();
+    let content_id = doc.new_object_id();
+
+    doc.objects.insert(
+        font_id,
+        dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+        }
+        .into(),
+    );
+
+    let mut operations = Vec::new();
+    for y in [740, 710, 680, 650] {
+        operations.push(Operation::new("m", vec![50.into(), y.into()]));
+        operations.push(Operation::new("l", vec![210.into(), y.into()]));
+    }
+    for x in [50, 130, 210] {
+        operations.push(Operation::new("m", vec![x.into(), 650.into()]));
+        operations.push(Operation::new("l", vec![x.into(), 740.into()]));
+    }
+    operations.push(Operation::new("S", vec![]));
+
+    operations.push(Operation::new("BT", vec![]));
+    operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
+    for (x, y, text) in [
+        (70, 724, "Branch"),
+        (150, 724, "Deposits"),
+        (70, 694, "Oak"),
+        (150, 694, "100"),
+        (70, 664, "Boardwalk"),
+        (150, 664, "200"),
+    ] {
+        operations.push(Operation::new(
+            "Tm",
+            vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
+        ));
+        operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
+    }
+    operations.push(Operation::new("ET", vec![]));
+
+    let content = Content { operations }.encode().unwrap();
+    doc.objects
+        .insert(content_id, Stream::new(dictionary! {}, content).into());
+    doc.objects.insert(
+        page_id,
+        dictionary! {
+            "Type" => "Page",
+            "Parent" => pages_id,
+            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => font_id,
+                },
+            },
+            "Contents" => content_id,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        pages_id,
+        dictionary! {
+            "Type" => "Pages",
+            "Kids" => vec![page_id.into()],
+            "Count" => 1,
+        }
+        .into(),
+    );
+    let catalog_id = doc.add_object(dictionary! {
+        "Type" => "Catalog",
+        "Pages" => pages_id,
+    });
+    doc.trailer.set("Root", catalog_id);
+
+    let mut bytes = Vec::new();
+    doc.save_to(&mut bytes).unwrap();
+    bytes
+}
+
+fn assert_close(actual: f32, expected: f32) {
+    assert!(
+        (actual - expected).abs() < 0.75,
+        "expected {actual} to be close to {expected}"
+    );
+}
+
+#[test]
+fn test_detect_vector_grid_in_region_line_pdf() {
+    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
+
+    let buf = synthetic_vector_grid_pdf(false);
+    let crop = [50.0_f32, 60.0, 210.0, 130.0];
+    let detected = detect_vector_grid_in_region_mem(&buf, 0, crop, 72.0)
+        .unwrap()
+        .expect("ruled vector table should be detected");
+
+    assert_eq!(detected.cell_bboxes.len(), 4);
+    assert_eq!(
+        detected
+            .structure_tokens
+            .iter()
+            .filter(|tok| tok.as_str() == "<td></td>")
+            .count(),
+        4
+    );
+    assert_eq!(detected.structure_tokens.first().unwrap(), "<table>");
+    assert_eq!(detected.structure_tokens.last().unwrap(), "</table>");
+
+    let first = &detected.cell_bboxes[0];
+    assert_close(first[0], 0.0);
+    assert_close(first[1], 0.0);
+    assert_close(first[2], 80.0);
+    assert_close(first[3], 30.0);
+
+    let markdown = extract_tables_with_structure_mem(
+        &buf,
+        &[TsrTableInput {
+            page: 0,
+            crop_pdf_pt_bbox: crop,
+            render_dpi: 72.0,
+            structure_tokens: detected.structure_tokens,
+            cell_bboxes: detected.cell_bboxes,
+        }],
+    )
+    .unwrap()
+    .remove(0);
+
+    assert!(markdown.contains("A1"));
+    assert!(markdown.contains("B1"));
+    assert!(markdown.contains("A2"));
+    assert!(markdown.contains("B2"));
+}
+
+#[test]
+fn test_detect_vector_grid_in_region_text_pdf_returns_none() {
+    let buf = make_minimal_text_pdf();
+    let detected =
+        detect_vector_grid_in_region_mem(&buf, 0, [0.0, 0.0, 300.0, 800.0], 72.0).unwrap();
+    assert!(detected.is_none());
+}
+
+#[test]
+fn test_detect_vector_grid_in_region_filters_to_requested_table() {
+    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
+
+    let buf = synthetic_vector_grid_pdf(true);
+    let second_table_crop = [50.0_f32, 240.0, 210.0, 310.0];
+    let detected = detect_vector_grid_in_region_mem(&buf, 0, second_table_crop, 72.0)
+        .unwrap()
+        .expect("second ruled table should be detected");
+
+    assert_eq!(detected.cell_bboxes.len(), 4);
+    let markdown = extract_tables_with_structure_mem(
+        &buf,
+        &[TsrTableInput {
+            page: 0,
+            crop_pdf_pt_bbox: second_table_crop,
+            render_dpi: 72.0,
+            structure_tokens: detected.structure_tokens,
+            cell_bboxes: detected.cell_bboxes,
+        }],
+    )
+    .unwrap()
+    .remove(0);
+
+    assert!(markdown.contains("C1"));
+    assert!(markdown.contains("D2"));
+    assert!(!markdown.contains("A1"));
+    assert!(!markdown.contains("B2"));
+}
+
 #[test]
 fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
     use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
@@ -2119,7 +2435,7 @@ fn test_auto_passes_through_clean_tsr_output() {
 }
 
 #[test]
-fn test_auto_falls_back_on_multi_row_in_cell() {
+fn test_auto_expands_multi_row_in_cell() {
     use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
 
     let buf = synthetic_dense_table_pdf();
@@ -2170,16 +2486,134 @@ fn test_auto_falls_back_on_multi_row_in_cell() {
     assert_eq!(results.len(), 1);
     assert_eq!(
         results[0].fallback_reason.as_deref(),
-        Some("multi_row_in_cell"),
-        "expected multi_row_in_cell fallback, got {:?}",
+        Some("multi_row_in_cell_expanded"),
+        "expected multi_row_in_cell_expanded, got {:?}",
         results[0].fallback_reason
     );
-    // The heuristic-fallback markdown should preserve all three PDF rows.
+    // The in-place expansion should preserve all three PDF rows.
     let md = &results[0].markdown;
     assert!(md.contains("Oak Street"), "missing Oak Street: {md}");
     assert!(md.contains("Boardwalk"), "missing Boardwalk: {md}");
     assert!(md.contains("100"), "missing 100: {md}");
     assert!(md.contains("200"), "missing 200: {md}");
+    assert!(
+        !md.contains("Oak Street Boardwalk"),
+        "rows should not remain compressed: {md}"
+    );
+}
+
+#[test]
+fn test_auto_expands_under_counted_vector_grid_rows() {
+    use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
+
+    let buf = synthetic_vector_grid_three_row_pdf();
+    let tokens: Vec<String> = [
+        "<table>",
+        "<thead>",
+        "<tr>",
+        "<th></th>",
+        "<th></th>",
+        "</tr>",
+        "</thead>",
+        "<tbody>",
+        "<tr>",
+        "<td></td>",
+        "<td></td>",
+        "</tr>",
+        "</tbody>",
+        "</table>",
+    ]
+    .into_iter()
+    .map(String::from)
+    .collect();
+    let crop = [50.0, 60.0, 210.0, 150.0];
+    let cell_bboxes = vec![
+        poly(0.0, 0.0, 80.0, 30.0),
+        poly(80.0, 0.0, 160.0, 30.0),
+        poly(0.0, 30.0, 80.0, 90.0),
+        poly(80.0, 30.0, 160.0, 90.0),
+    ];
+
+    let results = extract_tables_with_structure_auto_mem(
+        &buf,
+        &[TsrTableInput {
+            page: 0,
+            crop_pdf_pt_bbox: crop,
+            render_dpi: 72.0,
+            structure_tokens: tokens,
+            cell_bboxes,
+        }],
+    )
+    .unwrap();
+
+    assert_eq!(results.len(), 1);
+    assert_eq!(
+        results[0].fallback_reason.as_deref(),
+        Some("multi_row_in_cell_expanded")
+    );
+    let md = &results[0].markdown;
+    assert!(md.contains("|Branch|Deposits|"), "missing header: {md}");
+    assert!(md.contains("|Oak|100|"), "missing row 1: {md}");
+    assert!(md.contains("|Boardwalk|200|"), "missing row 2: {md}");
+    assert!(
+        !md.contains("Oak Boardwalk"),
+        "rows stayed compressed: {md}"
+    );
+}
+
+#[test]
+fn test_auto_keeps_wrapped_header_vector_grid_doc51() {
+    use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
+
+    let buf = std::fs::read("tests/fixtures/government_positions_women.pdf").unwrap();
+    let crop = [0.0, 0.0, 612.0, 792.0];
+    let grid = detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0)
+        .unwrap()
+        .expect("expected doc 51 vector grid");
+    assert_eq!(
+        grid.cell_bboxes.len(),
+        36,
+        "doc 51 should have a 9x4 vector grid"
+    );
+
+    let results = extract_tables_with_structure_auto_mem(
+        &buf,
+        &[TsrTableInput {
+            page: 0,
+            crop_pdf_pt_bbox: crop,
+            render_dpi: 200.0,
+            structure_tokens: grid.structure_tokens,
+            cell_bboxes: grid.cell_bboxes,
+        }],
+    )
+    .unwrap();
+
+    assert_eq!(results.len(), 1);
+    let r = &results[0];
+    assert!(
+        r.fallback_reason.is_none(),
+        "wrapped header/label text should not trigger heuristic fallback: {:?}\n{}",
+        r.fallback_reason,
+        r.markdown
+    );
+    let md = &r.markdown;
+    assert!(md.contains("Government Position"), "missing header: {md}");
+    assert!(
+        md.contains("Aquino Administration"),
+        "missing Aquino header: {md}"
+    );
+    assert!(
+        md.contains("Ramos Administration"),
+        "missing Ramos header: {md}"
+    );
+    assert!(
+        md.contains("City Municipal Councilor"),
+        "row label was truncated: {md}"
+    );
+    assert!(
+        !md.contains("|Position||Administration"),
+        "heuristic fallback split the header row: {md}"
+    );
 }
 
 #[test]
@@ -2256,15 +2690,14 @@ fn test_auto_does_not_fire_on_legit_rowspan_cell() {
 }
 
 #[test]
-fn test_auto_keeps_tsr_markdown_when_heuristic_returns_empty() {
+fn test_auto_expands_when_heuristic_region_is_empty() {
     use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
 
     let buf = synthetic_dense_table_pdf();
     // Same shape as the multi_row_in_cell regression — a tall data cell
-    // that catches Oak Street + Boardwalk. But the crop bbox we pass
-    // points at a strip of the page that has NO text items, so the
-    // heuristic's region will be empty when it tries to extract there.
-    // The auto wrapper must keep the TSR markdown rather than ship "".
+    // that catches Oak Street + Boardwalk. The crop bbox we pass points
+    // at a strip of the page that has NO text items, so the old heuristic
+    // fallback would be empty. Expansion uses the cell bboxes directly.
     let tokens: Vec<String> = [
         "<table>",
         "<thead>",
@@ -2310,20 +2743,19 @@ fn test_auto_keeps_tsr_markdown_when_heuristic_returns_empty() {
     let r = &results[0];
     assert_eq!(
         r.fallback_reason.as_deref(),
-        Some("multi_row_in_cell_heuristic_empty"),
-        "expected _heuristic_empty suffix, got {:?}",
+        Some("multi_row_in_cell_expanded"),
+        "expected expansion despite empty heuristic region, got {:?}",
         r.fallback_reason,
     );
-    // TSR markdown should be preserved — non-empty, contains the cell
-    // text we know was assigned by the TSR path.
     assert!(
-        !r.markdown.trim().is_empty(),
-        "expected TSR markdown to be preserved, got empty",
+        r.markdown.contains("|Oak Street|100|"),
+        "missing row 1: {}",
+        r.markdown
     );
     assert!(
-        r.markdown.contains("Oak Street") || r.markdown.contains("Boardwalk"),
-        "expected TSR markdown to contain at least one row, got: {}",
-        r.markdown,
+        r.markdown.contains("|Boardwalk|200|"),
+        "missing row 2: {}",
+        r.markdown
     );
 }
 
@@ -2597,3 +3029,338 @@ fn test_extract_pages_markdown_path_none_returns_all_pages() {
     let result = extract_pages_markdown(path, None).unwrap();
     assert_eq!(result.pages.len() as u32, page_count);
 }
+
+// ============================================================================
+// PROBE: investigate dense-cell text-assignment failure mode (failure mode 2)
+// ============================================================================
+
+fn synthetic_wide_row_pdf() -> Vec<u8> {
+    use lopdf::content::{Content, Operation};
+    use lopdf::{dictionary, Document, Object, Stream};
+
+    let mut doc = Document::with_version("1.5");
+    let pages_id = doc.new_object_id();
+    let page_id = doc.new_object_id();
+    let font_id = doc.new_object_id();
+    let content_id = doc.new_object_id();
+
+    doc.objects.insert(
+        font_id,
+        dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+        }
+        .into(),
+    );
+
+    let operations = vec![
+        Operation::new("BT", vec![]),
+        Operation::new("Tf", vec!["F1".into(), 10.into()]),
+        Operation::new("Td", vec![20.into(), 700.into()]),
+        // A single Tj that visually spans multiple cells. This mirrors PDFs
+        // where a row's address/role/email columns are emitted as one literal
+        // string with embedded spaces, producing one wide TextItem.
+        Operation::new(
+            "Tj",
+            vec![Object::string_literal("Name JobTitle Email Phone")],
+        ),
+        Operation::new("ET", vec![]),
+    ];
+    let content = Content { operations }.encode().unwrap();
+    doc.objects
+        .insert(content_id, Stream::new(dictionary! {}, content).into());
+
+    doc.objects.insert(
+        page_id,
+        dictionary! {
+            "Type" => "Page",
+            "Parent" => pages_id,
+            "MediaBox" => vec![0.into(), 0.into(), 200.into(), 800.into()],
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => font_id,
+                },
+            },
+            "Contents" => content_id,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        pages_id,
+        dictionary! {
+            "Type" => "Pages",
+            "Kids" => vec![page_id.into()],
+            "Count" => 1,
+        }
+        .into(),
+    );
+    let catalog_id = doc.add_object(dictionary! {
+        "Type" => "Catalog",
+        "Pages" => pages_id,
+    });
+    doc.trailer.set("Root", catalog_id);
+
+    let mut bytes = Vec::new();
+    doc.save_to(&mut bytes).unwrap();
+    bytes
+}
+
+#[test]
+fn test_extract_tables_with_structure_distributes_wide_item_across_cells() {
+    use pdf_inspector::{extract_tables_with_structure_cells_mem, TsrTableInput};
+
+    // Reproduces failure mode 2: a row of multi-token text rendered as one
+    // Tj produces a single wide TextItem that visually spans multiple cells.
+    // The current first-match-by-center routing parks the entire item in
+    // whichever cell holds the item's center, leaving the other cells empty.
+    // See production samples in scrape_id 019de788-ff41-... where 10-column
+    // grids ended up with row text packed into one cell.
+    let buf = synthetic_wide_row_pdf();
+
+    // Helvetica 10pt with width=0 falls back to char_count*font_size*0.5.
+    // "Name JobTitle Email Phone" is 25 chars → effective_width 125pt,
+    // text starts at PDF (20, 700), top-down y=[90, 100], char_w≈5pt.
+    // Tokens land at:
+    //   "Name"     chars 0-3   center≈x=30
+    //   "JobTitle" chars 5-12  center≈x=65
+    //   "Email"    chars 14-18 center≈x=100
+    //   "Phone"    chars 20-24 center≈x=130
+    let cell_bboxes = vec![
+        poly(15.0, 88.0, 50.0, 102.0),
+        poly(50.0, 88.0, 85.0, 102.0),
+        poly(85.0, 88.0, 120.0, 102.0),
+        poly(120.0, 88.0, 155.0, 102.0),
+    ];
+
+    let tokens: Vec<String> = [
+        "<table>",
+        "<tbody>",
+        "<tr>",
+        "<td></td>",
+        "<td></td>",
+        "<td></td>",
+        "<td></td>",
+        "</tr>",
+        "</tbody>",
+        "</table>",
+    ]
+    .into_iter()
+    .map(String::from)
+    .collect();
+
+    let cells_lists = extract_tables_with_structure_cells_mem(
+        &buf,
+        &[TsrTableInput {
+            page: 0,
+            crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
+            render_dpi: 72.0,
+            structure_tokens: tokens,
+            cell_bboxes,
+        }],
+    )
+    .unwrap();
+
+    let cells = &cells_lists[0];
+    assert_eq!(cells.len(), 4);
+    assert_eq!(
+        cells[0].text, "Name",
+        "cell 0 should hold 'Name', got {:?}",
+        cells[0].text
+    );
+    assert_eq!(
+        cells[1].text, "JobTitle",
+        "cell 1 should hold 'JobTitle', got {:?}",
+        cells[1].text
+    );
+    assert_eq!(
+        cells[2].text, "Email",
+        "cell 2 should hold 'Email', got {:?}",
+        cells[2].text
+    );
+    assert_eq!(
+        cells[3].text, "Phone",
+        "cell 3 should hold 'Phone', got {:?}",
+        cells[3].text
+    );
+}
+
+// ============================================================================
+// PROPER TEST: synthetic Type0/Identity-H PDF with malformed ToUnicode CMap
+// ============================================================================
+//
+// Complements the existing real-PDF fixture `shinagawa_identity_h.pdf` by
+// building a minimal Type0 / Identity-H font in process. We control:
+//   * the byte stream emitted by Tj (a 2-byte CID containing one high byte),
+//   * the malformed ToUnicode contents (junk bytes that won't parse), and
+//   * the DescendantFonts shape (just enough for `parse_type0_widths` to set
+//     `is_cid=true`, which is what the new guard in `extract_text_from_operand`
+//     keys off of).
+// No fixture file or external license to worry about.
+
+fn synthetic_type0_broken_tounicode_pdf() -> Vec<u8> {
+    use lopdf::content::{Content, Operation};
+    use lopdf::{dictionary, Document, Object, Stream};
+
+    let mut doc = Document::with_version("1.5");
+    let pages_id = doc.new_object_id();
+    let page_id = doc.new_object_id();
+    let font_id = doc.new_object_id();
+    let cid_font_id = doc.new_object_id();
+    let descriptor_id = doc.new_object_id();
+    let tounicode_id = doc.new_object_id();
+    let cid_system_info_id = doc.new_object_id();
+    let content_id = doc.new_object_id();
+
+    // Type0 font with Identity-H encoding and a broken ToUnicode reference.
+    doc.objects.insert(
+        font_id,
+        dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type0",
+            "BaseFont" => "AAAAAA+SyntheticCID",
+            "Encoding" => "Identity-H",
+            "DescendantFonts" => vec![cid_font_id.into()],
+            "ToUnicode" => tounicode_id,
+        }
+        .into(),
+    );
+
+    // CIDSystemInfo and a minimal CIDFontType2 descendant. parse_type0_widths
+    // walks DescendantFonts → returns FontWidthInfo with is_cid=true. That's
+    // the only thing the new Latin-1 guard needs to see.
+    doc.objects.insert(
+        cid_system_info_id,
+        dictionary! {
+            "Registry" => Object::string_literal("Adobe"),
+            "Ordering" => Object::string_literal("Identity"),
+            "Supplement" => 0,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        cid_font_id,
+        dictionary! {
+            "Type" => "Font",
+            "Subtype" => "CIDFontType2",
+            "BaseFont" => "AAAAAA+SyntheticCID",
+            "CIDSystemInfo" => cid_system_info_id,
+            "FontDescriptor" => descriptor_id,
+            "DW" => 1000,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        descriptor_id,
+        dictionary! {
+            "Type" => "FontDescriptor",
+            "FontName" => "AAAAAA+SyntheticCID",
+            "Flags" => 4,
+            "FontBBox" => vec![Object::Integer(-100), Object::Integer(-100), 1000.into(), 1000.into()],
+            "ItalicAngle" => 0,
+            "Ascent" => 800,
+            "Descent" => Object::Integer(-200),
+            "CapHeight" => 700,
+            "StemV" => 80,
+        }
+        .into(),
+    );
+
+    // Intentionally malformed ToUnicode stream — just junk bytes. ToUnicode
+    // CMap parsing will fail, so `font_cmaps.get_by_obj` returns None and
+    // `has_cmap` stays false. The reference still exists in the font dict,
+    // so `font_tounicode_refs` contains the entry — but the new guard now
+    // routes off `is_cid` from font_widths instead, which is robust to a
+    // failed CMap parse.
+    doc.objects.insert(
+        tounicode_id,
+        Stream::new(dictionary! {}, b"this is not a valid CMap stream".to_vec()).into(),
+    );
+
+    // Tj with a 2-byte CID stream containing a non-ASCII high byte.
+    // Pre-fix this would have decoded as Latin-1 to "\u{00CD}\u{00D9}" ("ÍÙ").
+    // Post-fix it should produce U+FFFD per CID.
+    let cid_bytes = vec![0xCD_u8, 0xD9, 0xCD, 0xD9];
+    let operations = vec![
+        Operation::new("BT", vec![]),
+        Operation::new("Tf", vec!["F0".into(), 12.into()]),
+        Operation::new("Td", vec![50.into(), 100.into()]),
+        Operation::new(
+            "Tj",
+            vec![Object::String(cid_bytes, lopdf::StringFormat::Hexadecimal)],
+        ),
+        Operation::new("ET", vec![]),
+    ];
+    let content = Content { operations }.encode().unwrap();
+    doc.objects
+        .insert(content_id, Stream::new(dictionary! {}, content).into());
+
+    doc.objects.insert(
+        page_id,
+        dictionary! {
+            "Type" => "Page",
+            "Parent" => pages_id,
+            "MediaBox" => vec![0.into(), 0.into(), 200.into(), 200.into()],
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F0" => font_id,
+                },
+            },
+            "Contents" => content_id,
+        }
+        .into(),
+    );
+    doc.objects.insert(
+        pages_id,
+        dictionary! {
+            "Type" => "Pages",
+            "Kids" => vec![page_id.into()],
+            "Count" => 1,
+        }
+        .into(),
+    );
+    let catalog_id = doc.add_object(dictionary! {
+        "Type" => "Catalog",
+        "Pages" => pages_id,
+    });
+    doc.trailer.set("Root", catalog_id);
+
+    let mut bytes = Vec::new();
+    doc.save_to(&mut bytes).unwrap();
+    bytes
+}
+
+#[test]
+fn test_synthetic_type0_broken_tounicode_emits_fffd_not_latin1_mojibake() {
+    let buf = synthetic_type0_broken_tounicode_pdf();
+
+    let items = pdf_inspector::extractor::extract_text_with_positions_mem(&buf).unwrap();
+    let combined: String = items.iter().map(|i| i.text.as_str()).collect();
+
+    // Mojibake leak check: 2-byte CID 0xCDD9 must NOT come out as "ÍÙ"
+    // (U+00CD U+00D9). That was the production scrape symptom.
+    assert!(
+        !combined.contains('\u{00CD}'),
+        "Latin-1 mojibake leaked from Type0 font: {combined:?}"
+    );
+    assert!(
+        !combined.contains('\u{00D9}'),
+        "Latin-1 mojibake leaked from Type0 font: {combined:?}"
+    );
+
+    // Marker presence: Type0/CID + non-ASCII bytes must produce U+FFFD so
+    // `detect_encoding_issues` can flag the page for OCR downstream.
+    assert!(
+        combined.contains('\u{FFFD}'),
+        "Type0 font with malformed ToUnicode CMap should emit U+FFFD per CID; got: {combined:?}"
+    );
+
+    // End-to-end check: the page is correctly routed to OCR.
+    let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
+    assert!(
+        result.pages_needing_ocr.contains(&1),
+        "Type0 page with broken ToUnicode + non-ASCII bytes must be flagged for OCR; \
+         pages_needing_ocr={:?}",
+        result.pages_needing_ocr
+    );
+}

tests/snapshots/td9264.md

@@ -54,9 +54,14 @@ company other than a life insurance company shall make a return on Form 1120PC.
 
 annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return.
 
-||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and|
-|---|---|
-||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of|
+(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and
+(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company.
+(4) Exception for insurance companies filing their Federal income tax returns
+electronically. If an insurance company described in paragraph (c)(1), (c)(2), or
+
+(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e).
+(5) Definition. For purposes of this section, the term annual statement means
+the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of
 
 Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement.
 
@@ -201,3 +206,4 @@ CFR part or section where Current OMB identified or described control No.
 Deputy Commissioner for Services and Enforcement.
 
 Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy).
+