fix: detect hierarchical-indent TOCs as tables

Validation 1 (≥25% of rows have first col) and validation 9 (paragraph
content) were rejecting TOC pages where top-level chapters indent at the
leftmost X but subsections cascade further right. The leftmost column ends
up sparse (only chapter rows land there) and most cells are empty, but the
structure is still an unambiguous TOC.

Skip both validations when cells form a TOC pattern AND the table is narrow
(≤5 cols). The width cap preserves existing handling of wide multi-column
TOCs (e.g. back-of-book 2-up indices) where format_toc_as_list would mash
adjacent visual entries together.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/workflows/publish.yml

@@ -2,14 +2,41 @@ name: Publish npm package
 
 on:
   push:
-    tags: ['v*']
+    branches: [main]
+    paths: ['napi/package.json']
 
 permissions:
   contents: read
   id-token: write
 
 jobs:
+  check-version:
+    name: Check version change
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      version: ${{ steps.check.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Check if version changed
+        id: check
+        run: |
+          NEW_VERSION=$(node -p "require('./napi/package.json').version")
+          OLD_VERSION=$(git show HEAD~1:napi/package.json | node -p "JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')).version")
+          echo "old=$OLD_VERSION new=$NEW_VERSION"
+          if [ "$NEW_VERSION" != "$OLD_VERSION" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
   build:
+    needs: check-version
+    if: needs.check-version.outputs.changed == 'true'
     name: Build ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -19,6 +46,8 @@ jobs:
             target: x86_64-unknown-linux-gnu
           - os: macos-14
             target: aarch64-apple-darwin
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
     steps:
       - uses: actions/checkout@v4
 
@@ -68,7 +97,7 @@ jobs:
 
   publish:
     name: Publish to npm
-    needs: build
+    needs: [check-version, build]
     runs-on: ubuntu-latest
     permissions:
       contents: read

AGENTS.md

@@ -0,0 +1,79 @@
+# pdf-inspector
+
+Fast PDF text extraction to structured Markdown. CLI binary: `pdf2md`. Detection binary: `detect-pdf`.
+
+## Build & Test
+
+```bash
+cargo fmt                                    # format
+cargo clippy -- -D warnings                  # lint (enforced, zero warnings)
+cargo test                                   # unit + integration tests (267+ unit, 73+ integration)
+cargo build --release                        # release binary for benchmarks
+```
+
+All three must pass before committing.
+
+## Binaries
+
+- `pdf2md` — extract PDF → Markdown. Supports `--json` for structured output.
+- `detect-pdf` — classify PDF type (TextBased/Scanned/Mixed/ImageBased). Supports `--analyze --json`.
+
+## Architecture
+
+```
+src/
+  lib.rs                        – public API, process_pdf_with_options, encoding issue detection
+  detector.rs                   – PDF type classification, tiled-scan detection, page sampling
+  types.rs                      – TextItem, TextLine, PdfRect, PdfLine
+  tounicode.rs                  – CMap/ToUnicode parsing, CID decoding
+  text_utils.rs                 – CJK/RTL handling, Otsu threshold, ligature expansion, NFKC
+  extractor/
+    mod.rs                      – top-level extraction orchestrator
+    content_stream.rs           – PDF operator state machine (Tj/TJ/Td/Tm/q/Q)
+    fonts.rs                    – font width/encoding, CMapDecisionCache, TrueType cmap fallback
+    layout.rs                   – column detection (histogram), newspaper/tabular classification,
+                                  spanning-line pre-masking, sidebar detection
+  tables/
+    detect_rects.rs             – rect-based table detection (union-find clustering)
+    detect_heuristic.rs         – heuristic table detection (gap-histogram, body-font tables)
+    detect_lines.rs             – line-based table detection (H/V line grids)
+    grid.rs                     – column/row boundaries, cell assignment
+    format.rs                   – table→Markdown formatting, continuation row merging
+  markdown/
+    convert.rs                  – core line→Markdown loop, struct-tree role support
+    analysis.rs                 – font stats, heading tiers, paragraph thresholds
+    classify.rs                 – line classification (header, list, code, caption)
+    preprocess.rs               – drop cap merging, heading line merging
+    postprocess.rs              – dot leaders, hyphenation, page numbers, URL formatting
+```
+
+## Key design decisions
+
+- **Primary audience is AI agents.** Output optimized for token efficiency and semantic quality, not visual formatting. No cosmetic padding.
+- **Three table detection strategies** run in priority order: rect-based → line-based → heuristic. First valid result wins.
+- **Column detection** uses horizontal projection histograms with valley detection. Multi-item spanning lines (titles, headers) are pre-masked using column-aware thresholds before column assignment.
+- **Newspaper vs tabular** classification determines reading order: newspaper reads columns sequentially, tabular Y-interleaves them.
+- **Tiled-scan detection** catches scanned PDFs with JBIG2/strip images where no single tile exceeds the template threshold but aggregate area does (≥2M pixels).
+- **Garbage text upgrade** reclassifies Mixed PDFs as Scanned when extracted text is <50% alphanumeric.
+- **Tagged PDF support** uses structure tree roles (H1-H6, P, L, Code, BlockQuote) when available, falling back to font-size heuristics.
+
+## Testing
+
+- **Unit tests**: inline `#[cfg(test)] mod tests` in each module with synthetic data.
+- **Integration tests**: `tests/integration_tests.rs` with fixture PDFs in `tests/fixtures/`.
+- **Regression suite**: sibling repo `pdf-evals` with 179+ snapshot PDFs. Run `cargo build --release` then `bench.py test` in that repo before committing.
+
+## Debugging
+
+```bash
+RUST_LOG=pdf_inspector::extractor::layout=debug cargo run --bin pdf2md -- file.pdf
+RUST_LOG=pdf_inspector::tables=debug cargo run --bin pdf2md -- file.pdf
+RUST_LOG=pdf_inspector::detector=debug cargo run --release --bin detect-pdf -- file.pdf
+```
+
+## Conventions
+
+- Clippy: use `is_some_and(...)` not `map_or(false, ...)`
+- lopdf quirk: `ParseError` is private — match by string for `InvalidFileHeader`
+- Column limit for tables: 25 (wide statistical tables)
+- `propagate_merged_cells` skipped for >10 columns (spanning rects = background fills)

README.md

@@ -55,12 +55,12 @@ print(result.markdown)   # Markdown string or None
 ### Node.js
 
 ```bash
-npm install @firecrawl/pdf-inspector-js
+npm install @firecrawl/pdf-inspector
 ```
 
 ```javascript
 import { readFileSync } from 'fs';
-import { processPdf, classifyPdf } from '@firecrawl/pdf-inspector-js';
+import { processPdf, classifyPdf } from '@firecrawl/pdf-inspector';
 
 const result = processPdf(readFileSync('document.pdf'));
 console.log(result.pdfType);   // "TextBased", "Scanned", "ImageBased", "Mixed"

napi/README.md

@@ -1,4 +1,4 @@
-# firecrawl-pdf-inspector
+# PDF Inspector
 
 Fast PDF classification and region-based text extraction for Node.js/Bun. Native Rust performance via [napi-rs](https://napi.rs).
 
@@ -7,9 +7,9 @@ Built by [Firecrawl](https://firecrawl.dev) for hybrid OCR pipelines — extract
 ## Install
 
 ```bash
-npm install firecrawl-pdf-inspector
+npm install @firecrawl/pdf-inspector
 # or
-bun add firecrawl-pdf-inspector
+bun add @firecrawl/pdf-inspector
 ```
 
 Prebuilt binaries included for **linux-x64** and **macOS ARM64**. No Rust toolchain needed.
@@ -21,7 +21,7 @@ Prebuilt binaries included for **linux-x64** and **macOS ARM64**. No Rust toolch
 Classify a PDF as TextBased, Scanned, Mixed, or ImageBased (~10-50ms). Returns which pages need OCR.
 
 ```typescript
-import { classifyPdf } from 'firecrawl-pdf-inspector'
+import { classifyPdf } from '@firecrawl/pdf-inspector'
 import { readFileSync } from 'fs'
 
 const pdf = readFileSync('document.pdf')
@@ -40,7 +40,7 @@ Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pip
 Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues).
 
 ```typescript
-import { extractTextInRegions } from 'firecrawl-pdf-inspector'
+import { extractTextInRegions } from '@firecrawl/pdf-inspector'
 
 const result = extractTextInRegions(pdf, [
   {

napi/bin/pdf-inspector.mjs

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+
+import { readFileSync, writeFileSync } from "fs";
+import { createRequire } from "module";
+
+const require = createRequire(import.meta.url);
+const { version } = require("../package.json");
+
+const HELP = `pdf-inspector v${version} — Fast PDF text extraction to Markdown
+
+Usage:
+  pdf-inspector <file>                  Extract markdown (default)
+  pdf-inspector detect <file>           Classify PDF type
+
+Options:
+  --json                  Output as JSON
+  --pages <pages>         Comma-separated page numbers (e.g. 1,3,5)
+  -o, --output <file>     Write output to file instead of stdout
+  -h, --help              Show this help
+  -v, --version           Show version
+
+Examples:
+  pdf-inspector document.pdf
+  pdf-inspector document.pdf --json
+  pdf-inspector document.pdf --pages 1,2,3
+  pdf-inspector detect document.pdf --json
+  cat document.pdf | pdf-inspector -`;
+
+function die(msg) {
+  process.stderr.write(`error: ${msg}\n`);
+  process.exit(1);
+}
+
+function parseArgs(argv) {
+  const opts = { json: false, pages: null, output: null, file: null, command: "extract" };
+  let i = 0;
+
+  // Check for subcommand
+  if (argv[0] === "detect") {
+    opts.command = "detect";
+    i = 1;
+  }
+
+  while (i < argv.length) {
+    const arg = argv[i];
+    if (arg === "-h" || arg === "--help") {
+      process.stdout.write(HELP + "\n");
+      process.exit(0);
+    } else if (arg === "-v" || arg === "--version") {
+      process.stdout.write(`${version}\n`);
+      process.exit(0);
+    } else if (arg === "--json") {
+      opts.json = true;
+    } else if (arg === "--pages") {
+      i++;
+      if (!argv[i]) die("--pages requires a value (e.g. 1,3,5)");
+      opts.pages = argv[i].split(",").map((p) => {
+        const n = parseInt(p.trim(), 10);
+        if (Number.isNaN(n) || n < 1) die(`invalid page number: ${p}`);
+        return n;
+      });
+    } else if (arg === "-o" || arg === "--output") {
+      i++;
+      if (!argv[i]) die("-o requires a filename");
+      opts.output = argv[i];
+    } else if (arg === "-" || !arg.startsWith("-")) {
+      if (opts.file) die(`unexpected argument: ${arg}`);
+      opts.file = arg;
+    } else {
+      die(`unknown option: ${arg}`);
+    }
+    i++;
+  }
+
+  return opts;
+}
+
+function readInput(file) {
+  if (file === "-") {
+    return readFileSync(0); // stdin fd
+  }
+  try {
+    return readFileSync(file);
+  } catch (err) {
+    if (err.code === "ENOENT") die(`file not found: ${file}`);
+    die(err.message);
+  }
+}
+
+function output(text, outputPath) {
+  if (outputPath) {
+    writeFileSync(outputPath, text);
+  } else {
+    process.stdout.write(text);
+  }
+}
+
+// ---- main ----
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (!opts.file) {
+  // Check if stdin is piped
+  if (process.stdin.isTTY !== false) {
+    process.stderr.write(HELP + "\n");
+    process.exit(1);
+  }
+  opts.file = "-";
+}
+
+const { processPdf, classifyPdf } = await import("../index.js");
+const buffer = readInput(opts.file);
+
+if (opts.command === "detect") {
+  const result = classifyPdf(buffer);
+  if (opts.json) {
+    output(JSON.stringify(result, null, 2) + "\n", opts.output);
+  } else {
+    const ocr = result.pagesNeedingOcr.length > 0
+      ? `, ${result.pagesNeedingOcr.length} pages need OCR`
+      : "";
+    output(`${result.pdfType} (${result.pageCount} pages, confidence: ${result.confidence.toFixed(2)}${ocr})\n`, opts.output);
+  }
+} else {
+  const result = processPdf(buffer, opts.pages ?? undefined);
+  if (opts.json) {
+    output(JSON.stringify(result, null, 2) + "\n", opts.output);
+  } else {
+    output((result.markdown ?? "") + "\n", opts.output);
+  }
+}

napi/package.json

@@ -1,9 +1,12 @@
 {
-  "name": "firecrawl-pdf-inspector",
+  "name": "@firecrawl/pdf-inspector",
-  "version": "0.7.0",
+  "version": "1.3.0",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",
+  "bin": {
+    "pdf-inspector": "bin/pdf-inspector.mjs"
+  },
   "license": "MIT",
   "keywords": [
     "pdf",
@@ -20,6 +23,7 @@
     "index.js",
     "index.d.ts",
     "*.node",
+    "bin/",
     "README.md"
   ],
   "repository": {
@@ -34,7 +38,8 @@
     "binaryName": "pdf-inspector",
     "targets": [
       "x86_64-unknown-linux-gnu",
-      "aarch64-apple-darwin"
+      "aarch64-apple-darwin",
+      "x86_64-pc-windows-msvc"
     ],
     "package": {
       "name": "@firecrawl/pdf-inspector-js"

src/extractor/content_stream.rs

@@ -216,6 +216,7 @@ pub(crate) fn extract_page_text_items(
     let mut marked_content_stack: Vec<MarkedContentEntry> = Vec::new();
     let mut suppress_glyph_extraction = false;
     let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
+    let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
     /// Get the innermost MCID from the marked content stack.
     fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
         stack.iter().rev().find_map(|e| e.mcid)
@@ -349,8 +350,15 @@ pub(crate) fn extract_page_text_items(
                             )
                         })
                     });
-                    // ActualText: suppress glyph extraction, just advance text matrix
+                    // ActualText: suppress glyph extraction, just advance text matrix.
+                    // Capture the FIRST glyph's text matrix as the rendering position
+                    // for the ActualText item. Td ops between BDC and the first Tj
+                    // may have moved the position to the correct line — the BDC-entry
+                    // position (actual_text_start_tm) can be on the previous line.
                     if suppress_glyph_extraction {
+                        if actual_text_glyph_tm.is_none() {
+                            actual_text_glyph_tm = Some(text_matrix);
+                        }
                         if let Some(w_ts) = w_ts_opt {
                             text_matrix[4] += w_ts * text_matrix[0];
                             text_matrix[5] += w_ts * text_matrix[1];
@@ -425,6 +433,10 @@ pub(crate) fn extract_page_text_items(
                         let font_info = font_widths.get(&current_font);
                         let is_invisible = (text_rendering_mode == 3 && !include_invisible)
                             || suppress_glyph_extraction;
+                        // Capture first-glyph position for ActualText
+                        if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
+                            actual_text_glyph_tm = Some(text_matrix);
+                        }
 
                         // Compute space threshold based on font metrics when available
                         let space_threshold = if let Some(font_info) = font_info {
@@ -700,6 +712,7 @@ pub(crate) fn extract_page_text_items(
                 if actual_text.is_some() {
                     suppress_glyph_extraction = true;
                     actual_text_start_tm = Some(text_matrix);
+                    actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
                 }
                 marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
             }
@@ -707,8 +720,13 @@ pub(crate) fn extract_page_text_items(
                 // End Marked Content — emit ActualText item with correct width
                 if let Some(entry) = marked_content_stack.pop() {
                     if let Some(at) = entry.actual_text {
-                        // Compute width from text matrix advancement during BDC..EMC
+                        // Use the first-glyph position (if available) instead of the
-                        if let Some(start_tm) = actual_text_start_tm.take() {
+                        // BDC-entry position. Td operators between BDC and the first
+                        // Tj may have moved the text position to the correct line —
+                        // the BDC-entry position can be on the previous line.
+                        let glyph_tm = actual_text_glyph_tm.take();
+                        let entry_tm = actual_text_start_tm.take();
+                        if let Some(start_tm) = glyph_tm.or(entry_tm) {
                             let combined = multiply_matrices(&start_tm, &ctm);
                             if combined[0].abs() >= combined[1].abs() {
                                 rotation_votes.horizontal += 1;

src/lib.rs

@@ -1,3 +1,9 @@
+// Rust 1.95 introduced collapsible_match for `if` inside match arms.
+// The content-stream parsers use this pattern extensively (match on operator
+// name, then check `in_text_block && !op.operands.is_empty()`). Collapsing
+// these into match guards would hurt readability. Allow crate-wide.
+#![allow(clippy::collapsible_match)]
+
 //! Smart PDF detection and text extraction using lopdf
 //!
 //! # Quick start

src/tables/detect_heuristic.rs

@@ -581,8 +581,14 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
     // Validation 1: some rows should have content in first column.
     // Use a lower threshold (25%) for tables with wrapped cells where
     // continuation lines leave the first column empty.
+    // Skip when cells form a narrow TOC pattern: hierarchical entries indented
+    // across multiple X levels leave the leftmost column sparse (only top-level
+    // chapters land there) but the structure is still a valid TOC. Narrow only
+    // (<=5 cols) — wide multi-column TOCs (e.g. 2-up indices) would render
+    // poorly through format_toc_as_list, which assumes one entry per row.
     let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
-    if rows_with_first_col < rows.len() / 4 {
+    let is_narrow_toc = columns.len() <= 5 && is_table_of_contents(&cells);
+    if rows_with_first_col < rows.len() / 4 && !is_narrow_toc {
         log::debug!(
             "  validation 1 fail: {}/{} rows have first col",
             rows_with_first_col,
@@ -653,15 +659,24 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
         return None;
     }
 
-    // Validation 8: Check for Table of Contents pattern
+    // Validation 8: Reject paragraph-like content falsely detected as tables.
-    if is_table_of_contents(&cells) {
+    // TOC pages with deep indentation (top-level chapters in col 0, subsections
-        log::debug!("  validation 8 fail: table of contents");
+    // in cols 1-3, page numbers in last col) leave most cells empty and trip
+    // the paragraph heuristic; TOC shape is a safer signal here. Narrow only
+    // — see narrow-TOC rationale at validation 1.
+    if is_paragraph_content(&cells) && !is_narrow_toc {
+        log::debug!("  validation 9 fail: paragraph content");
         return None;
     }
 
-    // Validation 9: Reject paragraph-like content falsely detected as tables
+    // Validation 9: Reject wide "index" layouts where every cell carries a
-    if is_paragraph_content(&cells) {
+    // full "label ... page" fragment (back-of-book IRS-style indices).
-        log::debug!("  validation 9 fail: paragraph content");
+    // These render poorly in any structured form; text flow is the best
+    // fallback.  Narrow dot-leader TOCs (2-3 cols) are kept so format.rs
+    // can emit them as a per-row flat list with titles tab-joined to page
+    // numbers.
+    if is_inline_leader_index(&cells) {
+        log::debug!("  validation 9 fail: inline-leader index");
         return None;
     }
 
@@ -898,77 +913,247 @@ fn looks_like_number(s: &str) -> bool {
         && s.chars().any(|c| c.is_ascii_digit())
 }
 
-/// Check if this looks like a Table of Contents
+/// Check if this looks like a Table of Contents (either style).
-/// TOCs have characteristic patterns: leader dots, page numbers, section names
+///
-fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
+/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
+pub(super) fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
+    is_dot_leader_toc(cells) || is_tabular_toc(cells)
+}
+
+/// Dot-leader TOC: any "Chapter 1 ........ 42" style with explicit leader
+/// dots.  Covers both narrow 2-3 col TOCs (where the leader is a dedicated
+/// cell) and wide indices (where each cell encodes a full "label ... page"
+/// fragment).  Used by format.rs to render as a flat list.
+pub(super) fn is_dot_leader_toc(cells: &[Vec<String>]) -> bool {
+    has_structural_dot_leader(cells) || is_inline_leader_index(cells)
+}
+
+/// Rows with a dedicated dots-only cell flanked by label + number (2-3 col
+/// TOC layout).  Format.rs handles these well via per-row flat-list
+/// rendering; they should NOT be rejected at detect time.
+fn has_structural_dot_leader(cells: &[Vec<String>]) -> bool {
     if cells.is_empty() {
         return false;
     }
+    let structural_rows = cells.iter().filter(|row| row_has_dot_leader(row)).count();
+    structural_rows as f32 / cells.len() as f32 >= 0.3
+}
 
-    let num_cols = cells[0].len();
+/// Wide index layout: each cell holds a full "label ... page" fragment
-    let mut dot_cells = 0;
+/// because the column detector kept multi-column indices as single cells.
-    let mut page_number_cells = 0;
+/// These render poorly both as markdown tables (column boundaries are
-    let mut total_cells = 0;
+/// arbitrary) and as flat lists (each row holds 3+ separate index
-    // Track which columns contain dots vs numbers to distinguish
+/// entries).  Reject these at detect time so they fall back to the page's
-    // TOC (dots span middle, page number at end) from data tables
+/// normal text flow.
-    // (dots only in label column, many number columns).
+pub(super) fn is_inline_leader_index(cells: &[Vec<String>]) -> bool {
-    let mut dot_cols = vec![0u32; num_cols];
+    let mut inline_cells = 0;
-    let mut numeric_cols = vec![0u32; num_cols];
+    let mut total_nonempty = 0;
-
     for row in cells {
-        for (ci, cell) in row.iter().enumerate() {
+        for cell in row {
             let trimmed = cell.trim();
             if trimmed.is_empty() {
                 continue;
             }
-            total_cells += 1;
+            total_nonempty += 1;
+            if cell_is_inline_leader(trimmed) {
+                inline_cells += 1;
+            }
+        }
+    }
+    total_nonempty >= 4 && inline_cells as f32 / total_nonempty as f32 >= 0.25
+}
 
-            // Check for leader dots (sequences of periods)
+/// A row with a dot-leader.  Accepts two layouts:
-            // TOCs often have "........" or ". . . ." patterns
+///   1. A dedicated dots-only cell ("....") with a text label somewhere
+///      to its left and a page number somewhere to its right.
+///   2. A "title ... " cell (trailing leader dots glued to the title)
+///      with a page number elsewhere in the same row.
+fn row_has_dot_leader(row: &[String]) -> bool {
+    let has_page_number = row.iter().any(|c| row_cell_is_page_number(c));
+
+    for (ci, cell) in row.iter().enumerate() {
+        let trimmed = cell.trim();
+
+        // Pattern 1: dedicated dots-only cell.
         let dot_count = trimmed.chars().filter(|&c| c == '.').count();
-            let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3;
+        let is_mostly_dots = dot_count >= 3
+            && dot_count > trimmed.len() / 2
+            && trimmed.chars().all(|c| c == '.' || c.is_whitespace());
         if is_mostly_dots {
-                dot_cells += 1;
+            let has_label_left = row[..ci].iter().any(|c| {
-                if ci < num_cols {
+                let t = c.trim();
-                    dot_cols[ci] += 1;
+                !t.is_empty() && t.chars().any(|ch| ch.is_alphabetic())
+            });
+            if has_label_left && has_page_number {
+                return true;
             }
+            continue;
         }
 
-            // Check for standalone page numbers (1-4 digits, possibly with spaces)
+        // Pattern 2: cell ends with a trailing " ... " run after a label.
-            let digits_only: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect();
+        if has_page_number && cell_has_trailing_leader(trimmed) {
-            if digits_only.len() <= 4
+            return true;
-                && !digits_only.is_empty()
-                && digits_only.chars().all(|c| c.is_ascii_digit())
-            {
-                page_number_cells += 1;
-                if ci < num_cols {
-                    numeric_cols[ci] += 1;
-                }
         }
     }
+    false
 }
 
-    if total_cells == 0 {
+/// Cell ends with a run of ≥3 dots preceded by alphabetic text and a
+/// space — the "Title ... " layout where the leader is glued to the name.
+/// Alphabetic (not alphanumeric) so that data-table row labels like
+/// "1973 ... " do not register as titles.
+fn cell_has_trailing_leader(cell: &str) -> bool {
+    let trimmed = cell.trim_end();
+    if !trimmed.ends_with('.') {
+        return false;
+    }
+    let without_dots = trimmed.trim_end_matches('.');
+    let dot_run = trimmed.len() - without_dots.len();
+    if dot_run < 3 {
+        return false;
+    }
+    // Require a space before the dot run (rules out "etc..." / "Mr...") and
+    // at least one alphabetic char (rules out "1973 ... " data-row labels).
+    without_dots.ends_with(' ') && without_dots.trim().chars().any(|c| c.is_alphabetic())
+}
+
+/// Page-number shape: single ≤4-digit integer, a ", "-separated list of
+/// ≤4-digit integers ("18, 36, 107"), or a dashed section-page ID
+/// ("A-1", "5-21").  Rejects decimal cells ("4. 0"), thousands-separated
+/// values ("189,164"), and other long numeric data that appears in
+/// statistical tables.
+fn row_cell_is_page_number(cell: &str) -> bool {
+    let t = cell.trim();
+    if t.is_empty() {
+        return false;
+    }
+    if looks_like_section_page_id(t) {
+        return true;
+    }
+    // Page list: ", " separator (with space) distinguishes real page lists
+    // from thousands-separated numbers like "189,164".
+    let parts: Vec<&str> = t.split(", ").collect();
+    parts
+        .iter()
+        .all(|p| !p.is_empty() && p.len() <= 4 && p.chars().all(|c| c.is_ascii_digit()))
+}
+
+/// A cell shaped like an index leader fragment.  Accepts two forms:
+///   - "text ... number" — label + dots + page number in one cell
+///   - "... number"      — bare leader + number (row where the label
+///     landed in a separate column)
+///
+/// Both only count if followed by pure numeric content (optionally
+/// comma-separated page lists like "127, 213").
+fn cell_is_inline_leader(cell: &str) -> bool {
+    let cell = cell.trim();
+
+    // Find the first "..." run.  Surrounding-whitespace checks below
+    // reject intra-word ellipses ("etc...").
+    let idx = match cell.match_indices("...").next() {
+        Some((i, _)) => i,
+        None => return false,
+    };
+
+    let before = &cell[..idx];
+    let after_dots = &cell[idx + 3..];
+    // Allow extra dots (e.g. "....") by skipping any additional '.'
+    let after = after_dots.trim_start_matches('.');
+
+    // Require space (or start-of-cell) before the dots and space/digit
+    // after — blocks intra-word ellipses.
+    let before_ok = before.is_empty() || before.ends_with(' ');
+    let after_ok = after.starts_with(' ') || after.is_empty();
+    if !before_ok || !after_ok {
         return false;
     }
 
-    // Data tables with dot leaders (e.g. "1973....") have dots concentrated
+    let after_trim = after.trim();
-    // in one column (the label column) while many other columns contain numbers.
+    if after_trim.is_empty() {
-    // True TOCs have dots spanning the middle and one page-number column at the end.
+        return false;
-    // If dots are confined to ≤1 column AND there are ≥3 columns with numbers,
+    }
-    // this is a data table, not a TOC.
+    // Tail must be purely numeric/page-list content.
-    let cols_with_dots = dot_cols.iter().filter(|&&c| c >= 2).count();
+    let tail_numeric = after_trim
-    let cols_with_numbers = numeric_cols.iter().filter(|&&c| c >= 2).count();
+        .chars()
-    if cols_with_dots <= 1 && cols_with_numbers >= 3 {
+        .all(|c| c.is_ascii_digit() || matches!(c, ',' | ' ' | '.' | '-' | '$'))
+        && after_trim.chars().any(|c| c.is_ascii_digit());
+    if !tail_numeric {
         return false;
     }
 
-    // If a significant portion of cells are dots or page numbers, it's likely a TOC
+    // Either we have a label before, or the leader is bare (starts the cell)
-    let dot_ratio = dot_cells as f32 / total_cells as f32;
+    // — both are legitimate index fragments.
-    let page_num_ratio = page_number_cells as f32 / total_cells as f32;
+    before.chars().any(|c| c.is_alphabetic()) || before.trim().is_empty()
+}
 
-    // TOC typically has >15% dot cells and >10% page number cells
+/// Dot-less tabular TOC: tagged PDFs emit entries as rows where the first
-    dot_ratio > 0.15 || (dot_ratio > 0.05 && page_num_ratio > 0.15)
+/// column starts with a dotted section number (e.g. "4.3.1 Something") and
+/// the last column is one or more page numbers.  These have no leader dots
+/// and benefit from flat-list formatting (page numbers aligned to titles).
+pub(super) fn is_tabular_toc(cells: &[Vec<String>]) -> bool {
+    if cells.is_empty() {
+        return false;
+    }
+    let num_cols = cells[0].len();
+    if num_cols < 2 || cells.len() < 4 {
+        return false;
+    }
+
+    let section_rows = cells
+        .iter()
+        .filter(|row| {
+            row.iter()
+                .find(|c| !c.trim().is_empty())
+                .is_some_and(|c| starts_with_section_number(c.trim()))
+        })
+        .count();
+
+    let last_col = num_cols - 1;
+    let (last_filled, last_page_num) = cells.iter().fold((0u32, 0u32), |(f, n), row| {
+        let cell = row.get(last_col).map(|s| s.trim()).unwrap_or("");
+        if cell.is_empty() {
+            return (f, n);
+        }
+        let is_page_nums = cell
+            .split_whitespace()
+            .all(|tok| !tok.is_empty() && tok.chars().all(|c| c.is_ascii_digit()));
+        (f + 1, n + if is_page_nums { 1 } else { 0 })
+    });
+
+    let section_ratio = section_rows as f32 / cells.len() as f32;
+    let page_num_last_ratio = if last_filled > 0 {
+        last_page_num as f32 / last_filled as f32
+    } else {
+        0.0
+    };
+
+    section_ratio >= 0.6 && last_filled >= 3 && page_num_last_ratio >= 0.7
+}
+
+/// Matches dashed section-page identifiers used in technical manuals:
+/// "5-21", "A-1", "B--3", "TC-2".  At least one ASCII digit is required.
+fn looks_like_section_page_id(s: &str) -> bool {
+    let ok = s
+        .chars()
+        .all(|c| c.is_ascii_digit() || c.is_ascii_uppercase() || c == '-');
+    ok && s.chars().any(|c| c.is_ascii_digit())
+}
+
+/// Returns true when the leading token looks like a dotted section number:
+/// "1", "1.2", "1.2.3", "4.3.1.2" — integer components joined by dots,
+/// with at least one dot (single-number prefixes are too ambiguous).
+fn starts_with_section_number(s: &str) -> bool {
+    let Some(first) = s.split_whitespace().next() else {
+        return false;
+    };
+    let first = first.trim_end_matches('.');
+    let parts: Vec<&str> = first.split('.').collect();
+    if parts.len() < 2 || parts.len() > 6 {
+        return false;
+    }
+    parts
+        .iter()
+        .all(|p| !p.is_empty() && p.len() <= 3 && p.chars().all(|c| c.is_ascii_digit()))
 }
 
 /// Check if detected "table" cells are actually paragraph text fragments.
@@ -1144,6 +1329,33 @@ pub(crate) fn find_first_table_row(
             continue;
         }
 
+        // Skip rows that have duplicate non-empty cells. These are spanning
+        // super-headers (e.g., "First Degree | First Degree | Higher Degree")
+        // that sit above the real column header row. Using them as the markdown
+        // header produces duplicate column names that downstream validation
+        // rejects. Only skip if a subsequent row looks like a better header
+        // (denser fill or has data).
+        if filled_count >= 2 && !has_data {
+            let mut text_counts: std::collections::HashMap<&str, usize> =
+                std::collections::HashMap::new();
+            for cell in &filled_cells {
+                *text_counts.entry(cell.trim()).or_insert(0) += 1;
+            }
+            let has_duplicates = text_counts.values().any(|&count| count >= 2);
+            if has_duplicates {
+                // Check if a later row is a better header candidate
+                let has_better_below = cells.iter().skip(row_idx + 1).take(3).any(|r| {
+                    let next_filled = r.iter().filter(|c| !c.trim().is_empty()).count();
+                    let next_fill = next_filled as f32 / total_cols as f32;
+                    let next_numeric = r.iter().filter(|c| looks_like_number(c.trim())).count();
+                    next_fill >= 0.4 || next_numeric >= 2
+                });
+                if has_better_below {
+                    continue;
+                }
+            }
+        }
+
         // Data rows are definitely table content
         if has_data {
             first_table_row = row_idx;
@@ -1398,4 +1610,283 @@ mod tests {
             "data table with dot-leader labels should not be rejected as TOC"
         );
     }
+
+    #[test]
+    fn is_table_of_contents_accepts_hierarchical_indented_toc() {
+        // Mythos system card pages 4-5: top-level chapters indent at col 0,
+        // subsections at cols 1-2, leaving col 0 mostly empty (only ~10% of
+        // rows). Validation 1 was rejecting these even though the structure
+        // is unambiguously a TOC.
+        let cells = vec![
+            vec!["Abstract".to_string(), String::new(), "3".to_string()],
+            vec![
+                "1 Introduction".to_string(),
+                String::new(),
+                "10".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1 Model training".to_string(),
+                "11".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1.1 Training data".to_string(),
+                "11".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1.2 Crowd workers".to_string(),
+                "12".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.2 Release decision".to_string(),
+                "13".to_string(),
+            ],
+            vec![
+                "2 RSP evaluations".to_string(),
+                String::new(),
+                "16".to_string(),
+            ],
+            vec![
+                String::new(),
+                "2.1 RSP risk assessment".to_string(),
+                "16".to_string(),
+            ],
+            vec![String::new(), "2.1.1 Context".to_string(), "16".to_string()],
+            vec![
+                String::new(),
+                "2.2 CB evaluations".to_string(),
+                "20".to_string(),
+            ],
+        ];
+        assert!(
+            is_table_of_contents(&cells),
+            "hierarchical TOC with sparse col 0 should still be detected"
+        );
+    }
+
+    #[test]
+    fn is_table_of_contents_rejects_dotless_toc() {
+        // Tabular TOC without leader dots: first column starts with dotted
+        // section numbers, last column is page numbers.  Pattern from
+        // Mythos system card pages 6-8.
+        let cells = vec![
+            vec![
+                "4.3 Case studies and targeted evaluations".to_string(),
+                String::new(),
+                "86".to_string(),
+            ],
+            vec![
+                "4.3.1 Destructive or reckless actions".to_string(),
+                "4.3.1.1 Synthetic-backend evaluation".to_string(),
+                "86 86".to_string(),
+            ],
+            vec![
+                "4.3.2 Adherence to constitution".to_string(),
+                "4.3.2.1 Overview".to_string(),
+                "89 89".to_string(),
+            ],
+            vec![
+                "4.3.3 Honesty and hallucinations".to_string(),
+                "4.3.3.1 Factual hallucinations".to_string(),
+                "93 94".to_string(),
+            ],
+            vec![
+                "4.4 Capability evaluations".to_string(),
+                String::new(),
+                "101".to_string(),
+            ],
+        ];
+        assert!(
+            is_table_of_contents(&cells),
+            "dot-less TOC with section numbers + page numbers should be rejected"
+        );
+    }
+
+    #[test]
+    fn dot_leader_toc_accepts_short_inline_leaders() {
+        // Index-style cells where the full "label ... number" pattern is
+        // preserved in a single cell (IRS Publication 17 back-of-book index).
+        let cells = vec![
+            vec!["Child tax credit ... 235".to_string(), String::new()],
+            vec!["Church employee ... 252".to_string(), String::new()],
+            vec!["Citizens outside the U.S ... 6".to_string(), String::new()],
+            vec![
+                "Claim for refund ... 18, 36, 107".to_string(),
+                String::new(),
+            ],
+            vec!["Clergy ... 7, 52".to_string(), String::new()],
+        ];
+        assert!(is_dot_leader_toc(&cells));
+    }
+
+    #[test]
+    fn dot_leader_toc_allows_ellipsis_data_table() {
+        // Data tables using "..." as a row-omission marker must not be
+        // mistaken for dot-leader TOCs.  Based on MCF5235RM QSPI RAM layout.
+        let cells = vec![
+            vec![
+                "0x00".to_string(),
+                "QTR0".to_string(),
+                "Transmit RAM".to_string(),
+            ],
+            vec!["0x01".to_string(), "QTR1".to_string(), String::new()],
+            vec![
+                "...".to_string(),
+                "...".to_string(),
+                "16 bits wide".to_string(),
+            ],
+            vec!["0x0F".to_string(), "QTR15".to_string(), String::new()],
+            vec![
+                "0x10".to_string(),
+                "QRR0".to_string(),
+                "Receive RAM".to_string(),
+            ],
+            vec!["0x11".to_string(), "QRR1".to_string(), String::new()],
+            vec![
+                "...".to_string(),
+                "...".to_string(),
+                "16 bits wide".to_string(),
+            ],
+            vec!["0x1F".to_string(), "QRR15".to_string(), String::new()],
+        ];
+        assert!(
+            !is_dot_leader_toc(&cells),
+            "ellipsis markers in a data table should not match TOC detection"
+        );
+    }
+
+    #[test]
+    fn dot_leader_toc_rejects_year_row_data_table() {
+        // ERP-2025 economic data tables: year labels with trailing " ... ",
+        // a final " ... " column, and decimal-looking numeric cells.  The
+        // detection previously classified these as dot-leader TOCs and
+        // routed them through flat-list formatting, destroying the grid.
+        let cells = vec![
+            vec![
+                "1973 ... ".to_string(),
+                "4. 0".to_string(),
+                "1. 8".to_string(),
+                "0. 4".to_string(),
+                "3. 2".to_string(),
+                " ... ".to_string(),
+            ],
+            vec![
+                "1974 ... ".to_string(),
+                "–1. 9".to_string(),
+                "–1. 6".to_string(),
+                "–5. 6".to_string(),
+                "2. 4".to_string(),
+                " ... ".to_string(),
+            ],
+            vec![
+                "1975 ... ".to_string(),
+                "2. 6".to_string(),
+                "5. 1".to_string(),
+                "6. 1".to_string(),
+                "4. 1".to_string(),
+                " ... ".to_string(),
+            ],
+            vec![
+                "1976 ... ".to_string(),
+                "4. 3".to_string(),
+                "5. 4".to_string(),
+                "6. 4".to_string(),
+                "4. 5".to_string(),
+                " ... ".to_string(),
+            ],
+        ];
+        assert!(
+            !is_dot_leader_toc(&cells),
+            "year-indexed data tables with decimal cells must not match TOC detection"
+        );
+    }
+
+    #[test]
+    fn dot_leader_toc_rejects_monthly_data_table() {
+        // ERP-2025 Table B-22: monthly labor-force rows with "Jan ... ",
+        // "Feb ... " labels and thousands-separated cells ("189,164").
+        // Previously matched TOC detection because "Jan ..." has alphabetic
+        // text and "189,164" passed the page-number shape check.
+        let cells = vec![
+            vec![
+                "2023: Jan ... ".to_string(),
+                "265,962".to_string(),
+                "165,871".to_string(),
+                "160,152".to_string(),
+                "62. 4".to_string(),
+            ],
+            vec![
+                "Feb ... ".to_string(),
+                "266,112".to_string(),
+                "166,263".to_string(),
+                "160,301".to_string(),
+                "62. 5".to_string(),
+            ],
+            vec![
+                "Mar ... ".to_string(),
+                "266,272".to_string(),
+                "166,690".to_string(),
+                "160,824".to_string(),
+                "62. 6".to_string(),
+            ],
+            vec![
+                "Apr ... ".to_string(),
+                "266,443".to_string(),
+                "166,678".to_string(),
+                "160,962".to_string(),
+                "62. 6".to_string(),
+            ],
+        ];
+        assert!(
+            !is_dot_leader_toc(&cells),
+            "monthly labor-force rows with thousands-separated data must not match TOC detection"
+        );
+    }
+
+    #[test]
+    fn tabular_toc_requires_section_numbers_and_pages() {
+        // Dot-less tabular TOC matches is_tabular_toc but not dot-leader.
+        let cells = vec![
+            vec![
+                "4.3 Case studies".to_string(),
+                String::new(),
+                "86".to_string(),
+            ],
+            vec![
+                "4.3.1 Destructive actions".to_string(),
+                String::new(),
+                "86".to_string(),
+            ],
+            vec![
+                "4.3.2 Adherence".to_string(),
+                String::new(),
+                "89".to_string(),
+            ],
+            vec!["4.3.3 Honesty".to_string(), String::new(), "93".to_string()],
+        ];
+        assert!(is_tabular_toc(&cells));
+        assert!(!is_dot_leader_toc(&cells));
+    }
+
+    #[test]
+    fn starts_with_section_number_matches_dotted() {
+        assert!(starts_with_section_number("1.2"));
+        assert!(starts_with_section_number("4.3.1"));
+        assert!(starts_with_section_number("4.3.1.2"));
+        assert!(starts_with_section_number("4.3 Case studies"));
+        assert!(starts_with_section_number("2.2.5.1 Expert red teaming"));
+    }
+
+    #[test]
+    fn starts_with_section_number_rejects_non_sections() {
+        assert!(!starts_with_section_number("Chapter 1"));
+        assert!(!starts_with_section_number("1973"));
+        assert!(!starts_with_section_number("1.5M"));
+        assert!(!starts_with_section_number("10.0%"));
+        assert!(!starts_with_section_number(""));
+        assert!(!starts_with_section_number("Hello world"));
+    }
 }

src/tables/format.rs

@@ -1,5 +1,6 @@
 //! Table-to-markdown formatting and cell cleanup.
 
+use super::detect_heuristic::is_table_of_contents;
 use super::Table;
 
 pub fn table_to_markdown(table: &Table) -> String {
@@ -7,6 +8,22 @@ pub fn table_to_markdown(table: &Table) -> String {
         return String::new();
     }
 
+    // Detect TOC on the raw cells: clean_table_cells merges rows in ways
+    // that can make genuine data tables superficially resemble a TOC
+    // (short numeric cells, few columns) — but the raw detection here
+    // preserves the original multi-column structure and only matches the
+    // true TOC pattern.
+    //
+    // Tables of contents render poorly as markdown tables — emit a flat
+    // per-row text list instead so the page numbers stay aligned with
+    // their section titles rather than drifting to a separate column.
+    // Format from raw cells: continuation-row merging collapses separate
+    // TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench") into a
+    // single line because sub-entries leave column 0 empty.
+    if is_table_of_contents(&table.cells) {
+        return format_toc_as_list(&table.cells, &[]);
+    }
+
     // Clean up the table: merge continuation rows, extract footnotes, remove empty rows
     let (cleaned_cells, footnotes) = clean_table_cells(&table.cells);
 
@@ -49,6 +66,101 @@ pub fn table_to_markdown(table: &Table) -> String {
     output
 }
 
+/// Render a table-of-contents as a flat per-row text block.
+///
+/// Each row becomes one line: non-empty cells joined with spaces, and the
+/// last cell (typically a page number) is separated by a tab so the page
+/// numbers stay aligned with their titles instead of being pulled into a
+/// separate column by the column-aware reader.
+fn format_toc_as_list(cells: &[Vec<String>], footnotes: &[String]) -> String {
+    let mut output = String::new();
+
+    for row in cells {
+        let trimmed: Vec<&str> = row.iter().map(|c| c.trim()).collect();
+        let last_idx = trimmed.iter().rposition(|c| !c.is_empty());
+        let Some(last_idx) = last_idx else {
+            continue;
+        };
+
+        let last_cell = trimmed[last_idx];
+        let last_is_page = is_page_number_cell(last_cell);
+
+        let (title_cells, trailing) = if last_is_page && last_idx > 0 {
+            (&trimmed[..last_idx], Some(last_cell))
+        } else {
+            (&trimmed[..=last_idx], None)
+        };
+
+        // Skip dots-only cells when joining the title — in a detected TOC
+        // layout, a "...." cell is a leader separator, not part of the
+        // entry name.
+        let title = title_cells
+            .iter()
+            .filter(|c| !c.is_empty() && !is_dots_only(c))
+            .copied()
+            .collect::<Vec<_>>()
+            .join(" ");
+
+        if title.is_empty() && trailing.is_none() {
+            continue;
+        }
+
+        if !title.is_empty() {
+            output.push_str(&title);
+        }
+        if let Some(page) = trailing {
+            if !title.is_empty() {
+                output.push('\t');
+            }
+            output.push_str(page);
+        }
+        output.push('\n');
+    }
+
+    if !footnotes.is_empty() {
+        output.push('\n');
+        for footnote in footnotes {
+            output.push_str(footnote);
+            output.push('\n');
+        }
+    }
+
+    output
+}
+
+/// True when the cell looks like a page number.  Accepts:
+///   - plain digit tokens: "42", "86 86"
+///   - dashed section-page IDs: "5-21", "A-1", "B--3", "TC-2" (common in
+///     technical manuals)
+fn is_page_number_cell(cell: &str) -> bool {
+    let tokens: Vec<&str> = cell.split_whitespace().collect();
+    if tokens.is_empty() {
+        return false;
+    }
+    tokens.iter().all(|t| {
+        if t.is_empty() || t.len() > 8 {
+            return false;
+        }
+        let all_digits = t.chars().all(|c| c.is_ascii_digit());
+        if all_digits {
+            return t.len() <= 4;
+        }
+        // Section-page form: uppercase letters, digits, dashes; at least
+        // one digit present.
+        t.chars()
+            .all(|c| c.is_ascii_digit() || c.is_ascii_uppercase() || c == '-')
+            && t.chars().any(|c| c.is_ascii_digit())
+    })
+}
+
+/// True when the cell is purely leader dots (any length ≥ 3) with optional
+/// whitespace.
+fn is_dots_only(cell: &str) -> bool {
+    let t = cell.trim();
+    let dots = t.chars().filter(|&c| c == '.').count();
+    dots >= 3 && t.chars().all(|c| c == '.' || c.is_whitespace())
+}
+
 /// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows
 fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
     let mut cleaned: Vec<Vec<String>> = Vec::new();
@@ -432,4 +544,42 @@ mod tests {
         };
         assert_eq!(table_to_markdown(&table), "");
     }
+
+    #[test]
+    fn test_table_to_markdown_toc_renders_as_flat_list() {
+        // A TOC-shaped table with section numbers in col 0 and page numbers
+        // in the last column should render as a flat list, not a markdown
+        // table, so the page numbers stay on the same line as their titles.
+        let table = Table {
+            columns: vec![50.0, 80.0, 300.0],
+            rows: vec![500.0; 5],
+            cells: vec![
+                vec![
+                    "4.3".into(),
+                    "Case studies and targeted evaluations".into(),
+                    "86".into(),
+                ],
+                vec![
+                    "4.3.1".into(),
+                    "Destructive or reckless actions".into(),
+                    "86".into(),
+                ],
+                vec![
+                    "4.3.2".into(),
+                    "Adherence to its constitution".into(),
+                    "89".into(),
+                ],
+                vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
+                vec!["4.5".into(), "White-box analyses".into(), "113".into()],
+            ],
+            item_indices: vec![],
+        };
+        let md = table_to_markdown(&table);
+        assert!(
+            !md.contains("|---|"),
+            "TOC should not render as a markdown table: {md}"
+        );
+        assert!(md.contains("4.3 Case studies and targeted evaluations\t86"));
+        assert!(md.contains("4.5 White-box analyses\t113"));
+    }
 }

src/tables/grid.rs

@@ -82,33 +82,42 @@ pub(crate) fn find_column_boundaries(
         }
     }
 
-    let mut columns = Vec::new();
+    // Track cluster membership: for each cluster, store the list of x positions
-    let mut cluster_items: Vec<f32> = vec![x_positions[0]];
+    let mut cluster_xs: Vec<Vec<f32>> = vec![vec![x_positions[0]]];
 
     for &x in &x_positions[1..] {
+        let last_cluster = cluster_xs.last().unwrap();
         // For dense columns (gap-histogram triggered), use edge-based clustering:
         // compare with the last item to avoid center-drift that merges adjacent
         // narrow columns.  For normal tables, use center-based (original behavior).
         let reference = if use_edge_clustering {
-            *cluster_items.last().unwrap()
+            *last_cluster.last().unwrap()
         } else {
-            cluster_items.iter().sum::<f32>() / cluster_items.len() as f32
+            last_cluster.iter().sum::<f32>() / last_cluster.len() as f32
         };
 
         if x - reference > cluster_threshold {
-            let cluster_center = cluster_items.iter().sum::<f32>() / cluster_items.len() as f32;
+            cluster_xs.push(vec![x]);
-            columns.push(cluster_center);
-            cluster_items = vec![x];
         } else {
-            cluster_items.push(x);
+            cluster_xs.last_mut().unwrap().push(x);
         }
     }
 
-    // Don't forget last cluster
+    // Numeric column merge pass: when a sparse cluster (few items, typically
-    if !cluster_items.is_empty() {
+    // header text) is adjacent to a dense numeric cluster and within 1.5×
-        columns.push(cluster_items.iter().sum::<f32>() / cluster_items.len() as f32);
+    // threshold, merge them. This fixes tables where multi-line wrapped
+    // headers have slightly different X positions than the data columns,
+    // causing the header and data to split into separate clusters.
+    let columns_before_merge = cluster_xs.len();
+    if columns_before_merge >= 3 {
+        cluster_xs = merge_numeric_adjacent_clusters(cluster_xs, items, cluster_threshold);
     }
 
+    let columns: Vec<f32> = cluster_xs
+        .iter()
+        .map(|xs| xs.iter().sum::<f32>() / xs.len() as f32)
+        .collect();
+
     // Filter columns - each should have multiple items
     let min_items_per_col = (items.len() / columns.len().max(1) / 4).max(2);
     let columns: Vec<f32> = columns
@@ -123,8 +132,9 @@ pub(crate) fn find_column_boundaries(
         .collect();
 
     log::debug!(
-        "  find_column_boundaries: {} columns before filter, threshold={:.1}, {} items",
+        "  find_column_boundaries: {} columns (merged from {}), threshold={:.1}, {} items",
         columns.len(),
+        columns_before_merge,
         cluster_threshold,
         items.len()
     );
@@ -148,6 +158,116 @@ pub(crate) fn find_column_boundaries(
     columns
 }
 
+/// Check if a text string looks like a number (digits, decimals, sign, comma).
+fn is_numeric_text(s: &str) -> bool {
+    let s = s.trim();
+    if s.is_empty() {
+        return false;
+    }
+    // Match patterns like: 8.23, -1.05, 9.99, 7.12, 100, 3,456.78, +5%, ---
+    // But NOT: BIO, Department, Core Courses
+    s.chars()
+        .all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+' || c == '%')
+        && s.chars().any(|c| c.is_ascii_digit())
+}
+
+/// Merge adjacent X-position clusters when one is a sparse header cluster
+/// and the other is a dense numeric data cluster. This prevents multi-line
+/// wrapped headers from splitting a logical column into two clusters.
+fn merge_numeric_adjacent_clusters(
+    mut clusters: Vec<Vec<f32>>,
+    items: &[(usize, &TextItem)],
+    threshold: f32,
+) -> Vec<Vec<f32>> {
+    // For each cluster, compute: center, item count, numeric fraction
+    struct ClusterInfo {
+        center: f32,
+        count: usize,
+        numeric_frac: f32,
+    }
+
+    let compute_info = |xs: &[f32]| -> ClusterInfo {
+        let center = xs.iter().sum::<f32>() / xs.len() as f32;
+        // Count items and numeric fraction for items near this cluster center
+        let mut total = 0;
+        let mut numeric = 0;
+        for (_, item) in items {
+            if (item.x - center).abs() < threshold {
+                total += 1;
+                if is_numeric_text(&item.text) {
+                    numeric += 1;
+                }
+            }
+        }
+        ClusterInfo {
+            center,
+            count: total,
+            numeric_frac: if total > 0 {
+                numeric as f32 / total as f32
+            } else {
+                0.0
+            },
+        }
+    };
+
+    // Merge distance: allow merging clusters that are slightly beyond the
+    // original threshold. Use 1.5× threshold to catch header-vs-data splits.
+    let merge_dist = threshold * 1.5;
+
+    // Iterate and merge adjacent pairs. Use a simple left-to-right scan.
+    let mut merged = true;
+    while merged {
+        merged = false;
+        let mut i = 0;
+        while i + 1 < clusters.len() {
+            let info_a = compute_info(&clusters[i]);
+            let info_b = compute_info(&clusters[i + 1]);
+            let dist = (info_b.center - info_a.center).abs();
+
+            if dist > merge_dist {
+                i += 1;
+                continue;
+            }
+
+            // Determine if one cluster is sparse (header) and the other
+            // is dense and numeric (data). A cluster is "sparse" if it has
+            // significantly fewer items than the other.
+            let (sparse, dense) = if info_a.count < info_b.count {
+                (&info_a, &info_b)
+            } else {
+                (&info_b, &info_a)
+            };
+
+            // Merge if the dense cluster is predominantly numeric (>50%)
+            // and the sparse cluster has at most 1/3 the items of the dense one.
+            let should_merge =
+                dense.numeric_frac > 0.50 && sparse.count <= dense.count / 2 && sparse.count <= 5;
+
+            if should_merge {
+                log::debug!(
+                    "  merging column clusters: center {:.1} ({} items, {:.0}% numeric) + {:.1} ({} items, {:.0}% numeric), dist={:.1}",
+                    info_a.center,
+                    info_a.count,
+                    info_a.numeric_frac * 100.0,
+                    info_b.center,
+                    info_b.count,
+                    info_b.numeric_frac * 100.0,
+                    dist,
+                );
+                // Merge cluster i+1 into cluster i
+                let next = clusters.remove(i + 1);
+                clusters[i].extend(next);
+                merged = true;
+                // Don't increment i — check if the merged cluster can merge further
+            } else {
+                i += 1;
+            }
+        }
+    }
+
+    clusters
+}
+
 /// Find row boundaries by clustering Y positions
 pub(crate) fn find_row_boundaries(items: &[(usize, &TextItem)]) -> Vec<f32> {
     let mut y_positions: Vec<f32> = items.iter().map(|(_, i)| i.y).collect();

src/tounicode.rs

@@ -520,6 +520,18 @@ impl ToUnicodeCMap {
         }
     }
 
+    /// Get the maximum source CID across all mappings (char_map + ranges).
+    fn max_source_cid(&self) -> Option<u16> {
+        let char_max = self.char_map.keys().copied().max();
+        let range_max = self.ranges.iter().map(|&(_, end, _)| end).max();
+        match (char_max, range_max) {
+            (Some(a), Some(b)) => Some(a.max(b)),
+            (a @ Some(_), None) => a,
+            (None, b @ Some(_)) => b,
+            (None, None) => None,
+        }
+    }
+
     /// Remap a CMap that references pre-subsetting GIDs to sequential post-subsetting GIDs.
     /// Collects all source CIDs, sorts them, and reassigns to 1, 2, 3, ...
     pub fn remap_to_sequential(&self) -> ToUnicodeCMap {
@@ -657,6 +669,81 @@ fn get_w_array_start_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> O
     }
 }
 
+/// Return true if the CIDFont's W (widths) array explicitly covers the given CID.
+///
+/// The W array uses two formats (PDF 32000-1:2008, §9.7.4.3):
+///   1. `c [w1 w2 ... wn]` — widths for CIDs c, c+1, ..., c+n-1
+///   2. `c_first c_last w` — CIDs c_first..c_last all have width w
+fn w_array_covers_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document, target: u16) -> bool {
+    let Ok(w_obj) = cid_font_dict.get(b"W") else {
+        return false;
+    };
+    let arr = match w_obj {
+        Object::Array(arr) => arr,
+        Object::Reference(r) => match doc.get_object(*r) {
+            Ok(Object::Array(arr)) => arr,
+            _ => return false,
+        },
+        _ => return false,
+    };
+
+    let resolve_int = |o: &Object| -> Option<i64> {
+        match o {
+            Object::Integer(n) => Some(*n),
+            Object::Reference(r) => match doc.get_object(*r) {
+                Ok(Object::Integer(n)) => Some(*n),
+                _ => None,
+            },
+            _ => None,
+        }
+    };
+
+    let resolve_arr = |o: &Object| -> Option<Vec<Object>> {
+        match o {
+            Object::Array(a) => Some(a.clone()),
+            Object::Reference(r) => match doc.get_object(*r) {
+                Ok(Object::Array(a)) => Some(a.clone()),
+                _ => None,
+            },
+            _ => None,
+        }
+    };
+
+    let target = target as i64;
+    let mut i = 0usize;
+    while i < arr.len() {
+        let Some(first) = resolve_int(&arr[i]) else {
+            break;
+        };
+        i += 1;
+        if i >= arr.len() {
+            break;
+        }
+        // Peek at arr[i] to decide format.
+        if let Some(widths) = resolve_arr(&arr[i]) {
+            // Format 1: c [w1 ... wn]
+            let last = first + widths.len() as i64 - 1;
+            if target >= first && target <= last {
+                return true;
+            }
+            i += 1;
+        } else if let Some(last) = resolve_int(&arr[i]) {
+            // Format 2: c_first c_last w
+            i += 1;
+            if i < arr.len() {
+                i += 1; // skip the width value
+            }
+            if target >= first && target <= last {
+                return true;
+            }
+        } else {
+            // Unknown token — abort parsing safely
+            break;
+        }
+    }
+    false
+}
+
 /// Extract CIDToGIDMap as a vector of GIDs (u16) indexed by CID.
 fn get_cid_to_gid_map(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> Option<Vec<u16>> {
     let obj = cid_font_dict.get(b"CIDToGIDMap").ok()?;
@@ -752,6 +839,20 @@ fn try_remap_subset_cmap(
         _ => return (cmap, None),
     };
 
+    // If the W array actually covers the CMap's max source CID, the CMap is
+    // aligned with the font — no sequential renumbering happened. A sparse W
+    // array starting at CID 0 (for .notdef) with additional high-CID entries
+    // matching the CMap is the normal subset layout, not a mismatch.
+    if let Some(max_cid) = cmap.max_source_cid() {
+        if w_array_covers_cid(cid_font_dict, doc, max_cid) {
+            debug!(
+                "Subset remap skipped for obj={}: W array covers CMap max CID {}",
+                obj_num, max_cid
+            );
+            return (cmap, None);
+        }
+    }
+
     debug!(
         "Subset GID mismatch detected for obj={}: W starts at CID {}, CMap min CID {}. Remapping to sequential.",
         obj_num, w_start, min_cid
@@ -2717,4 +2818,187 @@ endbfchar
         assert_eq!(remapped.unwrap().char_map.len(), 50);
         assert_eq!(fallback.unwrap().char_map.len(), 10);
     }
+
+    #[test]
+    fn test_max_source_cid() {
+        let cmap_content = r#"
+1 begincodespacerange
+<0000><FFFF>
+endcodespacerange
+2 beginbfchar
+<0003> <0020>
+<0031> <004E>
+endbfchar
+1 beginbfrange
+<0208> <0227> <0430>
+endbfrange
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+        assert_eq!(cmap.min_source_cid(), Some(0x0003));
+        assert_eq!(cmap.max_source_cid(), Some(0x0227));
+    }
+
+    /// Helper: build a minimal CIDFont dict with a W array and check coverage.
+    fn cid_font_dict_with_w(w_items: Vec<lopdf::Object>) -> lopdf::Dictionary {
+        let mut d = lopdf::Dictionary::new();
+        d.set("W", lopdf::Object::Array(w_items));
+        d
+    }
+
+    #[test]
+    fn test_w_array_covers_cid_format1() {
+        // Format 1: `c [w1 w2 ... wn]` — widths for CIDs c..c+n-1.
+        // Mimics the 16.pdf Tahoma W array: 0[1000] 3[313] 5[401] 11[383 383] 16[363 303 382]
+        let doc = Document::new();
+        let d = cid_font_dict_with_w(vec![
+            lopdf::Object::Integer(0),
+            lopdf::Object::Array(vec![lopdf::Object::Integer(1000)]),
+            lopdf::Object::Integer(3),
+            lopdf::Object::Array(vec![lopdf::Object::Integer(313)]),
+            lopdf::Object::Integer(5),
+            lopdf::Object::Array(vec![lopdf::Object::Integer(401)]),
+            lopdf::Object::Integer(11),
+            lopdf::Object::Array(vec![
+                lopdf::Object::Integer(383),
+                lopdf::Object::Integer(383),
+            ]),
+            lopdf::Object::Integer(16),
+            lopdf::Object::Array(vec![
+                lopdf::Object::Integer(363),
+                lopdf::Object::Integer(303),
+                lopdf::Object::Integer(382),
+            ]),
+            lopdf::Object::Integer(570),
+            lopdf::Object::Array(vec![lopdf::Object::Integer(667); 26]),
+        ]);
+
+        assert!(w_array_covers_cid(&d, &doc, 0));
+        assert!(w_array_covers_cid(&d, &doc, 3));
+        assert!(w_array_covers_cid(&d, &doc, 5));
+        assert!(w_array_covers_cid(&d, &doc, 11));
+        assert!(w_array_covers_cid(&d, &doc, 12));
+        assert!(w_array_covers_cid(&d, &doc, 16));
+        assert!(w_array_covers_cid(&d, &doc, 18));
+        assert!(w_array_covers_cid(&d, &doc, 570));
+        assert!(w_array_covers_cid(&d, &doc, 595));
+        // Gaps are NOT covered
+        assert!(!w_array_covers_cid(&d, &doc, 1));
+        assert!(!w_array_covers_cid(&d, &doc, 4));
+        assert!(!w_array_covers_cid(&d, &doc, 19));
+        assert!(!w_array_covers_cid(&d, &doc, 596));
+    }
+
+    #[test]
+    fn test_w_array_covers_cid_format2() {
+        // Format 2: `c_first c_last w` — CIDs c_first..c_last all have width w.
+        let doc = Document::new();
+        let d = cid_font_dict_with_w(vec![
+            lopdf::Object::Integer(100),
+            lopdf::Object::Integer(120),
+            lopdf::Object::Integer(500),
+        ]);
+
+        assert!(w_array_covers_cid(&d, &doc, 100));
+        assert!(w_array_covers_cid(&d, &doc, 110));
+        assert!(w_array_covers_cid(&d, &doc, 120));
+        assert!(!w_array_covers_cid(&d, &doc, 99));
+        assert!(!w_array_covers_cid(&d, &doc, 121));
+    }
+
+    #[test]
+    fn test_w_array_covers_cid_missing_w() {
+        let doc = Document::new();
+        let d = lopdf::Dictionary::new();
+        assert!(!w_array_covers_cid(&d, &doc, 3));
+    }
+
+    #[test]
+    fn test_try_remap_skipped_when_w_covers_cmap() {
+        // Simulates 16.pdf: CMap's max source CID (0x0279 = 633) is explicitly
+        // in the W array, so no subset-renumbering happened — remap must NOT fire.
+        let cmap_content = r#"
+1 begincodespacerange
+<0000><FFFF>
+endcodespacerange
+2 beginbfchar
+<0003> <0020>
+<0031> <004E>
+endbfchar
+2 beginbfrange
+<023A> <0253> <0410>
+<0255> <0279> <042B>
+endbfrange
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        let mut doc = Document::new();
+        // Build a CIDFont dict with Identity CIDToGIDMap and a W array that
+        // covers CID 633 via `597 [widths...]`.
+        let mut cid_font = lopdf::Dictionary::new();
+        cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
+        cid_font.set(
+            "W",
+            lopdf::Object::Array(vec![
+                lopdf::Object::Integer(0),
+                lopdf::Object::Array(vec![lopdf::Object::Integer(750)]),
+                lopdf::Object::Integer(597),
+                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 37]), // 597..633
+            ]),
+        );
+        let cid_font_id = doc.add_object(cid_font);
+
+        // Build the Type0 font dict with Identity-H + DescendantFonts ref.
+        let mut font_dict = lopdf::Dictionary::new();
+        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
+        font_dict.set(
+            "DescendantFonts",
+            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
+        );
+
+        let (primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 123);
+        assert!(
+            remapped.is_none(),
+            "Remap must be skipped when W covers CMap max CID (this is 16.pdf)"
+        );
+        assert_eq!(primary.lookup(0x0003), Some(" ".to_string()));
+    }
+
+    #[test]
+    fn test_try_remap_fires_for_true_subset_mismatch() {
+        // True mismatch: CMap has high CIDs (512-544) but W only lists low sequential CIDs.
+        let cmap_content = r#"
+1 begincodespacerange
+<0000><FFFF>
+endcodespacerange
+1 beginbfrange
+<0200> <0220> <0410>
+endbfrange
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        let mut doc = Document::new();
+        let mut cid_font = lopdf::Dictionary::new();
+        cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
+        cid_font.set(
+            "W",
+            lopdf::Object::Array(vec![
+                lopdf::Object::Integer(0),
+                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 34]), // 0..33
+            ]),
+        );
+        let cid_font_id = doc.add_object(cid_font);
+
+        let mut font_dict = lopdf::Dictionary::new();
+        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
+        font_dict.set(
+            "DescendantFonts",
+            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
+        );
+
+        let (_primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 456);
+        assert!(
+            remapped.is_some(),
+            "Remap must fire when CMap's CIDs are outside W array coverage"
+        );
+    }
 }

tests/integration_tests.rs

@@ -1438,6 +1438,42 @@ fn test_extract_tables_in_regions_nonexistent_page() {
     assert!(region.text.is_empty());
 }
 
+#[test]
+fn test_bits_pilani_page4_table_detection() {
+    // Page 4 (0-indexed 3) has a table with multi-line wrapped headers and
+    // numeric data columns. The heuristic detector previously failed because:
+    // 1. Header items at different X positions than data created extra column
+    //    clusters (6 cols instead of 4)
+    // 2. Spanning super-header row ("First Degree | First Degree") produced
+    //    duplicate header cells that looks_like_partial_table_ex rejected
+    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(3, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
+    assert_eq!(results.len(), 1);
+    let region = &results[0].regions[0];
+    assert!(
+        !region.needs_ocr,
+        "Page 4 table should be detected, got needs_ocr=true"
+    );
+    assert!(
+        region.text.contains("BIO"),
+        "Should contain department name BIO"
+    );
+    assert!(region.text.contains("8.23"), "Should contain numeric data");
+}
+
+#[test]
+fn test_bits_pilani_page8_table_detection() {
+    // Page 8 (0-indexed 7) has a numbered-row table that already worked.
+    // Verify it still works after changes.
+    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(7, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
+    assert_eq!(results.len(), 1);
+    let region = &results[0].regions[0];
+    assert!(!region.needs_ocr, "Page 8 table should still be detected");
+}
+
 // =========================================================================
 // extract_pages_markdown_mem tests
 // =========================================================================