fix: don't split tagged-LI continuation lines into separate bullets

Some tagged PDFs use a flat style where each wrapped visual line of a list
item gets its own MCID tagged directly under /LI. The LI branch was
unconditionally prefixing every such line with "- ", turning continuation
lines into their own bullets. Only emit a new bullet when we're not already
inside a list; otherwise fall through to the existing continuation logic so
the text is appended to the previous item.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/lib.rs

@@ -1782,19 +1782,26 @@ fn compute_layout_complexity(
                 markdown::filter_lines_to_band(lines, page, x_lo, x_hi)
             };
 
+            // TOC pages route through the table detector but render as flat
+            // lists. They aren't tables in any user-facing sense, so don't
+            // count them toward LayoutComplexity (would also trip the
+            // table-page guard in column detection below).
+            let has_data_table =
+                |tables: &[tables::Table]| tables.iter().any(|t| t.kind == tables::TableKind::Data);
+
             let (rect_tables, _) = tables::detect_tables_from_rects(&band_items, &band_rects, page);
-            if !rect_tables.is_empty() {
+            if has_data_table(&rect_tables) {
                 found_table = true;
                 break;
             }
             let line_tables = tables::detect_tables_from_lines(&band_items, &band_lines, page);
-            if !line_tables.is_empty() {
+            if has_data_table(&line_tables) {
                 found_table = true;
                 break;
             }
             // Heuristic fallback for borderless tables
             let heuristic_tables = tables::detect_tables(&band_items, base_size, false);
-            if !heuristic_tables.is_empty() {
+            if has_data_table(&heuristic_tables) {
                 found_table = true;
                 break;
             }

src/markdown/convert.rs

@@ -641,11 +641,16 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
             continue;
         }
 
-        // Structure-tree list item (LI only — LBody is a continuation, not a new item)
+        // Structure-tree list item (LI only — LBody is a continuation, not a new item).
+        // Some tagged PDFs use a "flat" style where every wrapped line in a list item
+        // gets its own MCID tagged directly under LI. When we're already inside a list
+        // and the line has no visible bullet marker, treat it as a continuation (falls
+        // through to the continuation logic below) rather than a new list item.
         if struct_role
             .as_ref()
             .is_some_and(|r| matches!(r, StructRole::LI))
             && !is_list_item(plain_trimmed)
+            && !in_list
         {
             if in_paragraph {
                 output.push_str("\n\n");
@@ -1090,6 +1095,59 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_struct_role_li_flat_continuation_lines_merge() {
+        // Regression: some tagged PDFs put each wrapped visual line of a list
+        // item under its own MCID, all tagged directly as LI. Continuation
+        // lines (no bullet marker) must merge into the bulleted parent item,
+        // not each become their own list item.
+        let make = |text: &str, mcid: i64, x: f32, y: f32| {
+            let mut item = make_item(text, 1, Some(mcid));
+            item.x = x;
+            item.y = y;
+            item
+        };
+        let lines = vec![
+            make_line(vec![make("● First item that wraps onto", 0, 90.0, 322.0)]),
+            make_line(vec![make("a continuation line.", 1, 108.0, 306.0)]),
+            make_line(vec![make("● Second bullet also wraps", 2, 90.0, 290.0)]),
+            make_line(vec![make("to a second line here.", 3, 108.0, 274.0)]),
+        ];
+
+        let mut page_roles = HashMap::new();
+        for mcid in 0..4 {
+            page_roles.insert(mcid, StructRole::LI);
+        }
+        let mut roles = HashMap::new();
+        roles.insert(1u32, page_roles);
+
+        let md = to_markdown_from_lines_with_tables_and_images(
+            lines,
+            MarkdownOptions::default(),
+            HashMap::new(),
+            HashMap::new(),
+            &std::collections::HashSet::new(),
+            Some(&roles),
+        );
+
+        assert!(
+            md.contains("- First item that wraps onto a continuation line."),
+            "continuation should merge into first bullet: {md}"
+        );
+        assert!(
+            md.contains("- Second bullet also wraps to a second line here."),
+            "continuation should merge into second bullet: {md}"
+        );
+        assert!(
+            !md.contains("- a continuation line."),
+            "continuation line should not get its own bullet: {md}"
+        );
+        assert!(
+            !md.contains("- to a second line here."),
+            "continuation line should not get its own bullet: {md}"
+        );
+    }
+
     #[test]
     fn test_struct_role_blockquote() {
         let lines = vec![make_line(vec![make_item("Quoted text", 1, Some(0))])];

src/tables/detect_heuristic.rs

@@ -581,8 +581,14 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
     // Validation 1: some rows should have content in first column.
     // Use a lower threshold (25%) for tables with wrapped cells where
     // continuation lines leave the first column empty.
+    // Skip when cells form a narrow TOC pattern: hierarchical entries indented
+    // across multiple X levels leave the leftmost column sparse (only top-level
+    // chapters land there) but the structure is still a valid TOC. Narrow only
+    // (<=5 cols) — wide multi-column TOCs (e.g. 2-up indices) would render
+    // poorly through format_toc_as_list, which assumes one entry per row.
     let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
-    if rows_with_first_col < rows.len() / 4 {
+    let is_narrow_toc = columns.len() <= 5 && is_table_of_contents(&cells);
+    if rows_with_first_col < rows.len() / 4 && !is_narrow_toc {
         log::debug!(
             "  validation 1 fail: {}/{} rows have first col",
             rows_with_first_col,
@@ -653,8 +659,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
         return None;
     }
 
-    // Validation 8: Reject paragraph-like content falsely detected as tables
-    if is_paragraph_content(&cells) {
+    // Validation 8: Reject paragraph-like content falsely detected as tables.
+    // TOC pages with deep indentation (top-level chapters in col 0, subsections
+    // in cols 1-3, page numbers in last col) leave most cells empty and trip
+    // the paragraph heuristic; TOC shape is a safer signal here. Narrow only
+    // — see narrow-TOC rationale at validation 1.
+    if is_paragraph_content(&cells) && !is_narrow_toc {
         log::debug!("  validation 9 fail: paragraph content");
         return None;
     }
@@ -677,12 +687,7 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
         item_indices.len()
     );
 
-    Some(Table {
-        columns,
-        rows,
-        cells,
-        item_indices,
-    })
+    Some(Table::new(columns, rows, cells, item_indices))
 }
 
 /// Check if this looks like a key-value pair layout rather than a table
@@ -906,7 +911,7 @@ fn looks_like_number(s: &str) -> bool {
 /// Check if this looks like a Table of Contents (either style).
 ///
 /// Used by format.rs to render TOCs as flat lists instead of markdown tables.
-pub(super) fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
+pub fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
     is_dot_leader_toc(cells) || is_tabular_toc(cells)
 }
 
@@ -1601,6 +1606,62 @@ mod tests {
         );
     }
 
+    #[test]
+    fn is_table_of_contents_accepts_hierarchical_indented_toc() {
+        // Mythos system card pages 4-5: top-level chapters indent at col 0,
+        // subsections at cols 1-2, leaving col 0 mostly empty (only ~10% of
+        // rows). Validation 1 was rejecting these even though the structure
+        // is unambiguously a TOC.
+        let cells = vec![
+            vec!["Abstract".to_string(), String::new(), "3".to_string()],
+            vec![
+                "1 Introduction".to_string(),
+                String::new(),
+                "10".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1 Model training".to_string(),
+                "11".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1.1 Training data".to_string(),
+                "11".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.1.2 Crowd workers".to_string(),
+                "12".to_string(),
+            ],
+            vec![
+                String::new(),
+                "1.2 Release decision".to_string(),
+                "13".to_string(),
+            ],
+            vec![
+                "2 RSP evaluations".to_string(),
+                String::new(),
+                "16".to_string(),
+            ],
+            vec![
+                String::new(),
+                "2.1 RSP risk assessment".to_string(),
+                "16".to_string(),
+            ],
+            vec![String::new(), "2.1.1 Context".to_string(), "16".to_string()],
+            vec![
+                String::new(),
+                "2.2 CB evaluations".to_string(),
+                "20".to_string(),
+            ],
+        ];
+        assert!(
+            is_table_of_contents(&cells),
+            "hierarchical TOC with sparse col 0 should still be detected"
+        );
+    }
+
     #[test]
     fn is_table_of_contents_rejects_dotless_toc() {
         // Tabular TOC without leader dots: first column starts with dotted

src/tables/detect_lines.rs

@@ -265,12 +265,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
         page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
     );
 
-    vec![Table {
-        columns: col_edges,
-        rows: row_edges_desc[..num_rows].to_vec(),
+    vec![Table::new(
+        col_edges,
+        row_edges_desc[..num_rows].to_vec(),
         cells,
         item_indices,
-    }]
+    )]
 }
 
 #[cfg(test)]

src/tables/detect_rects.rs

@@ -1037,12 +1037,7 @@ fn try_build_grid(
         (columns, cells)
     };
 
-    GridResult::Ok(Table {
-        columns,
-        rows,
-        cells,
-        item_indices,
-    })
+    GridResult::Ok(Table::new(columns, rows, cells, item_indices))
 }
 
 /// Deduplicate nearby edge values within a tolerance, returning sorted unique edges.
@@ -1442,12 +1437,7 @@ fn detect_row_stripe_table(
         content_ratio * 100.0
     );
 
-    Some(Table {
-        columns: column_centers,
-        rows: row_centers,
-        cells,
-        item_indices,
-    })
+    Some(Table::new(column_centers, row_centers, cells, item_indices))
 }
 
 /// Detect a table from cell-background rects that failed grid detection.
@@ -1691,12 +1681,7 @@ fn detect_row_stripe_table_from_cell_rects(
         non_empty_cells as f32 / total_cells * 100.0
     );
 
-    Some(Table {
-        columns: column_centers,
-        rows: row_centers,
-        cells,
-        item_indices,
-    })
+    Some(Table::new(column_centers, row_centers, cells, item_indices))
 }
 
 /// Detect a table by merging all cluster rects into one group.
@@ -1875,12 +1860,7 @@ fn detect_merged_cluster_table(
         content_ratio * 100.0
     );
 
-    Some(Table {
-        columns: column_centers,
-        rows: row_centers,
-        cells,
-        item_indices,
-    })
+    Some(Table::new(column_centers, row_centers, cells, item_indices))
 }
 
 /// Cluster text item X positions into column centers with a given minimum threshold.

src/tables/detect_struct.rs

@@ -188,12 +188,12 @@ pub fn detect_tables_from_struct_tree(
         all_item_indices.sort_unstable();
         all_item_indices.dedup();
 
-        tables.push(Table {
-            columns: col_positions,
-            rows: row_positions,
+        tables.push(Table::new(
+            col_positions,
+            row_positions,
             cells,
-            item_indices: all_item_indices,
-        });
+            all_item_indices,
+        ));
     }
 
     tables

src/tables/format.rs

@@ -1,26 +1,19 @@
 //! Table-to-markdown formatting and cell cleanup.
 
-use super::detect_heuristic::is_table_of_contents;
-use super::Table;
+use super::{Table, TableKind};
 
 pub fn table_to_markdown(table: &Table) -> String {
     if table.cells.is_empty() || table.cells[0].is_empty() {
         return String::new();
     }
 
-    // Detect TOC on the raw cells: clean_table_cells merges rows in ways
-    // that can make genuine data tables superficially resemble a TOC
-    // (short numeric cells, few columns) — but the raw detection here
-    // preserves the original multi-column structure and only matches the
-    // true TOC pattern.
-    //
-    // Tables of contents render poorly as markdown tables — emit a flat
-    // per-row text list instead so the page numbers stay aligned with
-    // their section titles rather than drifting to a separate column.
-    // Format from raw cells: continuation-row merging collapses separate
-    // TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench") into a
-    // single line because sub-entries leave column 0 empty.
-    if is_table_of_contents(&table.cells) {
+    // TOCs render poorly as markdown tables — emit a flat per-row text list
+    // instead so the page numbers stay aligned with their section titles
+    // rather than drifting to a separate column. Format from raw cells
+    // because continuation-row merging in clean_table_cells collapses
+    // separate TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench")
+    // into one line where sub-entries leave column 0 empty.
+    if table.kind == TableKind::Toc {
         return format_toc_as_list(&table.cells, &[]);
     }
 
@@ -470,6 +463,7 @@ mod tests {
                 vec!["Bob".into(), "25".into()],
             ],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         let md = table_to_markdown(&table);
         assert!(md.contains("|Name|"));
@@ -485,6 +479,7 @@ mod tests {
             rows: vec![500.0],
             cells: vec![vec!["Only".into(), "Row".into()]],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         let md = table_to_markdown(&table);
         assert!(md.contains("|Only|"));
@@ -498,6 +493,7 @@ mod tests {
             rows: vec![],
             cells: vec![],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         assert_eq!(table_to_markdown(&table), "");
     }
@@ -513,6 +509,7 @@ mod tests {
                 vec!["(1)".into(), "Footnote text".into()],
             ],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         let md = table_to_markdown(&table);
         assert!(md.contains("(1) Footnote text"));
@@ -528,6 +525,7 @@ mod tests {
                 vec!["太郎".into(), "25".into()],
             ],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         let md = table_to_markdown(&table);
         assert!(md.contains("名前"));
@@ -541,6 +539,7 @@ mod tests {
             rows: vec![500.0],
             cells: vec![vec![]],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
         assert_eq!(table_to_markdown(&table), "");
     }
@@ -550,10 +549,10 @@ mod tests {
         // A TOC-shaped table with section numbers in col 0 and page numbers
         // in the last column should render as a flat list, not a markdown
         // table, so the page numbers stay on the same line as their titles.
-        let table = Table {
-            columns: vec![50.0, 80.0, 300.0],
-            rows: vec![500.0; 5],
-            cells: vec![
+        let table = Table::new(
+            vec![50.0, 80.0, 300.0],
+            vec![500.0; 5],
+            vec![
                 vec![
                     "4.3".into(),
                     "Case studies and targeted evaluations".into(),
@@ -572,8 +571,9 @@ mod tests {
                 vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
                 vec!["4.5".into(), "White-box analyses".into(), "113".into()],
             ],
-            item_indices: vec![],
-        };
+            vec![],
+        );
+        assert_eq!(table.kind, TableKind::Toc);
         let md = table_to_markdown(&table);
         assert!(
             !md.contains("|---|"),

src/tables/grid.rs

@@ -499,6 +499,7 @@ pub(crate) fn recover_header_row(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::tables::TableKind;
     use crate::types::ItemType;
 
     fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem {
@@ -756,6 +757,7 @@ mod tests {
             rows: vec![500.0, 480.0],
             cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
             item_indices: vec![2, 3],
+            kind: TableKind::Data,
         };
 
         recover_header_row(&mut table, &all_items, 9.0);
@@ -774,6 +776,7 @@ mod tests {
             rows: vec![500.0],
             cells: vec![vec!["A".into(), "B".into()]],
             item_indices: vec![0, 1],
+            kind: TableKind::Data,
         };
 
         let rows_before = table.rows.len();
@@ -794,6 +797,7 @@ mod tests {
             rows: vec![500.0, 480.0],
             cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
             item_indices: vec![2, 3],
+            kind: TableKind::Data,
         };
 
         let rows_before = table.rows.len();
@@ -814,6 +818,7 @@ mod tests {
             rows: vec![500.0],
             cells: vec![vec!["A".into(), "B".into()]],
             item_indices: vec![1, 2],
+            kind: TableKind::Data,
         };
 
         let rows_before = table.rows.len();
@@ -829,6 +834,7 @@ mod tests {
             rows: vec![],
             cells: vec![],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
 
         recover_header_row(&mut table, &all_items, 9.0);

src/tables/mod.rs

@@ -11,6 +11,7 @@ mod format;
 mod grid;
 
 pub use detect_heuristic::detect_tables;
+pub(crate) use detect_heuristic::is_table_of_contents;
 pub use detect_lines::detect_tables_from_lines;
 pub(crate) use detect_rects::cluster_rects;
 pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
@@ -166,12 +167,12 @@ pub(crate) fn try_build_rect_guided_table(
     used_indices.sort_unstable();
     used_indices.dedup();
 
-    Some(Table {
-        columns: col_boundaries,
-        rows: row_boundaries,
+    Some(Table::new(
+        col_boundaries,
+        row_boundaries,
         cells,
-        item_indices: used_indices,
-    })
+        used_indices,
+    ))
 }
 
 /// Split a TextItem whose text contains multiple whitespace-separated tokens
@@ -541,12 +542,22 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
         multi_col_rows
     );
 
-    Some(Table {
-        columns: col_xs,
-        rows: row_ys,
-        cells,
-        item_indices,
-    })
+    Some(Table::new(col_xs, row_ys, cells, item_indices))
+}
+
+/// What kind of structure a detected `Table` represents. Classification is
+/// computed once at construction so consumers don't have to re-analyze the
+/// cells (and stay consistent across detection backends).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum TableKind {
+    /// A real data table — renders as markdown table syntax.
+    #[default]
+    Data,
+    /// A table of contents — renders as a flat list with tab-aligned page
+    /// numbers via `format_toc_as_list`. Detected through the table pipeline
+    /// because TOCs share row/column structure with tables, but they are not
+    /// data tables and shouldn't appear in `pages_with_tables` etc.
+    Toc,
 }
 
 /// A detected table.
@@ -560,6 +571,31 @@ pub struct Table {
     pub cells: Vec<Vec<String>>,
     /// Items that belong to this table
     pub item_indices: Vec<usize>,
+    /// Data table vs TOC. Set by `Table::new` from `cells`.
+    pub kind: TableKind,
+}
+
+impl Table {
+    /// Build a table and classify it (data vs TOC) from its cells.
+    pub fn new(
+        columns: Vec<f32>,
+        rows: Vec<f32>,
+        cells: Vec<Vec<String>>,
+        item_indices: Vec<usize>,
+    ) -> Self {
+        let kind = if is_table_of_contents(&cells) {
+            TableKind::Toc
+        } else {
+            TableKind::Data
+        };
+        Self {
+            columns,
+            rows,
+            cells,
+            item_indices,
+            kind,
+        }
+    }
 }
 
 #[cfg(test)]
@@ -642,6 +678,7 @@ mod tests {
                 vec!["Cell 1".into(), "Cell 2".into()],
             ],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
 
         let md = table_to_markdown(&table);
@@ -1002,6 +1039,7 @@ mod tests {
                 vec!["3".into(), "5/2".into(), "Item C".into(), "300".into()],
             ],
             item_indices: vec![],
+            kind: TableKind::Data,
         };
 
         let md = table_to_markdown(&table);