fix(layout): re-guard the ungated XY-cut fallthrough on table pages

Removing page_has_table from the relative-valley condition also
unlocked the XY-cut fallback inside that block, which has no prose
gate — a table page whose valley candidate was just rejected could
take an unvalidated split. The guard is restored on that call
specifically; the relative-valley path keeps its prose-gated access.
Also rewrite the stale dense-page comment for the 30-item floor.
Bench and corpus unchanged (462/884, corpus byte-identical).

src/extractor/layout.rs

@@ -225,11 +225,19 @@ pub(crate) fn detect_columns(
     // Justified text can leave gutter bins non-empty because item widths extend
     // to the column edge. Look for local minima that are significantly lower
     // than the peaks on either side.
-    // Only attempt this for dense pages (>=100 items) — sparse pages with shallow
-    // histogram dips are likely not multi-column.
-    // Skip on pages with detected tables — table column gaps look like gutters
-    // in the histogram but the table pipeline already handles reading order.
-    if valleys.is_empty() && page_items.len() >= 100 && !page_has_table {
+    //
+    // The 30-item floor admits sparse pages: OCR'd multi-column pages arrive
+    // as few long line-runs and were falling to single-column Y-sorting.
+    // Below 30 items the histogram is too shallow for even the prose gate
+    // to judge a dip.
+    //
+    // Pages with detected tables take the relative-valley path too: the
+    // table's items have already left the flow by the time grouping runs,
+    // so a table cannot fake a gutter here, and the prose gate below
+    // rejects any residual table-shaped split. Without this, the prose
+    // REMAINDER of a table-bearing two-column page falls to single-column
+    // Y-sorting and the columns interleave line by line.
+    if valleys.is_empty() && page_items.len() >= 30 {
         let rel_valleys = find_relative_valleys(
             &histogram,
             num_bins,
@@ -270,9 +278,14 @@ pub(crate) fn detect_columns(
                 }
             }
         }
-        // Try XY-cut fallback before giving up
-        if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
-            return columns;
+        // Try XY-cut fallback before giving up. Unlike the relative-valley
+        // path above, XY-cut has no prose gate, so the table-page guard
+        // stays here: without it a table page whose valley candidate was
+        // just rejected could take an unvalidated split.
+        if !page_has_table {
+            if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
+                return columns;
+            }
         }
         return vec![ColumnRegion { x_min, x_max }];
     }

src/markdown/mod.rs

@@ -9,6 +9,7 @@
 pub(crate) mod analysis;
 mod classify;
 mod convert;
+mod furniture;
 mod heading;
 mod postprocess;
 mod preprocess;
@@ -634,7 +635,12 @@ fn is_parallel_prose_table(table: &crate::tables::Table) -> bool {
         }
     }
 
-    let is_parallel = !has_compact_header
+    // A compact header row is evidence for a real table — unless cross-row
+    // prose continuations outnumber the rows, which no genuine table
+    // produces: the "header" is then just two short line fragments at the
+    // top of parallel prose columns.
+    let header_blocks = has_compact_header && continuation_fragments <= table.cells.len();
+    let is_parallel = !header_blocks
         && non_empty >= 5
         // Independent prose columns have asynchronous line/paragraph breaks;
         // a fully populated grid is positive evidence for a real descriptive
@@ -1149,7 +1155,7 @@ pub(crate) fn strip_repeated_header_footer_lines(
     lines: Vec<crate::types::TextLine>,
     page_count: u32,
 ) -> Vec<crate::types::TextLine> {
-    preprocess::strip_repeated_lines(lines, page_count)
+    furniture::strip_header_footer_lines(lines, page_count)
 }
 
 /// Convert positioned text items to markdown with structure detection
@@ -1374,7 +1380,6 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
             chart_page_prose_column_split(&page_layout_items)
                 .filter(|&split_x| chart_spans_prose_split(region, split_x))
         });
-        let chart_prose_columns = chart_prose_split.is_some();
 
         // Check for side-by-side table layout using the original items. Sparse
         // numeric cells need table context before they can be distinguished
@@ -1615,10 +1620,16 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
                     if subset_items.len() < min_items {
                         return;
                     }
-                    // Keep body-font detection available on chart pages: a real
-                    // table can share the prose anchors. Reject only candidates
-                    // whose cells prove they are parallel prose fragments.
-                    let reject_parallel_prose = chart_prose_columns && !was_split;
+                    // Reject candidates whose cells prove they are parallel
+                    // prose fragments — the shape produced when the body-font
+                    // pass projects a multi-column text page onto one table
+                    // grid (two-column reference sections are the classic
+                    // case). The check needs internal transition evidence
+                    // (unterminated cells flowing into lowercase starts in
+                    // the same column), so genuine tables with long cells
+                    // pass. Band-split retries stay exempt: they exist for
+                    // tables that only assemble after recombining bands.
+                    let reject_parallel_prose = !was_split;
                     let tables = detect_tables_with_page_width(
                         subset_items,
                         base_size,
@@ -2106,7 +2117,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
 
     // Strip repeated headers/footers before conversion
     let lines = if options.strip_headers_footers {
-        preprocess::strip_repeated_lines(lines, document_page_count)
+        furniture::strip_header_footer_lines(lines, document_page_count)
     } else {
         lines
     };
@@ -2673,6 +2684,35 @@ mod tests {
         );
         assert!(!is_parallel_prose_table(&data));
 
+        // A compact header row atop parallel prose columns: cross-row prose
+        // continuations outnumber the rows, so the header cannot save the
+        // candidate — this is page prose with two short fragments on top.
+        let headed_parallel_prose = crate::tables::Table::new(
+            vec![90.0, 340.0],
+            vec![340.0, 320.0, 300.0, 280.0, 260.0],
+            vec![
+                vec!["June 2023".into(), "Page 5".into()],
+                vec![
+                    "the committee reviewed the proposal and decided that the".into(),
+                    "funding for the second phase would continue subject to the".into(),
+                ],
+                vec![
+                    "implementation schedule should be extended by another".into(),
+                    "quarterly reviews established during the first phase of the".into(),
+                ],
+                vec![
+                    "six months to accommodate the revised procurement rules".into(),
+                    "".into(),
+                ],
+                vec![
+                    "adopted at the previous meeting of the governing board".into(),
+                    "participating institutions across the partner regions".into(),
+                ],
+            ],
+            (0..10).collect(),
+        );
+        assert!(is_parallel_prose_table(&headed_parallel_prose));
+
         let headed_text_table = crate::tables::Table::new(
             vec![90.0, 340.0],
             vec![320.0, 300.0, 280.0],

src/markdown/preprocess.rs

@@ -1,6 +1,6 @@
 //! Line preprocessing: heading merging, drop cap handling, and repeated line removal.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
 use crate::structure_tree::StructRole;
 use crate::types::{TextItem, TextLine};
@@ -229,342 +229,6 @@ pub(crate) fn merge_drop_caps(lines: Vec<TextLine>, base_size: f32) -> Vec<TextL
     result
 }
 
-/// Normalize whitespace in a string for comparison: trim and collapse internal runs of whitespace.
-fn normalize_whitespace(s: &str) -> String {
-    s.split_whitespace().collect::<Vec<_>>().join(" ")
-}
-
-/// Normalize text for frequency comparison: collapse whitespace and strip leading/trailing
-/// digit sequences (page numbers). E.g., "Chapter 3 — Page 5" and "Chapter 3 — Page 6"
-/// both normalize to "Chapter 3 — Page".
-fn normalize_for_comparison(s: &str) -> String {
-    let ws = normalize_whitespace(s);
-    let trimmed = ws
-        .trim_start_matches(|c: char| c.is_ascii_digit())
-        .trim_start();
-    let trimmed = trimmed
-        .trim_end_matches(|c: char| c.is_ascii_digit())
-        .trim_end();
-    trimmed.to_string()
-}
-
-/// Returns true if the line looks like a list item or heading (should not be stripped).
-fn is_structural_line(text: &str) -> bool {
-    let t = text.trim_start();
-    t.starts_with('#')
-        || t.starts_with("- ")
-        || t.starts_with("* ")
-        || t.starts_with("• ")
-        || t.chars()
-            .next()
-            .map(|c| c.is_ascii_digit())
-            .unwrap_or(false)
-            && (t.contains(". ") || t.contains(") "))
-}
-
-/// Returns true if a line consists entirely of a single repeated character
-/// (e.g., "----------", "**************", "============").
-fn is_decorative_separator(text: &str) -> bool {
-    let mut chars = text.chars();
-    let first = match chars.next() {
-        Some(c) => c,
-        None => return false,
-    };
-    chars.all(|c| c == first)
-}
-
-/// Strip lines that repeat on many distinct pages (running headers/footers).
-///
-/// A line is considered a repeated header/footer if:
-/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
-/// 2. It is at least 10 characters long
-/// 3. It doesn't look like a structural element (heading, list item)
-/// 4. It consistently appears in the top or bottom N distinct Y positions
-/// 5. Its Y positions across pages have low variance (consistent placement),
-///    distinguishing true headers/footers from table content that happens to
-///    land near page margins
-/// 6. It is not a decorative separator (repeated single character)
-///
-/// Additionally, TextLines at the same Y position on a page are grouped into
-/// "Y-bands." When any member of a Y-band is stripped, all siblings in that
-/// band are also stripped. This handles split column headers where individual
-/// fragments may not independently meet the frequency threshold.
-///
-/// Page numbers are stripped from line text before comparison, so headers like
-/// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
-pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
-    if lines.is_empty() || page_count < 3 {
-        return lines;
-    }
-
-    // Compute Y range per page (min_y, max_y)
-    let mut page_y_range: HashMap<u32, (f32, f32)> = HashMap::new();
-    for line in &lines {
-        let entry = page_y_range.entry(line.page).or_insert((line.y, line.y));
-        if line.y < entry.0 {
-            entry.0 = line.y;
-        }
-        if line.y > entry.1 {
-            entry.1 = line.y;
-        }
-    }
-
-    // Build sorted Y values per page, so we can check line rank (position from edge)
-    let mut page_sorted_ys: HashMap<u32, Vec<f32>> = HashMap::new();
-    for line in &lines {
-        page_sorted_ys.entry(line.page).or_default().push(line.y);
-    }
-    for ys in page_sorted_ys.values_mut() {
-        ys.sort_by(|a, b| a.total_cmp(b));
-        ys.dedup();
-    }
-
-    // A line is in the page margin if it's among the first or last N distinct
-    // Y positions on that page. This is more robust than a percentage-based zone
-    // because it catches actual edge lines regardless of how much content fills
-    // the page. N=5 accommodates multi-line headers/footers and repeated form
-    // column headers (e.g., 5-row IRS form headers) that sit just inside the
-    // page margin.
-    const EDGE_LINE_COUNT: usize = 5;
-
-    /// Returns true if the given Y position is among the first or last N distinct
-    /// Y positions on the specified page.
-    fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
-        let ys = match page_sorted_ys.get(&page) {
-            Some(ys) => ys,
-            None => return false,
-        };
-        if ys.len() <= n * 2 {
-            // Page has very few lines — everything is near the edge
-            return true;
-        }
-        // Check if this Y is among the first or last N
-        let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
-            Some(p) => p,
-            None => return false,
-        };
-        pos < n || pos >= ys.len() - n
-    }
-
-    // Average page span for normalizing Y variance
-    let avg_span = {
-        let total: f32 = page_y_range.values().map(|(lo, hi)| hi - lo).sum();
-        if page_y_range.is_empty() {
-            1.0
-        } else {
-            (total / page_y_range.len() as f32).max(1.0)
-        }
-    };
-
-    // Build Y-bands: group line indices by (page, quantized_y).
-    // Lines at the same Y position (within ~0.1pt) on the same page form a band.
-    let mut y_bands: HashMap<(u32, i32), Vec<usize>> = HashMap::new();
-    for (idx, line) in lines.iter().enumerate() {
-        let y_bucket = (line.y * 10.0).round() as i32;
-        y_bands.entry((line.page, y_bucket)).or_default().push(idx);
-    }
-
-    // Build frequency maps using normalize_for_comparison.
-    // Individual line text -> distinct pages
-    let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
-    let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
-    for line in &lines {
-        if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        let text = line.text();
-        let normalized = normalize_for_comparison(&text);
-        if normalized.len() < 10 || is_decorative_separator(&normalized) {
-            continue;
-        }
-        freq.entry(normalized.clone())
-            .or_default()
-            .insert(line.page);
-        y_positions.entry(normalized).or_default().push(line.y);
-    }
-
-    // Coalesced row text -> distinct pages (for multi-member Y-bands).
-    // This catches split column headers where individual fragments don't meet
-    // the frequency threshold but the combined row does.
-    let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
-    let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
-    for (&(page, _), indices) in &y_bands {
-        if indices.len() < 2 {
-            continue; // single-line bands are already in the individual map
-        }
-        let band_y = lines[indices[0]].y;
-        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        let mut sorted_indices = indices.clone();
-        sorted_indices.sort();
-        let coalesced: String = sorted_indices
-            .iter()
-            .map(|&i| lines[i].text())
-            .collect::<Vec<_>>()
-            .join(" ");
-        let normalized = normalize_for_comparison(&coalesced);
-        if normalized.len() < 10 || is_decorative_separator(&normalized) {
-            continue;
-        }
-        band_freq
-            .entry(normalized.clone())
-            .or_default()
-            .insert(page);
-        band_y_positions.entry(normalized).or_default().push(band_y);
-    }
-
-    // Compute threshold
-    let threshold = 3u32.max(page_count * 30 / 100);
-
-    // Check Y-position consistency: headers/footers appear at the same position
-    // on every page, table content varies. Require normalized stddev < 5% of
-    // average page span.
-    let has_consistent_y = |text: &str, positions: &HashMap<String, Vec<f32>>| -> bool {
-        let pos = match positions.get(text) {
-            Some(p) if p.len() >= 2 => p,
-            _ => return true, // single occurrence — allow
-        };
-        let n = pos.len() as f32;
-        let mean = pos.iter().sum::<f32>() / n;
-        let variance = pos.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
-        let stddev = variance.sqrt();
-        stddev / avg_span < 0.05
-    };
-
-    // Identify candidates from individual frequency map
-    let candidates: HashSet<String> = freq
-        .into_iter()
-        .filter(|(text, pages)| {
-            pages.len() as u32 >= threshold
-                && !is_structural_line(text)
-                && has_consistent_y(text, &y_positions)
-        })
-        .map(|(text, _)| text)
-        .collect();
-
-    // Identify candidates from coalesced band frequency map
-    let band_candidates: HashSet<String> = band_freq
-        .into_iter()
-        .filter(|(text, pages)| {
-            pages.len() as u32 >= threshold
-                && !is_structural_line(text)
-                && has_consistent_y(text, &band_y_positions)
-        })
-        .map(|(text, _)| text)
-        .collect();
-
-    if candidates.is_empty() && band_candidates.is_empty() {
-        return lines;
-    }
-
-    // Build removal set.
-    // A line is removed if it's at an edge position and:
-    //   (a) its individual text matches a candidate, OR
-    //   (b) its Y-band's coalesced text matches a band candidate, OR
-    //   (c) any sibling in its Y-band was removed (propagation).
-    //
-    // The first occurrence (lowest page number) of each repeated header/footer
-    // is kept so that document titles, column headers, etc. appear once.
-    let mut removal_set: HashSet<usize> = HashSet::new();
-
-    // Track which page first shows each candidate (to preserve first occurrence)
-    let mut first_page_individual: HashMap<String, u32> = HashMap::new();
-    for (idx, line) in lines.iter().enumerate() {
-        if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        let text = line.text();
-        let normalized = normalize_for_comparison(&text);
-        if candidates.contains(&normalized) {
-            let first = first_page_individual.entry(normalized).or_insert(line.page);
-            if line.page > *first {
-                removal_set.insert(idx);
-            } else if line.page == *first {
-                // Keep this occurrence (first page)
-            }
-        }
-    }
-
-    // Track first page for band candidates
-    let mut first_page_band: HashMap<String, u32> = HashMap::new();
-    // First pass: find first page for each band candidate
-    for (&(page, _), indices) in &y_bands {
-        if indices.len() < 2 {
-            continue;
-        }
-        let band_y = lines[indices[0]].y;
-        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        let mut sorted_indices = indices.clone();
-        sorted_indices.sort();
-        let coalesced: String = sorted_indices
-            .iter()
-            .map(|&i| lines[i].text())
-            .collect::<Vec<_>>()
-            .join(" ");
-        let normalized = normalize_for_comparison(&coalesced);
-        if band_candidates.contains(&normalized) {
-            let first = first_page_band.entry(normalized).or_insert(page);
-            if page < *first {
-                *first = page;
-            }
-        }
-    }
-    // Second pass: mark for removal (skip first page)
-    for (&(page, _), indices) in &y_bands {
-        if indices.len() < 2 {
-            continue;
-        }
-        let band_y = lines[indices[0]].y;
-        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        let mut sorted_indices = indices.clone();
-        sorted_indices.sort();
-        let coalesced: String = sorted_indices
-            .iter()
-            .map(|&i| lines[i].text())
-            .collect::<Vec<_>>()
-            .join(" ");
-        let normalized = normalize_for_comparison(&coalesced);
-        if band_candidates.contains(&normalized) {
-            let first = first_page_band.get(&normalized).copied().unwrap_or(0);
-            if page > first {
-                for &idx in &sorted_indices {
-                    removal_set.insert(idx);
-                }
-            }
-        }
-    }
-
-    // (c) Y-band sibling propagation: if any member is removed, remove all
-    //     members (provided the band is at an edge position).
-    for (&(page, _), indices) in &y_bands {
-        let band_y = lines[indices[0]].y;
-        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            continue;
-        }
-        if indices.iter().any(|idx| removal_set.contains(idx)) {
-            for &idx in indices {
-                removal_set.insert(idx);
-            }
-        }
-    }
-
-    if removal_set.is_empty() {
-        return lines;
-    }
-
-    lines
-        .into_iter()
-        .enumerate()
-        .filter(|(idx, _)| !removal_set.contains(idx))
-        .map(|(_, line)| line)
-        .collect()
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -679,53 +343,6 @@ mod tests {
         assert_eq!(result.len(), 2, "should merge font-based heading lines");
     }
 
-    #[test]
-    fn test_strip_repeated_keeps_first_occurrence() {
-        // Simulate a repeated page header on 10 pages.
-        // Each page has a running header at y=750 and many unique body lines.
-        let mut lines = Vec::new();
-        for page in 1..=10u32 {
-            // Header at top
-            lines.push(make_line(
-                "VOICE OF SOUTH MARION May fifteen twenty twenty five",
-                10.0,
-                page,
-                750.0,
-                None,
-            ));
-            // Body content — unique text per line per page (no digits to strip)
-            for j in 0..20u32 {
-                lines.push(make_line(
-                    &format!(
-                        "parcel r-{:04}-{:03} owner smith address oak street",
-                        page * 100 + j,
-                        page
-                    ),
-                    10.0,
-                    page,
-                    600.0 - j as f32 * 15.0,
-                    None,
-                ));
-            }
-        }
-
-        let result = strip_repeated_lines(lines, 10);
-
-        // The header should appear exactly once (page 1)
-        let header_count = result
-            .iter()
-            .filter(|l| l.text().contains("VOICE OF SOUTH MARION"))
-            .count();
-        assert_eq!(header_count, 1, "repeated header should be kept once");
-
-        // First occurrence should be on page 1
-        let first_header = result
-            .iter()
-            .find(|l| l.text().contains("VOICE OF SOUTH MARION"))
-            .unwrap();
-        assert_eq!(first_header.page, 1, "first occurrence should be on page 1");
-    }
-
     fn make_bold_line(text: &str, page: u32, y: f32) -> TextLine {
         let mut item = make_item(text, 12.0, None);
         item.is_bold = true;