feat: enhance header/footer stripping with Y-band coalescing and page number normalization

Strip leading/trailing digit sequences (page numbers) before frequency
comparison so headers like "Page 5" and "Page 6" are treated as identical.
Group TextLines at the same Y position into Y-bands and propagate removal
to all siblings when any member is stripped. Increase EDGE_LINE_COUNT from
4 to 5 to cover 5-row form column headers (e.g., IRS p1244).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/markdown/preprocess.rs

@@ -148,6 +148,20 @@ fn normalize_whitespace(s: &str) -> String {
     s.split_whitespace().collect::<Vec<_>>().join(" ")
 }
 
+/// Normalize text for frequency comparison: collapse whitespace and strip leading/trailing
+/// digit sequences (page numbers). E.g., "Chapter 3 — Page 5" and "Chapter 3 — Page 6"
+/// both normalize to "Chapter 3 — Page".
+fn normalize_for_comparison(s: &str) -> String {
+    let ws = normalize_whitespace(s);
+    let trimmed = ws
+        .trim_start_matches(|c: char| c.is_ascii_digit())
+        .trim_start();
+    let trimmed = trimmed
+        .trim_end_matches(|c: char| c.is_ascii_digit())
+        .trim_end();
+    trimmed.to_string()
+}
+
 /// Returns true if the line looks like a list item or heading (should not be stripped).
 fn is_structural_line(text: &str) -> bool {
     let t = text.trim_start();
@@ -179,11 +193,19 @@ fn is_decorative_separator(text: &str) -> bool {
 /// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
 /// 2. It is at least 10 characters long
 /// 3. It doesn't look like a structural element (heading, list item)
-/// 4. It consistently appears in the top or bottom 15% of the page's Y range
+/// 4. It consistently appears in the top or bottom N distinct Y positions
 /// 5. Its Y positions across pages have low variance (consistent placement),
 ///    distinguishing true headers/footers from table content that happens to
 ///    land near page margins
 /// 6. It is not a decorative separator (repeated single character)
+///
+/// Additionally, TextLines at the same Y position on a page are grouped into
+/// "Y-bands." When any member of a Y-band is stripped, all siblings in that
+/// band are also stripped. This handles split column headers where individual
+/// fragments may not independently meet the frequency threshold.
+///
+/// Page numbers are stripped from line text before comparison, so headers like
+/// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
 pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
     if lines.is_empty() || page_count < 3 {
         return lines;
@@ -214,14 +236,15 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     // A line is in the page margin if it's among the first or last N distinct
     // Y positions on that page. This is more robust than a percentage-based zone
     // because it catches actual edge lines regardless of how much content fills
-    // the page. N=4 accommodates multi-line headers/footers and repeated form
-    // column headers that sit just inside the page margin.
-    const EDGE_LINE_COUNT: usize = 4;
+    // the page. N=5 accommodates multi-line headers/footers and repeated form
+    // column headers (e.g., 5-row IRS form headers) that sit just inside the
+    // page margin.
+    const EDGE_LINE_COUNT: usize = 5;
 
-    /// Returns true if the line is among the first or last N distinct Y positions
-    /// on its page.
-    fn is_edge_line(line: &TextLine, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
-        let ys = match page_sorted_ys.get(&line.page) {
+    /// Returns true if the given Y position is among the first or last N distinct
+    /// Y positions on the specified page.
+    fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
+        let ys = match page_sorted_ys.get(&page) {
             Some(ys) => ys,
             None => return false,
         };
@@ -230,7 +253,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
             return true;
         }
         // Check if this Y is among the first or last N
-        let pos = match ys.iter().position(|&y| (y - line.y).abs() < 0.1) {
+        let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
             Some(p) => p,
             None => return false,
         };
@@ -247,20 +270,25 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
         }
     };
 
-    // Build frequency map: only count lines at the page edges
-    // Also collect Y positions for variance check
+    // Build Y-bands: group line indices by (page, quantized_y).
+    // Lines at the same Y position (within ~0.1pt) on the same page form a band.
+    let mut y_bands: HashMap<(u32, i32), Vec<usize>> = HashMap::new();
+    for (idx, line) in lines.iter().enumerate() {
+        let y_bucket = (line.y * 10.0).round() as i32;
+        y_bands.entry((line.page, y_bucket)).or_default().push(idx);
+    }
+
+    // Build frequency maps using normalize_for_comparison.
+    // Individual line text -> distinct pages
     let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
     let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
     for line in &lines {
+        if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            continue;
+        }
         let text = line.text();
-        let normalized = normalize_whitespace(&text);
-        if normalized.len() < 10 {
-            continue;
-        }
-        if is_decorative_separator(&normalized) {
-            continue;
-        }
-        if !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) {
+        let normalized = normalize_for_comparison(&text);
+        if normalized.len() < 10 || is_decorative_separator(&normalized) {
             continue;
         }
         freq.entry(normalized.clone())
@@ -269,51 +297,146 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
         y_positions.entry(normalized).or_default().push(line.y);
     }
 
+    // Coalesced row text -> distinct pages (for multi-member Y-bands).
+    // This catches split column headers where individual fragments don't meet
+    // the frequency threshold but the combined row does.
+    let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
+    let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
+    for (&(page, _), indices) in &y_bands {
+        if indices.len() < 2 {
+            continue; // single-line bands are already in the individual map
+        }
+        let band_y = lines[indices[0]].y;
+        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            continue;
+        }
+        let mut sorted_indices = indices.clone();
+        sorted_indices.sort();
+        let coalesced: String = sorted_indices
+            .iter()
+            .map(|&i| lines[i].text())
+            .collect::<Vec<_>>()
+            .join(" ");
+        let normalized = normalize_for_comparison(&coalesced);
+        if normalized.len() < 10 || is_decorative_separator(&normalized) {
+            continue;
+        }
+        band_freq
+            .entry(normalized.clone())
+            .or_default()
+            .insert(page);
+        band_y_positions.entry(normalized).or_default().push(band_y);
+    }
+
     // Compute threshold
     let threshold = 3u32.max(page_count * 30 / 100);
 
     // Check Y-position consistency: headers/footers appear at the same position
     // on every page, table content varies. Require normalized stddev < 5% of
     // average page span.
-    let has_consistent_y = |text: &str| -> bool {
-        let positions = match y_positions.get(text) {
+    let has_consistent_y = |text: &str, positions: &HashMap<String, Vec<f32>>| -> bool {
+        let pos = match positions.get(text) {
             Some(p) if p.len() >= 2 => p,
             _ => return true, // single occurrence — allow
         };
-        let n = positions.len() as f32;
-        let mean = positions.iter().sum::<f32>() / n;
-        let variance = positions.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
+        let n = pos.len() as f32;
+        let mean = pos.iter().sum::<f32>() / n;
+        let variance = pos.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
         let stddev = variance.sqrt();
         stddev / avg_span < 0.05
     };
 
-    // Collect candidate texts to remove
+    // Identify candidates from individual frequency map
     let candidates: HashSet<String> = freq
         .into_iter()
         .filter(|(text, pages)| {
-            pages.len() as u32 >= threshold && !is_structural_line(text) && has_consistent_y(text)
+            pages.len() as u32 >= threshold
+                && !is_structural_line(text)
+                && has_consistent_y(text, &y_positions)
         })
         .map(|(text, _)| text)
         .collect();
 
-    if candidates.is_empty() {
+    // Identify candidates from coalesced band frequency map
+    let band_candidates: HashSet<String> = band_freq
+        .into_iter()
+        .filter(|(text, pages)| {
+            pages.len() as u32 >= threshold
+                && !is_structural_line(text)
+                && has_consistent_y(text, &band_y_positions)
+        })
+        .map(|(text, _)| text)
+        .collect();
+
+    if candidates.is_empty() && band_candidates.is_empty() {
+        return lines;
+    }
+
+    // Build removal set.
+    // A line is removed if it's at an edge position and:
+    //   (a) its individual text matches a candidate, OR
+    //   (b) its Y-band's coalesced text matches a band candidate, OR
+    //   (c) any sibling in its Y-band was removed (propagation).
+    let mut removal_set: HashSet<usize> = HashSet::new();
+
+    // (a) Lines matching individual candidates at edge positions
+    for (idx, line) in lines.iter().enumerate() {
+        if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            continue;
+        }
+        let text = line.text();
+        let normalized = normalize_for_comparison(&text);
+        if candidates.contains(&normalized) {
+            removal_set.insert(idx);
+        }
+    }
+
+    // (b) Lines in Y-bands whose coalesced text matches a band candidate
+    for (&(page, _), indices) in &y_bands {
+        if indices.len() < 2 {
+            continue;
+        }
+        let band_y = lines[indices[0]].y;
+        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            continue;
+        }
+        let mut sorted_indices = indices.clone();
+        sorted_indices.sort();
+        let coalesced: String = sorted_indices
+            .iter()
+            .map(|&i| lines[i].text())
+            .collect::<Vec<_>>()
+            .join(" ");
+        let normalized = normalize_for_comparison(&coalesced);
+        if band_candidates.contains(&normalized) {
+            for &idx in &sorted_indices {
+                removal_set.insert(idx);
+            }
+        }
+    }
+
+    // (c) Y-band sibling propagation: if any member is removed, remove all
+    //     members (provided the band is at an edge position).
+    for (&(page, _), indices) in &y_bands {
+        let band_y = lines[indices[0]].y;
+        if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            continue;
+        }
+        if indices.iter().any(|idx| removal_set.contains(idx)) {
+            for &idx in indices {
+                removal_set.insert(idx);
+            }
+        }
+    }
+
+    if removal_set.is_empty() {
         return lines;
     }
 
-    // Only strip candidate instances that are in edge positions on their page.
-    // This preserves body content that happens to match a running header/footer
-    // (e.g., "New Britannia Mill" appearing both as a page header and in the
-    // document's subject line).
     lines
         .into_iter()
-        .filter(|line| {
-            let text = line.text();
-            let normalized = normalize_whitespace(&text);
-            if !candidates.contains(&normalized) {
-                return true; // not a candidate — keep
-            }
-            // Candidate: only strip if this instance is in an edge position
-            !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT)
-        })
+        .enumerate()
+        .filter(|(idx, _)| !removal_set.contains(idx))
+        .map(|(_, line)| line)
         .collect()
 }