From 8adfacd3e3c026c898c2a8348613805fad5f3ded Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 3 Mar 2026 09:46:51 -0800 Subject: [PATCH] feat: enhance header/footer stripping with Y-band coalescing and page number normalization Strip leading/trailing digit sequences (page numbers) before frequency comparison so headers like "Page 5" and "Page 6" are treated as identical. Group TextLines at the same Y position into Y-bands and propagate removal to all siblings when any member is stripped. Increase EDGE_LINE_COUNT from 4 to 5 to cover 5-row form column headers (e.g., IRS p1244). Co-Authored-By: Claude Opus 4.6 --- src/markdown/preprocess.rs | 203 +++++++++++++++++++++++++++------- tests/snapshots/p1244-1996.md | 6 - 2 files changed, 163 insertions(+), 46 deletions(-) diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index d0393dd..fa89efa 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -148,6 +148,20 @@ fn normalize_whitespace(s: &str) -> String { s.split_whitespace().collect::>().join(" ") } +/// Normalize text for frequency comparison: collapse whitespace and strip leading/trailing +/// digit sequences (page numbers). E.g., "Chapter 3 — Page 5" and "Chapter 3 — Page 6" +/// both normalize to "Chapter 3 — Page". +fn normalize_for_comparison(s: &str) -> String { + let ws = normalize_whitespace(s); + let trimmed = ws + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start(); + let trimmed = trimmed + .trim_end_matches(|c: char| c.is_ascii_digit()) + .trim_end(); + trimmed.to_string() +} + /// Returns true if the line looks like a list item or heading (should not be stripped). fn is_structural_line(text: &str) -> bool { let t = text.trim_start(); @@ -179,11 +193,19 @@ fn is_decorative_separator(text: &str) -> bool { /// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages /// 2. It is at least 10 characters long /// 3. It doesn't look like a structural element (heading, list item) -/// 4. It consistently appears in the top or bottom 15% of the page's Y range +/// 4. It consistently appears in the top or bottom N distinct Y positions /// 5. Its Y positions across pages have low variance (consistent placement), /// distinguishing true headers/footers from table content that happens to /// land near page margins /// 6. It is not a decorative separator (repeated single character) +/// +/// Additionally, TextLines at the same Y position on a page are grouped into +/// "Y-bands." When any member of a Y-band is stripped, all siblings in that +/// band are also stripped. This handles split column headers where individual +/// fragments may not independently meet the frequency threshold. +/// +/// Page numbers are stripped from line text before comparison, so headers like +/// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text. pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec { if lines.is_empty() || page_count < 3 { return lines; @@ -214,14 +236,15 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec // A line is in the page margin if it's among the first or last N distinct // Y positions on that page. This is more robust than a percentage-based zone // because it catches actual edge lines regardless of how much content fills - // the page. N=4 accommodates multi-line headers/footers and repeated form - // column headers that sit just inside the page margin. - const EDGE_LINE_COUNT: usize = 4; + // the page. N=5 accommodates multi-line headers/footers and repeated form + // column headers (e.g., 5-row IRS form headers) that sit just inside the + // page margin. + const EDGE_LINE_COUNT: usize = 5; - /// Returns true if the line is among the first or last N distinct Y positions - /// on its page. - fn is_edge_line(line: &TextLine, page_sorted_ys: &HashMap>, n: usize) -> bool { - let ys = match page_sorted_ys.get(&line.page) { + /// Returns true if the given Y position is among the first or last N distinct + /// Y positions on the specified page. + fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap>, n: usize) -> bool { + let ys = match page_sorted_ys.get(&page) { Some(ys) => ys, None => return false, }; @@ -230,7 +253,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec return true; } // Check if this Y is among the first or last N - let pos = match ys.iter().position(|&y| (y - line.y).abs() < 0.1) { + let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) { Some(p) => p, None => return false, }; @@ -247,20 +270,25 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec } }; - // Build frequency map: only count lines at the page edges - // Also collect Y positions for variance check + // Build Y-bands: group line indices by (page, quantized_y). + // Lines at the same Y position (within ~0.1pt) on the same page form a band. + let mut y_bands: HashMap<(u32, i32), Vec> = HashMap::new(); + for (idx, line) in lines.iter().enumerate() { + let y_bucket = (line.y * 10.0).round() as i32; + y_bands.entry((line.page, y_bucket)).or_default().push(idx); + } + + // Build frequency maps using normalize_for_comparison. + // Individual line text -> distinct pages let mut freq: HashMap> = HashMap::new(); let mut y_positions: HashMap> = HashMap::new(); for line in &lines { + if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } let text = line.text(); - let normalized = normalize_whitespace(&text); - if normalized.len() < 10 { - continue; - } - if is_decorative_separator(&normalized) { - continue; - } - if !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) { + let normalized = normalize_for_comparison(&text); + if normalized.len() < 10 || is_decorative_separator(&normalized) { continue; } freq.entry(normalized.clone()) @@ -269,51 +297,146 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec y_positions.entry(normalized).or_default().push(line.y); } + // Coalesced row text -> distinct pages (for multi-member Y-bands). + // This catches split column headers where individual fragments don't meet + // the frequency threshold but the combined row does. + let mut band_freq: HashMap> = HashMap::new(); + let mut band_y_positions: HashMap> = HashMap::new(); + for (&(page, _), indices) in &y_bands { + if indices.len() < 2 { + continue; // single-line bands are already in the individual map + } + let band_y = lines[indices[0]].y; + if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + let mut sorted_indices = indices.clone(); + sorted_indices.sort(); + let coalesced: String = sorted_indices + .iter() + .map(|&i| lines[i].text()) + .collect::>() + .join(" "); + let normalized = normalize_for_comparison(&coalesced); + if normalized.len() < 10 || is_decorative_separator(&normalized) { + continue; + } + band_freq + .entry(normalized.clone()) + .or_default() + .insert(page); + band_y_positions.entry(normalized).or_default().push(band_y); + } + // Compute threshold let threshold = 3u32.max(page_count * 30 / 100); // Check Y-position consistency: headers/footers appear at the same position // on every page, table content varies. Require normalized stddev < 5% of // average page span. - let has_consistent_y = |text: &str| -> bool { - let positions = match y_positions.get(text) { + let has_consistent_y = |text: &str, positions: &HashMap>| -> bool { + let pos = match positions.get(text) { Some(p) if p.len() >= 2 => p, _ => return true, // single occurrence — allow }; - let n = positions.len() as f32; - let mean = positions.iter().sum::() / n; - let variance = positions.iter().map(|y| (y - mean).powi(2)).sum::() / n; + let n = pos.len() as f32; + let mean = pos.iter().sum::() / n; + let variance = pos.iter().map(|y| (y - mean).powi(2)).sum::() / n; let stddev = variance.sqrt(); stddev / avg_span < 0.05 }; - // Collect candidate texts to remove + // Identify candidates from individual frequency map let candidates: HashSet = freq .into_iter() .filter(|(text, pages)| { - pages.len() as u32 >= threshold && !is_structural_line(text) && has_consistent_y(text) + pages.len() as u32 >= threshold + && !is_structural_line(text) + && has_consistent_y(text, &y_positions) }) .map(|(text, _)| text) .collect(); - if candidates.is_empty() { + // Identify candidates from coalesced band frequency map + let band_candidates: HashSet = band_freq + .into_iter() + .filter(|(text, pages)| { + pages.len() as u32 >= threshold + && !is_structural_line(text) + && has_consistent_y(text, &band_y_positions) + }) + .map(|(text, _)| text) + .collect(); + + if candidates.is_empty() && band_candidates.is_empty() { + return lines; + } + + // Build removal set. + // A line is removed if it's at an edge position and: + // (a) its individual text matches a candidate, OR + // (b) its Y-band's coalesced text matches a band candidate, OR + // (c) any sibling in its Y-band was removed (propagation). + let mut removal_set: HashSet = HashSet::new(); + + // (a) Lines matching individual candidates at edge positions + for (idx, line) in lines.iter().enumerate() { + if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + let text = line.text(); + let normalized = normalize_for_comparison(&text); + if candidates.contains(&normalized) { + removal_set.insert(idx); + } + } + + // (b) Lines in Y-bands whose coalesced text matches a band candidate + for (&(page, _), indices) in &y_bands { + if indices.len() < 2 { + continue; + } + let band_y = lines[indices[0]].y; + if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + let mut sorted_indices = indices.clone(); + sorted_indices.sort(); + let coalesced: String = sorted_indices + .iter() + .map(|&i| lines[i].text()) + .collect::>() + .join(" "); + let normalized = normalize_for_comparison(&coalesced); + if band_candidates.contains(&normalized) { + for &idx in &sorted_indices { + removal_set.insert(idx); + } + } + } + + // (c) Y-band sibling propagation: if any member is removed, remove all + // members (provided the band is at an edge position). + for (&(page, _), indices) in &y_bands { + let band_y = lines[indices[0]].y; + if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + if indices.iter().any(|idx| removal_set.contains(idx)) { + for &idx in indices { + removal_set.insert(idx); + } + } + } + + if removal_set.is_empty() { return lines; } - // Only strip candidate instances that are in edge positions on their page. - // This preserves body content that happens to match a running header/footer - // (e.g., "New Britannia Mill" appearing both as a page header and in the - // document's subject line). lines .into_iter() - .filter(|line| { - let text = line.text(); - let normalized = normalize_whitespace(&text); - if !candidates.contains(&normalized) { - return true; // not a candidate — keep - } - // Candidate: only strip if this instance is in an edge position - !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) - }) + .enumerate() + .filter(|(idx, _)| !removal_set.contains(idx)) + .map(|(_, line)| line) .collect() } diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index 3979198..a80af44 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -33,20 +33,14 @@ Date Date **a. Tips received** **b. Credit card tips c. Tips paid out to d. Names of employees to whom you** tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1** -rec’d. entry and other employees - 7 8 9 10 11 12 13 14 15 **Subtotals** **Page 2** -rec’d. entry and other employees - 17 18 19 20 21 22 23 24 25 **Subtotals** **Page 3** -rec’d. entry and other employees - 27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals** **1.** Report total cash tips (col. a) on Form 4070, line 1.