mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance header/footer stripping with Y-band coalescing and page number normalization

Browse Source 
Strip leading/trailing digit sequences (page numbers) before frequency
comparison so headers like "Page 5" and "Page 6" are treated as identical.
Group TextLines at the same Y position into Y-bands and propagate removal
to all siblings when any member is stripped. Increase EDGE_LINE_COUNT from
4 to 5 to cover 5-row form column headers (e.g., IRS p1244).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-03 09:46:51 -08:00
co-authored by Claude Opus 4.6
parent eca2db119c
commit 8adfacd3e3
2 changed files with 163 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/markdown/preprocess.rs
+163 -40
View File
@@ -148,6 +148,20 @@ fn normalize_whitespace(s: &str) -> String {
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
/// Normalize text for frequency comparison: collapse whitespace and strip leading/trailing
/// digit sequences (page numbers). E.g., "Chapter 3 — Page 5" and "Chapter 3 — Page 6"
/// both normalize to "Chapter 3 — Page".
fn normalize_for_comparison(s: &str) -> String {
let ws = normalize_whitespace(s);
let trimmed = ws
.trim_start_matches(|c: char| c.is_ascii_digit())
.trim_start();
let trimmed = trimmed
.trim_end_matches(|c: char| c.is_ascii_digit())
.trim_end();
trimmed.to_string()
}
/// Returns true if the line looks like a list item or heading (should not be stripped).
fn is_structural_line(text: &str) -> bool {
let t = text.trim_start();
@@ -179,11 +193,19 @@ fn is_decorative_separator(text: &str) -> bool {
/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
/// 2. It is at least 10 characters long
/// 3. It doesn't look like a structural element (heading, list item)
/// 4. It consistently appears in the top or bottom 15% of the page's Y range
/// 4. It consistently appears in the top or bottom N distinct Y positions
/// 5. Its Y positions across pages have low variance (consistent placement),
/// distinguishing true headers/footers from table content that happens to
/// land near page margins
/// 6. It is not a decorative separator (repeated single character)
///
/// Additionally, TextLines at the same Y position on a page are grouped into
/// "Y-bands." When any member of a Y-band is stripped, all siblings in that
/// band are also stripped. This handles split column headers where individual
/// fragments may not independently meet the frequency threshold.
///
/// Page numbers are stripped from line text before comparison, so headers like
/// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
if lines.is_empty() || page_count < 3 {
return lines;
@@ -214,14 +236,15 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
// A line is in the page margin if it's among the first or last N distinct
// Y positions on that page. This is more robust than a percentage-based zone
// because it catches actual edge lines regardless of how much content fills
// the page. N=4 accommodates multi-line headers/footers and repeated form
// column headers that sit just inside the page margin.
const EDGE_LINE_COUNT: usize = 4;
// the page. N=5 accommodates multi-line headers/footers and repeated form
// column headers (e.g., 5-row IRS form headers) that sit just inside the
// page margin.
const EDGE_LINE_COUNT: usize = 5;
/// Returns true if the line is among the first or last N distinct Y positions
/// on its page.
fn is_edge_line(line: &TextLine, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
let ys = match page_sorted_ys.get(&line.page) {
/// Returns true if the given Y position is among the first or last N distinct
/// Y positions on the specified page.
fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
let ys = match page_sorted_ys.get(&page) {
Some(ys) => ys,
None => return false,
};
@@ -230,7 +253,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
return true;
}
// Check if this Y is among the first or last N
let pos = match ys.iter().position(|&y| (y - line.y).abs() < 0.1) {
let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
Some(p) => p,
None => return false,
};
@@ -247,20 +270,25 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
}
};
// Build frequency map: only count lines at the page edges
// Also collect Y positions for variance check
// Build Y-bands: group line indices by (page, quantized_y).
// Lines at the same Y position (within ~0.1pt) on the same page form a band.
let mut y_bands: HashMap<(u32, i32), Vec<usize>> = HashMap::new();
for (idx, line) in lines.iter().enumerate() {
let y_bucket = (line.y * 10.0).round() as i32;
y_bands.entry((line.page, y_bucket)).or_default().push(idx);
}
// Build frequency maps using normalize_for_comparison.
// Individual line text -> distinct pages
let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
for line in &lines {
if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
continue;
}
let text = line.text();
let normalized = normalize_whitespace(&text);
if normalized.len() < 10 {
continue;
}
if is_decorative_separator(&normalized) {
continue;
}
if !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) {
let normalized = normalize_for_comparison(&text);
if normalized.len() < 10 || is_decorative_separator(&normalized) {
continue;
}
freq.entry(normalized.clone())
@@ -269,51 +297,146 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
y_positions.entry(normalized).or_default().push(line.y);
}
// Coalesced row text -> distinct pages (for multi-member Y-bands).
// This catches split column headers where individual fragments don't meet
// the frequency threshold but the combined row does.
let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
for (&(page, _), indices) in &y_bands {
if indices.len() < 2 {
continue; // single-line bands are already in the individual map
}
let band_y = lines[indices[0]].y;
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
continue;
}
let mut sorted_indices = indices.clone();
sorted_indices.sort();
let coalesced: String = sorted_indices
.iter()
.map(|&i| lines[i].text())
.collect::<Vec<_>>()
.join(" ");
let normalized = normalize_for_comparison(&coalesced);
if normalized.len() < 10 || is_decorative_separator(&normalized) {
continue;
}
band_freq
.entry(normalized.clone())
.or_default()
.insert(page);
band_y_positions.entry(normalized).or_default().push(band_y);
}
// Compute threshold
let threshold = 3u32.max(page_count * 30 / 100);
// Check Y-position consistency: headers/footers appear at the same position
// on every page, table content varies. Require normalized stddev < 5% of
// average page span.
let has_consistent_y = |text: &str| -> bool {
let positions = match y_positions.get(text) {
let has_consistent_y = |text: &str, positions: &HashMap<String, Vec<f32>>| -> bool {
let pos = match positions.get(text) {
Some(p) if p.len() >= 2 => p,
_ => return true, // single occurrence — allow
};
let n = positions.len() as f32;
let mean = positions.iter().sum::<f32>() / n;
let variance = positions.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
let n = pos.len() as f32;
let mean = pos.iter().sum::<f32>() / n;
let variance = pos.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
let stddev = variance.sqrt();
stddev / avg_span < 0.05
};
// Collect candidate texts to remove
// Identify candidates from individual frequency map
let candidates: HashSet<String> = freq
.into_iter()
.filter(|(text, pages)| {
pages.len() as u32 >= threshold && !is_structural_line(text) && has_consistent_y(text)
pages.len() as u32 >= threshold
&& !is_structural_line(text)
&& has_consistent_y(text, &y_positions)
})
.map(|(text, _)| text)
.collect();
if candidates.is_empty() {
// Identify candidates from coalesced band frequency map
let band_candidates: HashSet<String> = band_freq
.into_iter()
.filter(|(text, pages)| {
pages.len() as u32 >= threshold
&& !is_structural_line(text)
&& has_consistent_y(text, &band_y_positions)
})
.map(|(text, _)| text)
.collect();
if candidates.is_empty() && band_candidates.is_empty() {
return lines;
}
// Build removal set.
// A line is removed if it's at an edge position and:
// (a) its individual text matches a candidate, OR
// (b) its Y-band's coalesced text matches a band candidate, OR
// (c) any sibling in its Y-band was removed (propagation).
let mut removal_set: HashSet<usize> = HashSet::new();
// (a) Lines matching individual candidates at edge positions
for (idx, line) in lines.iter().enumerate() {
if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
continue;
}
let text = line.text();
let normalized = normalize_for_comparison(&text);
if candidates.contains(&normalized) {
removal_set.insert(idx);
}
}
// (b) Lines in Y-bands whose coalesced text matches a band candidate
for (&(page, _), indices) in &y_bands {
if indices.len() < 2 {
continue;
}
let band_y = lines[indices[0]].y;
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
continue;
}
let mut sorted_indices = indices.clone();
sorted_indices.sort();
let coalesced: String = sorted_indices
.iter()
.map(|&i| lines[i].text())
.collect::<Vec<_>>()
.join(" ");
let normalized = normalize_for_comparison(&coalesced);
if band_candidates.contains(&normalized) {
for &idx in &sorted_indices {
removal_set.insert(idx);
}
}
}
// (c) Y-band sibling propagation: if any member is removed, remove all
// members (provided the band is at an edge position).
for (&(page, _), indices) in &y_bands {
let band_y = lines[indices[0]].y;
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
continue;
}
if indices.iter().any(|idx| removal_set.contains(idx)) {
for &idx in indices {
removal_set.insert(idx);
}
}
}
if removal_set.is_empty() {
return lines;
}
// Only strip candidate instances that are in edge positions on their page.
// This preserves body content that happens to match a running header/footer
// (e.g., "New Britannia Mill" appearing both as a page header and in the
// document's subject line).
lines
.into_iter()
.filter(|line| {
let text = line.text();
let normalized = normalize_whitespace(&text);
if !candidates.contains(&normalized) {
return true; // not a candidate — keep
}
// Candidate: only strip if this instance is in an edge position
!is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT)
})
.enumerate()
.filter(|(idx, _)| !removal_set.contains(idx))
.map(|(_, line)| line)
.collect()
}
tests/snapshots/p1244-1996.md
-6
View File
@@ -33,20 +33,14 @@ Date Date **a. Tips received**
**b. Credit card tips c. Tips paid out to d. Names of employees to whom you**
tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1**
rec’d. entry and other employees
7 8 9 10 11 12 13 14 15 **Subtotals**
**Page 2**
rec’d. entry and other employees
17 18 19 20 21 22 23 24 25 **Subtotals**
**Page 3**
rec’d. entry and other employees
27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals**
**1.** Report total cash tips (col. a) on Form 4070, line 1.