From 7b6bb69feb9b6edc7eb90a9e14dadd6259905e59 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 3 Mar 2026 00:04:17 -0800 Subject: [PATCH] feat: strip repeated headers/footers from markdown output Add strip_repeated_lines() preprocessing step that removes running headers and footers before markdown conversion. Uses multiple guard rails to avoid false positives: - Edge-line detection: only considers lines among the first/last 3 distinct Y positions on each page (not percentage-based margins) - Y-position consistency: requires low variance across pages (stddev < 5% of page span) to distinguish headers from scattered table content - Position-aware removal: only strips instances at page edges, preserving body content that happens to match a header/footer text - Skips structural lines (headings, list items), short lines (<10 chars), and decorative separators (repeated single characters) - Frequency threshold: >= max(3, page_count * 30%) distinct pages New strip_headers_footers option on MarkdownOptions (default: true). Co-Authored-By: Claude Opus 4.6 --- src/markdown/mod.rs | 11 +++ src/markdown/preprocess.rs | 178 +++++++++++++++++++++++++++++++++- tests/snapshots/p1244-1996.md | 15 +-- 3 files changed, 191 insertions(+), 13 deletions(-) diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 5ddabda..0bbf2ce 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -50,6 +50,8 @@ pub struct MarkdownOptions { pub include_links: bool, /// Insert page break markers () between pages pub include_page_numbers: bool, + /// Strip repeated headers/footers that appear on many pages + pub strip_headers_footers: bool, } impl Default for MarkdownOptions { @@ -67,6 +69,7 @@ impl Default for MarkdownOptions { include_images: true, include_links: true, include_page_numbers: false, + strip_headers_footers: true, } } } @@ -206,6 +209,7 @@ pub fn to_markdown_from_items_with_rects( let mut pages: Vec = page_groups.keys().copied().collect(); pages.sort(); + let page_count = pages.last().copied().unwrap_or(0) + 1; for page in pages { let group = page_groups.get(&page).unwrap(); @@ -323,6 +327,13 @@ pub fn to_markdown_from_items_with_rects( let lines = group_into_lines(non_table_items); + // Strip repeated headers/footers before conversion + let lines = if options.strip_headers_footers { + preprocess::strip_repeated_lines(lines, page_count) + } else { + lines + }; + // Convert to markdown, inserting tables and images at appropriate positions to_markdown_from_lines_with_tables_and_images(lines, options, page_tables, page_images) } diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index 9092c81..7feae81 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -1,4 +1,6 @@ -//! Line preprocessing: heading merging and drop cap handling. +//! Line preprocessing: heading merging, drop cap handling, and repeated line removal. + +use std::collections::{HashMap, HashSet}; use crate::types::TextLine; @@ -140,3 +142,177 @@ pub(crate) fn merge_drop_caps(lines: Vec, base_size: f32) -> Vec String { + s.split_whitespace().collect::>().join(" ") +} + +/// Returns true if the line looks like a list item or heading (should not be stripped). +fn is_structural_line(text: &str) -> bool { + let t = text.trim_start(); + t.starts_with('#') + || t.starts_with("- ") + || t.starts_with("* ") + || t.starts_with("• ") + || t.chars() + .next() + .map(|c| c.is_ascii_digit()) + .unwrap_or(false) + && (t.contains(". ") || t.contains(") ")) +} + +/// Returns true if a line consists entirely of a single repeated character +/// (e.g., "----------", "**************", "============"). +fn is_decorative_separator(text: &str) -> bool { + let mut chars = text.chars(); + let first = match chars.next() { + Some(c) => c, + None => return false, + }; + chars.all(|c| c == first) +} + +/// Strip lines that repeat on many distinct pages (running headers/footers). +/// +/// A line is considered a repeated header/footer if: +/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages +/// 2. It is at least 10 characters long +/// 3. It doesn't look like a structural element (heading, list item) +/// 4. It consistently appears in the top or bottom 15% of the page's Y range +/// 5. Its Y positions across pages have low variance (consistent placement), +/// distinguishing true headers/footers from table content that happens to +/// land near page margins +/// 6. It is not a decorative separator (repeated single character) +pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec { + if lines.is_empty() || page_count < 3 { + return lines; + } + + // Compute Y range per page (min_y, max_y) + let mut page_y_range: HashMap = HashMap::new(); + for line in &lines { + let entry = page_y_range.entry(line.page).or_insert((line.y, line.y)); + if line.y < entry.0 { + entry.0 = line.y; + } + if line.y > entry.1 { + entry.1 = line.y; + } + } + + // Build sorted Y values per page, so we can check line rank (position from edge) + let mut page_sorted_ys: HashMap> = HashMap::new(); + for line in &lines { + page_sorted_ys.entry(line.page).or_default().push(line.y); + } + for ys in page_sorted_ys.values_mut() { + ys.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + ys.dedup(); + } + + // A line is in the page margin if it's among the first or last N distinct + // Y positions on that page. This is more robust than a percentage-based zone + // because it catches actual edge lines regardless of how much content fills + // the page. N=3 accommodates multi-line headers/footers (e.g., nav bar + title). + const EDGE_LINE_COUNT: usize = 3; + + /// Returns true if the line is among the first or last N distinct Y positions + /// on its page. + fn is_edge_line(line: &TextLine, page_sorted_ys: &HashMap>, n: usize) -> bool { + let ys = match page_sorted_ys.get(&line.page) { + Some(ys) => ys, + None => return false, + }; + if ys.len() <= n * 2 { + // Page has very few lines — everything is near the edge + return true; + } + // Check if this Y is among the first or last N + let pos = match ys.iter().position(|&y| (y - line.y).abs() < 0.1) { + Some(p) => p, + None => return false, + }; + pos < n || pos >= ys.len() - n + } + + // Average page span for normalizing Y variance + let avg_span = { + let total: f32 = page_y_range.values().map(|(lo, hi)| hi - lo).sum(); + if page_y_range.is_empty() { + 1.0 + } else { + (total / page_y_range.len() as f32).max(1.0) + } + }; + + // Build frequency map: only count lines at the page edges + // Also collect Y positions for variance check + let mut freq: HashMap> = HashMap::new(); + let mut y_positions: HashMap> = HashMap::new(); + for line in &lines { + let text = line.text(); + let normalized = normalize_whitespace(&text); + if normalized.len() < 10 { + continue; + } + if is_decorative_separator(&normalized) { + continue; + } + if !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + freq.entry(normalized.clone()) + .or_default() + .insert(line.page); + y_positions.entry(normalized).or_default().push(line.y); + } + + // Compute threshold + let threshold = 3u32.max(page_count * 30 / 100); + + // Check Y-position consistency: headers/footers appear at the same position + // on every page, table content varies. Require normalized stddev < 5% of + // average page span. + let has_consistent_y = |text: &str| -> bool { + let positions = match y_positions.get(text) { + Some(p) if p.len() >= 2 => p, + _ => return true, // single occurrence — allow + }; + let n = positions.len() as f32; + let mean = positions.iter().sum::() / n; + let variance = positions.iter().map(|y| (y - mean).powi(2)).sum::() / n; + let stddev = variance.sqrt(); + stddev / avg_span < 0.05 + }; + + // Collect candidate texts to remove + let candidates: HashSet = freq + .into_iter() + .filter(|(text, pages)| { + pages.len() as u32 >= threshold && !is_structural_line(text) && has_consistent_y(text) + }) + .map(|(text, _)| text) + .collect(); + + if candidates.is_empty() { + return lines; + } + + // Only strip candidate instances that are in edge positions on their page. + // This preserves body content that happens to match a running header/footer + // (e.g., "New Britannia Mill" appearing both as a page header and in the + // document's subject line). + lines + .into_iter() + .filter(|line| { + let text = line.text(); + let normalized = normalize_whitespace(&text); + if !candidates.contains(&normalized) { + return true; // not a candidate — keep + } + // Candidate: only strip if this instance is in an edge position + !is_edge_line(line, &page_sorted_ys, EDGE_LINE_COUNT) + }) + .collect() +} diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index 68fe38c..edb8616 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -33,28 +33,19 @@ Date Date **a. Tips received** **b. Credit card tips c. Tips paid out to d. Names of employees to whom you** tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1** -Date Date **a. Tips received** - -**b. Credit card tips c. Tips paid out to d. Names of employees to whom you** -tips of directly from customers received other employees paid tips rec’d. entry and other employees +received other employees paid tips rec’d. entry and other employees 7 8 9 10 11 12 13 14 15 **Subtotals** **Page 2** -Date Date **a. Tips received** - -**b. Credit card tips c. Tips paid out to d. Names of employees to whom you** -tips of directly from customers received other employees paid tips rec’d. entry and other employees +received other employees paid tips rec’d. entry and other employees 17 18 19 20 21 22 23 24 25 **Subtotals** **Page 3** -Date Date **a. Tips received** - -**b. Credit card tips c. Tips paid out to d. Names of employees to whom you** -tips of directly from customers received other employees paid tips rec’d. entry and other employees +received other employees paid tips rec’d. entry and other employees 27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals**