Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
9bb1349759 | ||
|
Abimael Martell
|
64c5b6dc1a | ||
|
Abimael Martell
|
ac5b1a303f | ||
|
Abimael Martell
|
95af5350b7 | ||
|
Abimael Martell
|
0cf7907c63 | ||
|
Abimael Martell
|
666ff0f9e1 | ||
|
Abimael Martell
|
c93728e803 | ||
|
Abimael Martell
|
9e8947b9d5 | ||
|
Abimael Martell
|
d27632ead6 |
File diff suppressed because it is too large
Load Diff
+3
-2
@@ -9,6 +9,7 @@
|
||||
pub(crate) mod analysis;
|
||||
mod classify;
|
||||
mod convert;
|
||||
mod furniture;
|
||||
mod heading;
|
||||
mod postprocess;
|
||||
mod preprocess;
|
||||
@@ -1149,7 +1150,7 @@ pub(crate) fn strip_repeated_header_footer_lines(
|
||||
lines: Vec<crate::types::TextLine>,
|
||||
page_count: u32,
|
||||
) -> Vec<crate::types::TextLine> {
|
||||
preprocess::strip_repeated_lines(lines, page_count)
|
||||
furniture::strip_header_footer_lines(lines, page_count)
|
||||
}
|
||||
|
||||
/// Convert positioned text items to markdown with structure detection
|
||||
@@ -2106,7 +2107,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
|
||||
// Strip repeated headers/footers before conversion
|
||||
let lines = if options.strip_headers_footers {
|
||||
preprocess::strip_repeated_lines(lines, document_page_count)
|
||||
furniture::strip_header_footer_lines(lines, document_page_count)
|
||||
} else {
|
||||
lines
|
||||
};
|
||||
|
||||
+1
-384
@@ -1,6 +1,6 @@
|
||||
//! Line preprocessing: heading merging, drop cap handling, and repeated line removal.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::structure_tree::StructRole;
|
||||
use crate::types::{TextItem, TextLine};
|
||||
@@ -229,342 +229,6 @@ pub(crate) fn merge_drop_caps(lines: Vec<TextLine>, base_size: f32) -> Vec<TextL
|
||||
result
|
||||
}
|
||||
|
||||
/// Normalize whitespace in a string for comparison: trim and collapse internal runs of whitespace.
|
||||
fn normalize_whitespace(s: &str) -> String {
|
||||
s.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Normalize text for frequency comparison: collapse whitespace and strip leading/trailing
|
||||
/// digit sequences (page numbers). E.g., "Chapter 3 — Page 5" and "Chapter 3 — Page 6"
|
||||
/// both normalize to "Chapter 3 — Page".
|
||||
fn normalize_for_comparison(s: &str) -> String {
|
||||
let ws = normalize_whitespace(s);
|
||||
let trimmed = ws
|
||||
.trim_start_matches(|c: char| c.is_ascii_digit())
|
||||
.trim_start();
|
||||
let trimmed = trimmed
|
||||
.trim_end_matches(|c: char| c.is_ascii_digit())
|
||||
.trim_end();
|
||||
trimmed.to_string()
|
||||
}
|
||||
|
||||
/// Returns true if the line looks like a list item or heading (should not be stripped).
|
||||
fn is_structural_line(text: &str) -> bool {
|
||||
let t = text.trim_start();
|
||||
t.starts_with('#')
|
||||
|| t.starts_with("- ")
|
||||
|| t.starts_with("* ")
|
||||
|| t.starts_with("• ")
|
||||
|| t.chars()
|
||||
.next()
|
||||
.map(|c| c.is_ascii_digit())
|
||||
.unwrap_or(false)
|
||||
&& (t.contains(". ") || t.contains(") "))
|
||||
}
|
||||
|
||||
/// Returns true if a line consists entirely of a single repeated character
|
||||
/// (e.g., "----------", "**************", "============").
|
||||
fn is_decorative_separator(text: &str) -> bool {
|
||||
let mut chars = text.chars();
|
||||
let first = match chars.next() {
|
||||
Some(c) => c,
|
||||
None => return false,
|
||||
};
|
||||
chars.all(|c| c == first)
|
||||
}
|
||||
|
||||
/// Strip lines that repeat on many distinct pages (running headers/footers).
|
||||
///
|
||||
/// A line is considered a repeated header/footer if:
|
||||
/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
|
||||
/// 2. It is at least 10 characters long
|
||||
/// 3. It doesn't look like a structural element (heading, list item)
|
||||
/// 4. It consistently appears in the top or bottom N distinct Y positions
|
||||
/// 5. Its Y positions across pages have low variance (consistent placement),
|
||||
/// distinguishing true headers/footers from table content that happens to
|
||||
/// land near page margins
|
||||
/// 6. It is not a decorative separator (repeated single character)
|
||||
///
|
||||
/// Additionally, TextLines at the same Y position on a page are grouped into
|
||||
/// "Y-bands." When any member of a Y-band is stripped, all siblings in that
|
||||
/// band are also stripped. This handles split column headers where individual
|
||||
/// fragments may not independently meet the frequency threshold.
|
||||
///
|
||||
/// Page numbers are stripped from line text before comparison, so headers like
|
||||
/// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
|
||||
pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
|
||||
if lines.is_empty() || page_count < 3 {
|
||||
return lines;
|
||||
}
|
||||
|
||||
// Compute Y range per page (min_y, max_y)
|
||||
let mut page_y_range: HashMap<u32, (f32, f32)> = HashMap::new();
|
||||
for line in &lines {
|
||||
let entry = page_y_range.entry(line.page).or_insert((line.y, line.y));
|
||||
if line.y < entry.0 {
|
||||
entry.0 = line.y;
|
||||
}
|
||||
if line.y > entry.1 {
|
||||
entry.1 = line.y;
|
||||
}
|
||||
}
|
||||
|
||||
// Build sorted Y values per page, so we can check line rank (position from edge)
|
||||
let mut page_sorted_ys: HashMap<u32, Vec<f32>> = HashMap::new();
|
||||
for line in &lines {
|
||||
page_sorted_ys.entry(line.page).or_default().push(line.y);
|
||||
}
|
||||
for ys in page_sorted_ys.values_mut() {
|
||||
ys.sort_by(|a, b| a.total_cmp(b));
|
||||
ys.dedup();
|
||||
}
|
||||
|
||||
// A line is in the page margin if it's among the first or last N distinct
|
||||
// Y positions on that page. This is more robust than a percentage-based zone
|
||||
// because it catches actual edge lines regardless of how much content fills
|
||||
// the page. N=5 accommodates multi-line headers/footers and repeated form
|
||||
// column headers (e.g., 5-row IRS form headers) that sit just inside the
|
||||
// page margin.
|
||||
const EDGE_LINE_COUNT: usize = 5;
|
||||
|
||||
/// Returns true if the given Y position is among the first or last N distinct
|
||||
/// Y positions on the specified page.
|
||||
fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
|
||||
let ys = match page_sorted_ys.get(&page) {
|
||||
Some(ys) => ys,
|
||||
None => return false,
|
||||
};
|
||||
if ys.len() <= n * 2 {
|
||||
// Page has very few lines — everything is near the edge
|
||||
return true;
|
||||
}
|
||||
// Check if this Y is among the first or last N
|
||||
let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
pos < n || pos >= ys.len() - n
|
||||
}
|
||||
|
||||
// Average page span for normalizing Y variance
|
||||
let avg_span = {
|
||||
let total: f32 = page_y_range.values().map(|(lo, hi)| hi - lo).sum();
|
||||
if page_y_range.is_empty() {
|
||||
1.0
|
||||
} else {
|
||||
(total / page_y_range.len() as f32).max(1.0)
|
||||
}
|
||||
};
|
||||
|
||||
// Build Y-bands: group line indices by (page, quantized_y).
|
||||
// Lines at the same Y position (within ~0.1pt) on the same page form a band.
|
||||
let mut y_bands: HashMap<(u32, i32), Vec<usize>> = HashMap::new();
|
||||
for (idx, line) in lines.iter().enumerate() {
|
||||
let y_bucket = (line.y * 10.0).round() as i32;
|
||||
y_bands.entry((line.page, y_bucket)).or_default().push(idx);
|
||||
}
|
||||
|
||||
// Build frequency maps using normalize_for_comparison.
|
||||
// Individual line text -> distinct pages
|
||||
let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
|
||||
let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
|
||||
for line in &lines {
|
||||
if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
let text = line.text();
|
||||
let normalized = normalize_for_comparison(&text);
|
||||
if normalized.len() < 10 || is_decorative_separator(&normalized) {
|
||||
continue;
|
||||
}
|
||||
freq.entry(normalized.clone())
|
||||
.or_default()
|
||||
.insert(line.page);
|
||||
y_positions.entry(normalized).or_default().push(line.y);
|
||||
}
|
||||
|
||||
// Coalesced row text -> distinct pages (for multi-member Y-bands).
|
||||
// This catches split column headers where individual fragments don't meet
|
||||
// the frequency threshold but the combined row does.
|
||||
let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
|
||||
let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
|
||||
for (&(page, _), indices) in &y_bands {
|
||||
if indices.len() < 2 {
|
||||
continue; // single-line bands are already in the individual map
|
||||
}
|
||||
let band_y = lines[indices[0]].y;
|
||||
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
let mut sorted_indices = indices.clone();
|
||||
sorted_indices.sort();
|
||||
let coalesced: String = sorted_indices
|
||||
.iter()
|
||||
.map(|&i| lines[i].text())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let normalized = normalize_for_comparison(&coalesced);
|
||||
if normalized.len() < 10 || is_decorative_separator(&normalized) {
|
||||
continue;
|
||||
}
|
||||
band_freq
|
||||
.entry(normalized.clone())
|
||||
.or_default()
|
||||
.insert(page);
|
||||
band_y_positions.entry(normalized).or_default().push(band_y);
|
||||
}
|
||||
|
||||
// Compute threshold
|
||||
let threshold = 3u32.max(page_count * 30 / 100);
|
||||
|
||||
// Check Y-position consistency: headers/footers appear at the same position
|
||||
// on every page, table content varies. Require normalized stddev < 5% of
|
||||
// average page span.
|
||||
let has_consistent_y = |text: &str, positions: &HashMap<String, Vec<f32>>| -> bool {
|
||||
let pos = match positions.get(text) {
|
||||
Some(p) if p.len() >= 2 => p,
|
||||
_ => return true, // single occurrence — allow
|
||||
};
|
||||
let n = pos.len() as f32;
|
||||
let mean = pos.iter().sum::<f32>() / n;
|
||||
let variance = pos.iter().map(|y| (y - mean).powi(2)).sum::<f32>() / n;
|
||||
let stddev = variance.sqrt();
|
||||
stddev / avg_span < 0.05
|
||||
};
|
||||
|
||||
// Identify candidates from individual frequency map
|
||||
let candidates: HashSet<String> = freq
|
||||
.into_iter()
|
||||
.filter(|(text, pages)| {
|
||||
pages.len() as u32 >= threshold
|
||||
&& !is_structural_line(text)
|
||||
&& has_consistent_y(text, &y_positions)
|
||||
})
|
||||
.map(|(text, _)| text)
|
||||
.collect();
|
||||
|
||||
// Identify candidates from coalesced band frequency map
|
||||
let band_candidates: HashSet<String> = band_freq
|
||||
.into_iter()
|
||||
.filter(|(text, pages)| {
|
||||
pages.len() as u32 >= threshold
|
||||
&& !is_structural_line(text)
|
||||
&& has_consistent_y(text, &band_y_positions)
|
||||
})
|
||||
.map(|(text, _)| text)
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() && band_candidates.is_empty() {
|
||||
return lines;
|
||||
}
|
||||
|
||||
// Build removal set.
|
||||
// A line is removed if it's at an edge position and:
|
||||
// (a) its individual text matches a candidate, OR
|
||||
// (b) its Y-band's coalesced text matches a band candidate, OR
|
||||
// (c) any sibling in its Y-band was removed (propagation).
|
||||
//
|
||||
// The first occurrence (lowest page number) of each repeated header/footer
|
||||
// is kept so that document titles, column headers, etc. appear once.
|
||||
let mut removal_set: HashSet<usize> = HashSet::new();
|
||||
|
||||
// Track which page first shows each candidate (to preserve first occurrence)
|
||||
let mut first_page_individual: HashMap<String, u32> = HashMap::new();
|
||||
for (idx, line) in lines.iter().enumerate() {
|
||||
if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
let text = line.text();
|
||||
let normalized = normalize_for_comparison(&text);
|
||||
if candidates.contains(&normalized) {
|
||||
let first = first_page_individual.entry(normalized).or_insert(line.page);
|
||||
if line.page > *first {
|
||||
removal_set.insert(idx);
|
||||
} else if line.page == *first {
|
||||
// Keep this occurrence (first page)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Track first page for band candidates
|
||||
let mut first_page_band: HashMap<String, u32> = HashMap::new();
|
||||
// First pass: find first page for each band candidate
|
||||
for (&(page, _), indices) in &y_bands {
|
||||
if indices.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
let band_y = lines[indices[0]].y;
|
||||
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
let mut sorted_indices = indices.clone();
|
||||
sorted_indices.sort();
|
||||
let coalesced: String = sorted_indices
|
||||
.iter()
|
||||
.map(|&i| lines[i].text())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let normalized = normalize_for_comparison(&coalesced);
|
||||
if band_candidates.contains(&normalized) {
|
||||
let first = first_page_band.entry(normalized).or_insert(page);
|
||||
if page < *first {
|
||||
*first = page;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Second pass: mark for removal (skip first page)
|
||||
for (&(page, _), indices) in &y_bands {
|
||||
if indices.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
let band_y = lines[indices[0]].y;
|
||||
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
let mut sorted_indices = indices.clone();
|
||||
sorted_indices.sort();
|
||||
let coalesced: String = sorted_indices
|
||||
.iter()
|
||||
.map(|&i| lines[i].text())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let normalized = normalize_for_comparison(&coalesced);
|
||||
if band_candidates.contains(&normalized) {
|
||||
let first = first_page_band.get(&normalized).copied().unwrap_or(0);
|
||||
if page > first {
|
||||
for &idx in &sorted_indices {
|
||||
removal_set.insert(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (c) Y-band sibling propagation: if any member is removed, remove all
|
||||
// members (provided the band is at an edge position).
|
||||
for (&(page, _), indices) in &y_bands {
|
||||
let band_y = lines[indices[0]].y;
|
||||
if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
if indices.iter().any(|idx| removal_set.contains(idx)) {
|
||||
for &idx in indices {
|
||||
removal_set.insert(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if removal_set.is_empty() {
|
||||
return lines;
|
||||
}
|
||||
|
||||
lines
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter(|(idx, _)| !removal_set.contains(idx))
|
||||
.map(|(_, line)| line)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -679,53 +343,6 @@ mod tests {
|
||||
assert_eq!(result.len(), 2, "should merge font-based heading lines");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_strip_repeated_keeps_first_occurrence() {
|
||||
// Simulate a repeated page header on 10 pages.
|
||||
// Each page has a running header at y=750 and many unique body lines.
|
||||
let mut lines = Vec::new();
|
||||
for page in 1..=10u32 {
|
||||
// Header at top
|
||||
lines.push(make_line(
|
||||
"VOICE OF SOUTH MARION May fifteen twenty twenty five",
|
||||
10.0,
|
||||
page,
|
||||
750.0,
|
||||
None,
|
||||
));
|
||||
// Body content — unique text per line per page (no digits to strip)
|
||||
for j in 0..20u32 {
|
||||
lines.push(make_line(
|
||||
&format!(
|
||||
"parcel r-{:04}-{:03} owner smith address oak street",
|
||||
page * 100 + j,
|
||||
page
|
||||
),
|
||||
10.0,
|
||||
page,
|
||||
600.0 - j as f32 * 15.0,
|
||||
None,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let result = strip_repeated_lines(lines, 10);
|
||||
|
||||
// The header should appear exactly once (page 1)
|
||||
let header_count = result
|
||||
.iter()
|
||||
.filter(|l| l.text().contains("VOICE OF SOUTH MARION"))
|
||||
.count();
|
||||
assert_eq!(header_count, 1, "repeated header should be kept once");
|
||||
|
||||
// First occurrence should be on page 1
|
||||
let first_header = result
|
||||
.iter()
|
||||
.find(|l| l.text().contains("VOICE OF SOUTH MARION"))
|
||||
.unwrap();
|
||||
assert_eq!(first_header.page, 1, "first occurrence should be on page 1");
|
||||
}
|
||||
|
||||
fn make_bold_line(text: &str, page: u32, y: f32) -> TextLine {
|
||||
let mut item = make_item(text, 12.0, None);
|
||||
item.is_bold = true;
|
||||
|
||||
Reference in New Issue
Block a user