fix(postprocess): collapse consecutive spaces in extracted text

OCR text layers and some PDF producers emit trailing spaces on each
text item, which combine with gap-based joining to produce double
spaces ("Vice  President"). Now collapses runs of 2+ spaces to single
space within lines, preserving leading indentation.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-18 11:49:43 -07:00
co-authored by Claude Opus 4.6
parent b8211db4b0
commit 258210a821
6 changed files with 118 additions and 83 deletions
+35
View File
@@ -24,6 +24,12 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
text = format_urls(&text);
}
// Collapse consecutive spaces within text lines.
// OCR text layers and some PDF producers emit trailing spaces on each
// text item, which combine with gap-based space insertion to produce
// double spaces ("Vice President" instead of "Vice President").
collapse_consecutive_spaces(&mut text);
// Remove excessive newlines (more than 2 in a row)
while text.contains("\n\n\n") {
text = text.replace("\n\n\n", "\n\n");
@@ -36,6 +42,35 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
text
}
/// Collapse runs of 2+ spaces to a single space within each line.
/// Preserves leading indentation and markdown table pipe alignment.
fn collapse_consecutive_spaces(text: &mut String) {
let mut result = String::with_capacity(text.len());
for line in text.split('\n') {
if !result.is_empty() {
result.push('\n');
}
// Preserve leading whitespace
let trimmed = line.trim_start();
let leading = &line[..line.len() - trimmed.len()];
result.push_str(leading);
// Collapse inner runs of spaces to single space
let mut prev_space = false;
for ch in trimmed.chars() {
if ch == ' ' {
if !prev_space {
result.push(' ');
}
prev_space = true;
} else {
prev_space = false;
result.push(ch);
}
}
}
*text = result;
}
/// Collapse dot leaders (runs of 4+ dots) into " ... "
/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
fn collapse_dot_leaders(text: &str) -> String {