feat(text): adaptive join threshold for Canva-style letter-spaced PDFs

Canva-generated PDFs render text character-by-character with CSS-style
letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold
caused every character to get a space inserted ("K a r i b i b").

Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c"
pattern), compute an IQR-based threshold (median × 1.55) on the gap
distribution BEFORE space removal, then propagate per-page thresholds
through PageThresholds → group_into_lines_with_thresholds → TextLine
→ should_join_items.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-12 17:54:49 -07:00
co-authored by Claude Opus 4.6
parent 80a3b81ff9
commit 152de8b56d
7 changed files with 537 additions and 21 deletions
+18 -4
View File
@@ -125,6 +125,10 @@ pub struct TextLine {
pub items: Vec<TextItem>,
pub y: f32,
pub page: u32,
/// Adaptive join threshold from page-level letter-spacing detection.
/// Default 0.10 for normal PDFs; higher for Canva-style PDFs.
#[doc(hidden)]
pub adaptive_threshold: f32,
}
impl TextLine {
@@ -138,6 +142,8 @@ impl TextLine {
return self.text_plain();
}
let single_char_threshold = self.adaptive_threshold;
let mut result = String::new();
let mut current_bold = false;
let mut current_italic = false;
@@ -156,7 +162,7 @@ impl TextLine {
false
} else {
let prev_item = &self.items[i - 1];
self.needs_space_between(prev_item, item, &result)
self.needs_space_between(prev_item, item, &result, single_char_threshold)
};
// Preserve leading whitespace from the item text.
@@ -211,6 +217,8 @@ impl TextLine {
/// Get plain text without formatting
fn text_plain(&self) -> String {
let single_char_threshold = self.adaptive_threshold;
let mut result = String::new();
for (i, item) in self.items.iter().enumerate() {
let text = item.text.as_str();
@@ -218,7 +226,7 @@ impl TextLine {
result.push_str(text);
} else {
let prev_item = &self.items[i - 1];
if self.needs_space_between(prev_item, item, &result) {
if self.needs_space_between(prev_item, item, &result, single_char_threshold) {
result.push(' ');
}
result.push_str(text);
@@ -228,7 +236,13 @@ impl TextLine {
}
/// Determine if a space is needed between two items
fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
fn needs_space_between(
&self,
prev_item: &TextItem,
item: &TextItem,
result: &str,
single_char_threshold: f32,
) -> bool {
let text = item.text.as_str();
// Don't add space before/after hyphens for hyphenated words
@@ -245,7 +259,7 @@ impl TextLine {
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
// Use position-based spacing detection
let should_join = should_join_items(prev_item, item);
let should_join = should_join_items(prev_item, item, single_char_threshold);
// Check if space already exists
let prev_ends_with_space = result.ends_with(' ');