feat(text): adaptive join threshold for Canva-style letter-spaced PDFs

Canva-generated PDFs render text character-by-character with CSS-style
letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold
caused every character to get a space inserted ("K a r i b i b").

Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c"
pattern), compute an IQR-based threshold (median × 1.55) on the gap
distribution BEFORE space removal, then propagate per-page thresholds
through PageThresholds → group_into_lines_with_thresholds → TextLine
→ should_join_items.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-12 17:54:49 -07:00
co-authored by Claude Opus 4.6
parent 80a3b81ff9
commit 152de8b56d
7 changed files with 537 additions and 21 deletions
+24 -5
View File
@@ -25,6 +25,7 @@ pub use crate::text_utils::{is_bold_font, is_italic_font};
pub use crate::types::{ItemType, TextLine};
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;
pub(crate) use layout::group_into_lines_with_thresholds;
// ---------------------------------------------------------------------------
// Public API
@@ -96,7 +97,9 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
/// Extract text with positions from memory buffer
@@ -127,23 +130,31 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
// ---------------------------------------------------------------------------
// Orchestration
// ---------------------------------------------------------------------------
/// Per-page adaptive join thresholds from Canva-style letter-spacing detection.
pub(crate) type PageThresholds = HashMap<u32, f32>;
/// Extract positioned text, rectangles, and line segments from a pre-loaded document.
///
/// Also returns per-page adaptive join thresholds for Canva-style pages.
pub(crate) fn extract_positioned_text_from_doc(
doc: &Document,
font_cmaps: &FontCMaps,
page_filter: Option<&HashSet<u32>>,
) -> Result<PageExtraction, PdfError> {
) -> Result<(PageExtraction, PageThresholds), PdfError> {
let pages = doc.get_pages();
let mut all_items = Vec::new();
let mut all_rects = Vec::new();
let mut all_lines = Vec::new();
let mut page_thresholds: PageThresholds = HashMap::new();
// Build page ObjectId → page number map for form field extraction
let page_id_to_num: HashMap<ObjectId, u32> =
@@ -155,7 +166,12 @@ pub(crate) fn extract_positioned_text_from_doc(
continue;
}
}
let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let (mut items, rects, lines) =
extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let threshold = crate::text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
page_thresholds.insert(*page_num, threshold);
}
debug!(
"page {}: {} text items, {} rects, {} lines",
page_num,
@@ -194,7 +210,7 @@ pub(crate) fn extract_positioned_text_from_doc(
let form_items = extract_form_fields(doc, &page_id_to_num);
all_items.extend(form_items);
Ok((all_items, all_rects, all_lines))
Ok(((all_items, all_rects, all_lines), page_thresholds))
}
// ---------------------------------------------------------------------------
@@ -824,6 +840,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -856,6 +873,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -888,6 +906,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,