mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(text): adaptive join threshold for Canva-style letter-spaced PDFs

Browse Source 
Canva-generated PDFs render text character-by-character with CSS-style
letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold
caused every character to get a space inserted ("K a r i b i b").

Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c"
pattern), compute an IQR-based threshold (median × 1.55) on the gap
distribution BEFORE space removal, then propagate per-page thresholds
through PageThresholds → group_into_lines_with_thresholds → TextLine
→ should_join_items.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-12 17:54:49 -07:00
co-authored by Claude Opus 4.6
parent 80a3b81ff9
commit 152de8b56d
7 changed files with 537 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor/layout.rs
+22 -4
View File
@@ -1,5 +1,7 @@
//! Column detection, line grouping, and reading-order layout.
use std::collections::HashMap;
use crate::text_utils::{effective_width, sort_line_items};
use crate::types::{TextItem, TextLine};
use log::debug;
@@ -367,6 +369,16 @@ fn split_column_stragglers(lines: Vec<TextLine>) -> (Vec<TextLine>, Vec<TextLine
}
pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
group_into_lines_with_thresholds(items, &HashMap::new())
}
/// Group text items into lines, using pre-computed per-page adaptive thresholds
/// from Canva-style letter-spacing detection. Falls back to computing the
/// threshold from item gaps when no pre-computed value is available.
pub(crate) fn group_into_lines_with_thresholds(
items: Vec<TextItem>,
page_thresholds: &HashMap<u32, f32>,
) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
@@ -387,12 +399,17 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
for page in pages {
let page_items: Vec<TextItem> = items.iter().filter(|i| i.page == page).cloned().collect();
// Use pre-computed threshold from fix_letterspaced_items if available
// (computed before embedded-space removal, with full signal).
// Non-Canva pages use the default 0.10 threshold.
let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10);
// Detect columns for this page
let columns = detect_columns(&page_items, page);
if columns.len() <= 1 {
// Single column - use simple sorting
let lines = group_single_column(page_items);
let lines = group_single_column(page_items, adaptive_threshold);
all_lines.extend(lines);
} else {
// Multi-column - separate spanning items from column items
@@ -461,12 +478,12 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
let mut per_column_lines: Vec<Vec<TextLine>> = Vec::new();
for col_items in col_buckets {
let lines = group_single_column(col_items);
let lines = group_single_column(col_items, adaptive_threshold);
per_column_lines.push(lines);
}
// Process spanning items as their own group
let spanning_lines = group_single_column(spanning_items);
let spanning_lines = group_single_column(spanning_items, adaptive_threshold);
let is_newspaper = is_newspaper_layout(&per_column_lines);
debug!(
@@ -618,7 +635,7 @@ fn should_use_y_sorting(items: &[TextItem]) -> bool {
/// Group items from a single column into lines
/// Uses heuristics to decide between PDF stream order and Y-position sorting.
fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
fn group_single_column(items: Vec<TextItem>, adaptive_threshold: f32) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
@@ -687,6 +704,7 @@ fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
items: vec![item],
y,
page,
adaptive_threshold,
});
}
}
src/extractor/mod.rs
+24 -5
View File
@@ -25,6 +25,7 @@ pub use crate::text_utils::{is_bold_font, is_italic_font};
pub use crate::types::{ItemType, TextLine};
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;
pub(crate) use layout::group_into_lines_with_thresholds;
// ---------------------------------------------------------------------------
// Public API
@@ -96,7 +97,9 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
/// Extract text with positions from memory buffer
@@ -127,23 +130,31 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
// ---------------------------------------------------------------------------
// Orchestration
// ---------------------------------------------------------------------------
/// Per-page adaptive join thresholds from Canva-style letter-spacing detection.
pub(crate) type PageThresholds = HashMap<u32, f32>;
/// Extract positioned text, rectangles, and line segments from a pre-loaded document.
///
/// Also returns per-page adaptive join thresholds for Canva-style pages.
pub(crate) fn extract_positioned_text_from_doc(
doc: &Document,
font_cmaps: &FontCMaps,
page_filter: Option<&HashSet<u32>>,
) -> Result<PageExtraction, PdfError> {
) -> Result<(PageExtraction, PageThresholds), PdfError> {
let pages = doc.get_pages();
let mut all_items = Vec::new();
let mut all_rects = Vec::new();
let mut all_lines = Vec::new();
let mut page_thresholds: PageThresholds = HashMap::new();
// Build page ObjectId → page number map for form field extraction
let page_id_to_num: HashMap<ObjectId, u32> =
@@ -155,7 +166,12 @@ pub(crate) fn extract_positioned_text_from_doc(
continue;
}
}
let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let (mut items, rects, lines) =
extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let threshold = crate::text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
page_thresholds.insert(*page_num, threshold);
}
debug!(
"page {}: {} text items, {} rects, {} lines",
page_num,
@@ -194,7 +210,7 @@ pub(crate) fn extract_positioned_text_from_doc(
let form_items = extract_form_fields(doc, &page_id_to_num);
all_items.extend(form_items);
Ok((all_items, all_rects, all_lines))
Ok(((all_items, all_rects, all_lines), page_thresholds))
}
// ---------------------------------------------------------------------------
@@ -824,6 +840,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -856,6 +873,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -888,6 +906,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,