feat(text): adaptive join threshold for Canva-style letter-spaced PDFs

Canva-generated PDFs render text character-by-character with CSS-style
letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold
caused every character to get a space inserted ("K a r i b i b").

Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c"
pattern), compute an IQR-based threshold (median × 1.55) on the gap
distribution BEFORE space removal, then propagate per-page thresholds
through PageThresholds → group_into_lines_with_thresholds → TextLine
→ should_join_items.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-12 17:54:49 -07:00
co-authored by Claude Opus 4.6
parent 80a3b81ff9
commit 152de8b56d
7 changed files with 537 additions and 21 deletions
+22 -4
View File
@@ -1,5 +1,7 @@
//! Column detection, line grouping, and reading-order layout.
use std::collections::HashMap;
use crate::text_utils::{effective_width, sort_line_items};
use crate::types::{TextItem, TextLine};
use log::debug;
@@ -367,6 +369,16 @@ fn split_column_stragglers(lines: Vec<TextLine>) -> (Vec<TextLine>, Vec<TextLine
}
pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
group_into_lines_with_thresholds(items, &HashMap::new())
}
/// Group text items into lines, using pre-computed per-page adaptive thresholds
/// from Canva-style letter-spacing detection. Falls back to computing the
/// threshold from item gaps when no pre-computed value is available.
pub(crate) fn group_into_lines_with_thresholds(
items: Vec<TextItem>,
page_thresholds: &HashMap<u32, f32>,
) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
@@ -387,12 +399,17 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
for page in pages {
let page_items: Vec<TextItem> = items.iter().filter(|i| i.page == page).cloned().collect();
// Use pre-computed threshold from fix_letterspaced_items if available
// (computed before embedded-space removal, with full signal).
// Non-Canva pages use the default 0.10 threshold.
let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10);
// Detect columns for this page
let columns = detect_columns(&page_items, page);
if columns.len() <= 1 {
// Single column - use simple sorting
let lines = group_single_column(page_items);
let lines = group_single_column(page_items, adaptive_threshold);
all_lines.extend(lines);
} else {
// Multi-column - separate spanning items from column items
@@ -461,12 +478,12 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
let mut per_column_lines: Vec<Vec<TextLine>> = Vec::new();
for col_items in col_buckets {
let lines = group_single_column(col_items);
let lines = group_single_column(col_items, adaptive_threshold);
per_column_lines.push(lines);
}
// Process spanning items as their own group
let spanning_lines = group_single_column(spanning_items);
let spanning_lines = group_single_column(spanning_items, adaptive_threshold);
let is_newspaper = is_newspaper_layout(&per_column_lines);
debug!(
@@ -618,7 +635,7 @@ fn should_use_y_sorting(items: &[TextItem]) -> bool {
/// Group items from a single column into lines
/// Uses heuristics to decide between PDF stream order and Y-position sorting.
fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
fn group_single_column(items: Vec<TextItem>, adaptive_threshold: f32) -> Vec<TextLine> {
if items.is_empty() {
return Vec::new();
}
@@ -687,6 +704,7 @@ fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
items: vec![item],
y,
page,
adaptive_threshold,
});
}
}
+24 -5
View File
@@ -25,6 +25,7 @@ pub use crate::text_utils::{is_bold_font, is_italic_font};
pub use crate::types::{ItemType, TextLine};
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;
pub(crate) use layout::group_into_lines_with_thresholds;
// ---------------------------------------------------------------------------
// Public API
@@ -96,7 +97,9 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
/// Extract text with positions from memory buffer
@@ -127,23 +130,31 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
let (extraction, _thresholds) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
Ok(extraction)
}
// ---------------------------------------------------------------------------
// Orchestration
// ---------------------------------------------------------------------------
/// Per-page adaptive join thresholds from Canva-style letter-spacing detection.
pub(crate) type PageThresholds = HashMap<u32, f32>;
/// Extract positioned text, rectangles, and line segments from a pre-loaded document.
///
/// Also returns per-page adaptive join thresholds for Canva-style pages.
pub(crate) fn extract_positioned_text_from_doc(
doc: &Document,
font_cmaps: &FontCMaps,
page_filter: Option<&HashSet<u32>>,
) -> Result<PageExtraction, PdfError> {
) -> Result<(PageExtraction, PageThresholds), PdfError> {
let pages = doc.get_pages();
let mut all_items = Vec::new();
let mut all_rects = Vec::new();
let mut all_lines = Vec::new();
let mut page_thresholds: PageThresholds = HashMap::new();
// Build page ObjectId → page number map for form field extraction
let page_id_to_num: HashMap<ObjectId, u32> =
@@ -155,7 +166,12 @@ pub(crate) fn extract_positioned_text_from_doc(
continue;
}
}
let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let (mut items, rects, lines) =
extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
let threshold = crate::text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
page_thresholds.insert(*page_num, threshold);
}
debug!(
"page {}: {} text items, {} rects, {} lines",
page_num,
@@ -194,7 +210,7 @@ pub(crate) fn extract_positioned_text_from_doc(
let form_items = extract_form_fields(doc, &page_id_to_num);
all_items.extend(form_items);
Ok((all_items, all_rects, all_lines))
Ok(((all_items, all_rects, all_lines), page_thresholds))
}
// ---------------------------------------------------------------------------
@@ -824,6 +840,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -856,6 +873,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
@@ -888,6 +906,7 @@ mod tests {
let make_line = |y: f32, x: f32, page: u32| TextLine {
y,
page,
adaptive_threshold: 0.10,
items: vec![TextItem {
text: "text".into(),
x,
+2 -1
View File
@@ -351,7 +351,7 @@ fn process_document(
};
let (markdown, layout, has_encoding_issues) = match extracted {
Some((items, rects, lines)) => {
Some(((items, rects, lines), page_thresholds)) => {
let layout = compute_layout_complexity(&items, &rects, &lines);
let md = if options.mode == ProcessMode::Analyze {
@@ -362,6 +362,7 @@ fn process_document(
options.markdown,
&rects,
&lines,
&page_thresholds,
))
};
+9 -5
View File
@@ -16,7 +16,7 @@ pub use convert::to_markdown_from_lines;
use std::collections::{HashMap, HashSet};
use crate::extractor::group_into_lines;
use crate::extractor::group_into_lines_with_thresholds;
use crate::types::{PdfLine, PdfRect, TextItem};
use analysis::calculate_font_stats_from_items;
@@ -452,7 +452,7 @@ pub fn to_markdown_from_items_with_rects(
options: MarkdownOptions,
rects: &[crate::types::PdfRect],
) -> String {
to_markdown_from_items_with_rects_and_lines(items, options, rects, &[])
to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new())
}
/// Convert positioned text items to markdown, using rectangles and line segments for table detection.
@@ -464,6 +464,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
options: MarkdownOptions,
rects: &[crate::types::PdfRect],
pdf_lines: &[crate::types::PdfLine],
page_thresholds: &HashMap<u32, f32>,
) -> String {
use crate::tables::{
detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown,
@@ -781,7 +782,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
// items from different side-by-side zones (e.g. left/right month columns
// in a calendar) don't merge into the same line.
let lines = if page_band_splits.is_empty() {
group_into_lines(non_table_items)
group_into_lines_with_thresholds(non_table_items, page_thresholds)
} else {
// Separate items into band-split pages and non-split pages
let mut split_page_items: HashMap<u32, Vec<TextItem>> = HashMap::new();
@@ -794,7 +795,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
}
}
// Process unsplit pages normally
let mut all_lines = group_into_lines(unsplit_items);
let mut all_lines = group_into_lines_with_thresholds(unsplit_items, page_thresholds);
// Process each split page's bands independently, then interleave
// by Y position so paired zones (e.g. left/right months) appear together.
let mut split_pages: Vec<u32> = split_page_items.keys().copied().collect();
@@ -811,7 +812,10 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
.cloned()
.collect();
if !band_items.is_empty() {
page_lines.extend(group_into_lines(band_items));
page_lines.extend(group_into_lines_with_thresholds(
band_items,
page_thresholds,
));
}
}
// Sort by Y descending (top to bottom) so left and right
+457 -2
View File
@@ -278,12 +278,267 @@ pub(crate) fn is_cid_font(font: &str) -> bool {
font.starts_with("C2_") || font.starts_with("C0_")
}
/// Detect and fix Canva-style letter-spacing within text items.
///
/// Canva-generated PDFs render text character-by-character with CSS-style
/// letter-spacing. The TJ handler inserts spaces between each character,
/// producing items like `"a r i b"` instead of `"arib"`. This function
/// detects such items by checking if the text follows a strict pattern of
/// alternating single characters and spaces, then removes the spurious spaces.
///
/// Only activates when ≥50% of items on the page are letter-spaced, to avoid
/// false positives on normal PDFs with short items like `"a b"`.
///
/// Returns the adaptive join threshold for this page: DEFAULT (0.10) for normal
/// pages, or a higher Otsu-derived threshold for Canva-style pages.
pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 {
const DEFAULT: f32 = 0.10;
if items.is_empty() {
return DEFAULT;
}
// Check if the item text matches "x y z" pattern (single chars separated by spaces)
fn is_letterspaced(text: &str) -> bool {
let trimmed = text.trim();
let chars: Vec<char> = trimmed.chars().collect();
// Need at least 3 chars: "a b" = ['a', ' ', 'b']
if chars.len() < 3 {
return false;
}
// Pattern: non-space, space, non-space, space, ...
chars
.iter()
.enumerate()
.all(|(i, &c)| if i % 2 == 0 { c != ' ' } else { c == ' ' })
}
// Count how many items are letter-spaced vs total non-trivial items
let mut letterspaced_count = 0u32;
let mut total_text_items = 0u32;
for item in items.iter() {
let trimmed = item.text.trim();
if trimmed.is_empty() || trimmed.len() < 3 {
continue;
}
total_text_items += 1;
if is_letterspaced(&item.text) {
letterspaced_count += 1;
}
}
// Only fix if ≥50% of substantial items are letter-spaced
if total_text_items < 4 || letterspaced_count * 2 < total_text_items {
return DEFAULT;
}
// Compute threshold BEFORE removing spaces. Since we've confirmed this
// is a Canva-style page (≥50% letterspaced), use the ungated variant
// that includes all pairs — the char-count guard in the normal function
// would filter out long letterspaced items like "i s s i o n" (11 chars).
let threshold = compute_canva_join_threshold(items);
// Remove spaces from letter-spaced items
for item in items.iter_mut() {
if is_letterspaced(&item.text) {
let fixed: String = item.text.chars().filter(|&c| c != ' ').collect();
item.text = fixed;
}
}
threshold
}
/// Compute Otsu join threshold for a confirmed Canva-style page.
///
/// Like [`compute_single_char_join_threshold`] but without the per-pair
/// char-count guard, since we already know the page has Canva-style
/// letter-spacing. Uses all adjacent pairs for maximum sample size.
fn compute_canva_join_threshold(items: &[TextItem]) -> f32 {
const DEFAULT: f32 = 0.10;
const MIN_SAMPLES: usize = 8;
let mut ratios: Vec<f32> = Vec::new();
for pair in items.windows(2) {
let prev = &pair[0];
let curr = &pair[1];
// Skip CJK pairs
let prev_c = prev.text.trim().chars().last();
let curr_c = curr.text.trim().chars().next();
if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) {
continue;
}
if prev.width <= 0.0 || prev.font_size <= 0.0 {
continue;
}
let gap = if prev.x <= curr.x {
curr.x - (prev.x + prev.width)
} else {
prev.x - (curr.x + curr.width)
};
let ratio = gap / prev.font_size;
if !(0.0..=3.0).contains(&ratio) {
continue;
}
ratios.push(ratio);
}
if ratios.len() < MIN_SAMPLES {
return DEFAULT;
}
ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
// If all gaps are tight (max < 0.40), use default — normal PDF
let max_ratio = ratios[ratios.len() - 1];
if max_ratio < 0.40 {
return DEFAULT;
}
// If the minimum gap is below 0.40, there's a mix of tight and wide gaps,
// meaning this isn't a uniform letter-spacing PDF — use default.
if ratios[0] < 0.40 {
return DEFAULT;
}
// Median-based threshold: for Canva letter-spacing, the median gap ratio
// is dominated by the letter-spacing value (~0.55× font_size). Word gaps
// are consistently ~1.8× the letter-spacing. Using median × 1.55 places
// the threshold between the widest intra-word gaps and the narrowest
// inter-word gaps.
let median = ratios[ratios.len() / 2];
(median * 1.55).clamp(0.50, 2.0)
}
/// Compute an adaptive join threshold for text items on a line.
///
/// Uses Otsu's method on the gap/font_size ratio distribution to find the
/// natural split between intra-word and inter-word gaps. With per-pair
/// char-count guard (both items ≥ 5 chars → skip). Used only in tests;
/// production code uses `compute_canva_join_threshold` via `fix_letterspaced_items`.
#[cfg(test)]
fn compute_single_char_join_threshold(items: &[TextItem]) -> f32 {
const DEFAULT: f32 = 0.10;
const MIN_SAMPLES: usize = 8;
// Collect gap/font_size ratios for adjacent pairs involving at least one
// short fragment (< 5 chars). This detects per-character rendering
// (Canva-style) without being fooled by uniform word-level spacing.
let mut ratios: Vec<f32> = Vec::new();
for pair in items.windows(2) {
let prev = &pair[0];
let curr = &pair[1];
let prev_chars = prev.text.trim().chars().count();
let curr_chars = curr.text.trim().chars().count();
// Require at least one item to be a short fragment.
// Pairs of long words (both ≥ 5 chars) indicate normal text.
if prev_chars >= 5 && curr_chars >= 5 {
continue;
}
// Skip CJK pairs
let prev_c = prev.text.trim().chars().last();
let curr_c = curr.text.trim().chars().next();
if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) {
continue;
}
if prev.width <= 0.0 || prev.font_size <= 0.0 {
continue;
}
let gap = if prev.x <= curr.x {
curr.x - (prev.x + prev.width)
} else {
prev.x - (curr.x + curr.width)
};
let ratio = gap / prev.font_size;
// Skip negative gaps and huge gaps (> 3× font_size)
if !(0.0..=3.0).contains(&ratio) {
continue;
}
ratios.push(ratio);
}
if ratios.len() < MIN_SAMPLES {
return DEFAULT;
}
ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
// If all gaps are tight (max < 0.40), use default — normal PDF
let max_ratio = ratios[ratios.len() - 1];
if max_ratio < 0.40 {
return DEFAULT;
}
// If the minimum gap is below 0.40, there's a mix of tight and wide gaps,
// meaning this isn't a uniform letter-spacing PDF — use default.
// Canva-style letter-spacing has min gaps ≈ 0.5× font_size; normal
// justified text gaps are ≈ 0.150.30× font_size.
if ratios[0] < 0.40 {
return DEFAULT;
}
// All gaps are wide (≥0.25× font_size) — Canva-style letter-spacing.
// Use Otsu to find the split between intra-word and inter-word gaps.
let n = ratios.len() as f32;
let total_sum: f32 = ratios.iter().sum();
let mut best_threshold = DEFAULT;
let mut best_variance = f32::NEG_INFINITY;
let mut w0: f32 = 0.0;
let mut sum0: f32 = 0.0;
for i in 0..ratios.len() - 1 {
w0 += 1.0;
sum0 += ratios[i];
let w1 = n - w0;
if w1 == 0.0 {
break;
}
let mean0 = sum0 / w0;
let mean1 = (total_sum - sum0) / w1;
let variance = w0 * w1 * (mean0 - mean1).powi(2);
// Only consider thresholds at value boundaries (skip duplicates)
if i + 1 < ratios.len() && (ratios[i + 1] - ratios[i]).abs() < 1e-6 {
continue;
}
if variance > best_variance {
best_variance = variance;
// Place threshold midway between the two classes
best_threshold = (ratios[i] + ratios[i + 1]) / 2.0;
}
}
best_threshold.clamp(0.05, 2.0)
}
/// Determine if two adjacent text items should be joined without a space
/// based on their physical positions on the page and character case.
/// Uses a hybrid approach: position-based with case-aware thresholds.
/// CID fonts emit one word per text operator with gaps ≈ 0 between words.
/// Non-CID (Type1/TrueType) fonts emit phrases or fragments.
pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool {
pub(crate) fn should_join_items(
prev_item: &TextItem,
curr_item: &TextItem,
single_char_threshold: f32,
) -> bool {
// If either text explicitly has leading/trailing spaces, respect them
if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') {
return false;
@@ -365,6 +620,12 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
}
}
// When the adaptive threshold indicates Canva-style letter-spacing
// (all gaps wide), use it uniformly for all pair types.
if single_char_threshold > 0.20 {
return gap < font_size * single_char_threshold;
}
// Single-character fragment joined to a multi-character item: use a
// moderately generous threshold to rejoin split words like "b" + "illion"
// or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words.
@@ -385,7 +646,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
return gap < font_size * 0.25;
}
}
return gap < font_size * 0.10;
return gap < font_size * single_char_threshold;
}
// With accurate widths, a gap < 15% of font size means glyphs are
@@ -473,6 +734,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ItemType;
#[test]
fn strip_soft_hyphen() {
@@ -581,4 +843,197 @@ mod tests {
// Latin
assert!(!is_arabic_presentation_form('A'));
}
/// Helper to create a single-char TextItem at a given x position with width.
fn make_char_item(ch: char, x: f32, width: f32, font_size: f32) -> TextItem {
TextItem {
text: ch.to_string(),
x,
y: 100.0,
width,
height: font_size,
font: "TestFont".to_string(),
font_size,
page: 1,
is_bold: false,
is_italic: false,
item_type: ItemType::Text,
}
}
#[test]
fn otsu_threshold_sec_style_tight_gaps() {
// SEC-style: intra-word gaps ≈ 0, word gap ≈ 0.15× font_size
// All gaps tight → should return default 0.10
let fs = 12.0;
let char_w = fs * 0.5;
let mut items = Vec::new();
// 15 chars with gap ≈ 0 (intra-word)
for i in 0..15 {
let x = 100.0 + i as f32 * (char_w + fs * 0.01);
items.push(make_char_item('a', x, char_w, fs));
}
// Word gap
let word_x = items.last().unwrap().x + char_w + fs * 0.15;
items.push(make_char_item('b', word_x, char_w, fs));
// 5 more tight chars
for i in 1..5 {
let x = word_x + i as f32 * (char_w + fs * 0.01);
items.push(make_char_item('c', x, char_w, fs));
}
let threshold = compute_single_char_join_threshold(&items);
// Max gap is 0.15, but most are 0.01 → max < 0.20 → default
assert!(
(threshold - 0.10).abs() < 0.01,
"SEC-style should return default ~0.10, got {threshold}"
);
}
#[test]
fn otsu_threshold_canva_style_wide_gaps() {
// Canva-style: intra-word gaps ≈ 0.6× font_size, word gaps ≈ 1.2× font_size
let fs = 12.0;
let char_w = fs * 0.5;
let intra_gap = fs * 0.6;
let word_gap = fs * 1.2;
let mut items = Vec::new();
// Word 1: 8 chars with intra-word spacing
for i in 0..8 {
let x = 100.0 + i as f32 * (char_w + intra_gap);
items.push(make_char_item('K', x, char_w, fs));
}
// Word gap
let word_x = items.last().unwrap().x + char_w + word_gap;
items.push(make_char_item('T', word_x, char_w, fs));
// Word 2: 7 more chars
for i in 1..7 {
let x = word_x + i as f32 * (char_w + intra_gap);
items.push(make_char_item('o', x, char_w, fs));
}
let threshold = compute_single_char_join_threshold(&items);
// Should find threshold between 0.6 and 1.2 → roughly 0.9
assert!(
threshold > 0.5 && threshold < 1.1,
"Canva-style should find threshold ~0.9, got {threshold}"
);
}
#[test]
fn otsu_threshold_few_samples_returns_default() {
// < 8 single-char pairs → default
let fs = 12.0;
let char_w = fs * 0.5;
let items: Vec<TextItem> = (0..5)
.map(|i| make_char_item('x', 100.0 + i as f32 * (char_w + 1.0), char_w, fs))
.collect();
let threshold = compute_single_char_join_threshold(&items);
assert!(
(threshold - 0.10).abs() < 0.01,
"few samples should return default 0.10, got {threshold}"
);
}
#[test]
fn fix_letterspaced_items_returns_adaptive_threshold() {
// Simulate Canva page with many letter-spaced items and word gaps.
// Needs ≥8 inter-item gaps for the threshold to be computed.
let fs = 12.0;
let char_w = fs * 0.5;
let letter_gap = fs * 0.6; // 0.6× font_size between items
let word_gap = fs * 1.2; // 1.2× font_size between words
let words: Vec<&str> = vec![
"H e l l o",
"W o r l d",
"F o o",
"B a r",
"B a z",
"Q u x",
"T e s t",
"D a t a",
"M o r e",
"T e x t",
];
let mut items = Vec::new();
let mut x = 100.0;
for (wi, word) in words.iter().enumerate() {
let char_count = word.chars().filter(|c| !c.is_whitespace()).count();
let w = char_count as f32 * char_w + (char_count - 1) as f32 * letter_gap;
items.push(TextItem {
text: word.to_string(),
x,
y: 100.0,
width: w,
height: fs,
font: "TestFont".to_string(),
font_size: fs,
page: 1,
is_bold: false,
is_italic: false,
item_type: ItemType::Text,
});
// Alternate between letter-gap and word-gap to create bimodal distribution
x += w + if wi % 3 == 2 { word_gap } else { letter_gap };
}
let threshold = fix_letterspaced_items(&mut items);
// Threshold should be above default (Canva-style detected)
assert!(
threshold > 0.50,
"Canva page should get threshold > 0.50, got {threshold}"
);
// Spaces should be removed from letter-spaced items
assert_eq!(items[0].text, "Hello");
assert_eq!(items[1].text, "World");
assert_eq!(items[2].text, "Foo");
assert_eq!(items[9].text, "Text");
}
#[test]
fn canva_style_items_join_correctly() {
// Simulate Canva PDF: "Hello" with 0.6× font_size letter-spacing
let fs = 12.0;
let char_w = fs * 0.5;
let intra_gap = fs * 0.6;
let word_gap = fs * 1.2;
let mut items = Vec::new();
let chars = ['H', 'e', 'l', 'l', 'o'];
for (i, &ch) in chars.iter().enumerate() {
let x = 100.0 + i as f32 * (char_w + intra_gap);
items.push(make_char_item(ch, x, char_w, fs));
}
// Space then "W"
let w_x = items.last().unwrap().x + char_w + word_gap;
items.push(make_char_item('W', w_x, char_w, fs));
let chars2 = ['o', 'r', 'l', 'd'];
for (i, &ch) in chars2.iter().enumerate() {
let x = w_x + (i + 1) as f32 * (char_w + intra_gap);
items.push(make_char_item(ch, x, char_w, fs));
}
let threshold = compute_single_char_join_threshold(&items);
// Intra-word pairs should join
assert!(
should_join_items(&items[0], &items[1], threshold),
"H+e should join with threshold {threshold}"
);
assert!(
should_join_items(&items[3], &items[4], threshold),
"l+o should join with threshold {threshold}"
);
// Word boundary should NOT join
assert!(
!should_join_items(&items[4], &items[5], threshold),
"o+W (word boundary) should NOT join with threshold {threshold}"
);
}
}
+18 -4
View File
@@ -125,6 +125,10 @@ pub struct TextLine {
pub items: Vec<TextItem>,
pub y: f32,
pub page: u32,
/// Adaptive join threshold from page-level letter-spacing detection.
/// Default 0.10 for normal PDFs; higher for Canva-style PDFs.
#[doc(hidden)]
pub adaptive_threshold: f32,
}
impl TextLine {
@@ -138,6 +142,8 @@ impl TextLine {
return self.text_plain();
}
let single_char_threshold = self.adaptive_threshold;
let mut result = String::new();
let mut current_bold = false;
let mut current_italic = false;
@@ -156,7 +162,7 @@ impl TextLine {
false
} else {
let prev_item = &self.items[i - 1];
self.needs_space_between(prev_item, item, &result)
self.needs_space_between(prev_item, item, &result, single_char_threshold)
};
// Preserve leading whitespace from the item text.
@@ -211,6 +217,8 @@ impl TextLine {
/// Get plain text without formatting
fn text_plain(&self) -> String {
let single_char_threshold = self.adaptive_threshold;
let mut result = String::new();
for (i, item) in self.items.iter().enumerate() {
let text = item.text.as_str();
@@ -218,7 +226,7 @@ impl TextLine {
result.push_str(text);
} else {
let prev_item = &self.items[i - 1];
if self.needs_space_between(prev_item, item, &result) {
if self.needs_space_between(prev_item, item, &result, single_char_threshold) {
result.push(' ');
}
result.push_str(text);
@@ -228,7 +236,13 @@ impl TextLine {
}
/// Determine if a space is needed between two items
fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
fn needs_space_between(
&self,
prev_item: &TextItem,
item: &TextItem,
result: &str,
single_char_threshold: f32,
) -> bool {
let text = item.text.as_str();
// Don't add space before/after hyphens for hyphenated words
@@ -245,7 +259,7 @@ impl TextLine {
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
// Use position-based spacing detection
let should_join = should_join_items(prev_item, item);
let should_join = should_join_items(prev_item, item, single_char_threshold);
// Check if space already exists
let prev_ends_with_space = result.ends_with(' ');
+5
View File
@@ -138,6 +138,7 @@ fn test_text_line_text_method() {
items,
y: 700.0,
page: 1,
adaptive_threshold: 0.10,
};
assert_eq!(line.text(), "Hello World");
}
@@ -149,6 +150,7 @@ fn test_text_line_single_item() {
items,
y: 700.0,
page: 1,
adaptive_threshold: 0.10,
};
assert_eq!(line.text(), "Single");
}
@@ -159,6 +161,7 @@ fn test_text_line_empty() {
items: vec![],
y: 700.0,
page: 1,
adaptive_threshold: 0.10,
};
assert_eq!(line.text(), "");
}
@@ -473,11 +476,13 @@ fn test_markdown_from_lines_basic() {
items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)],
y: 700.0,
page: 1,
adaptive_threshold: 0.10,
},
TextLine {
items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)],
y: 680.0,
page: 1,
adaptive_threshold: 0.10,
},
];
let md = to_markdown_from_lines(lines, MarkdownOptions::default());