feat(text): adaptive join threshold for Canva-style letter-spaced PDFs
Canva-generated PDFs render text character-by-character with CSS-style
letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold
caused every character to get a space inserted ("K a r i b i b").
Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c"
pattern), compute an IQR-based threshold (median × 1.55) on the gap
distribution BEFORE space removal, then propagate per-page thresholds
through PageThresholds → group_into_lines_with_thresholds → TextLine
→ should_join_items.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
80a3b81ff9
commit
152de8b56d
+22
-4
@@ -1,5 +1,7 @@
|
||||
//! Column detection, line grouping, and reading-order layout.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::text_utils::{effective_width, sort_line_items};
|
||||
use crate::types::{TextItem, TextLine};
|
||||
use log::debug;
|
||||
@@ -367,6 +369,16 @@ fn split_column_stragglers(lines: Vec<TextLine>) -> (Vec<TextLine>, Vec<TextLine
|
||||
}
|
||||
|
||||
pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
group_into_lines_with_thresholds(items, &HashMap::new())
|
||||
}
|
||||
|
||||
/// Group text items into lines, using pre-computed per-page adaptive thresholds
|
||||
/// from Canva-style letter-spacing detection. Falls back to computing the
|
||||
/// threshold from item gaps when no pre-computed value is available.
|
||||
pub(crate) fn group_into_lines_with_thresholds(
|
||||
items: Vec<TextItem>,
|
||||
page_thresholds: &HashMap<u32, f32>,
|
||||
) -> Vec<TextLine> {
|
||||
if items.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -387,12 +399,17 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
for page in pages {
|
||||
let page_items: Vec<TextItem> = items.iter().filter(|i| i.page == page).cloned().collect();
|
||||
|
||||
// Use pre-computed threshold from fix_letterspaced_items if available
|
||||
// (computed before embedded-space removal, with full signal).
|
||||
// Non-Canva pages use the default 0.10 threshold.
|
||||
let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10);
|
||||
|
||||
// Detect columns for this page
|
||||
let columns = detect_columns(&page_items, page);
|
||||
|
||||
if columns.len() <= 1 {
|
||||
// Single column - use simple sorting
|
||||
let lines = group_single_column(page_items);
|
||||
let lines = group_single_column(page_items, adaptive_threshold);
|
||||
all_lines.extend(lines);
|
||||
} else {
|
||||
// Multi-column - separate spanning items from column items
|
||||
@@ -461,12 +478,12 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
|
||||
let mut per_column_lines: Vec<Vec<TextLine>> = Vec::new();
|
||||
for col_items in col_buckets {
|
||||
let lines = group_single_column(col_items);
|
||||
let lines = group_single_column(col_items, adaptive_threshold);
|
||||
per_column_lines.push(lines);
|
||||
}
|
||||
|
||||
// Process spanning items as their own group
|
||||
let spanning_lines = group_single_column(spanning_items);
|
||||
let spanning_lines = group_single_column(spanning_items, adaptive_threshold);
|
||||
|
||||
let is_newspaper = is_newspaper_layout(&per_column_lines);
|
||||
debug!(
|
||||
@@ -618,7 +635,7 @@ fn should_use_y_sorting(items: &[TextItem]) -> bool {
|
||||
|
||||
/// Group items from a single column into lines
|
||||
/// Uses heuristics to decide between PDF stream order and Y-position sorting.
|
||||
fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
fn group_single_column(items: Vec<TextItem>, adaptive_threshold: f32) -> Vec<TextLine> {
|
||||
if items.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -687,6 +704,7 @@ fn group_single_column(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
items: vec![item],
|
||||
y,
|
||||
page,
|
||||
adaptive_threshold,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+24
-5
@@ -25,6 +25,7 @@ pub use crate::text_utils::{is_bold_font, is_italic_font};
|
||||
pub use crate::types::{ItemType, TextLine};
|
||||
pub(crate) use layout::detect_columns;
|
||||
pub use layout::group_into_lines;
|
||||
pub(crate) use layout::group_into_lines_with_thresholds;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
@@ -96,7 +97,9 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
|
||||
let (extraction, _thresholds) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
Ok(extraction)
|
||||
}
|
||||
|
||||
/// Extract text with positions from memory buffer
|
||||
@@ -127,23 +130,31 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
|
||||
let (extraction, _thresholds) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
Ok(extraction)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Orchestration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-page adaptive join thresholds from Canva-style letter-spacing detection.
|
||||
pub(crate) type PageThresholds = HashMap<u32, f32>;
|
||||
|
||||
/// Extract positioned text, rectangles, and line segments from a pre-loaded document.
|
||||
///
|
||||
/// Also returns per-page adaptive join thresholds for Canva-style pages.
|
||||
pub(crate) fn extract_positioned_text_from_doc(
|
||||
doc: &Document,
|
||||
font_cmaps: &FontCMaps,
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<PageExtraction, PdfError> {
|
||||
) -> Result<(PageExtraction, PageThresholds), PdfError> {
|
||||
let pages = doc.get_pages();
|
||||
let mut all_items = Vec::new();
|
||||
let mut all_rects = Vec::new();
|
||||
let mut all_lines = Vec::new();
|
||||
let mut page_thresholds: PageThresholds = HashMap::new();
|
||||
|
||||
// Build page ObjectId → page number map for form field extraction
|
||||
let page_id_to_num: HashMap<ObjectId, u32> =
|
||||
@@ -155,7 +166,12 @@ pub(crate) fn extract_positioned_text_from_doc(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
|
||||
let (mut items, rects, lines) =
|
||||
extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
|
||||
let threshold = crate::text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
debug!(
|
||||
"page {}: {} text items, {} rects, {} lines",
|
||||
page_num,
|
||||
@@ -194,7 +210,7 @@ pub(crate) fn extract_positioned_text_from_doc(
|
||||
let form_items = extract_form_fields(doc, &page_id_to_num);
|
||||
all_items.extend(form_items);
|
||||
|
||||
Ok((all_items, all_rects, all_lines))
|
||||
Ok(((all_items, all_rects, all_lines), page_thresholds))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -824,6 +840,7 @@ mod tests {
|
||||
let make_line = |y: f32, x: f32, page: u32| TextLine {
|
||||
y,
|
||||
page,
|
||||
adaptive_threshold: 0.10,
|
||||
items: vec![TextItem {
|
||||
text: "text".into(),
|
||||
x,
|
||||
@@ -856,6 +873,7 @@ mod tests {
|
||||
let make_line = |y: f32, x: f32, page: u32| TextLine {
|
||||
y,
|
||||
page,
|
||||
adaptive_threshold: 0.10,
|
||||
items: vec![TextItem {
|
||||
text: "text".into(),
|
||||
x,
|
||||
@@ -888,6 +906,7 @@ mod tests {
|
||||
let make_line = |y: f32, x: f32, page: u32| TextLine {
|
||||
y,
|
||||
page,
|
||||
adaptive_threshold: 0.10,
|
||||
items: vec![TextItem {
|
||||
text: "text".into(),
|
||||
x,
|
||||
|
||||
+2
-1
@@ -351,7 +351,7 @@ fn process_document(
|
||||
};
|
||||
|
||||
let (markdown, layout, has_encoding_issues) = match extracted {
|
||||
Some((items, rects, lines)) => {
|
||||
Some(((items, rects, lines), page_thresholds)) => {
|
||||
let layout = compute_layout_complexity(&items, &rects, &lines);
|
||||
|
||||
let md = if options.mode == ProcessMode::Analyze {
|
||||
@@ -362,6 +362,7 @@ fn process_document(
|
||||
options.markdown,
|
||||
&rects,
|
||||
&lines,
|
||||
&page_thresholds,
|
||||
))
|
||||
};
|
||||
|
||||
|
||||
+9
-5
@@ -16,7 +16,7 @@ pub use convert::to_markdown_from_lines;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::extractor::group_into_lines;
|
||||
use crate::extractor::group_into_lines_with_thresholds;
|
||||
use crate::types::{PdfLine, PdfRect, TextItem};
|
||||
|
||||
use analysis::calculate_font_stats_from_items;
|
||||
@@ -452,7 +452,7 @@ pub fn to_markdown_from_items_with_rects(
|
||||
options: MarkdownOptions,
|
||||
rects: &[crate::types::PdfRect],
|
||||
) -> String {
|
||||
to_markdown_from_items_with_rects_and_lines(items, options, rects, &[])
|
||||
to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new())
|
||||
}
|
||||
|
||||
/// Convert positioned text items to markdown, using rectangles and line segments for table detection.
|
||||
@@ -464,6 +464,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
options: MarkdownOptions,
|
||||
rects: &[crate::types::PdfRect],
|
||||
pdf_lines: &[crate::types::PdfLine],
|
||||
page_thresholds: &HashMap<u32, f32>,
|
||||
) -> String {
|
||||
use crate::tables::{
|
||||
detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown,
|
||||
@@ -781,7 +782,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
// items from different side-by-side zones (e.g. left/right month columns
|
||||
// in a calendar) don't merge into the same line.
|
||||
let lines = if page_band_splits.is_empty() {
|
||||
group_into_lines(non_table_items)
|
||||
group_into_lines_with_thresholds(non_table_items, page_thresholds)
|
||||
} else {
|
||||
// Separate items into band-split pages and non-split pages
|
||||
let mut split_page_items: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
@@ -794,7 +795,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
}
|
||||
}
|
||||
// Process unsplit pages normally
|
||||
let mut all_lines = group_into_lines(unsplit_items);
|
||||
let mut all_lines = group_into_lines_with_thresholds(unsplit_items, page_thresholds);
|
||||
// Process each split page's bands independently, then interleave
|
||||
// by Y position so paired zones (e.g. left/right months) appear together.
|
||||
let mut split_pages: Vec<u32> = split_page_items.keys().copied().collect();
|
||||
@@ -811,7 +812,10 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
||||
.cloned()
|
||||
.collect();
|
||||
if !band_items.is_empty() {
|
||||
page_lines.extend(group_into_lines(band_items));
|
||||
page_lines.extend(group_into_lines_with_thresholds(
|
||||
band_items,
|
||||
page_thresholds,
|
||||
));
|
||||
}
|
||||
}
|
||||
// Sort by Y descending (top to bottom) so left and right
|
||||
|
||||
+457
-2
@@ -278,12 +278,267 @@ pub(crate) fn is_cid_font(font: &str) -> bool {
|
||||
font.starts_with("C2_") || font.starts_with("C0_")
|
||||
}
|
||||
|
||||
/// Detect and fix Canva-style letter-spacing within text items.
|
||||
///
|
||||
/// Canva-generated PDFs render text character-by-character with CSS-style
|
||||
/// letter-spacing. The TJ handler inserts spaces between each character,
|
||||
/// producing items like `"a r i b"` instead of `"arib"`. This function
|
||||
/// detects such items by checking if the text follows a strict pattern of
|
||||
/// alternating single characters and spaces, then removes the spurious spaces.
|
||||
///
|
||||
/// Only activates when ≥50% of items on the page are letter-spaced, to avoid
|
||||
/// false positives on normal PDFs with short items like `"a b"`.
|
||||
///
|
||||
/// Returns the adaptive join threshold for this page: DEFAULT (0.10) for normal
|
||||
/// pages, or a higher Otsu-derived threshold for Canva-style pages.
|
||||
pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 {
|
||||
const DEFAULT: f32 = 0.10;
|
||||
|
||||
if items.is_empty() {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
// Check if the item text matches "x y z" pattern (single chars separated by spaces)
|
||||
fn is_letterspaced(text: &str) -> bool {
|
||||
let trimmed = text.trim();
|
||||
let chars: Vec<char> = trimmed.chars().collect();
|
||||
// Need at least 3 chars: "a b" = ['a', ' ', 'b']
|
||||
if chars.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
// Pattern: non-space, space, non-space, space, ...
|
||||
chars
|
||||
.iter()
|
||||
.enumerate()
|
||||
.all(|(i, &c)| if i % 2 == 0 { c != ' ' } else { c == ' ' })
|
||||
}
|
||||
|
||||
// Count how many items are letter-spaced vs total non-trivial items
|
||||
let mut letterspaced_count = 0u32;
|
||||
let mut total_text_items = 0u32;
|
||||
for item in items.iter() {
|
||||
let trimmed = item.text.trim();
|
||||
if trimmed.is_empty() || trimmed.len() < 3 {
|
||||
continue;
|
||||
}
|
||||
total_text_items += 1;
|
||||
if is_letterspaced(&item.text) {
|
||||
letterspaced_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Only fix if ≥50% of substantial items are letter-spaced
|
||||
if total_text_items < 4 || letterspaced_count * 2 < total_text_items {
|
||||
return DEFAULT;
|
||||
}
|
||||
// Compute threshold BEFORE removing spaces. Since we've confirmed this
|
||||
// is a Canva-style page (≥50% letterspaced), use the ungated variant
|
||||
// that includes all pairs — the char-count guard in the normal function
|
||||
// would filter out long letterspaced items like "i s s i o n" (11 chars).
|
||||
let threshold = compute_canva_join_threshold(items);
|
||||
|
||||
// Remove spaces from letter-spaced items
|
||||
for item in items.iter_mut() {
|
||||
if is_letterspaced(&item.text) {
|
||||
let fixed: String = item.text.chars().filter(|&c| c != ' ').collect();
|
||||
item.text = fixed;
|
||||
}
|
||||
}
|
||||
|
||||
threshold
|
||||
}
|
||||
|
||||
/// Compute Otsu join threshold for a confirmed Canva-style page.
|
||||
///
|
||||
/// Like [`compute_single_char_join_threshold`] but without the per-pair
|
||||
/// char-count guard, since we already know the page has Canva-style
|
||||
/// letter-spacing. Uses all adjacent pairs for maximum sample size.
|
||||
fn compute_canva_join_threshold(items: &[TextItem]) -> f32 {
|
||||
const DEFAULT: f32 = 0.10;
|
||||
const MIN_SAMPLES: usize = 8;
|
||||
|
||||
let mut ratios: Vec<f32> = Vec::new();
|
||||
for pair in items.windows(2) {
|
||||
let prev = &pair[0];
|
||||
let curr = &pair[1];
|
||||
|
||||
// Skip CJK pairs
|
||||
let prev_c = prev.text.trim().chars().last();
|
||||
let curr_c = curr.text.trim().chars().next();
|
||||
if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if prev.width <= 0.0 || prev.font_size <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let gap = if prev.x <= curr.x {
|
||||
curr.x - (prev.x + prev.width)
|
||||
} else {
|
||||
prev.x - (curr.x + curr.width)
|
||||
};
|
||||
|
||||
let ratio = gap / prev.font_size;
|
||||
|
||||
if !(0.0..=3.0).contains(&ratio) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ratios.push(ratio);
|
||||
}
|
||||
|
||||
if ratios.len() < MIN_SAMPLES {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
// If all gaps are tight (max < 0.40), use default — normal PDF
|
||||
let max_ratio = ratios[ratios.len() - 1];
|
||||
if max_ratio < 0.40 {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
// If the minimum gap is below 0.40, there's a mix of tight and wide gaps,
|
||||
// meaning this isn't a uniform letter-spacing PDF — use default.
|
||||
if ratios[0] < 0.40 {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
// Median-based threshold: for Canva letter-spacing, the median gap ratio
|
||||
// is dominated by the letter-spacing value (~0.55× font_size). Word gaps
|
||||
// are consistently ~1.8× the letter-spacing. Using median × 1.55 places
|
||||
// the threshold between the widest intra-word gaps and the narrowest
|
||||
// inter-word gaps.
|
||||
let median = ratios[ratios.len() / 2];
|
||||
(median * 1.55).clamp(0.50, 2.0)
|
||||
}
|
||||
|
||||
/// Compute an adaptive join threshold for text items on a line.
|
||||
///
|
||||
/// Uses Otsu's method on the gap/font_size ratio distribution to find the
|
||||
/// natural split between intra-word and inter-word gaps. With per-pair
|
||||
/// char-count guard (both items ≥ 5 chars → skip). Used only in tests;
|
||||
/// production code uses `compute_canva_join_threshold` via `fix_letterspaced_items`.
|
||||
#[cfg(test)]
|
||||
fn compute_single_char_join_threshold(items: &[TextItem]) -> f32 {
|
||||
const DEFAULT: f32 = 0.10;
|
||||
const MIN_SAMPLES: usize = 8;
|
||||
|
||||
// Collect gap/font_size ratios for adjacent pairs involving at least one
|
||||
// short fragment (< 5 chars). This detects per-character rendering
|
||||
// (Canva-style) without being fooled by uniform word-level spacing.
|
||||
let mut ratios: Vec<f32> = Vec::new();
|
||||
for pair in items.windows(2) {
|
||||
let prev = &pair[0];
|
||||
let curr = &pair[1];
|
||||
|
||||
let prev_chars = prev.text.trim().chars().count();
|
||||
let curr_chars = curr.text.trim().chars().count();
|
||||
|
||||
// Require at least one item to be a short fragment.
|
||||
// Pairs of long words (both ≥ 5 chars) indicate normal text.
|
||||
if prev_chars >= 5 && curr_chars >= 5 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip CJK pairs
|
||||
let prev_c = prev.text.trim().chars().last();
|
||||
let curr_c = curr.text.trim().chars().next();
|
||||
if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if prev.width <= 0.0 || prev.font_size <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let gap = if prev.x <= curr.x {
|
||||
curr.x - (prev.x + prev.width)
|
||||
} else {
|
||||
prev.x - (curr.x + curr.width)
|
||||
};
|
||||
|
||||
let ratio = gap / prev.font_size;
|
||||
|
||||
// Skip negative gaps and huge gaps (> 3× font_size)
|
||||
if !(0.0..=3.0).contains(&ratio) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ratios.push(ratio);
|
||||
}
|
||||
|
||||
if ratios.len() < MIN_SAMPLES {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
// If all gaps are tight (max < 0.40), use default — normal PDF
|
||||
let max_ratio = ratios[ratios.len() - 1];
|
||||
if max_ratio < 0.40 {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
// If the minimum gap is below 0.40, there's a mix of tight and wide gaps,
|
||||
// meaning this isn't a uniform letter-spacing PDF — use default.
|
||||
// Canva-style letter-spacing has min gaps ≈ 0.5× font_size; normal
|
||||
// justified text gaps are ≈ 0.15–0.30× font_size.
|
||||
if ratios[0] < 0.40 {
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
// All gaps are wide (≥0.25× font_size) — Canva-style letter-spacing.
|
||||
// Use Otsu to find the split between intra-word and inter-word gaps.
|
||||
let n = ratios.len() as f32;
|
||||
let total_sum: f32 = ratios.iter().sum();
|
||||
|
||||
let mut best_threshold = DEFAULT;
|
||||
let mut best_variance = f32::NEG_INFINITY;
|
||||
|
||||
let mut w0: f32 = 0.0;
|
||||
let mut sum0: f32 = 0.0;
|
||||
|
||||
for i in 0..ratios.len() - 1 {
|
||||
w0 += 1.0;
|
||||
sum0 += ratios[i];
|
||||
|
||||
let w1 = n - w0;
|
||||
if w1 == 0.0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mean0 = sum0 / w0;
|
||||
let mean1 = (total_sum - sum0) / w1;
|
||||
let variance = w0 * w1 * (mean0 - mean1).powi(2);
|
||||
|
||||
// Only consider thresholds at value boundaries (skip duplicates)
|
||||
if i + 1 < ratios.len() && (ratios[i + 1] - ratios[i]).abs() < 1e-6 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if variance > best_variance {
|
||||
best_variance = variance;
|
||||
// Place threshold midway between the two classes
|
||||
best_threshold = (ratios[i] + ratios[i + 1]) / 2.0;
|
||||
}
|
||||
}
|
||||
|
||||
best_threshold.clamp(0.05, 2.0)
|
||||
}
|
||||
|
||||
/// Determine if two adjacent text items should be joined without a space
|
||||
/// based on their physical positions on the page and character case.
|
||||
/// Uses a hybrid approach: position-based with case-aware thresholds.
|
||||
/// CID fonts emit one word per text operator with gaps ≈ 0 between words.
|
||||
/// Non-CID (Type1/TrueType) fonts emit phrases or fragments.
|
||||
pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool {
|
||||
pub(crate) fn should_join_items(
|
||||
prev_item: &TextItem,
|
||||
curr_item: &TextItem,
|
||||
single_char_threshold: f32,
|
||||
) -> bool {
|
||||
// If either text explicitly has leading/trailing spaces, respect them
|
||||
if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') {
|
||||
return false;
|
||||
@@ -365,6 +620,12 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
|
||||
}
|
||||
}
|
||||
|
||||
// When the adaptive threshold indicates Canva-style letter-spacing
|
||||
// (all gaps wide), use it uniformly for all pair types.
|
||||
if single_char_threshold > 0.20 {
|
||||
return gap < font_size * single_char_threshold;
|
||||
}
|
||||
|
||||
// Single-character fragment joined to a multi-character item: use a
|
||||
// moderately generous threshold to rejoin split words like "b" + "illion"
|
||||
// or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words.
|
||||
@@ -385,7 +646,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
|
||||
return gap < font_size * 0.25;
|
||||
}
|
||||
}
|
||||
return gap < font_size * 0.10;
|
||||
return gap < font_size * single_char_threshold;
|
||||
}
|
||||
|
||||
// With accurate widths, a gap < 15% of font size means glyphs are
|
||||
@@ -473,6 +734,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::ItemType;
|
||||
|
||||
#[test]
|
||||
fn strip_soft_hyphen() {
|
||||
@@ -581,4 +843,197 @@ mod tests {
|
||||
// Latin
|
||||
assert!(!is_arabic_presentation_form('A'));
|
||||
}
|
||||
|
||||
/// Helper to create a single-char TextItem at a given x position with width.
|
||||
fn make_char_item(ch: char, x: f32, width: f32, font_size: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: ch.to_string(),
|
||||
x,
|
||||
y: 100.0,
|
||||
width,
|
||||
height: font_size,
|
||||
font: "TestFont".to_string(),
|
||||
font_size,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn otsu_threshold_sec_style_tight_gaps() {
|
||||
// SEC-style: intra-word gaps ≈ 0, word gap ≈ 0.15× font_size
|
||||
// All gaps tight → should return default 0.10
|
||||
let fs = 12.0;
|
||||
let char_w = fs * 0.5;
|
||||
let mut items = Vec::new();
|
||||
// 15 chars with gap ≈ 0 (intra-word)
|
||||
for i in 0..15 {
|
||||
let x = 100.0 + i as f32 * (char_w + fs * 0.01);
|
||||
items.push(make_char_item('a', x, char_w, fs));
|
||||
}
|
||||
// Word gap
|
||||
let word_x = items.last().unwrap().x + char_w + fs * 0.15;
|
||||
items.push(make_char_item('b', word_x, char_w, fs));
|
||||
// 5 more tight chars
|
||||
for i in 1..5 {
|
||||
let x = word_x + i as f32 * (char_w + fs * 0.01);
|
||||
items.push(make_char_item('c', x, char_w, fs));
|
||||
}
|
||||
|
||||
let threshold = compute_single_char_join_threshold(&items);
|
||||
// Max gap is 0.15, but most are 0.01 → max < 0.20 → default
|
||||
assert!(
|
||||
(threshold - 0.10).abs() < 0.01,
|
||||
"SEC-style should return default ~0.10, got {threshold}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn otsu_threshold_canva_style_wide_gaps() {
|
||||
// Canva-style: intra-word gaps ≈ 0.6× font_size, word gaps ≈ 1.2× font_size
|
||||
let fs = 12.0;
|
||||
let char_w = fs * 0.5;
|
||||
let intra_gap = fs * 0.6;
|
||||
let word_gap = fs * 1.2;
|
||||
let mut items = Vec::new();
|
||||
|
||||
// Word 1: 8 chars with intra-word spacing
|
||||
for i in 0..8 {
|
||||
let x = 100.0 + i as f32 * (char_w + intra_gap);
|
||||
items.push(make_char_item('K', x, char_w, fs));
|
||||
}
|
||||
// Word gap
|
||||
let word_x = items.last().unwrap().x + char_w + word_gap;
|
||||
items.push(make_char_item('T', word_x, char_w, fs));
|
||||
// Word 2: 7 more chars
|
||||
for i in 1..7 {
|
||||
let x = word_x + i as f32 * (char_w + intra_gap);
|
||||
items.push(make_char_item('o', x, char_w, fs));
|
||||
}
|
||||
|
||||
let threshold = compute_single_char_join_threshold(&items);
|
||||
// Should find threshold between 0.6 and 1.2 → roughly 0.9
|
||||
assert!(
|
||||
threshold > 0.5 && threshold < 1.1,
|
||||
"Canva-style should find threshold ~0.9, got {threshold}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn otsu_threshold_few_samples_returns_default() {
|
||||
// < 8 single-char pairs → default
|
||||
let fs = 12.0;
|
||||
let char_w = fs * 0.5;
|
||||
let items: Vec<TextItem> = (0..5)
|
||||
.map(|i| make_char_item('x', 100.0 + i as f32 * (char_w + 1.0), char_w, fs))
|
||||
.collect();
|
||||
|
||||
let threshold = compute_single_char_join_threshold(&items);
|
||||
assert!(
|
||||
(threshold - 0.10).abs() < 0.01,
|
||||
"few samples should return default 0.10, got {threshold}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fix_letterspaced_items_returns_adaptive_threshold() {
|
||||
// Simulate Canva page with many letter-spaced items and word gaps.
|
||||
// Needs ≥8 inter-item gaps for the threshold to be computed.
|
||||
let fs = 12.0;
|
||||
let char_w = fs * 0.5;
|
||||
let letter_gap = fs * 0.6; // 0.6× font_size between items
|
||||
let word_gap = fs * 1.2; // 1.2× font_size between words
|
||||
|
||||
let words: Vec<&str> = vec![
|
||||
"H e l l o",
|
||||
"W o r l d",
|
||||
"F o o",
|
||||
"B a r",
|
||||
"B a z",
|
||||
"Q u x",
|
||||
"T e s t",
|
||||
"D a t a",
|
||||
"M o r e",
|
||||
"T e x t",
|
||||
];
|
||||
|
||||
let mut items = Vec::new();
|
||||
let mut x = 100.0;
|
||||
for (wi, word) in words.iter().enumerate() {
|
||||
let char_count = word.chars().filter(|c| !c.is_whitespace()).count();
|
||||
let w = char_count as f32 * char_w + (char_count - 1) as f32 * letter_gap;
|
||||
items.push(TextItem {
|
||||
text: word.to_string(),
|
||||
x,
|
||||
y: 100.0,
|
||||
width: w,
|
||||
height: fs,
|
||||
font: "TestFont".to_string(),
|
||||
font_size: fs,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
});
|
||||
// Alternate between letter-gap and word-gap to create bimodal distribution
|
||||
x += w + if wi % 3 == 2 { word_gap } else { letter_gap };
|
||||
}
|
||||
|
||||
let threshold = fix_letterspaced_items(&mut items);
|
||||
|
||||
// Threshold should be above default (Canva-style detected)
|
||||
assert!(
|
||||
threshold > 0.50,
|
||||
"Canva page should get threshold > 0.50, got {threshold}"
|
||||
);
|
||||
|
||||
// Spaces should be removed from letter-spaced items
|
||||
assert_eq!(items[0].text, "Hello");
|
||||
assert_eq!(items[1].text, "World");
|
||||
assert_eq!(items[2].text, "Foo");
|
||||
assert_eq!(items[9].text, "Text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canva_style_items_join_correctly() {
|
||||
// Simulate Canva PDF: "Hello" with 0.6× font_size letter-spacing
|
||||
let fs = 12.0;
|
||||
let char_w = fs * 0.5;
|
||||
let intra_gap = fs * 0.6;
|
||||
let word_gap = fs * 1.2;
|
||||
|
||||
let mut items = Vec::new();
|
||||
let chars = ['H', 'e', 'l', 'l', 'o'];
|
||||
for (i, &ch) in chars.iter().enumerate() {
|
||||
let x = 100.0 + i as f32 * (char_w + intra_gap);
|
||||
items.push(make_char_item(ch, x, char_w, fs));
|
||||
}
|
||||
// Space then "W"
|
||||
let w_x = items.last().unwrap().x + char_w + word_gap;
|
||||
items.push(make_char_item('W', w_x, char_w, fs));
|
||||
let chars2 = ['o', 'r', 'l', 'd'];
|
||||
for (i, &ch) in chars2.iter().enumerate() {
|
||||
let x = w_x + (i + 1) as f32 * (char_w + intra_gap);
|
||||
items.push(make_char_item(ch, x, char_w, fs));
|
||||
}
|
||||
|
||||
let threshold = compute_single_char_join_threshold(&items);
|
||||
|
||||
// Intra-word pairs should join
|
||||
assert!(
|
||||
should_join_items(&items[0], &items[1], threshold),
|
||||
"H+e should join with threshold {threshold}"
|
||||
);
|
||||
assert!(
|
||||
should_join_items(&items[3], &items[4], threshold),
|
||||
"l+o should join with threshold {threshold}"
|
||||
);
|
||||
// Word boundary should NOT join
|
||||
assert!(
|
||||
!should_join_items(&items[4], &items[5], threshold),
|
||||
"o+W (word boundary) should NOT join with threshold {threshold}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+18
-4
@@ -125,6 +125,10 @@ pub struct TextLine {
|
||||
pub items: Vec<TextItem>,
|
||||
pub y: f32,
|
||||
pub page: u32,
|
||||
/// Adaptive join threshold from page-level letter-spacing detection.
|
||||
/// Default 0.10 for normal PDFs; higher for Canva-style PDFs.
|
||||
#[doc(hidden)]
|
||||
pub adaptive_threshold: f32,
|
||||
}
|
||||
|
||||
impl TextLine {
|
||||
@@ -138,6 +142,8 @@ impl TextLine {
|
||||
return self.text_plain();
|
||||
}
|
||||
|
||||
let single_char_threshold = self.adaptive_threshold;
|
||||
|
||||
let mut result = String::new();
|
||||
let mut current_bold = false;
|
||||
let mut current_italic = false;
|
||||
@@ -156,7 +162,7 @@ impl TextLine {
|
||||
false
|
||||
} else {
|
||||
let prev_item = &self.items[i - 1];
|
||||
self.needs_space_between(prev_item, item, &result)
|
||||
self.needs_space_between(prev_item, item, &result, single_char_threshold)
|
||||
};
|
||||
|
||||
// Preserve leading whitespace from the item text.
|
||||
@@ -211,6 +217,8 @@ impl TextLine {
|
||||
|
||||
/// Get plain text without formatting
|
||||
fn text_plain(&self) -> String {
|
||||
let single_char_threshold = self.adaptive_threshold;
|
||||
|
||||
let mut result = String::new();
|
||||
for (i, item) in self.items.iter().enumerate() {
|
||||
let text = item.text.as_str();
|
||||
@@ -218,7 +226,7 @@ impl TextLine {
|
||||
result.push_str(text);
|
||||
} else {
|
||||
let prev_item = &self.items[i - 1];
|
||||
if self.needs_space_between(prev_item, item, &result) {
|
||||
if self.needs_space_between(prev_item, item, &result, single_char_threshold) {
|
||||
result.push(' ');
|
||||
}
|
||||
result.push_str(text);
|
||||
@@ -228,7 +236,13 @@ impl TextLine {
|
||||
}
|
||||
|
||||
/// Determine if a space is needed between two items
|
||||
fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
|
||||
fn needs_space_between(
|
||||
&self,
|
||||
prev_item: &TextItem,
|
||||
item: &TextItem,
|
||||
result: &str,
|
||||
single_char_threshold: f32,
|
||||
) -> bool {
|
||||
let text = item.text.as_str();
|
||||
|
||||
// Don't add space before/after hyphens for hyphenated words
|
||||
@@ -245,7 +259,7 @@ impl TextLine {
|
||||
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
|
||||
|
||||
// Use position-based spacing detection
|
||||
let should_join = should_join_items(prev_item, item);
|
||||
let should_join = should_join_items(prev_item, item, single_char_threshold);
|
||||
|
||||
// Check if space already exists
|
||||
let prev_ends_with_space = result.ends_with(' ');
|
||||
|
||||
@@ -138,6 +138,7 @@ fn test_text_line_text_method() {
|
||||
items,
|
||||
y: 700.0,
|
||||
page: 1,
|
||||
adaptive_threshold: 0.10,
|
||||
};
|
||||
assert_eq!(line.text(), "Hello World");
|
||||
}
|
||||
@@ -149,6 +150,7 @@ fn test_text_line_single_item() {
|
||||
items,
|
||||
y: 700.0,
|
||||
page: 1,
|
||||
adaptive_threshold: 0.10,
|
||||
};
|
||||
assert_eq!(line.text(), "Single");
|
||||
}
|
||||
@@ -159,6 +161,7 @@ fn test_text_line_empty() {
|
||||
items: vec![],
|
||||
y: 700.0,
|
||||
page: 1,
|
||||
adaptive_threshold: 0.10,
|
||||
};
|
||||
assert_eq!(line.text(), "");
|
||||
}
|
||||
@@ -473,11 +476,13 @@ fn test_markdown_from_lines_basic() {
|
||||
items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)],
|
||||
y: 700.0,
|
||||
page: 1,
|
||||
adaptive_threshold: 0.10,
|
||||
},
|
||||
TextLine {
|
||||
items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)],
|
||||
y: 680.0,
|
||||
page: 1,
|
||||
adaptive_threshold: 0.10,
|
||||
},
|
||||
];
|
||||
let md = to_markdown_from_lines(lines, MarkdownOptions::default());
|
||||
|
||||
Reference in New Issue
Block a user