diff --git a/src/extractor.rs b/src/extractor.rs index a5c24a1..5382e55 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -547,6 +547,13 @@ impl TextLine { self.needs_space_between(prev_item, item, &result) }; + // Preserve leading whitespace from the item text. + // Items like " means any person" have a leading space that indicates + // a word boundary. needs_space_between returns false for these (because + // space_already_exists), but we still need to emit the space since + // we push text_trimmed below (which strips it). + let has_leading_space = text.starts_with(' '); + // Check for style changes let item_bold = format_bold && item.is_bold; let item_italic = format_italic && item.is_italic; @@ -561,8 +568,8 @@ impl TextLine { current_bold = false; } - // Add space after closing markers if needed - if needs_space { + // Add space: either from spacing logic or preserved from item text + if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) { result.push(' '); } @@ -680,17 +687,55 @@ fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { let font_size = prev_item.font_size; // When items perfectly touch (gap ≈ 0) and both are multi-character, - // they are likely separate words from different text operators. + // they are likely separate words from different text operators (CID fonts). // Single-character items (per-glyph positioning) should still be joined. // Skip for CJK text — CJK languages don't use spaces between words. + // + // Use a very tight gap threshold (0.01 = 0.12pt for 12pt font) to only + // catch truly-touching CID items (gap ≈ 0.0). Type1 font fragments have + // gaps of 0.2-0.5pt from positioning imprecision and should NOT trigger this. + // + // Also skip if the last word of prev or first word of curr is very short + // (≤ 2 chars), which indicates a word split across text operators. let prev_chars = prev_item.text.trim().chars().count(); let curr_chars = curr_item.text.trim().chars().count(); let prev_last_char = prev_item.text.trim().chars().last(); let curr_first_char = curr_item.text.trim().chars().next(); let is_cjk = - prev_last_char.map_or(false, is_cjk_char) || curr_first_char.map_or(false, is_cjk_char); - if !is_cjk && gap >= 0.0 && gap < font_size * 0.05 && prev_chars >= 3 && curr_chars >= 2 { - return false; // Don't join — separate words + prev_last_char.is_some_and(is_cjk_char) || curr_first_char.is_some_and(is_cjk_char); + + if !is_cjk && gap >= 0.0 && gap < font_size * 0.01 && prev_chars >= 3 && curr_chars >= 2 { + // Count words in the prev item to distinguish line-level vs word-level + // text operators. Line-level operators (Type1 fonts) emit long phrases + // like "should specifi" that get split mid-word at operator boundaries. + // Word-level operators (CID fonts like C2_0) emit single words. + let prev_word_count = prev_item.text.split_whitespace().count(); + + if prev_word_count >= 3 { + // Multi-word phrase (3+ words) from a line-level operator. + // The boundary is likely mid-word, not a word boundary. + return gap < font_size * 0.15; + } + + // Prev is a short item (1-2 words), typical of CID fonts. + // Additional safety: skip if last word of prev or first word of curr + // is very short (≤ 2 chars), indicating a split word. + let prev_trimmed = prev_item.text.trim_end(); + let last_word_len = prev_trimmed + .rsplit(|c: char| c.is_whitespace()) + .next() + .map(|w| w.chars().count()) + .unwrap_or(prev_chars); + let curr_trimmed = curr_item.text.trim_start(); + let first_word_len = curr_trimmed + .split(|c: char| c.is_whitespace()) + .next() + .map(|w| w.chars().count()) + .unwrap_or(curr_chars); + + if last_word_len > 2 && first_word_len > 2 { + return false; // Don't join — separate words from CID font + } } // With accurate widths, a gap < 15% of font size means glyphs are @@ -2124,9 +2169,10 @@ fn is_page_number(item: &TextItem) -> bool { return false; } - // Must be at top (y > 800) or bottom (y < 100) of page - // These thresholds work for standard page sizes - item.y > 800.0 || item.y < 100.0 + // Must be at top or bottom of page. + // US Letter = 792pt, A4 = 841pt. Page numbers are typically in the + // top ~5% or bottom ~12% of the page. + item.y > 720.0 || item.y < 100.0 } /// Group text items into lines, with multi-column support diff --git a/src/markdown.rs b/src/markdown.rs index 7f20214..85cd12b 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -271,6 +271,12 @@ fn to_markdown_from_lines_with_tables_and_images( // Discover heading tiers for this document let heading_tiers = compute_heading_tiers(&lines, base_size); + // Compute the typical line spacing for paragraph break detection. + // For double-spaced documents (like legal/government PDFs), the normal + // line spacing can be 2.3x base_size, which would exceed a fixed 1.8x + // threshold and cause every line to be treated as a paragraph break. + let para_threshold = compute_paragraph_threshold(&lines, base_size); + let mut output = String::new(); let mut current_page = 0u32; let mut prev_y = f32::MAX; @@ -317,7 +323,7 @@ fn to_markdown_from_lines_with_tables_and_images( output.push_str("\n\n"); in_paragraph = false; } - output.push_str("---\n\n"); + output.push_str("\n\n"); } current_page = line.page; prev_y = f32::MAX; @@ -357,9 +363,9 @@ fn to_markdown_from_lines_with_tables_and_images( } } - // Paragraph break (large Y gap) + // Paragraph break (large Y gap relative to document's typical line spacing) let y_gap = prev_y - line.y; - let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold + let is_para_break = y_gap > para_threshold; if is_para_break && in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -533,6 +539,9 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Discover heading tiers for this document let heading_tiers = compute_heading_tiers(&lines, base_size); + // Compute the typical line spacing for paragraph break detection + let para_threshold = compute_paragraph_threshold(&lines, base_size); + let mut output = String::new(); let mut current_page = 0u32; let mut prev_y = f32::MAX; @@ -548,7 +557,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> output.push_str("\n\n"); in_paragraph = false; } - output.push_str("---\n\n"); + output.push_str("\n\n"); } current_page = line.page; prev_y = f32::MAX; @@ -556,9 +565,9 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> last_list_x = None; } - // Paragraph break (large Y gap) + // Paragraph break (large Y gap relative to document's typical line spacing) let y_gap = prev_y - line.y; - let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold + let is_para_break = y_gap > para_threshold; if is_para_break && in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -793,6 +802,52 @@ fn calculate_font_stats(lines: &[TextLine]) -> FontStats { FontStats { most_common_size } } +/// Compute the Y-gap threshold for paragraph break detection. +/// +/// Instead of using a fixed multiple of base_size (which fails for double-spaced +/// documents), we compute the document's typical (median) line spacing and use +/// a multiplier on that. A gap significantly larger than typical indicates a +/// paragraph break. +/// +/// Fallback: if we can't compute typical spacing, use base_size * 1.8. +fn compute_paragraph_threshold(lines: &[TextLine], base_size: f32) -> f32 { + let fallback = base_size * 1.8; + + // Collect Y gaps between consecutive lines on the same page + let mut gaps: Vec = Vec::new(); + let mut prev_y: Option<(u32, f32)> = None; + + for line in lines { + if let Some((prev_page, py)) = prev_y { + if line.page == prev_page { + let gap = py - line.y; + // Only consider positive gaps within a reasonable range + // (skip huge gaps from page headers/footers) + if gap > 0.0 && gap < base_size * 10.0 { + gaps.push(gap); + } + } + } + prev_y = Some((line.page, line.y)); + } + + if gaps.len() < 5 { + return fallback; + } + + gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let median = gaps[gaps.len() / 2]; + + // The paragraph threshold should be larger than the typical line spacing. + // Use 1.3x the median gap. This means: + // - Single-spaced (median ~14pt for 12pt font): threshold = 18.2pt + // - Double-spaced (median ~28pt for 12pt font): threshold = 36.4pt + // Also ensure it's at least base_size * 1.5 to avoid false paragraph breaks + // in tightly-spaced documents. + (median * 1.3).max(base_size * 1.5) +} + /// Discover distinct heading font-size tiers in the document. /// Returns tiers sorted largest-first (tier 0 = H1, tier 1 = H2, …). /// Sizes within 0.5pt are clustered into the same tier. Capped at 4 tiers. diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e030acc..1cf89e1 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -440,11 +440,14 @@ fn test_markdown_from_items_monospace_code() { fn test_markdown_from_items_page_breaks() { use pdf_inspector::markdown::to_markdown_from_items; let items = vec![ - make_text_item("Page 1", 100.0, 700.0, 12.0, 1), - make_text_item("Page 2", 100.0, 700.0, 12.0, 2), + make_text_item("Content on first page", 100.0, 700.0, 12.0, 1), + make_text_item("Content on second page", 100.0, 700.0, 12.0, 2), ]; let md = to_markdown_from_items(items, MarkdownOptions::default()); - assert!(md.contains("---")); // Page break marker + // Pages should be separated by blank lines (no --- markers) + assert!(!md.contains("---")); + assert!(md.contains("Content on first page")); + assert!(md.contains("Content on second page")); } // ============================================================================