fix(text): Improve word spacing, paragraph joining, and page breaks

- Preserve leading whitespace in text_with_formatting() to fix ~80% of
  missing-space issues (e.g. "holdermeans" -> "holder means")
- Tighten word boundary gap threshold from 0.05 to 0.01 and add
  word-count heuristic to distinguish CID word-level operators from
  Type1 line-level operators, fixing spurious spaces (e.g. "t emporary")
- Replace fixed paragraph threshold (1.8x base_size) with dynamic
  median-based computation for double-spaced documents
- Remove --- page break markers between pages
- Fix page number y-threshold for US Letter (720pt vs 800pt)

Eval: +1.4% mean word_sim, zero char_sim regressions across 63 PDFs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/extractor.rs

@@ -547,6 +547,13 @@ impl TextLine {
                 self.needs_space_between(prev_item, item, &result)
             };
 
+            // Preserve leading whitespace from the item text.
+            // Items like " means any person" have a leading space that indicates
+            // a word boundary. needs_space_between returns false for these (because
+            // space_already_exists), but we still need to emit the space since
+            // we push text_trimmed below (which strips it).
+            let has_leading_space = text.starts_with(' ');
+
             // Check for style changes
             let item_bold = format_bold && item.is_bold;
             let item_italic = format_italic && item.is_italic;
@@ -561,8 +568,8 @@ impl TextLine {
                 current_bold = false;
             }
 
-            // Add space after closing markers if needed
-            if needs_space {
+            // Add space: either from spacing logic or preserved from item text
+            if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) {
                 result.push(' ');
             }
 
@@ -680,17 +687,55 @@ fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool {
         let font_size = prev_item.font_size;
 
         // When items perfectly touch (gap ≈ 0) and both are multi-character,
-        // they are likely separate words from different text operators.
+        // they are likely separate words from different text operators (CID fonts).
         // Single-character items (per-glyph positioning) should still be joined.
         // Skip for CJK text — CJK languages don't use spaces between words.
+        //
+        // Use a very tight gap threshold (0.01 = 0.12pt for 12pt font) to only
+        // catch truly-touching CID items (gap ≈ 0.0). Type1 font fragments have
+        // gaps of 0.2-0.5pt from positioning imprecision and should NOT trigger this.
+        //
+        // Also skip if the last word of prev or first word of curr is very short
+        // (≤ 2 chars), which indicates a word split across text operators.
         let prev_chars = prev_item.text.trim().chars().count();
         let curr_chars = curr_item.text.trim().chars().count();
         let prev_last_char = prev_item.text.trim().chars().last();
         let curr_first_char = curr_item.text.trim().chars().next();
         let is_cjk =
-            prev_last_char.map_or(false, is_cjk_char) || curr_first_char.map_or(false, is_cjk_char);
-        if !is_cjk && gap >= 0.0 && gap < font_size * 0.05 && prev_chars >= 3 && curr_chars >= 2 {
-            return false; // Don't join — separate words
+            prev_last_char.is_some_and(is_cjk_char) || curr_first_char.is_some_and(is_cjk_char);
+
+        if !is_cjk && gap >= 0.0 && gap < font_size * 0.01 && prev_chars >= 3 && curr_chars >= 2 {
+            // Count words in the prev item to distinguish line-level vs word-level
+            // text operators. Line-level operators (Type1 fonts) emit long phrases
+            // like "should specifi" that get split mid-word at operator boundaries.
+            // Word-level operators (CID fonts like C2_0) emit single words.
+            let prev_word_count = prev_item.text.split_whitespace().count();
+
+            if prev_word_count >= 3 {
+                // Multi-word phrase (3+ words) from a line-level operator.
+                // The boundary is likely mid-word, not a word boundary.
+                return gap < font_size * 0.15;
+            }
+
+            // Prev is a short item (1-2 words), typical of CID fonts.
+            // Additional safety: skip if last word of prev or first word of curr
+            // is very short (≤ 2 chars), indicating a split word.
+            let prev_trimmed = prev_item.text.trim_end();
+            let last_word_len = prev_trimmed
+                .rsplit(|c: char| c.is_whitespace())
+                .next()
+                .map(|w| w.chars().count())
+                .unwrap_or(prev_chars);
+            let curr_trimmed = curr_item.text.trim_start();
+            let first_word_len = curr_trimmed
+                .split(|c: char| c.is_whitespace())
+                .next()
+                .map(|w| w.chars().count())
+                .unwrap_or(curr_chars);
+
+            if last_word_len > 2 && first_word_len > 2 {
+                return false; // Don't join — separate words from CID font
+            }
         }
 
         // With accurate widths, a gap < 15% of font size means glyphs are
@@ -2124,9 +2169,10 @@ fn is_page_number(item: &TextItem) -> bool {
         return false;
     }
 
-    // Must be at top (y > 800) or bottom (y < 100) of page
-    // These thresholds work for standard page sizes
-    item.y > 800.0 || item.y < 100.0
+    // Must be at top or bottom of page.
+    // US Letter = 792pt, A4 = 841pt. Page numbers are typically in the
+    // top ~5% or bottom ~12% of the page.
+    item.y > 720.0 || item.y < 100.0
 }
 
 /// Group text items into lines, with multi-column support

src/markdown.rs

@@ -271,6 +271,12 @@ fn to_markdown_from_lines_with_tables_and_images(
     // Discover heading tiers for this document
     let heading_tiers = compute_heading_tiers(&lines, base_size);
 
+    // Compute the typical line spacing for paragraph break detection.
+    // For double-spaced documents (like legal/government PDFs), the normal
+    // line spacing can be 2.3x base_size, which would exceed a fixed 1.8x
+    // threshold and cause every line to be treated as a paragraph break.
+    let para_threshold = compute_paragraph_threshold(&lines, base_size);
+
     let mut output = String::new();
     let mut current_page = 0u32;
     let mut prev_y = f32::MAX;
@@ -317,7 +323,7 @@ fn to_markdown_from_lines_with_tables_and_images(
                     output.push_str("\n\n");
                     in_paragraph = false;
                 }
-                output.push_str("---\n\n");
+                output.push_str("\n\n");
             }
             current_page = line.page;
             prev_y = f32::MAX;
@@ -357,9 +363,9 @@ fn to_markdown_from_lines_with_tables_and_images(
             }
         }
 
-        // Paragraph break (large Y gap)
+        // Paragraph break (large Y gap relative to document's typical line spacing)
         let y_gap = prev_y - line.y;
-        let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold
+        let is_para_break = y_gap > para_threshold;
         if is_para_break && in_paragraph {
             output.push_str("\n\n");
             in_paragraph = false;
@@ -533,6 +539,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
     // Discover heading tiers for this document
     let heading_tiers = compute_heading_tiers(&lines, base_size);
 
+    // Compute the typical line spacing for paragraph break detection
+    let para_threshold = compute_paragraph_threshold(&lines, base_size);
+
     let mut output = String::new();
     let mut current_page = 0u32;
     let mut prev_y = f32::MAX;
@@ -548,7 +557,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
                     output.push_str("\n\n");
                     in_paragraph = false;
                 }
-                output.push_str("---\n\n");
+                output.push_str("\n\n");
             }
             current_page = line.page;
             prev_y = f32::MAX;
@@ -556,9 +565,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
             last_list_x = None;
         }
 
-        // Paragraph break (large Y gap)
+        // Paragraph break (large Y gap relative to document's typical line spacing)
         let y_gap = prev_y - line.y;
-        let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold
+        let is_para_break = y_gap > para_threshold;
         if is_para_break && in_paragraph {
             output.push_str("\n\n");
             in_paragraph = false;
@@ -793,6 +802,52 @@ fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
     FontStats { most_common_size }
 }
 
+/// Compute the Y-gap threshold for paragraph break detection.
+///
+/// Instead of using a fixed multiple of base_size (which fails for double-spaced
+/// documents), we compute the document's typical (median) line spacing and use
+/// a multiplier on that. A gap significantly larger than typical indicates a
+/// paragraph break.
+///
+/// Fallback: if we can't compute typical spacing, use base_size * 1.8.
+fn compute_paragraph_threshold(lines: &[TextLine], base_size: f32) -> f32 {
+    let fallback = base_size * 1.8;
+
+    // Collect Y gaps between consecutive lines on the same page
+    let mut gaps: Vec<f32> = Vec::new();
+    let mut prev_y: Option<(u32, f32)> = None;
+
+    for line in lines {
+        if let Some((prev_page, py)) = prev_y {
+            if line.page == prev_page {
+                let gap = py - line.y;
+                // Only consider positive gaps within a reasonable range
+                // (skip huge gaps from page headers/footers)
+                if gap > 0.0 && gap < base_size * 10.0 {
+                    gaps.push(gap);
+                }
+            }
+        }
+        prev_y = Some((line.page, line.y));
+    }
+
+    if gaps.len() < 5 {
+        return fallback;
+    }
+
+    gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+
+    let median = gaps[gaps.len() / 2];
+
+    // The paragraph threshold should be larger than the typical line spacing.
+    // Use 1.3x the median gap. This means:
+    // - Single-spaced (median ~14pt for 12pt font): threshold = 18.2pt
+    // - Double-spaced (median ~28pt for 12pt font): threshold = 36.4pt
+    // Also ensure it's at least base_size * 1.5 to avoid false paragraph breaks
+    // in tightly-spaced documents.
+    (median * 1.3).max(base_size * 1.5)
+}
+
 /// Discover distinct heading font-size tiers in the document.
 /// Returns tiers sorted largest-first (tier 0 = H1, tier 1 = H2, …).
 /// Sizes within 0.5pt are clustered into the same tier. Capped at 4 tiers.

tests/integration_tests.rs

@@ -440,11 +440,14 @@ fn test_markdown_from_items_monospace_code() {
 fn test_markdown_from_items_page_breaks() {
     use pdf_inspector::markdown::to_markdown_from_items;
     let items = vec![
-        make_text_item("Page 1", 100.0, 700.0, 12.0, 1),
-        make_text_item("Page 2", 100.0, 700.0, 12.0, 2),
+        make_text_item("Content on first page", 100.0, 700.0, 12.0, 1),
+        make_text_item("Content on second page", 100.0, 700.0, 12.0, 2),
     ];
     let md = to_markdown_from_items(items, MarkdownOptions::default());
-    assert!(md.contains("---")); // Page break marker
+    // Pages should be separated by blank lines (no --- markers)
+    assert!(!md.contains("---"));
+    assert!(md.contains("Content on first page"));
+    assert!(md.contains("Content on second page"));
 }
 
 // ============================================================================