Improve text extraction with visual reading order and header detection

Major changes:
- Switch from lopdf.extract_text() to position-aware extraction
- Text is now sorted by visual reading order (top→bottom, left→right)
- Fixed font size calculation to account for text matrix scaling
- Headers are now detected based on font size ratios
- Skip very short text (≤3 chars) for header detection to avoid drop caps

This significantly improves output quality for PDFs with complex layouts.
Before: Title appeared at end, no structure detected
After: Correct reading order, headers marked with # syntax

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/extractor.rs

@@ -175,14 +175,15 @@ fn extract_page_text_items(
                         extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font)
                     {
                         if !text.trim().is_empty() {
+                            let rendered_size = effective_font_size(current_font_size, &text_matrix);
                             items.push(TextItem {
                                 text,
                                 x: text_matrix[4],
                                 y: text_matrix[5],
                                 width: 0.0, // Would need glyph widths
-                                height: current_font_size,
+                                height: rendered_size,
                                 font: current_font.clone(),
-                                font_size: current_font_size,
+                                font_size: rendered_size,
                                 page: page_num,
                             });
                         }
@@ -202,14 +203,15 @@ fn extract_page_text_items(
                             }
                         }
                         if !combined_text.trim().is_empty() {
+                            let rendered_size = effective_font_size(current_font_size, &text_matrix);
                             items.push(TextItem {
                                 text: combined_text,
                                 x: text_matrix[4],
                                 y: text_matrix[5],
                                 width: 0.0,
-                                height: current_font_size,
+                                height: rendered_size,
                                 font: current_font.clone(),
-                                font_size: current_font_size,
+                                font_size: rendered_size,
                                 page: page_num,
                             });
                         }
@@ -225,14 +227,15 @@ fn extract_page_text_items(
                         extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font)
                     {
                         if !text.trim().is_empty() {
+                            let rendered_size = effective_font_size(current_font_size, &text_matrix);
                             items.push(TextItem {
                                 text,
                                 x: text_matrix[4],
                                 y: text_matrix[5],
                                 width: 0.0,
-                                height: current_font_size,
+                                height: rendered_size,
                                 font: current_font.clone(),
-                                font_size: current_font_size,
+                                font_size: rendered_size,
                                 page: page_num,
                             });
                         }
@@ -255,6 +258,19 @@ fn get_number(obj: &Object) -> Option<f32> {
     }
 }
 
+/// Compute effective font size from base size and text matrix
+/// Text matrix is [a, b, c, d, tx, ty] where a,d are scale factors
+fn effective_font_size(base_size: f32, text_matrix: &[f32; 6]) -> f32 {
+    // The scale factor is typically the magnitude of the transformation
+    // For most PDFs, text_matrix[0] (a) is the horizontal scale
+    // and text_matrix[3] (d) is the vertical scale
+    let scale_x = (text_matrix[0].powi(2) + text_matrix[1].powi(2)).sqrt();
+    let scale_y = (text_matrix[2].powi(2) + text_matrix[3].powi(2)).sqrt();
+    // Use the larger of the two scales (usually they're equal for non-rotated text)
+    let scale = scale_x.max(scale_y);
+    base_size * scale
+}
+
 /// Extract text from a text operand, handling encoding
 fn extract_text_from_operand(
     obj: &Object,

src/lib.rs

@@ -11,7 +11,7 @@ pub mod markdown;
 
 pub use detector::{detect_pdf_type, PdfType, PdfTypeResult};
 pub use extractor::{extract_text, extract_text_with_positions, TextItem};
-pub use markdown::{to_markdown, MarkdownOptions};
+pub use markdown::{to_markdown, to_markdown_from_items, MarkdownOptions};
 
 use std::path::Path;
 
@@ -44,13 +44,13 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
 
     let result = match detection.pdf_type {
         PdfType::TextBased => {
-            // Step 2: Full extraction for text-based PDFs
-            let text = extract_text(&path)?;
-            let markdown = to_markdown(&text, MarkdownOptions::default());
+            // Step 2: Full extraction with position-aware reading order
+            let items = extract_text_with_positions(&path)?;
+            let markdown = to_markdown_from_items(items, MarkdownOptions::default());
 
             PdfProcessResult {
                 pdf_type: PdfType::TextBased,
-                text: Some(text),
+                text: None, // We now produce markdown directly
                 markdown: Some(markdown),
                 page_count: detection.page_count,
                 processing_time_ms: start.elapsed().as_millis() as u64,
@@ -67,15 +67,13 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
             }
         }
         PdfType::Mixed => {
-            // Try to extract what we can
-            let text = extract_text(&path).ok();
-            let markdown = text
-                .as_ref()
-                .map(|t| to_markdown(t, MarkdownOptions::default()));
+            // Try to extract what we can with position-aware reading order
+            let items = extract_text_with_positions(&path).ok();
+            let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default()));
 
             PdfProcessResult {
                 pdf_type: PdfType::Mixed,
-                text,
+                text: None,
                 markdown,
                 page_count: detection.page_count,
                 processing_time_ms: start.elapsed().as_millis() as u64,
@@ -95,13 +93,13 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
 
     let result = match detection.pdf_type {
         PdfType::TextBased => {
-            // Step 2: Full extraction for text-based PDFs
-            let text = extractor::extract_text_mem(buffer)?;
-            let markdown = to_markdown(&text, MarkdownOptions::default());
+            // Step 2: Full extraction with position-aware reading order
+            let items = extractor::extract_text_with_positions_mem(buffer)?;
+            let markdown = to_markdown_from_items(items, MarkdownOptions::default());
 
             PdfProcessResult {
                 pdf_type: PdfType::TextBased,
-                text: Some(text),
+                text: None,
                 markdown: Some(markdown),
                 page_count: detection.page_count,
                 processing_time_ms: start.elapsed().as_millis() as u64,
@@ -115,14 +113,12 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
             processing_time_ms: start.elapsed().as_millis() as u64,
         },
         PdfType::Mixed => {
-            let text = extractor::extract_text_mem(buffer).ok();
-            let markdown = text
-                .as_ref()
-                .map(|t| to_markdown(t, MarkdownOptions::default()));
+            let items = extractor::extract_text_with_positions_mem(buffer).ok();
+            let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default()));
 
             PdfProcessResult {
                 pdf_type: PdfType::Mixed,
-                text,
+                text: None,
                 markdown,
                 page_count: detection.page_count,
                 processing_time_ms: start.elapsed().as_millis() as u64,

src/markdown.rs

@@ -140,7 +140,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
         }
 
         // Detect headers by font size
-        if options.detect_headers {
+        // Skip very short text (likely drop caps or labels)
+        if options.detect_headers && trimmed.len() > 3 {
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
             if let Some(header_level) = detect_header_level(line_font_size, base_size) {
                 let prefix = "#".repeat(header_level);