From 5e44e468ac262ee0dc005fd1e8ab261e61117d21 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Sun, 8 Feb 2026 21:58:02 -0800 Subject: [PATCH] Formatting Improvements 1. Word Fragment Joining (extractor.rs:63-94) Added is_word_continuation() to detect when text items are fragments of the same word and should be joined without spaces: 2. Caption Detection (markdown.rs:621-658) Added is_caption_line() to detect figures, tables, and source citations: - Ensures captions are on their own line with paragraph breaks 3. Paragraph Threshold Adjustment Changed from base_size * 2.0 to base_size * 1.8 for better paragraph detection. --- src/extractor.rs | 42 +++++++++++++++++++++++++++- src/markdown.rs | 72 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/src/extractor.rs b/src/extractor.rs index 045608a..2bc6a6d 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -62,13 +62,19 @@ impl TextLine { // Previous item was subscript/superscript (returning to normal size) let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; + // Detect word fragments that should be joined without space + // This happens when a word is broken across text elements + // e.g., "ve" + "ntos" should become "ventos" not "ve ntos" + let is_word_fragment = is_word_continuation(&result, text); + if prev_ends_with_hyphen || curr_is_hyphen || curr_starts_with_hyphen || is_sub_super || was_sub_super + || is_word_fragment { - // No space for hyphenated words or subscript/superscript + // No space for hyphenated words, subscript/superscript, or word fragments result.push_str(text); } else { result.push(' '); @@ -80,6 +86,40 @@ impl TextLine { } } +/// Check if the current text is a continuation of a word from the previous text +/// Returns true if the items should be joined without a space +fn is_word_continuation(prev_text: &str, curr_text: &str) -> bool { + // Get the last character of previous text (excluding trailing spaces) + let prev_trimmed = prev_text.trim_end(); + let last_char = match prev_trimmed.chars().last() { + Some(c) => c, + None => return false, + }; + + // Get the first character of current text (excluding leading spaces) + let curr_trimmed = curr_text.trim_start(); + let first_char = match curr_trimmed.chars().next() { + Some(c) => c, + None => return false, + }; + + // If previous ends with a letter and current starts with a lowercase letter, + // this is likely a word fragment that should be joined + // e.g., "ve" + "ntos" -> "ventos" + if last_char.is_alphabetic() && first_char.is_lowercase() { + // Additional check: previous should not end with a space + // and current should not start with a space in the original + let prev_ends_with_space = prev_text.ends_with(' '); + let curr_starts_with_space = curr_text.starts_with(' '); + + if !prev_ends_with_space && !curr_starts_with_space { + return true; + } + } + + false +} + /// Extract text from PDF file as plain string pub fn extract_text>(path: P) -> Result { let doc = Document::load(path)?; diff --git a/src/markdown.rs b/src/markdown.rs index e59e9b5..7cfd4e7 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -264,7 +264,7 @@ fn to_markdown_from_lines_with_tables( // Paragraph break (large Y gap) let y_gap = prev_y - line.y; - let is_para_break = y_gap > base_size * 2.0; + let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold if is_para_break { if in_paragraph { output.push_str("\n\n"); @@ -283,6 +283,18 @@ fn to_markdown_from_lines_with_tables( continue; } + // Detect figure/table captions and source citations + // These should be on their own line followed by a paragraph break + if is_caption_line(trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(trimmed); + output.push_str("\n\n"); + continue; + } + // Detect headers by font size if options.detect_headers && trimmed.len() > 3 { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); @@ -395,7 +407,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Paragraph break (large Y gap) let y_gap = prev_y - line.y; - let is_para_break = y_gap > base_size * 2.0; + let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold if is_para_break { if in_paragraph { output.push_str("\n\n"); @@ -414,6 +426,18 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> continue; } + // Detect figure/table captions and source citations + // These should be on their own line followed by a paragraph break + if is_caption_line(trimmed) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(trimmed); + output.push_str("\n\n"); + continue; + } + // Detect headers by font size // Skip very short text (likely drop caps or labels) if options.detect_headers && trimmed.len() > 3 { @@ -605,6 +629,50 @@ fn detect_header_level(font_size: f32, base_size: f32) -> Option { } } +/// Check if text is a figure/table caption or source citation +fn is_caption_line(text: &str) -> bool { + let trimmed = text.trim(); + + // Common caption prefixes in multiple languages + let caption_prefixes = [ + "Figure ", + "Figura ", + "Fig. ", + "Fig ", + "Table ", + "Tabela ", + "Source:", + "Fonte:", + "Source ", + "Fonte ", + "Note:", + "Nota:", + "Chart ", + "Gráfico ", + "Graph ", + "Diagram ", + "Image ", + "Imagem ", + "Photo ", + "Foto ", + ]; + + // Check if line starts with a caption prefix + for prefix in &caption_prefixes { + if trimmed.starts_with(prefix) { + return true; + } + } + + // Check case-insensitive patterns + let lower = trimmed.to_lowercase(); + if lower.starts_with("figure ") || lower.starts_with("table ") || lower.starts_with("source:") { + return true; + } + + false +} + /// Check if text looks like a list item fn is_list_item(text: &str) -> bool { let trimmed = text.trim_start();