Abimael MartellandClaude Opus 4.6
e9c0737bd8
Detect newspaper-style two-column layouts (dense independent text flows) and emit columns sequentially instead of Y-interleaving them. This fixes academic papers where column text was being merged across columns at the same Y position. Key changes: - Add is_newspaper_layout() heuristic (≥15 lines/col, >50% Y-collision) - Add split_column_stragglers() to separate core column clusters from header remnants and per-word items that landed in column buckets - Filter wide items (>60% page width) from column detection histogram so full-width text doesn't fill the gutter and prevent detection - Emit: above spanning → core columns sequentially → column stragglers sequentially → below spanning items - Fix markdown paragraph break to detect backward Y jumps (y_gap.abs()) so sequential columns on the same page get proper paragraph breaks Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1806 lines
60 KiB
Rust
1806 lines
60 KiB
Rust
//! Markdown conversion with structure detection
|
|
//!
|
|
//! This module converts extracted text to markdown, detecting:
|
|
//! - Headers (by font size)
|
|
//! - Lists (bullet points, numbered lists)
|
|
//! - Code blocks (monospace fonts, indentation)
|
|
//! - Paragraphs
|
|
|
|
use crate::extractor::{group_into_lines, TextItem, TextLine};
|
|
use std::collections::{HashMap, HashSet};
|
|
|
|
use regex::Regex;
|
|
|
|
/// Options for markdown conversion
|
|
#[derive(Debug, Clone)]
|
|
pub struct MarkdownOptions {
|
|
/// Detect headers by font size
|
|
pub detect_headers: bool,
|
|
/// Detect list items
|
|
pub detect_lists: bool,
|
|
/// Detect code blocks
|
|
pub detect_code: bool,
|
|
/// Base font size for comparison
|
|
pub base_font_size: Option<f32>,
|
|
/// Remove standalone page numbers
|
|
pub remove_page_numbers: bool,
|
|
/// Convert URLs to markdown links
|
|
pub format_urls: bool,
|
|
/// Fix hyphenation (broken words across lines)
|
|
pub fix_hyphenation: bool,
|
|
/// Detect and format bold text from font names
|
|
pub detect_bold: bool,
|
|
/// Detect and format italic text from font names
|
|
pub detect_italic: bool,
|
|
/// Include image placeholders in output
|
|
pub include_images: bool,
|
|
/// Include extracted hyperlinks
|
|
pub include_links: bool,
|
|
/// Insert page break markers (<!-- Page N -->) between pages
|
|
pub include_page_numbers: bool,
|
|
}
|
|
|
|
impl Default for MarkdownOptions {
|
|
fn default() -> Self {
|
|
Self {
|
|
detect_headers: true,
|
|
detect_lists: true,
|
|
detect_code: true,
|
|
base_font_size: None,
|
|
remove_page_numbers: true,
|
|
format_urls: true,
|
|
fix_hyphenation: true,
|
|
detect_bold: true,
|
|
detect_italic: true,
|
|
include_images: true,
|
|
include_links: true,
|
|
include_page_numbers: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert plain text to markdown (basic conversion)
|
|
pub fn to_markdown(text: &str, options: MarkdownOptions) -> String {
|
|
let mut output = String::new();
|
|
let mut in_list = false;
|
|
let mut in_code_block = false;
|
|
|
|
for line in text.lines() {
|
|
let trimmed = line.trim();
|
|
|
|
if trimmed.is_empty() {
|
|
if in_list {
|
|
in_list = false;
|
|
}
|
|
if in_code_block {
|
|
output.push_str("```\n");
|
|
in_code_block = false;
|
|
}
|
|
output.push('\n');
|
|
continue;
|
|
}
|
|
|
|
// Detect list items
|
|
if options.detect_lists && is_list_item(trimmed) {
|
|
let formatted = format_list_item(trimmed);
|
|
output.push_str(&formatted);
|
|
output.push('\n');
|
|
in_list = true;
|
|
continue;
|
|
}
|
|
|
|
// Detect code blocks (indented lines)
|
|
if options.detect_code && is_code_like(trimmed) {
|
|
if !in_code_block {
|
|
output.push_str("```\n");
|
|
in_code_block = true;
|
|
}
|
|
output.push_str(trimmed);
|
|
output.push('\n');
|
|
continue;
|
|
} else if in_code_block {
|
|
output.push_str("```\n");
|
|
in_code_block = false;
|
|
}
|
|
|
|
// Regular paragraph text
|
|
output.push_str(trimmed);
|
|
output.push('\n');
|
|
}
|
|
|
|
if in_code_block {
|
|
output.push_str("```\n");
|
|
}
|
|
|
|
output
|
|
}
|
|
|
|
/// Convert positioned text items to markdown with structure detection
|
|
pub fn to_markdown_from_items(items: Vec<TextItem>, options: MarkdownOptions) -> String {
|
|
to_markdown_from_items_with_rects(items, options, &[])
|
|
}
|
|
|
|
/// Convert positioned text items to markdown, using rectangle data for table detection
|
|
pub fn to_markdown_from_items_with_rects(
|
|
items: Vec<TextItem>,
|
|
options: MarkdownOptions,
|
|
rects: &[crate::extractor::PdfRect],
|
|
) -> String {
|
|
use crate::extractor::ItemType;
|
|
use crate::tables::{detect_tables, detect_tables_from_rects, table_to_markdown};
|
|
use std::collections::HashSet;
|
|
|
|
if items.is_empty() {
|
|
return String::new();
|
|
}
|
|
|
|
// Separate images and links from text items
|
|
let mut images: Vec<TextItem> = Vec::new();
|
|
let mut links: Vec<TextItem> = Vec::new();
|
|
let mut text_items: Vec<TextItem> = Vec::new();
|
|
|
|
for item in items {
|
|
match &item.item_type {
|
|
ItemType::Image => {
|
|
if options.include_images {
|
|
images.push(item);
|
|
}
|
|
}
|
|
ItemType::Link(_) => {
|
|
if options.include_links {
|
|
links.push(item);
|
|
}
|
|
}
|
|
ItemType::Text | ItemType::FormField => {
|
|
text_items.push(item);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Calculate base font size for table detection
|
|
let font_stats = calculate_font_stats_from_items(&text_items);
|
|
let base_size = options
|
|
.base_font_size
|
|
.unwrap_or(font_stats.most_common_size);
|
|
|
|
// Detect tables on each page
|
|
let mut table_items: HashSet<usize> = HashSet::new();
|
|
let mut page_tables: std::collections::HashMap<u32, Vec<(f32, String)>> =
|
|
std::collections::HashMap::new();
|
|
|
|
// Store images by page and Y position for insertion
|
|
let mut page_images: std::collections::HashMap<u32, Vec<(f32, String)>> =
|
|
std::collections::HashMap::new();
|
|
|
|
for img in &images {
|
|
// Extract image name from "[Image: Im0]" format
|
|
let img_name = img
|
|
.text
|
|
.strip_prefix("[Image: ")
|
|
.and_then(|s| s.strip_suffix(']'))
|
|
.unwrap_or(&img.text);
|
|
let img_md = format!("\n", img_name);
|
|
page_images
|
|
.entry(img.page)
|
|
.or_default()
|
|
.push((img.y, img_md));
|
|
}
|
|
|
|
// Pre-group items by page with their global indices (O(n) instead of O(pages*n))
|
|
let mut page_groups: HashMap<u32, Vec<(usize, &TextItem)>> = HashMap::new();
|
|
for (global_idx, item) in text_items.iter().enumerate() {
|
|
page_groups
|
|
.entry(item.page)
|
|
.or_default()
|
|
.push((global_idx, item));
|
|
}
|
|
|
|
let mut pages: Vec<u32> = page_groups.keys().copied().collect();
|
|
pages.sort();
|
|
|
|
for page in pages {
|
|
let group = page_groups.get(&page).unwrap();
|
|
let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();
|
|
|
|
// Track which local indices are claimed by rect-based tables
|
|
let mut rect_claimed: HashSet<usize> = HashSet::new();
|
|
|
|
// Try rectangle-based table detection first
|
|
let rect_tables = detect_tables_from_rects(&page_items, rects, page);
|
|
for table in &rect_tables {
|
|
for &idx in &table.item_indices {
|
|
rect_claimed.insert(idx);
|
|
if let Some(&(global_idx, _)) = group.get(idx) {
|
|
table_items.insert(global_idx);
|
|
}
|
|
}
|
|
let table_y = table.rows.first().copied().unwrap_or(0.0);
|
|
let table_md = table_to_markdown(table);
|
|
page_tables
|
|
.entry(page)
|
|
.or_default()
|
|
.push((table_y, table_md));
|
|
}
|
|
|
|
// Run heuristic detection on unclaimed items only
|
|
if rect_claimed.is_empty() {
|
|
// No rect tables — run heuristic on all items
|
|
let tables = detect_tables(&page_items, base_size, false);
|
|
for table in tables {
|
|
for &idx in &table.item_indices {
|
|
if let Some(&(global_idx, _)) = group.get(idx) {
|
|
table_items.insert(global_idx);
|
|
}
|
|
}
|
|
let table_y = table.rows.first().copied().unwrap_or(0.0);
|
|
let table_md = table_to_markdown(&table);
|
|
page_tables
|
|
.entry(page)
|
|
.or_default()
|
|
.push((table_y, table_md));
|
|
}
|
|
} else {
|
|
// Rect tables found — run heuristic on unclaimed items
|
|
let unclaimed_items: Vec<TextItem> = page_items
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(idx, _)| !rect_claimed.contains(idx))
|
|
.map(|(_, item)| item.clone())
|
|
.collect();
|
|
if unclaimed_items.len() >= 6 {
|
|
let tables = detect_tables(&unclaimed_items, base_size, false);
|
|
for table in tables {
|
|
// Remap indices from unclaimed-space back to page-space
|
|
let unclaimed_map: Vec<usize> = page_items
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(idx, _)| !rect_claimed.contains(idx))
|
|
.map(|(idx, _)| idx)
|
|
.collect();
|
|
for &idx in &table.item_indices {
|
|
if let Some(&page_idx) = unclaimed_map.get(idx) {
|
|
if let Some(&(global_idx, _)) = group.get(page_idx) {
|
|
table_items.insert(global_idx);
|
|
}
|
|
}
|
|
}
|
|
let table_y = table.rows.first().copied().unwrap_or(0.0);
|
|
let table_md = table_to_markdown(&table);
|
|
page_tables
|
|
.entry(page)
|
|
.or_default()
|
|
.push((table_y, table_md));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Filter out table items and process the rest
|
|
let non_table_items: Vec<TextItem> = text_items
|
|
.into_iter()
|
|
.enumerate()
|
|
.filter(|(idx, _)| !table_items.contains(idx))
|
|
.map(|(_, item)| item)
|
|
.collect();
|
|
|
|
// Find pages that are table-only (no remaining non-table text)
|
|
let table_only_pages: HashSet<u32> = {
|
|
let pages_with_text: HashSet<u32> = non_table_items.iter().map(|i| i.page).collect();
|
|
page_tables
|
|
.keys()
|
|
.filter(|p| !pages_with_text.contains(p))
|
|
.copied()
|
|
.collect()
|
|
};
|
|
|
|
// Merge continuation tables across page breaks, but only for table-only pages
|
|
merge_continuation_tables(&mut page_tables, &table_only_pages);
|
|
|
|
let lines = group_into_lines(non_table_items);
|
|
|
|
// Convert to markdown, inserting tables and images at appropriate positions
|
|
to_markdown_from_lines_with_tables_and_images(lines, options, page_tables, page_images)
|
|
}
|
|
|
|
/// Calculate font stats directly from items (before grouping into lines)
|
|
fn calculate_font_stats_from_items(items: &[TextItem]) -> FontStats {
|
|
let mut size_counts: HashMap<i32, usize> = HashMap::new();
|
|
|
|
for item in items {
|
|
if item.font_size >= 9.0 {
|
|
let size_key = (item.font_size * 10.0) as i32;
|
|
*size_counts.entry(size_key).or_insert(0) += 1;
|
|
}
|
|
}
|
|
|
|
let most_common_size = size_counts
|
|
.iter()
|
|
.max_by_key(|(_, count)| *count)
|
|
.map(|(size, _)| *size as f32 / 10.0)
|
|
.unwrap_or(12.0);
|
|
|
|
FontStats { most_common_size }
|
|
}
|
|
|
|
/// Merge continuation tables that span across page breaks.
|
|
///
|
|
/// When consecutive pages each have exactly one table with the same number of columns
|
|
/// AND both pages are table-only (no non-table text), treat them as a single table.
|
|
/// We strip their header+separator rows and append their data rows to the first page's
|
|
/// table, then remove them from later pages.
|
|
fn merge_continuation_tables(
|
|
page_tables: &mut std::collections::HashMap<u32, Vec<(f32, String)>>,
|
|
table_only_pages: &HashSet<u32>,
|
|
) {
|
|
let mut sorted_pages: Vec<u32> = page_tables.keys().copied().collect();
|
|
sorted_pages.sort();
|
|
|
|
if sorted_pages.len() < 2 {
|
|
return;
|
|
}
|
|
|
|
// Find runs of consecutive pages that each have exactly one table with matching columns
|
|
let mut i = 0;
|
|
while i < sorted_pages.len() {
|
|
let first_page = sorted_pages[i];
|
|
let first_tables = match page_tables.get(&first_page) {
|
|
Some(t) if t.len() == 1 => t,
|
|
_ => {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
};
|
|
|
|
// First page must be table-only to start a merge chain
|
|
if !table_only_pages.contains(&first_page) {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
|
|
let first_col_count = count_table_columns(&first_tables[0].1);
|
|
if first_col_count == 0 {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
|
|
// Collect continuation pages (must also be table-only)
|
|
let mut continuation_pages = Vec::new();
|
|
let mut j = i + 1;
|
|
while j < sorted_pages.len() {
|
|
let next_page = sorted_pages[j];
|
|
// Must be consecutive page numbers
|
|
let prev_page = if continuation_pages.is_empty() {
|
|
first_page
|
|
} else {
|
|
*continuation_pages.last().unwrap()
|
|
};
|
|
if next_page != prev_page + 1 {
|
|
break;
|
|
}
|
|
|
|
// Continuation page must be table-only
|
|
if !table_only_pages.contains(&next_page) {
|
|
break;
|
|
}
|
|
|
|
let next_tables = match page_tables.get(&next_page) {
|
|
Some(t) if t.len() == 1 => t,
|
|
_ => break,
|
|
};
|
|
|
|
let next_col_count = count_table_columns(&next_tables[0].1);
|
|
if next_col_count != first_col_count {
|
|
break;
|
|
}
|
|
|
|
continuation_pages.push(next_page);
|
|
j += 1;
|
|
}
|
|
|
|
if !continuation_pages.is_empty() {
|
|
// Collect data rows from continuation pages
|
|
let mut extra_rows = String::new();
|
|
for &cont_page in &continuation_pages {
|
|
if let Some(tables) = page_tables.get(&cont_page) {
|
|
let table_md = &tables[0].1;
|
|
// Skip header row (line 1) and separator row (line 2), keep the rest
|
|
for (line_idx, line) in table_md.lines().enumerate() {
|
|
if line_idx >= 2 {
|
|
extra_rows.push_str(line);
|
|
extra_rows.push('\n');
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Append continuation rows to the first page's table
|
|
if let Some(tables) = page_tables.get_mut(&first_page) {
|
|
tables[0].1.push_str(&extra_rows);
|
|
}
|
|
|
|
// Remove continuation pages from the map
|
|
for &cont_page in &continuation_pages {
|
|
page_tables.remove(&cont_page);
|
|
}
|
|
|
|
// Skip past the merged pages
|
|
i = j;
|
|
} else {
|
|
i += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Count the number of columns in a markdown table by counting `|` in the separator row.
|
|
fn count_table_columns(table_md: &str) -> usize {
|
|
// The separator row is the second line, containing "| --- | --- |"
|
|
if let Some(sep_line) = table_md.lines().nth(1) {
|
|
if sep_line.contains("---") {
|
|
// Count cells: number of | minus 1 (leading |), but handle edge cases
|
|
let pipes = sep_line.chars().filter(|&c| c == '|').count();
|
|
return if pipes >= 2 { pipes - 1 } else { 0 };
|
|
}
|
|
}
|
|
0
|
|
}
|
|
|
|
/// Flush any remaining tables and images for a given page
|
|
fn flush_page_tables_and_images(
|
|
page: u32,
|
|
page_tables: &std::collections::HashMap<u32, Vec<(f32, String)>>,
|
|
page_images: &std::collections::HashMap<u32, Vec<(f32, String)>>,
|
|
inserted_tables: &mut HashSet<(u32, usize)>,
|
|
inserted_images: &mut HashSet<(u32, usize)>,
|
|
output: &mut String,
|
|
in_paragraph: &mut bool,
|
|
) {
|
|
if let Some(tables) = page_tables.get(&page) {
|
|
for (idx, (_, table_md)) in tables.iter().enumerate() {
|
|
if !inserted_tables.contains(&(page, idx)) {
|
|
if *in_paragraph {
|
|
output.push_str("\n\n");
|
|
*in_paragraph = false;
|
|
}
|
|
output.push('\n');
|
|
output.push_str(table_md);
|
|
output.push('\n');
|
|
inserted_tables.insert((page, idx));
|
|
}
|
|
}
|
|
}
|
|
if let Some(images) = page_images.get(&page) {
|
|
for (idx, (_, image_md)) in images.iter().enumerate() {
|
|
if !inserted_images.contains(&(page, idx)) {
|
|
if *in_paragraph {
|
|
output.push_str("\n\n");
|
|
*in_paragraph = false;
|
|
}
|
|
output.push('\n');
|
|
output.push_str(image_md);
|
|
output.push('\n');
|
|
inserted_images.insert((page, idx));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert text lines to markdown, inserting tables and images at appropriate Y positions
|
|
fn to_markdown_from_lines_with_tables_and_images(
|
|
lines: Vec<TextLine>,
|
|
options: MarkdownOptions,
|
|
page_tables: std::collections::HashMap<u32, Vec<(f32, String)>>,
|
|
page_images: std::collections::HashMap<u32, Vec<(f32, String)>>,
|
|
) -> String {
|
|
if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() {
|
|
return String::new();
|
|
}
|
|
|
|
// Calculate font statistics
|
|
let font_stats = calculate_font_stats(&lines);
|
|
let base_size = options
|
|
.base_font_size
|
|
.unwrap_or(font_stats.most_common_size);
|
|
|
|
// Merge drop caps with following text
|
|
let lines = merge_drop_caps(lines, base_size);
|
|
|
|
// Discover heading tiers for this document
|
|
let heading_tiers = compute_heading_tiers(&lines, base_size);
|
|
|
|
// Merge consecutive heading lines at the same level (e.g., wrapped titles)
|
|
let lines = merge_heading_lines(lines, base_size, &heading_tiers);
|
|
|
|
// Compute the typical line spacing for paragraph break detection.
|
|
// For double-spaced documents (like legal/government PDFs), the normal
|
|
// line spacing can be 2.3x base_size, which would exceed a fixed 1.8x
|
|
// threshold and cause every line to be treated as a paragraph break.
|
|
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
|
|
|
let mut output = String::new();
|
|
let mut current_page = 0u32;
|
|
let mut prev_y = f32::MAX;
|
|
let mut in_list = false;
|
|
let mut in_paragraph = false;
|
|
let mut last_list_x: Option<f32> = None;
|
|
let mut prev_had_dot_leaders = false;
|
|
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
|
|
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
|
|
|
|
// Collect all pages that have tables or images (including image-only pages)
|
|
let mut all_content_pages: Vec<u32> = page_tables
|
|
.keys()
|
|
.chain(page_images.keys())
|
|
.copied()
|
|
.collect();
|
|
all_content_pages.sort();
|
|
all_content_pages.dedup();
|
|
|
|
for line in lines {
|
|
// Page break
|
|
if line.page != current_page {
|
|
// Flush current page's remaining tables and images
|
|
if current_page > 0 {
|
|
flush_page_tables_and_images(
|
|
current_page,
|
|
&page_tables,
|
|
&page_images,
|
|
&mut inserted_tables,
|
|
&mut inserted_images,
|
|
&mut output,
|
|
&mut in_paragraph,
|
|
);
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push_str("\n\n");
|
|
}
|
|
|
|
// Flush any intermediate pages (image-only or table-only) between
|
|
// current_page and line.page that have no text lines
|
|
for &p in &all_content_pages {
|
|
if p <= current_page {
|
|
continue;
|
|
}
|
|
if p >= line.page {
|
|
break;
|
|
}
|
|
flush_page_tables_and_images(
|
|
p,
|
|
&page_tables,
|
|
&page_images,
|
|
&mut inserted_tables,
|
|
&mut inserted_images,
|
|
&mut output,
|
|
&mut in_paragraph,
|
|
);
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push_str("\n\n");
|
|
}
|
|
|
|
current_page = line.page;
|
|
prev_y = f32::MAX;
|
|
|
|
if options.include_page_numbers {
|
|
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
|
}
|
|
}
|
|
|
|
// Check if we should insert a table before this line
|
|
if let Some(tables) = page_tables.get(¤t_page) {
|
|
for (idx, (table_y, table_md)) in tables.iter().enumerate() {
|
|
// Insert table when we pass its Y position
|
|
if *table_y > line.y && !inserted_tables.contains(&(current_page, idx)) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push('\n');
|
|
output.push_str(table_md);
|
|
output.push('\n');
|
|
inserted_tables.insert((current_page, idx));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if we should insert an image before this line
|
|
if let Some(images) = page_images.get(¤t_page) {
|
|
for (idx, (image_y, image_md)) in images.iter().enumerate() {
|
|
// Insert image when we pass its Y position
|
|
if *image_y > line.y && !inserted_images.contains(&(current_page, idx)) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push('\n');
|
|
output.push_str(image_md);
|
|
output.push('\n');
|
|
inserted_images.insert((current_page, idx));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Paragraph break: large forward Y gap (normal) or large backward jump
|
|
// (newspaper columns emitted sequentially on the same page).
|
|
let y_gap = prev_y - line.y;
|
|
let is_para_break = y_gap.abs() > para_threshold;
|
|
if is_para_break && in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
// Don't immediately end list on paragraph break
|
|
// Let the continuation check below decide if we're still in a list
|
|
prev_y = line.y;
|
|
|
|
// Get text with optional bold/italic formatting
|
|
let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
|
|
let trimmed = text.trim();
|
|
|
|
// Also get plain text for pattern matching (list detection, captions, etc.)
|
|
let plain_text = line.text();
|
|
let plain_trimmed = plain_text.trim();
|
|
|
|
if trimmed.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Detect figure/table captions and source citations
|
|
// These should be on their own line followed by a paragraph break
|
|
if is_caption_line(plain_trimmed) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push_str(trimmed);
|
|
output.push_str("\n\n");
|
|
continue;
|
|
}
|
|
|
|
// Detect headers by font size
|
|
// Note: Headers typically shouldn't have bold markers since they're already emphasized
|
|
// Skip very short text (drop caps/labels) and very long text (body paragraphs)
|
|
if options.detect_headers
|
|
&& plain_trimmed.len() > 3
|
|
&& plain_trimmed.split_whitespace().count() <= 15
|
|
{
|
|
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
|
if let Some(header_level) =
|
|
detect_header_level(line_font_size, base_size, &heading_tiers)
|
|
{
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
let prefix = "#".repeat(header_level);
|
|
// Use plain text for headers to avoid redundant formatting
|
|
output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
|
|
in_list = false;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Detect list items
|
|
if options.detect_lists && is_list_item(plain_trimmed) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
let formatted = format_list_item(trimmed);
|
|
output.push_str(&formatted);
|
|
output.push('\n');
|
|
in_list = true;
|
|
last_list_x = line.items.first().map(|i| i.x);
|
|
continue;
|
|
} else if in_list {
|
|
// Check if this line is a continuation of the previous list item
|
|
// Continuations have similar X position and reasonable Y gap
|
|
let line_x = line.items.first().map(|i| i.x);
|
|
let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) {
|
|
// Continuation criteria:
|
|
// 1. X is at or past the list text position
|
|
// 2. Y gap is not too large (max ~5 line heights)
|
|
// 3. Not a new list item
|
|
let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0;
|
|
let y_ok = y_gap < base_size * 7.0;
|
|
x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed)
|
|
} else {
|
|
false
|
|
};
|
|
|
|
if is_continuation {
|
|
// Append to previous list item with a space
|
|
if output.ends_with('\n') {
|
|
output.pop();
|
|
output.push(' ');
|
|
}
|
|
output.push_str(trimmed);
|
|
output.push('\n');
|
|
continue;
|
|
} else {
|
|
in_list = false;
|
|
last_list_x = None;
|
|
}
|
|
}
|
|
|
|
// Detect code blocks by font
|
|
if options.detect_code {
|
|
let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font));
|
|
if is_mono {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
// Use plain text for code blocks
|
|
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Regular text - join lines within same paragraph with space
|
|
let cur_dot_leaders = has_dot_leaders(plain_trimmed);
|
|
if in_paragraph {
|
|
if cur_dot_leaders || prev_had_dot_leaders {
|
|
output.push('\n');
|
|
} else {
|
|
output.push(' ');
|
|
}
|
|
}
|
|
output.push_str(trimmed);
|
|
in_paragraph = true;
|
|
prev_had_dot_leaders = cur_dot_leaders;
|
|
}
|
|
|
|
// Flush current page and any remaining pages with tables/images
|
|
// (handles table-only pages after the last text line, and trailing image-only pages)
|
|
flush_page_tables_and_images(
|
|
current_page,
|
|
&page_tables,
|
|
&page_images,
|
|
&mut inserted_tables,
|
|
&mut inserted_images,
|
|
&mut output,
|
|
&mut in_paragraph,
|
|
);
|
|
for &p in &all_content_pages {
|
|
if p <= current_page {
|
|
continue;
|
|
}
|
|
flush_page_tables_and_images(
|
|
p,
|
|
&page_tables,
|
|
&page_images,
|
|
&mut inserted_tables,
|
|
&mut inserted_images,
|
|
&mut output,
|
|
&mut in_paragraph,
|
|
);
|
|
}
|
|
|
|
// Close final paragraph
|
|
if in_paragraph {
|
|
output.push('\n');
|
|
}
|
|
|
|
// Clean up and post-process
|
|
clean_markdown(output, &options)
|
|
}
|
|
|
|
/// Convert text lines to markdown
|
|
pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) -> String {
|
|
if lines.is_empty() {
|
|
return String::new();
|
|
}
|
|
|
|
// Calculate font statistics
|
|
let font_stats = calculate_font_stats(&lines);
|
|
let base_size = options
|
|
.base_font_size
|
|
.unwrap_or(font_stats.most_common_size);
|
|
|
|
// Merge drop caps with following text
|
|
let lines = merge_drop_caps(lines, base_size);
|
|
|
|
// Discover heading tiers for this document
|
|
let heading_tiers = compute_heading_tiers(&lines, base_size);
|
|
|
|
// Merge consecutive heading lines at the same level (e.g., wrapped titles)
|
|
let lines = merge_heading_lines(lines, base_size, &heading_tiers);
|
|
|
|
// Compute the typical line spacing for paragraph break detection
|
|
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
|
|
|
let mut output = String::new();
|
|
let mut current_page = 0u32;
|
|
let mut prev_y = f32::MAX;
|
|
let mut in_list = false;
|
|
let mut in_paragraph = false;
|
|
let mut last_list_x: Option<f32> = None;
|
|
let mut prev_had_dot_leaders = false;
|
|
|
|
for line in lines {
|
|
// Page break
|
|
if line.page != current_page {
|
|
if current_page > 0 {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push_str("\n\n");
|
|
}
|
|
current_page = line.page;
|
|
prev_y = f32::MAX;
|
|
in_list = false;
|
|
last_list_x = None;
|
|
prev_had_dot_leaders = false;
|
|
|
|
if options.include_page_numbers {
|
|
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
|
}
|
|
}
|
|
|
|
// Paragraph break: large forward Y gap (normal) or large backward jump
|
|
// (newspaper columns emitted sequentially on the same page).
|
|
let y_gap = prev_y - line.y;
|
|
let is_para_break = y_gap.abs() > para_threshold;
|
|
if is_para_break && in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
// Don't immediately end list on paragraph break
|
|
// Let the continuation check below decide if we're still in a list
|
|
prev_y = line.y;
|
|
|
|
// Get text with optional bold/italic formatting
|
|
let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
|
|
let trimmed = text.trim();
|
|
|
|
// Also get plain text for pattern matching
|
|
let plain_text = line.text();
|
|
let plain_trimmed = plain_text.trim();
|
|
|
|
if trimmed.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Detect figure/table captions and source citations
|
|
// These should be on their own line followed by a paragraph break
|
|
if is_caption_line(plain_trimmed) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
output.push_str(trimmed);
|
|
output.push_str("\n\n");
|
|
continue;
|
|
}
|
|
|
|
// Detect headers by font size
|
|
// Skip very short text (drop caps/labels) and very long text (body paragraphs)
|
|
if options.detect_headers
|
|
&& plain_trimmed.len() > 3
|
|
&& plain_trimmed.split_whitespace().count() <= 15
|
|
{
|
|
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
|
if let Some(header_level) =
|
|
detect_header_level(line_font_size, base_size, &heading_tiers)
|
|
{
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
let prefix = "#".repeat(header_level);
|
|
// Use plain text for headers to avoid redundant formatting
|
|
output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
|
|
in_list = false;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Detect list items
|
|
if options.detect_lists && is_list_item(plain_trimmed) {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
let formatted = format_list_item(trimmed);
|
|
output.push_str(&formatted);
|
|
output.push('\n');
|
|
in_list = true;
|
|
last_list_x = line.items.first().map(|i| i.x);
|
|
continue;
|
|
} else if in_list {
|
|
// Check if this line is a continuation of the previous list item
|
|
let line_x = line.items.first().map(|i| i.x);
|
|
let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) {
|
|
// Continuation criteria:
|
|
// 1. X is at or past the list text position
|
|
// 2. Y gap is not too large (max ~5 line heights)
|
|
// 3. Not a new list item
|
|
let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0;
|
|
let y_ok = y_gap < base_size * 7.0;
|
|
x_ok && y_ok && !is_list_item(plain_trimmed) && !has_dot_leaders(plain_trimmed)
|
|
} else {
|
|
false
|
|
};
|
|
|
|
if is_continuation {
|
|
// Append to previous list item with a space
|
|
if output.ends_with('\n') {
|
|
output.pop();
|
|
output.push(' ');
|
|
}
|
|
output.push_str(trimmed);
|
|
output.push('\n');
|
|
continue;
|
|
} else {
|
|
in_list = false;
|
|
last_list_x = None;
|
|
}
|
|
}
|
|
|
|
// Detect code blocks by font
|
|
if options.detect_code {
|
|
let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font));
|
|
if is_mono {
|
|
if in_paragraph {
|
|
output.push_str("\n\n");
|
|
in_paragraph = false;
|
|
}
|
|
// Use plain text for code blocks
|
|
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Regular text - join lines within same paragraph with space
|
|
let cur_dot_leaders = has_dot_leaders(plain_trimmed);
|
|
if in_paragraph {
|
|
if cur_dot_leaders || prev_had_dot_leaders {
|
|
output.push('\n');
|
|
} else {
|
|
output.push(' ');
|
|
}
|
|
}
|
|
output.push_str(trimmed);
|
|
in_paragraph = true;
|
|
prev_had_dot_leaders = cur_dot_leaders;
|
|
}
|
|
|
|
// Close final paragraph
|
|
if in_paragraph {
|
|
output.push('\n');
|
|
}
|
|
|
|
// Clean up and post-process
|
|
clean_markdown(output, &options)
|
|
}
|
|
|
|
/// Merge drop caps with the appropriate line
|
|
/// A drop cap is a single large letter at the start of a paragraph
|
|
/// Due to PDF coordinate sorting, the drop cap may appear AFTER the line it belongs to
|
|
/// Merge consecutive heading lines at the same level into a single line.
|
|
///
|
|
/// When a heading wraps across multiple text lines (e.g., "About Glenair, the Mission-Critical"
|
|
/// and "Interconnect Company"), each fragment becomes a separate `# Header` in the output.
|
|
/// This function detects consecutive lines at the same heading tier on the same page
|
|
/// with a small Y gap and merges them into one line.
|
|
fn merge_heading_lines(
|
|
lines: Vec<TextLine>,
|
|
base_size: f32,
|
|
heading_tiers: &[f32],
|
|
) -> Vec<TextLine> {
|
|
if lines.is_empty() {
|
|
return lines;
|
|
}
|
|
|
|
let mut result: Vec<TextLine> = Vec::with_capacity(lines.len());
|
|
|
|
for line in lines {
|
|
let line_font = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
|
let line_level = detect_header_level(line_font, base_size, heading_tiers);
|
|
|
|
// Check if the previous line is a heading at the same level on the same page
|
|
let should_merge = if let (Some(prev), Some(curr_level)) = (result.last(), line_level) {
|
|
let prev_font = prev.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
|
let prev_level = detect_header_level(prev_font, base_size, heading_tiers);
|
|
let same_page = prev.page == line.page;
|
|
let same_level = prev_level == Some(curr_level);
|
|
let y_gap = prev.y - line.y;
|
|
// Merge if gap is within ~2x the font size (normal line wrap spacing)
|
|
let close_enough = y_gap > 0.0 && y_gap < line_font * 2.0;
|
|
same_page && same_level && close_enough
|
|
} else {
|
|
false
|
|
};
|
|
|
|
if should_merge {
|
|
// Append this line's items to the previous line
|
|
let prev = result.last_mut().unwrap();
|
|
// Add a space-bearing TextItem to separate the merged text
|
|
if let Some(first_item) = line.items.first() {
|
|
let mut space_item = first_item.clone();
|
|
space_item.text = format!(" {}", space_item.text.trim_start());
|
|
prev.items.push(space_item);
|
|
}
|
|
for item in line.items.into_iter().skip(1) {
|
|
prev.items.push(item);
|
|
}
|
|
} else {
|
|
result.push(line);
|
|
}
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
fn merge_drop_caps(lines: Vec<TextLine>, base_size: f32) -> Vec<TextLine> {
|
|
let mut result: Vec<TextLine> = Vec::with_capacity(lines.len());
|
|
|
|
for line in &lines {
|
|
let text = line.text();
|
|
let trimmed = text.trim();
|
|
|
|
// Check if this looks like a drop cap:
|
|
// 1. Single character (or single char + space)
|
|
// 2. Much larger than base font (3x or more)
|
|
// 3. The character is uppercase
|
|
let is_drop_cap = trimmed.len() <= 2
|
|
&& line.items.first().map(|i| i.font_size).unwrap_or(0.0) >= base_size * 2.5
|
|
&& trimmed
|
|
.chars()
|
|
.next()
|
|
.map(|c| c.is_uppercase())
|
|
.unwrap_or(false);
|
|
|
|
if is_drop_cap {
|
|
let drop_char = trimmed.chars().next().unwrap();
|
|
|
|
// Find the first line that starts with lowercase and is at the START of a paragraph
|
|
// (i.e., preceded by a header or non-lowercase-starting line)
|
|
let mut target_idx: Option<usize> = None;
|
|
|
|
for (idx, prev_line) in result.iter().enumerate() {
|
|
if prev_line.page != line.page {
|
|
continue;
|
|
}
|
|
|
|
let prev_text = prev_line.text();
|
|
let prev_trimmed = prev_text.trim();
|
|
|
|
// Check if this line starts with lowercase
|
|
if prev_trimmed
|
|
.chars()
|
|
.next()
|
|
.map(|c| c.is_lowercase())
|
|
.unwrap_or(false)
|
|
{
|
|
// Check if previous line exists and doesn't start with lowercase
|
|
// (meaning this is the start of a paragraph)
|
|
let is_para_start = if idx == 0 {
|
|
true
|
|
} else {
|
|
let before = result[idx - 1].text();
|
|
let before_trimmed = before.trim();
|
|
!before_trimmed
|
|
.chars()
|
|
.next()
|
|
.map(|c| c.is_lowercase())
|
|
.unwrap_or(true)
|
|
};
|
|
|
|
if is_para_start {
|
|
target_idx = Some(idx);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Merge with the target line
|
|
if let Some(idx) = target_idx {
|
|
if let Some(first_item) = result[idx].items.first_mut() {
|
|
let prev_text = first_item.text.trim().to_string();
|
|
first_item.text = format!("{}{}", drop_char, prev_text);
|
|
}
|
|
}
|
|
// Don't add the drop cap line itself
|
|
continue;
|
|
}
|
|
|
|
result.push(line.clone());
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
/// Font statistics for a document
|
|
struct FontStats {
|
|
most_common_size: f32,
|
|
}
|
|
|
|
fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
|
|
let mut size_counts: HashMap<i32, usize> = HashMap::new();
|
|
|
|
for line in lines {
|
|
// Count once per line (first item) to give each line equal weight
|
|
// Prevents small captions/footnotes from skewing the base
|
|
if let Some(first) = line.items.first() {
|
|
if first.font_size >= 9.0 {
|
|
let size_key = (first.font_size * 10.0) as i32;
|
|
*size_counts.entry(size_key).or_insert(0) += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let most_common_size = size_counts
|
|
.iter()
|
|
.max_by_key(|(_, count)| *count)
|
|
.map(|(size, _)| *size as f32 / 10.0)
|
|
.unwrap_or(12.0);
|
|
|
|
FontStats { most_common_size }
|
|
}
|
|
|
|
/// Detect TOC-style lines that contain dot leaders (e.g., "Section Name .... 42").
|
|
/// These lines should never be joined with adjacent lines into a paragraph.
|
|
/// Handles both consecutive dots ("....") and spaced dots ("... ...").
|
|
fn has_dot_leaders(text: &str) -> bool {
|
|
// Consecutive dots (4+)
|
|
if text.contains("....") {
|
|
return true;
|
|
}
|
|
// Spaced dot leaders: "..." followed by whitespace and more dots
|
|
// Count occurrences of "..." (3+ dots) — if 2+ groups, it's a dot leader
|
|
let mut dot_groups = 0;
|
|
let mut consecutive_dots = 0;
|
|
for ch in text.chars() {
|
|
if ch == '.' {
|
|
consecutive_dots += 1;
|
|
} else {
|
|
if consecutive_dots >= 3 {
|
|
dot_groups += 1;
|
|
}
|
|
consecutive_dots = 0;
|
|
}
|
|
}
|
|
if consecutive_dots >= 3 {
|
|
dot_groups += 1;
|
|
}
|
|
dot_groups >= 2
|
|
}
|
|
|
|
/// Compute the Y-gap threshold for paragraph break detection.
|
|
///
|
|
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
|
/// documents), we compute the document's typical (median) line spacing and use
|
|
/// a multiplier on that. A gap significantly larger than typical indicates a
|
|
/// paragraph break.
|
|
///
|
|
/// Fallback: if we can't compute typical spacing, use base_size * 1.8.
|
|
fn compute_paragraph_threshold(lines: &[TextLine], base_size: f32) -> f32 {
|
|
let fallback = base_size * 1.8;
|
|
|
|
// Collect Y gaps between consecutive lines on the same page
|
|
let mut gaps: Vec<f32> = Vec::new();
|
|
let mut prev_y: Option<(u32, f32)> = None;
|
|
|
|
for line in lines {
|
|
if let Some((prev_page, py)) = prev_y {
|
|
if line.page == prev_page {
|
|
let gap = py - line.y;
|
|
// Only consider positive gaps within a reasonable range
|
|
// (skip huge gaps from page headers/footers)
|
|
if gap > 0.0 && gap < base_size * 10.0 {
|
|
gaps.push(gap);
|
|
}
|
|
}
|
|
}
|
|
prev_y = Some((line.page, line.y));
|
|
}
|
|
|
|
if gaps.len() < 5 {
|
|
return fallback;
|
|
}
|
|
|
|
gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
|
|
|
let median = gaps[gaps.len() / 2];
|
|
|
|
// The paragraph threshold should be larger than the typical line spacing.
|
|
// Use 1.3x the median gap. This means:
|
|
// - Single-spaced (median ~14pt for 12pt font): threshold = 18.2pt
|
|
// - Double-spaced (median ~28pt for 12pt font): threshold = 36.4pt
|
|
// Also ensure it's at least base_size * 1.5 to avoid false paragraph breaks
|
|
// in tightly-spaced documents.
|
|
(median * 1.3).max(base_size * 1.5)
|
|
}
|
|
|
|
/// Discover distinct heading font-size tiers in the document.
|
|
/// Returns tiers sorted largest-first (tier 0 = H1, tier 1 = H2, …).
|
|
/// Sizes within 0.5pt are clustered into the same tier. Capped at 4 tiers.
|
|
fn compute_heading_tiers(lines: &[TextLine], base_size: f32) -> Vec<f32> {
|
|
let mut heading_sizes: Vec<f32> = Vec::new();
|
|
|
|
for line in lines {
|
|
if let Some(first) = line.items.first() {
|
|
if first.font_size / base_size >= 1.2 {
|
|
heading_sizes.push(first.font_size);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort descending
|
|
heading_sizes.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
|
|
|
|
// Cluster sizes within 0.5pt into same tier (use first value as representative)
|
|
let mut tiers: Vec<f32> = Vec::new();
|
|
for size in heading_sizes {
|
|
let already_in_tier = tiers.iter().any(|&t| (t - size).abs() < 0.5);
|
|
if !already_in_tier {
|
|
tiers.push(size);
|
|
}
|
|
}
|
|
|
|
// Cap at 4 tiers
|
|
tiers.truncate(4);
|
|
tiers
|
|
}
|
|
|
|
/// Detect header level from font size using document-specific heading tiers.
|
|
/// When tiers are available, maps tier 0→H1, tier 1→H2, etc.
|
|
/// Falls back to ratio-based thresholds when no tiers exist.
|
|
fn detect_header_level(font_size: f32, base_size: f32, heading_tiers: &[f32]) -> Option<usize> {
|
|
let ratio = font_size / base_size;
|
|
|
|
if ratio < 1.2 {
|
|
return None; // Regular text
|
|
}
|
|
|
|
if !heading_tiers.is_empty() {
|
|
// Match font_size to a tier (within 0.5pt tolerance)
|
|
for (i, &tier_size) in heading_tiers.iter().enumerate() {
|
|
if (font_size - tier_size).abs() < 0.5 {
|
|
return Some(i + 1); // tier 0 → H1, tier 1 → H2, etc.
|
|
}
|
|
}
|
|
// No tier match but large ratio — assign level after last tier
|
|
if ratio >= 1.5 {
|
|
let level = (heading_tiers.len() + 1).min(4);
|
|
return Some(level);
|
|
}
|
|
// No tier match and small ratio — not a heading
|
|
return None;
|
|
}
|
|
|
|
// Fallback: original ratio-based thresholds (no tiers discovered)
|
|
if ratio >= 2.0 {
|
|
Some(1)
|
|
} else if ratio >= 1.5 {
|
|
Some(2)
|
|
} else if ratio >= 1.25 {
|
|
Some(3)
|
|
} else {
|
|
Some(4)
|
|
}
|
|
}
|
|
|
|
/// Check if text is a figure/table caption or source citation
|
|
fn is_caption_line(text: &str) -> bool {
|
|
let trimmed = text.trim();
|
|
|
|
// Common caption prefixes in multiple languages
|
|
let caption_prefixes = [
|
|
"Figure ",
|
|
"Figura ",
|
|
"Fig. ",
|
|
"Fig ",
|
|
"Table ",
|
|
"Tabela ",
|
|
"Source:",
|
|
"Fonte:",
|
|
"Source ",
|
|
"Fonte ",
|
|
"Note:",
|
|
"Nota:",
|
|
"Chart ",
|
|
"Gráfico ",
|
|
"Graph ",
|
|
"Diagram ",
|
|
"Image ",
|
|
"Imagem ",
|
|
"Photo ",
|
|
"Foto ",
|
|
];
|
|
|
|
// Check if line starts with a caption prefix
|
|
for prefix in &caption_prefixes {
|
|
if trimmed.starts_with(prefix) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Check case-insensitive patterns
|
|
let lower = trimmed.to_lowercase();
|
|
if lower.starts_with("figure ") || lower.starts_with("table ") || lower.starts_with("source:") {
|
|
return true;
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Check if text looks like a list item
|
|
fn is_list_item(text: &str) -> bool {
|
|
let trimmed = text.trim_start();
|
|
|
|
// Bullet patterns
|
|
if trimmed.starts_with("• ")
|
|
|| trimmed.starts_with("- ")
|
|
|| trimmed.starts_with("* ")
|
|
|| trimmed.starts_with("○ ")
|
|
|| trimmed.starts_with("● ")
|
|
|| trimmed.starts_with("◦ ")
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Numbered list patterns: "1.", "1)", "(1)", "a.", "a)"
|
|
let first_chars: String = trimmed.chars().take(5).collect();
|
|
if first_chars.contains(|c: char| c.is_ascii_digit()) {
|
|
// Check for "1.", "1)", "10."
|
|
if let Some(idx) = first_chars.find(['.', ')']) {
|
|
let prefix = &first_chars[..idx];
|
|
if prefix.chars().all(|c| c.is_ascii_digit()) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Letter list: "a.", "a)", "(a)"
|
|
let mut chars = trimmed.chars();
|
|
if let (Some(first), Some(second)) = (chars.next(), chars.next()) {
|
|
if first.is_ascii_alphabetic() && (second == '.' || second == ')') {
|
|
return true;
|
|
}
|
|
if first == '(' && chars.next() == Some(')') {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Format list item to markdown
|
|
fn format_list_item(text: &str) -> String {
|
|
let trimmed = text.trim_start();
|
|
|
|
// Convert various bullet styles to markdown
|
|
// Note: bullet characters like • are multi-byte in UTF-8, use char indices
|
|
for bullet in &['•', '○', '●', '◦'] {
|
|
if let Some(rest) = trimmed.strip_prefix(*bullet) {
|
|
return format!("- {}", rest.trim_start());
|
|
}
|
|
}
|
|
|
|
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
|
|
return trimmed.to_string();
|
|
}
|
|
|
|
// Keep numbered lists as-is (markdown supports them)
|
|
trimmed.to_string()
|
|
}
|
|
|
|
/// Check if text looks like code
|
|
fn is_code_like(text: &str) -> bool {
|
|
let trimmed = text.trim();
|
|
|
|
// Code patterns
|
|
let code_patterns = [
|
|
// Language keywords
|
|
"import ",
|
|
"export ",
|
|
"from ",
|
|
"const ",
|
|
"let ",
|
|
"var ",
|
|
"function ",
|
|
"class ",
|
|
"def ",
|
|
"pub fn ",
|
|
"fn ",
|
|
"async fn ",
|
|
"impl ",
|
|
// Syntax patterns
|
|
"=> ",
|
|
"-> ",
|
|
":: ",
|
|
":= ",
|
|
// Common code endings
|
|
];
|
|
|
|
for pattern in &code_patterns {
|
|
if trimmed.starts_with(pattern) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Check for code-like syntax
|
|
let special_chars: usize = trimmed
|
|
.chars()
|
|
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
|
|
.count();
|
|
|
|
if special_chars >= 3 && trimmed.len() < 200 {
|
|
return true;
|
|
}
|
|
|
|
// Ends with semicolon or braces
|
|
if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') {
|
|
return true;
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Check if font name indicates monospace
|
|
fn is_monospace_font(font_name: &str) -> bool {
|
|
let lower = font_name.to_lowercase();
|
|
let patterns = [
|
|
"courier",
|
|
"consolas",
|
|
"monaco",
|
|
"menlo",
|
|
"mono",
|
|
"fixed",
|
|
"terminal",
|
|
"typewriter",
|
|
"source code",
|
|
"fira code",
|
|
"jetbrains",
|
|
"inconsolata",
|
|
"dejavu sans mono",
|
|
"liberation mono",
|
|
];
|
|
|
|
patterns.iter().any(|p| lower.contains(p))
|
|
}
|
|
|
|
/// Clean up markdown output with post-processing
|
|
fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String {
|
|
// Collapse dot leaders (e.g. TOC entries: "Introduction...............................1")
|
|
text = collapse_dot_leaders(&text);
|
|
|
|
// Fix hyphenation first (before other processing)
|
|
if options.fix_hyphenation {
|
|
text = fix_hyphenation(&text);
|
|
}
|
|
|
|
// Remove standalone page numbers
|
|
if options.remove_page_numbers {
|
|
text = remove_page_numbers(&text);
|
|
}
|
|
|
|
// Format URLs as markdown links
|
|
if options.format_urls {
|
|
text = format_urls(&text);
|
|
}
|
|
|
|
// Remove excessive newlines (more than 2 in a row)
|
|
while text.contains("\n\n\n") {
|
|
text = text.replace("\n\n\n", "\n\n");
|
|
}
|
|
|
|
// Trim leading and trailing whitespace, ensure ends with single newline
|
|
text = text.trim().to_string();
|
|
text.push('\n');
|
|
|
|
text
|
|
}
|
|
|
|
/// Collapse dot leaders (runs of 4+ dots) into " ... "
|
|
/// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
|
|
fn collapse_dot_leaders(text: &str) -> String {
|
|
use once_cell::sync::Lazy;
|
|
static DOT_LEADER_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"\.{4,}").unwrap());
|
|
|
|
DOT_LEADER_RE.replace_all(text, " ... ").to_string()
|
|
}
|
|
|
|
/// Fix words broken across lines with spaces before the continuation
|
|
/// e.g., "Limoeiro do Nort e" -> "Limoeiro do Norte"
|
|
fn fix_hyphenation(text: &str) -> String {
|
|
use once_cell::sync::Lazy;
|
|
|
|
// Fix "word - word" patterns that should be "word-word" (compound words)
|
|
// But be careful not to break list items (which start with "- ")
|
|
static SPACED_HYPHEN_RE: Lazy<Regex> = Lazy::new(|| {
|
|
Regex::new(r"([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ]) - ([a-zA-ZáàâãéèêíïóôõöúçñÁÀÂÃÉÈÊÍÏÓÔÕÖÚÇÑ])").unwrap()
|
|
});
|
|
|
|
let result = SPACED_HYPHEN_RE
|
|
.replace_all(text, |caps: ®ex::Captures| {
|
|
format!("{}-{}", &caps[1], &caps[2])
|
|
})
|
|
.to_string();
|
|
|
|
result
|
|
}
|
|
|
|
/// Remove standalone page numbers (lines that are just 1-4 digit numbers)
|
|
fn remove_page_numbers(text: &str) -> String {
|
|
let mut result = Vec::new();
|
|
let lines: Vec<&str> = text.lines().collect();
|
|
|
|
for (i, line) in lines.iter().enumerate() {
|
|
let trimmed = line.trim();
|
|
|
|
// Check for page number patterns
|
|
if is_page_number_line(trimmed) {
|
|
// Check context to determine if this is isolated
|
|
let prev_is_break = i > 0 && lines[i - 1].trim() == "---";
|
|
let next_is_break = i + 1 < lines.len() && lines[i + 1].trim() == "---";
|
|
let prev_is_empty = i > 0 && lines[i - 1].trim().is_empty();
|
|
let next_is_empty = i + 1 < lines.len() && lines[i + 1].trim().is_empty();
|
|
|
|
// Check if it's on its own line (surrounded by empty lines or page breaks)
|
|
let is_isolated = (prev_is_break || prev_is_empty || i == 0)
|
|
&& (next_is_break || next_is_empty || i + 1 == lines.len());
|
|
|
|
// Also remove numbers that appear right before a page break
|
|
let before_break = i + 1 < lines.len()
|
|
&& (lines[i + 1].trim() == "---"
|
|
|| (i + 2 < lines.len()
|
|
&& lines[i + 1].trim().is_empty()
|
|
&& lines[i + 2].trim() == "---"));
|
|
|
|
if is_isolated || before_break {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
result.push(*line);
|
|
}
|
|
|
|
result.join("\n")
|
|
}
|
|
|
|
/// Check if a line looks like a page number
|
|
fn is_page_number_line(trimmed: &str) -> bool {
|
|
// Empty lines are not page numbers
|
|
if trimmed.is_empty() {
|
|
return false;
|
|
}
|
|
|
|
// Pattern 1: Just a number (1-4 digits)
|
|
if trimmed.len() <= 4 && trimmed.chars().all(|c| c.is_ascii_digit()) {
|
|
return true;
|
|
}
|
|
|
|
// Pattern 2: "Page X of Y" or "Page X" or "Page of" (placeholder)
|
|
let lower = trimmed.to_lowercase();
|
|
if let Some(rest) = lower.strip_prefix("page") {
|
|
let rest = rest.trim();
|
|
// "Page of" (empty page numbers)
|
|
if rest == "of" || rest.starts_with("of ") {
|
|
return true;
|
|
}
|
|
// "Page X" or "Page X of Y"
|
|
if rest
|
|
.chars()
|
|
.next()
|
|
.map(|c| c.is_ascii_digit())
|
|
.unwrap_or(false)
|
|
{
|
|
return true;
|
|
}
|
|
// Just "Page" followed by whitespace and maybe "of"
|
|
if rest.is_empty()
|
|
|| rest
|
|
.split_whitespace()
|
|
.all(|w| w == "of" || w.chars().all(|c| c.is_ascii_digit()))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Pattern 3: "X of Y" where X and Y are numbers
|
|
if let Some(of_idx) = trimmed.find(" of ") {
|
|
let before = trimmed[..of_idx].trim();
|
|
let after = trimmed[of_idx + 4..].trim();
|
|
if before.chars().all(|c| c.is_ascii_digit())
|
|
&& after.chars().all(|c| c.is_ascii_digit())
|
|
&& !before.is_empty()
|
|
&& !after.is_empty()
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Pattern 4: "- X -" centered page number
|
|
if trimmed.len() >= 3 && trimmed.starts_with('-') && trimmed.ends_with('-') {
|
|
let inner = trimmed[1..trimmed.len() - 1].trim();
|
|
if inner.chars().all(|c| c.is_ascii_digit()) && !inner.is_empty() {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Convert URLs to markdown links
|
|
fn format_urls(text: &str) -> String {
|
|
use once_cell::sync::Lazy;
|
|
|
|
// Match URLs - we'll check context manually to avoid formatting already-linked URLs
|
|
static URL_RE: Lazy<Regex> =
|
|
Lazy::new(|| Regex::new(r"https?://[^\s<>\)\]]+[^\s<>\)\]\.\,;]").unwrap());
|
|
|
|
let mut result = String::with_capacity(text.len());
|
|
let mut last_end = 0;
|
|
|
|
for mat in URL_RE.find_iter(text) {
|
|
let start = mat.start();
|
|
let url = mat.as_str();
|
|
|
|
// Check if this URL is already in a markdown link by looking at preceding chars
|
|
// Use safe character boundary checking for multi-byte UTF-8
|
|
let before = {
|
|
let mut check_start = start.saturating_sub(2);
|
|
// Find a valid character boundary
|
|
while check_start > 0 && !text.is_char_boundary(check_start) {
|
|
check_start -= 1;
|
|
}
|
|
if check_start < start && text.is_char_boundary(start) {
|
|
&text[check_start..start]
|
|
} else {
|
|
""
|
|
}
|
|
};
|
|
let already_linked = before.ends_with("](") || before.ends_with("](");
|
|
|
|
// Also check if it's inside square brackets (link text)
|
|
// Ensure we're slicing at a valid char boundary
|
|
let prefix = if text.is_char_boundary(start) {
|
|
&text[..start]
|
|
} else {
|
|
// Find the nearest valid boundary before start
|
|
let mut safe_start = start;
|
|
while safe_start > 0 && !text.is_char_boundary(safe_start) {
|
|
safe_start -= 1;
|
|
}
|
|
&text[..safe_start]
|
|
};
|
|
let open_brackets = prefix.matches('[').count();
|
|
let close_brackets = prefix.matches(']').count();
|
|
let inside_link_text = open_brackets > close_brackets;
|
|
|
|
// Ensure mat boundaries are valid char boundaries
|
|
let safe_last_end = if text.is_char_boundary(last_end) {
|
|
last_end
|
|
} else {
|
|
let mut pos = last_end;
|
|
while pos < text.len() && !text.is_char_boundary(pos) {
|
|
pos += 1;
|
|
}
|
|
pos
|
|
};
|
|
let safe_start = if text.is_char_boundary(start) {
|
|
start
|
|
} else {
|
|
let mut pos = start;
|
|
while pos < text.len() && !text.is_char_boundary(pos) {
|
|
pos += 1;
|
|
}
|
|
pos
|
|
};
|
|
let safe_end = if text.is_char_boundary(mat.end()) {
|
|
mat.end()
|
|
} else {
|
|
let mut pos = mat.end();
|
|
while pos < text.len() && !text.is_char_boundary(pos) {
|
|
pos += 1;
|
|
}
|
|
pos
|
|
};
|
|
|
|
if already_linked || inside_link_text {
|
|
// Already formatted, keep as-is
|
|
if safe_last_end <= safe_end {
|
|
result.push_str(&text[safe_last_end..safe_end]);
|
|
}
|
|
} else {
|
|
// Add text before this URL
|
|
if safe_last_end <= safe_start {
|
|
result.push_str(&text[safe_last_end..safe_start]);
|
|
}
|
|
// Format as markdown link
|
|
result.push_str(&format!("[{}]({})", url, url));
|
|
}
|
|
last_end = safe_end;
|
|
}
|
|
|
|
// Add remaining text (ensure valid char boundary)
|
|
let safe_last_end = if text.is_char_boundary(last_end) {
|
|
last_end
|
|
} else {
|
|
let mut pos = last_end;
|
|
while pos < text.len() && !text.is_char_boundary(pos) {
|
|
pos += 1;
|
|
}
|
|
pos
|
|
};
|
|
if safe_last_end < text.len() {
|
|
result.push_str(&text[safe_last_end..]);
|
|
}
|
|
result
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_is_list_item() {
|
|
assert!(is_list_item("• Item one"));
|
|
assert!(is_list_item("- Item two"));
|
|
assert!(is_list_item("* Item three"));
|
|
assert!(is_list_item("1. First"));
|
|
assert!(is_list_item("2) Second"));
|
|
assert!(is_list_item("a. Letter item"));
|
|
assert!(!is_list_item("Regular text"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_format_list_item() {
|
|
assert_eq!(format_list_item("• Item"), "- Item");
|
|
assert_eq!(format_list_item("- Item"), "- Item");
|
|
assert_eq!(format_list_item("1. First"), "1. First");
|
|
}
|
|
|
|
#[test]
|
|
fn test_is_code_like() {
|
|
assert!(is_code_like("const x = 5;"));
|
|
assert!(is_code_like("function foo() {"));
|
|
assert!(is_code_like("import React from 'react'"));
|
|
assert!(!is_code_like("This is regular text."));
|
|
}
|
|
|
|
#[test]
|
|
fn test_detect_header_level() {
|
|
// With three tiers: 24→H1, 18→H2, 15→H3, 12→None
|
|
let tiers = vec![24.0, 18.0, 15.0];
|
|
assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1));
|
|
assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2));
|
|
assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3));
|
|
assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
|
|
|
|
// Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2)
|
|
let tiers = vec![15.0];
|
|
assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(1));
|
|
assert_eq!(detect_header_level(14.0, 12.0, &tiers), None);
|
|
assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
|
|
|
|
// No tiers (empty): falls back to ratio thresholds
|
|
let tiers: Vec<f32> = vec![];
|
|
assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1));
|
|
assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2));
|
|
assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3));
|
|
assert_eq!(detect_header_level(14.5, 12.0, &tiers), Some(4));
|
|
assert_eq!(detect_header_level(14.0, 12.0, &tiers), None);
|
|
assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
|
|
|
|
// Body text excluded when tiers exist: 13pt (ratio 1.08) → None
|
|
let tiers = vec![20.0];
|
|
assert_eq!(detect_header_level(13.0, 12.0, &tiers), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_to_markdown() {
|
|
let text = "• First item\n• Second item\n\nRegular paragraph.";
|
|
let md = to_markdown(text, MarkdownOptions::default());
|
|
assert!(md.contains("- First item"));
|
|
assert!(md.contains("- Second item"));
|
|
}
|
|
}
|