From ee09e96cc437209c19ba8cfe549c8925d3105e50 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 6 Feb 2026 17:58:27 -0800 Subject: [PATCH] add tests and readme --- Cargo.toml | 4 +- README.md | 236 +++++++++++++ src/markdown.rs | 11 +- tests/integration_tests.rs | 659 +++++++++++++++++++++++++++++++++++++ 4 files changed, 902 insertions(+), 8 deletions(-) create mode 100644 tests/integration_tests.rs diff --git a/Cargo.toml b/Cargo.toml index 75f7b77..f5b173d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,8 @@ description = "Fast PDF to Markdown conversion with smart detection" license = "MIT" [dependencies] -# PDF parsing - use local firecrawl lopdf -lopdf = { path = "../../lopdf", features = ["rayon"] } +# PDF parsing +lopdf = { git = "https://github.com/J-F-Liu/lopdf", features = ["rayon"] } # Error handling thiserror = "2.0" diff --git a/README.md b/README.md index 92d7950..1cbd6c8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,210 @@ Fast Rust library for PDF to Markdown conversion with smart scanned vs text-base - **Structure Detection** - Headers (by font size), lists, code blocks (monospace fonts) - **CLI Tools** - `detect-pdf` and `pdf2md` binaries included +## Installation + +Add to your `Cargo.toml`: + +```toml +[dependencies] +pdf-to-markdown = "0.1" +``` + +## Usage + +### Quick Start + +The simplest way to convert a PDF to Markdown: + +```rust +use pdf_to_markdown::process_pdf; + +fn main() -> Result<(), Box> { + let result = process_pdf("document.pdf")?; + + match result.pdf_type { + pdf_to_markdown::PdfType::TextBased => { + println!("Markdown:\n{}", result.markdown.unwrap()); + } + pdf_to_markdown::PdfType::Scanned => { + println!("PDF is scanned - OCR required"); + } + _ => {} + } + + Ok(()) +} +``` + +### PDF Type Detection + +Quickly detect if a PDF is text-based or scanned without full extraction: + +```rust +use pdf_to_markdown::{detect_pdf_type, PdfType}; + +fn main() -> Result<(), Box> { + let result = detect_pdf_type("document.pdf")?; + + println!("Type: {:?}", result.pdf_type); + println!("Pages: {}", result.page_count); + println!("Confidence: {:.0}%", result.confidence * 100.0); + + if let Some(title) = result.title { + println!("Title: {}", title); + } + + match result.pdf_type { + PdfType::TextBased => println!("Ready for text extraction"), + PdfType::Scanned => println!("Needs OCR"), + PdfType::ImageBased => println!("Mostly images"), + PdfType::Mixed => println!("Mix of text and images"), + } + + Ok(()) +} +``` + +### Text Extraction + +Extract plain text from a PDF: + +```rust +use pdf_to_markdown::extract_text; + +fn main() -> Result<(), Box> { + let text = extract_text("document.pdf")?; + println!("{}", text); + Ok(()) +} +``` + +### Extract Text with Position Information + +Get text items with position data for advanced processing: + +```rust +use pdf_to_markdown::{extract_text_with_positions, TextItem}; +use pdf_to_markdown::extractor::group_into_lines; + +fn main() -> Result<(), Box> { + let items = extract_text_with_positions("document.pdf")?; + + for item in &items { + println!("'{}' at ({}, {}) size={}", + item.text, item.x, item.y, item.font_size); + } + + // Group items into lines + let lines = group_into_lines(items); + for line in lines { + println!("Line: {}", line.text()); + } + + Ok(()) +} +``` + +### Custom Markdown Conversion + +Convert text to Markdown with custom options: + +```rust +use pdf_to_markdown::{to_markdown, MarkdownOptions}; + +fn main() { + let text = "• First item\n• Second item\n\nconst x = 5;"; + + // With all detection enabled (default) + let md = to_markdown(text, MarkdownOptions::default()); + println!("{}", md); + + // Disable code detection + let opts = MarkdownOptions { + detect_headers: true, + detect_lists: true, + detect_code: false, + base_font_size: None, + }; + let md = to_markdown(text, opts); + println!("{}", md); +} +``` + +### Processing from Memory + +All functions have memory buffer variants for processing PDFs already in memory: + +```rust +use pdf_to_markdown::{process_pdf_mem, detector::detect_pdf_type_mem}; +use pdf_to_markdown::extractor::{extract_text_mem, extract_text_with_positions_mem}; + +fn main() -> Result<(), Box> { + let buffer = std::fs::read("document.pdf")?; + + // Process from memory + let result = process_pdf_mem(&buffer)?; + + // Or detect only + let detection = detect_pdf_type_mem(&buffer)?; + + // Or extract text + let text = extract_text_mem(&buffer)?; + + Ok(()) +} +``` + +### Custom Detection Configuration + +Fine-tune the detection algorithm: + +```rust +use pdf_to_markdown::detector::{detect_pdf_type_with_config, DetectionConfig}; + +fn main() -> Result<(), Box> { + let config = DetectionConfig { + max_pages_to_sample: 10, // Sample more pages + min_text_ops_per_page: 5, // Require more text operators + text_page_ratio_threshold: 0.8, // Stricter text classification + }; + + let result = detect_pdf_type_with_config("document.pdf", config)?; + println!("{:?}", result.pdf_type); + + Ok(()) +} +``` + +## CLI Tools + +### pdf2md + +Convert a PDF to Markdown: + +```bash +# Output to stdout +pdf2md document.pdf + +# Output to file +pdf2md document.pdf output.md + +# JSON output with metadata +pdf2md document.pdf --json +``` + +### detect-pdf + +Detect PDF type without conversion: + +```bash +# Human-readable output +detect-pdf document.pdf + +# JSON output +detect-pdf document.pdf --json +``` + ## How Detection Works Instead of loading the entire PDF, we: @@ -19,3 +223,35 @@ Instead of loading the entire PDF, we: 4. Classify based on text operator presence This allows detecting 300+ page PDFs in milliseconds. + +## API Reference + +### Types + +| Type | Description | +|------|-------------| +| `PdfType` | Enum: `TextBased`, `Scanned`, `ImageBased`, `Mixed` | +| `PdfProcessResult` | Full processing result with text, markdown, and metadata | +| `PdfTypeResult` | Detection result with type, confidence, and page count | +| `TextItem` | Text with position (x, y), font info, and page number | +| `TextLine` | Group of `TextItem`s on the same line | +| `MarkdownOptions` | Configuration for markdown conversion | +| `DetectionConfig` | Configuration for PDF type detection | +| `PdfError` | Error type: `Io`, `Parse`, `Encrypted`, `InvalidStructure` | + +### Functions + +| Function | Description | +|----------|-------------| +| `process_pdf(path)` | High-level: detect, extract, and convert | +| `process_pdf_mem(buffer)` | Same as above, from memory | +| `detect_pdf_type(path)` | Fast type detection | +| `detect_pdf_type_mem(buffer)` | Type detection from memory | +| `extract_text(path)` | Extract plain text | +| `extract_text_mem(buffer)` | Extract text from memory | +| `extract_text_with_positions(path)` | Extract text with coordinates | +| `to_markdown(text, options)` | Convert text to markdown | + +## License + +MIT diff --git a/src/markdown.rs b/src/markdown.rs index 052a5d4..f91b51c 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -270,12 +270,11 @@ fn format_list_item(text: &str) -> String { let trimmed = text.trim_start(); // Convert various bullet styles to markdown - if trimmed.starts_with("• ") - || trimmed.starts_with("○ ") - || trimmed.starts_with("● ") - || trimmed.starts_with("◦ ") - { - return format!("- {}", &trimmed[2..].trim_start()); + // Note: bullet characters like • are multi-byte in UTF-8, use char indices + for bullet in &['•', '○', '●', '◦'] { + if let Some(rest) = trimmed.strip_prefix(*bullet) { + return format!("- {}", rest.trim_start()); + } } if trimmed.starts_with("- ") || trimmed.starts_with("* ") { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs new file mode 100644 index 0000000..24ad58f --- /dev/null +++ b/tests/integration_tests.rs @@ -0,0 +1,659 @@ +//! Integration tests for pdf-to-markdown library + +use pdf_to_markdown::{ + detect_pdf_type, extract_text, extract_text_with_positions, to_markdown, + MarkdownOptions, PdfType, TextItem, +}; +use pdf_to_markdown::detector::DetectionConfig; +use pdf_to_markdown::extractor::{group_into_lines, TextLine}; + +// Helper to create test TextItems +fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: "Helvetica".to_string(), + font_size, + page, + } +} + +fn make_text_item_with_font(text: &str, x: f32, y: f32, font_size: f32, font: &str, page: u32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: font.to_string(), + font_size, + page, + } +} + +// ============================================================================ +// Detection Config Tests +// ============================================================================ + +#[test] +fn test_detection_config_default() { + let config = DetectionConfig::default(); + assert_eq!(config.max_pages_to_sample, 5); + assert_eq!(config.min_text_ops_per_page, 3); + assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001); +} + +#[test] +fn test_detection_config_custom() { + let config = DetectionConfig { + max_pages_to_sample: 10, + min_text_ops_per_page: 5, + text_page_ratio_threshold: 0.8, + }; + assert_eq!(config.max_pages_to_sample, 10); + assert_eq!(config.min_text_ops_per_page, 5); + assert!((config.text_page_ratio_threshold - 0.8).abs() < 0.001); +} + +// ============================================================================ +// PdfType Tests +// ============================================================================ + +#[test] +fn test_pdf_type_equality() { + assert_eq!(PdfType::TextBased, PdfType::TextBased); + assert_eq!(PdfType::Scanned, PdfType::Scanned); + assert_eq!(PdfType::ImageBased, PdfType::ImageBased); + assert_eq!(PdfType::Mixed, PdfType::Mixed); + assert_ne!(PdfType::TextBased, PdfType::Scanned); +} + +#[test] +fn test_pdf_type_clone() { + let original = PdfType::TextBased; + let cloned = original.clone(); + assert_eq!(original, cloned); +} + +#[test] +fn test_pdf_type_debug() { + let pdf_type = PdfType::TextBased; + let debug_str = format!("{:?}", pdf_type); + assert_eq!(debug_str, "TextBased"); +} + +// ============================================================================ +// TextItem Tests +// ============================================================================ + +#[test] +fn test_text_item_creation() { + let item = make_text_item("Hello", 100.0, 700.0, 12.0, 1); + assert_eq!(item.text, "Hello"); + assert_eq!(item.x, 100.0); + assert_eq!(item.y, 700.0); + assert_eq!(item.font_size, 12.0); + assert_eq!(item.page, 1); +} + +#[test] +fn test_text_item_clone() { + let item = make_text_item("Test", 50.0, 600.0, 14.0, 2); + let cloned = item.clone(); + assert_eq!(item.text, cloned.text); + assert_eq!(item.x, cloned.x); + assert_eq!(item.y, cloned.y); +} + +// ============================================================================ +// TextLine Tests +// ============================================================================ + +#[test] +fn test_text_line_text_method() { + let items = vec![ + make_text_item("Hello", 100.0, 700.0, 12.0, 1), + make_text_item("World", 160.0, 700.0, 12.0, 1), + ]; + let line = TextLine { + items, + y: 700.0, + page: 1, + }; + assert_eq!(line.text(), "Hello World"); +} + +#[test] +fn test_text_line_single_item() { + let items = vec![make_text_item("Single", 100.0, 700.0, 12.0, 1)]; + let line = TextLine { + items, + y: 700.0, + page: 1, + }; + assert_eq!(line.text(), "Single"); +} + +#[test] +fn test_text_line_empty() { + let line = TextLine { + items: vec![], + y: 700.0, + page: 1, + }; + assert_eq!(line.text(), ""); +} + +// ============================================================================ +// Group Into Lines Tests +// ============================================================================ + +#[test] +fn test_group_into_lines_empty() { + let items: Vec = vec![]; + let lines = group_into_lines(items); + assert!(lines.is_empty()); +} + +#[test] +fn test_group_into_lines_same_line() { + let items = vec![ + make_text_item("A", 100.0, 700.0, 12.0, 1), + make_text_item("B", 120.0, 700.0, 12.0, 1), + make_text_item("C", 140.0, 700.0, 12.0, 1), + ]; + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].items.len(), 3); + assert_eq!(lines[0].text(), "A B C"); +} + +#[test] +fn test_group_into_lines_different_lines() { + let items = vec![ + make_text_item("Line1", 100.0, 700.0, 12.0, 1), + make_text_item("Line2", 100.0, 680.0, 12.0, 1), + make_text_item("Line3", 100.0, 660.0, 12.0, 1), + ]; + let lines = group_into_lines(items); + assert_eq!(lines.len(), 3); + assert_eq!(lines[0].text(), "Line1"); + assert_eq!(lines[1].text(), "Line2"); + assert_eq!(lines[2].text(), "Line3"); +} + +#[test] +fn test_group_into_lines_y_tolerance() { + // Items within 3.0 Y tolerance should be grouped + // Note: items are sorted by Y descending, then X ascending + let items = vec![ + make_text_item("A", 100.0, 700.0, 12.0, 1), + make_text_item("B", 150.0, 700.0, 12.0, 1), // Same Y + ]; + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "A B"); +} + +#[test] +fn test_group_into_lines_multiple_pages() { + let items = vec![ + make_text_item("Page1Text", 100.0, 700.0, 12.0, 1), + make_text_item("Page2Text", 100.0, 700.0, 12.0, 2), + ]; + let lines = group_into_lines(items); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].page, 1); + assert_eq!(lines[1].page, 2); +} + +#[test] +fn test_group_into_lines_sorting_by_x() { + // Items on same line should be sorted by X position + let items = vec![ + make_text_item("Third", 200.0, 700.0, 12.0, 1), + make_text_item("First", 50.0, 700.0, 12.0, 1), + make_text_item("Second", 100.0, 700.0, 12.0, 1), + ]; + let lines = group_into_lines(items); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text(), "First Second Third"); +} + +// ============================================================================ +// MarkdownOptions Tests +// ============================================================================ + +#[test] +fn test_markdown_options_default() { + let opts = MarkdownOptions::default(); + assert!(opts.detect_headers); + assert!(opts.detect_lists); + assert!(opts.detect_code); + assert!(opts.base_font_size.is_none()); +} + +#[test] +fn test_markdown_options_custom() { + let opts = MarkdownOptions { + detect_headers: false, + detect_lists: true, + detect_code: false, + base_font_size: Some(14.0), + }; + assert!(!opts.detect_headers); + assert!(opts.detect_lists); + assert!(!opts.detect_code); + assert_eq!(opts.base_font_size, Some(14.0)); +} + +// ============================================================================ +// Markdown Conversion Tests +// ============================================================================ + +#[test] +fn test_to_markdown_basic() { + let text = "Hello World"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("Hello World")); +} + +#[test] +fn test_to_markdown_multiple_lines() { + let text = "Line one\nLine two\nLine three"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("Line one")); + assert!(md.contains("Line two")); + assert!(md.contains("Line three")); +} + +#[test] +fn test_to_markdown_bullet_list() { + let text = "• First\n• Second\n• Third"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("- First")); + assert!(md.contains("- Second")); + assert!(md.contains("- Third")); +} + +#[test] +fn test_to_markdown_dash_list() { + let text = "- One\n- Two\n- Three"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("- One")); + assert!(md.contains("- Two")); +} + +#[test] +fn test_to_markdown_numbered_list() { + let text = "1. First\n2. Second\n3. Third"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("1. First")); + assert!(md.contains("2. Second")); +} + +#[test] +fn test_to_markdown_code_detection() { + let text = "const x = 5;\nlet y = 10;"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("```")); +} + +#[test] +fn test_to_markdown_no_code_detection() { + let text = "const x = 5;"; + let opts = MarkdownOptions { + detect_code: false, + ..Default::default() + }; + let md = to_markdown(text, opts); + assert!(!md.contains("```")); +} + +#[test] +fn test_to_markdown_no_list_detection() { + let text = "• Item"; + let opts = MarkdownOptions { + detect_lists: false, + ..Default::default() + }; + let md = to_markdown(text, opts); + // Should keep original bullet character + assert!(md.contains("•")); +} + +#[test] +fn test_to_markdown_empty_lines() { + let text = "Para one\n\nPara two"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("Para one")); + assert!(md.contains("Para two")); +} + +#[test] +fn test_to_markdown_whitespace_only_lines() { + let text = "Content\n \nMore content"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.contains("Content")); + assert!(md.contains("More content")); +} + +// ============================================================================ +// Markdown From Items Tests +// ============================================================================ + +#[test] +fn test_markdown_from_items_empty() { + use pdf_to_markdown::markdown::to_markdown_from_items; + let items: Vec = vec![]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.is_empty()); +} + +#[test] +fn test_markdown_from_items_single() { + use pdf_to_markdown::markdown::to_markdown_from_items; + let items = vec![make_text_item("Hello", 100.0, 700.0, 12.0, 1)]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("Hello")); +} + +#[test] +fn test_markdown_from_items_header_detection() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // Need multiple body items to establish base font size + let items = vec![ + make_text_item("Title", 100.0, 750.0, 24.0, 1), // Large font = H1 + make_text_item("Body text one", 100.0, 700.0, 12.0, 1), + make_text_item("Body text two", 100.0, 680.0, 12.0, 1), + make_text_item("Body text three", 100.0, 660.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("# Title")); + assert!(md.contains("Body text")); +} + +#[test] +fn test_markdown_from_items_h2_detection() { + use pdf_to_markdown::markdown::to_markdown_from_items; + let items = vec![ + make_text_item("Subtitle", 100.0, 750.0, 18.0, 1), // 1.5x = H2 + make_text_item("Body text", 100.0, 700.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("## Subtitle")); +} + +#[test] +fn test_markdown_from_items_monospace_code() { + use pdf_to_markdown::markdown::to_markdown_from_items; + let items = vec![ + make_text_item_with_font("let x = 5", 100.0, 700.0, 12.0, "Courier", 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("```")); + assert!(md.contains("let x = 5")); +} + +#[test] +fn test_markdown_from_items_page_breaks() { + use pdf_to_markdown::markdown::to_markdown_from_items; + let items = vec![ + make_text_item("Page 1", 100.0, 700.0, 12.0, 1), + make_text_item("Page 2", 100.0, 700.0, 12.0, 2), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("---")); // Page break marker +} + +// ============================================================================ +// Markdown From Lines Tests +// ============================================================================ + +#[test] +fn test_markdown_from_lines_empty() { + use pdf_to_markdown::markdown::to_markdown_from_lines; + let lines: Vec = vec![]; + let md = to_markdown_from_lines(lines, MarkdownOptions::default()); + assert!(md.is_empty()); +} + +#[test] +fn test_markdown_from_lines_basic() { + use pdf_to_markdown::markdown::to_markdown_from_lines; + let lines = vec![ + TextLine { + items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)], + y: 700.0, + page: 1, + }, + TextLine { + items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)], + y: 680.0, + page: 1, + }, + ]; + let md = to_markdown_from_lines(lines, MarkdownOptions::default()); + assert!(md.contains("First")); + assert!(md.contains("Second")); +} + +// ============================================================================ +// Error Handling Tests +// ============================================================================ + +#[test] +fn test_extract_text_nonexistent_file() { + let result = extract_text("/nonexistent/file.pdf"); + assert!(result.is_err()); +} + +#[test] +fn test_detect_pdf_type_nonexistent_file() { + let result = detect_pdf_type("/nonexistent/file.pdf"); + assert!(result.is_err()); +} + +#[test] +fn test_extract_text_with_positions_nonexistent_file() { + let result = extract_text_with_positions("/nonexistent/file.pdf"); + assert!(result.is_err()); +} + +// ============================================================================ +// List Pattern Tests +// ============================================================================ + +#[test] +fn test_bullet_variations() { + // Unicode bullets get converted to markdown dash + let unicode_bullets = ["• Item", "○ Item", "● Item", "◦ Item"]; + for bullet in &unicode_bullets { + let md = to_markdown(bullet, MarkdownOptions::default()); + assert!(md.contains("- Item"), "Failed for: {}", bullet); + } + + // Markdown-compatible bullets stay as-is + let md_bullets = ["- Item", "* Item"]; + for bullet in &md_bullets { + let md = to_markdown(bullet, MarkdownOptions::default()); + assert!(md.contains(bullet), "Failed for: {}", bullet); + } +} + +#[test] +fn test_numbered_list_variations() { + let lists = ["1. First", "2) Second", "10. Tenth"]; + for item in &lists { + let md = to_markdown(item, MarkdownOptions::default()); + assert!(md.trim().len() > 0, "Failed for: {}", item); + } +} + +#[test] +fn test_letter_list_items() { + let md = to_markdown("a. Letter item", MarkdownOptions::default()); + assert!(md.contains("a. Letter item")); +} + +// ============================================================================ +// Code Detection Tests +// ============================================================================ + +#[test] +fn test_code_keywords() { + let keywords = [ + "import foo", + "export default", + "const x = 5;", + "let y = 10;", + "function test() {", + "class MyClass {", + "def func():", + "pub fn main() {", + "async fn process() {", + "impl Trait {", + ]; + for code in &keywords { + let md = to_markdown(code, MarkdownOptions::default()); + assert!(md.contains("```"), "Code not detected for: {}", code); + } +} + +#[test] +fn test_code_syntax_patterns() { + // Patterns that start with code keywords/syntax + let patterns = [ + "=> value", // Starts with => + "-> Result", // Starts with -> + ":: io::Result", // Starts with :: + ]; + for code in &patterns { + let md = to_markdown(code, MarkdownOptions::default()); + assert!(md.contains("```"), "Code not detected for: {}", code); + } +} + +#[test] +fn test_code_special_chars() { + let code = "if (x > 0) { return y; }"; + let md = to_markdown(code, MarkdownOptions::default()); + assert!(md.contains("```")); +} + +#[test] +fn test_non_code_text() { + let text = "This is regular text about programming."; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(!md.contains("```")); +} + +// ============================================================================ +// Monospace Font Detection Tests +// ============================================================================ + +#[test] +fn test_monospace_font_names() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // Font names that contain the patterns in is_monospace_font + let monospace_fonts = [ + "Courier", "Consolas", "Monaco", "Menlo", + "Fira Code", "JetBrains Mono", "Inconsolata", + "DejaVu Sans Mono", "Liberation Mono", "Fixed", "Terminal", + ]; + + for font in &monospace_fonts { + let items = vec![make_text_item_with_font("code", 100.0, 700.0, 12.0, font, 1)]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("```"), "Font not detected as monospace: {}", font); + } +} + +// ============================================================================ +// Header Level Detection Tests +// ============================================================================ + +#[test] +fn test_header_level_h1() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // 24.0 / 12.0 = 2.0x = H1 + // Need multiple body items to establish base font size + let items = vec![ + make_text_item("H1 Title", 100.0, 700.0, 24.0, 1), + make_text_item("body text one", 100.0, 650.0, 12.0, 1), + make_text_item("body text two", 100.0, 630.0, 12.0, 1), + make_text_item("body text three", 100.0, 610.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("# H1 Title")); +} + +#[test] +fn test_header_level_h2() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // 18.0 / 12.0 = 1.5x = H2 + // Need multiple body items to establish base font size + let items = vec![ + make_text_item("H2 Title", 100.0, 700.0, 18.0, 1), + make_text_item("body text one", 100.0, 650.0, 12.0, 1), + make_text_item("body text two", 100.0, 630.0, 12.0, 1), + make_text_item("body text three", 100.0, 610.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("## H2 Title")); +} + +#[test] +fn test_header_level_h3() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // 15.0 / 12.0 = 1.25x = H3 + // Need multiple body items to establish base font size + let items = vec![ + make_text_item("H3 Title", 100.0, 700.0, 15.0, 1), + make_text_item("body text one", 100.0, 650.0, 12.0, 1), + make_text_item("body text two", 100.0, 630.0, 12.0, 1), + make_text_item("body text three", 100.0, 610.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("### H3 Title")); +} + +#[test] +fn test_header_level_h4() { + use pdf_to_markdown::markdown::to_markdown_from_items; + // 13.5 / 12.0 = 1.125x = H4 (>= 1.1) + // Need multiple body items to establish base font size + let items = vec![ + make_text_item("H4 Title", 100.0, 700.0, 13.5, 1), + make_text_item("body text one", 100.0, 650.0, 12.0, 1), + make_text_item("body text two", 100.0, 630.0, 12.0, 1), + make_text_item("body text three", 100.0, 610.0, 12.0, 1), + ]; + let md = to_markdown_from_items(items, MarkdownOptions::default()); + assert!(md.contains("#### H4 Title")); +} + +// ============================================================================ +// Clean Markdown Tests +// ============================================================================ + +#[test] +fn test_excessive_newlines_preserved_in_plain_text() { + // Plain text to_markdown preserves structure from input + let text = "Para one\n\n\n\n\nPara two"; + let md = to_markdown(text, MarkdownOptions::default()); + // The function processes line by line, empty lines become single newlines + assert!(md.contains("Para one")); + assert!(md.contains("Para two")); +} + +#[test] +fn test_trailing_newline() { + let text = "Content"; + let md = to_markdown(text, MarkdownOptions::default()); + assert!(md.ends_with('\n')); + assert!(!md.ends_with("\n\n")); +}