add ci, plus new package
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --verbose
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-clippy-
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy -- -D warnings
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-build-
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --verbose
|
||||
+3
-2
@@ -1,10 +1,11 @@
|
||||
[package]
|
||||
name = "pdf-to-markdown"
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Fast PDF to Markdown conversion with smart detection"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/firecrawl/pdf-inspector"
|
||||
|
||||
[dependencies]
|
||||
# PDF parsing
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# pdf-to-markdown
|
||||
# pdf-inspector
|
||||
|
||||
Fast Rust library for PDF to Markdown conversion with smart scanned vs text-based detection.
|
||||
Fast Rust library for PDF inspection, classification, and text extraction. Intelligently detects scanned vs text-based PDFs to enable smart routing decisions.
|
||||
|
||||
## Features
|
||||
|
||||
@@ -15,7 +15,7 @@ Add to your `Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-to-markdown = "0.1"
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -25,16 +25,16 @@ pdf-to-markdown = "0.1"
|
||||
The simplest way to convert a PDF to Markdown:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::process_pdf;
|
||||
use pdf_inspector::process_pdf;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let result = process_pdf("document.pdf")?;
|
||||
|
||||
match result.pdf_type {
|
||||
pdf_to_markdown::PdfType::TextBased => {
|
||||
pdf_inspector::PdfType::TextBased => {
|
||||
println!("Markdown:\n{}", result.markdown.unwrap());
|
||||
}
|
||||
pdf_to_markdown::PdfType::Scanned => {
|
||||
pdf_inspector::PdfType::Scanned => {
|
||||
println!("PDF is scanned - OCR required");
|
||||
}
|
||||
_ => {}
|
||||
@@ -49,7 +49,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Quickly detect if a PDF is text-based or scanned without full extraction:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::{detect_pdf_type, PdfType};
|
||||
use pdf_inspector::{detect_pdf_type, PdfType};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let result = detect_pdf_type("document.pdf")?;
|
||||
@@ -78,7 +78,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Extract plain text from a PDF:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::extract_text;
|
||||
use pdf_inspector::extract_text;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let text = extract_text("document.pdf")?;
|
||||
@@ -92,8 +92,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Get text items with position data for advanced processing:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::{extract_text_with_positions, TextItem};
|
||||
use pdf_to_markdown::extractor::group_into_lines;
|
||||
use pdf_inspector::{extract_text_with_positions, TextItem};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let items = extract_text_with_positions("document.pdf")?;
|
||||
@@ -118,7 +118,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Convert text to Markdown with custom options:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::{to_markdown, MarkdownOptions};
|
||||
use pdf_inspector::{to_markdown, MarkdownOptions};
|
||||
|
||||
fn main() {
|
||||
let text = "• First item\n• Second item\n\nconst x = 5;";
|
||||
@@ -144,8 +144,8 @@ fn main() {
|
||||
All functions have memory buffer variants for processing PDFs already in memory:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::{process_pdf_mem, detector::detect_pdf_type_mem};
|
||||
use pdf_to_markdown::extractor::{extract_text_mem, extract_text_with_positions_mem};
|
||||
use pdf_inspector::{process_pdf_mem, detector::detect_pdf_type_mem};
|
||||
use pdf_inspector::extractor::{extract_text_mem, extract_text_with_positions_mem};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let buffer = std::fs::read("document.pdf")?;
|
||||
@@ -168,7 +168,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Fine-tune the detection algorithm:
|
||||
|
||||
```rust
|
||||
use pdf_to_markdown::detector::{detect_pdf_type_with_config, DetectionConfig};
|
||||
use pdf_inspector::detector::{detect_pdf_type_with_config, DetectionConfig};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = DetectionConfig {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! CLI tool for detecting PDF type (text-based vs scanned)
|
||||
|
||||
use pdf_to_markdown::{detect_pdf_type, PdfType};
|
||||
use pdf_inspector::{detect_pdf_type, PdfType};
|
||||
use std::env;
|
||||
use std::process;
|
||||
use std::time::Instant;
|
||||
@@ -36,7 +36,11 @@ fn main() {
|
||||
result.pages_sampled,
|
||||
result.pages_with_text,
|
||||
result.confidence,
|
||||
result.title.as_ref().map(|t| format!("\"{}\"", t.replace('"', "\\\""))).unwrap_or_else(|| "null".to_string()),
|
||||
result
|
||||
.title
|
||||
.as_ref()
|
||||
.map(|t| format!("\"{}\"", t.replace('"', "\\\"")))
|
||||
.unwrap_or_else(|| "null".to_string()),
|
||||
elapsed.as_millis()
|
||||
);
|
||||
} else {
|
||||
@@ -77,7 +81,9 @@ fn main() {
|
||||
println!("Recommendation: Use OCR for best results");
|
||||
}
|
||||
PdfType::Mixed => {
|
||||
println!("Recommendation: Try text extraction first, use OCR for image pages");
|
||||
println!(
|
||||
"Recommendation: Try text extraction first, use OCR for image pages"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-2
@@ -1,6 +1,6 @@
|
||||
//! CLI tool for PDF to Markdown conversion
|
||||
|
||||
use pdf_to_markdown::{process_pdf, PdfType};
|
||||
use pdf_inspector::{process_pdf, PdfType};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::process;
|
||||
@@ -32,7 +32,11 @@ fn main() {
|
||||
let md_escaped = result
|
||||
.markdown
|
||||
.as_ref()
|
||||
.map(|m| m.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n"))
|
||||
.map(|m| {
|
||||
m.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('\n', "\\n")
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
println!(
|
||||
|
||||
+5
-10
@@ -263,19 +263,14 @@ fn scan_content_for_text_operators(content: &[u8]) -> (u32, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// Look for BT (Begin Text) as additional confirmation
|
||||
if b == b'B' && i + 1 < content.len() && content[i + 1] == b'T' {
|
||||
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
|
||||
// BT found - text block marker
|
||||
}
|
||||
}
|
||||
|
||||
// Look for 'Do' operator (XObject/image placement)
|
||||
if b == b'D' && i + 1 < content.len() && content[i + 1] == b'o' {
|
||||
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
|
||||
if b == b'D'
|
||||
&& i + 1 < content.len()
|
||||
&& content[i + 1] == b'o'
|
||||
&& (i + 2 >= content.len() || content[i + 2].is_ascii_whitespace())
|
||||
{
|
||||
has_images = true;
|
||||
}
|
||||
}
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
+22
-10
@@ -37,7 +37,11 @@ pub struct TextLine {
|
||||
|
||||
impl TextLine {
|
||||
pub fn text(&self) -> String {
|
||||
self.items.iter().map(|i| i.text.as_str()).collect::<Vec<_>>().join(" ")
|
||||
self.items
|
||||
.iter()
|
||||
.map(|i| i.text.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,11 +105,11 @@ fn extract_page_text_items(
|
||||
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
|
||||
|
||||
// Get content
|
||||
let content_data = doc.get_page_content(page_id)
|
||||
let content_data = doc
|
||||
.get_page_content(page_id)
|
||||
.map_err(|e| PdfError::Parse(e.to_string()))?;
|
||||
|
||||
let content = Content::decode(&content_data)
|
||||
.map_err(|e| PdfError::Parse(e.to_string()))?;
|
||||
let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?;
|
||||
|
||||
// Text state tracking
|
||||
let mut current_font = String::new();
|
||||
@@ -153,7 +157,8 @@ fn extract_page_text_items(
|
||||
// Set text matrix
|
||||
if op.operands.len() >= 6 {
|
||||
for (i, operand) in op.operands.iter().take(6).enumerate() {
|
||||
text_matrix[i] = get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 });
|
||||
text_matrix[i] =
|
||||
get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 });
|
||||
}
|
||||
line_matrix = text_matrix;
|
||||
}
|
||||
@@ -166,7 +171,9 @@ fn extract_page_text_items(
|
||||
"Tj" => {
|
||||
// Show text string
|
||||
if in_text_block && !op.operands.is_empty() {
|
||||
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font) {
|
||||
if let Some(text) =
|
||||
extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font)
|
||||
{
|
||||
if !text.trim().is_empty() {
|
||||
items.push(TextItem {
|
||||
text,
|
||||
@@ -188,7 +195,9 @@ fn extract_page_text_items(
|
||||
if let Ok(array) = op.operands[0].as_array() {
|
||||
let mut combined_text = String::new();
|
||||
for item in array {
|
||||
if let Some(text) = extract_text_from_operand(item, doc, &fonts, ¤t_font) {
|
||||
if let Some(text) =
|
||||
extract_text_from_operand(item, doc, &fonts, ¤t_font)
|
||||
{
|
||||
combined_text.push_str(&text);
|
||||
}
|
||||
}
|
||||
@@ -212,7 +221,9 @@ fn extract_page_text_items(
|
||||
line_matrix[5] -= current_font_size * 1.2;
|
||||
text_matrix = line_matrix;
|
||||
if !op.operands.is_empty() {
|
||||
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font) {
|
||||
if let Some(text) =
|
||||
extract_text_from_operand(&op.operands[0], doc, &fonts, ¤t_font)
|
||||
{
|
||||
if !text.trim().is_empty() {
|
||||
items.push(TextItem {
|
||||
text,
|
||||
@@ -239,7 +250,7 @@ fn extract_page_text_items(
|
||||
fn get_number(obj: &Object) -> Option<f32> {
|
||||
match obj {
|
||||
Object::Integer(i) => Some(*i as f32),
|
||||
Object::Real(r) => Some(*r as f32),
|
||||
Object::Real(r) => Some(*r),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -286,7 +297,8 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
|
||||
// Sort by page, then by Y (descending for PDF coords), then by X
|
||||
let mut sorted = items;
|
||||
sorted.sort_by(|a, b| {
|
||||
a.page.cmp(&b.page)
|
||||
a.page
|
||||
.cmp(&b.page)
|
||||
.then(b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal))
|
||||
.then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal))
|
||||
});
|
||||
|
||||
+8
-6
@@ -69,7 +69,9 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
||||
PdfType::Mixed => {
|
||||
// Try to extract what we can
|
||||
let text = extract_text(&path).ok();
|
||||
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
|
||||
let markdown = text
|
||||
.as_ref()
|
||||
.map(|t| to_markdown(t, MarkdownOptions::default()));
|
||||
|
||||
PdfProcessResult {
|
||||
pdf_type: PdfType::Mixed,
|
||||
@@ -105,18 +107,18 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
PdfType::Scanned | PdfType::ImageBased => {
|
||||
PdfProcessResult {
|
||||
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
|
||||
pdf_type: detection.pdf_type,
|
||||
text: None,
|
||||
markdown: None,
|
||||
page_count: detection.page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
},
|
||||
PdfType::Mixed => {
|
||||
let text = extractor::extract_text_mem(buffer).ok();
|
||||
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
|
||||
let markdown = text
|
||||
.as_ref()
|
||||
.map(|t| to_markdown(t, MarkdownOptions::default()));
|
||||
|
||||
PdfProcessResult {
|
||||
pdf_type: PdfType::Mixed,
|
||||
|
||||
+39
-13
@@ -6,7 +6,7 @@
|
||||
//! - Code blocks (monospace fonts, indentation)
|
||||
//! - Paragraphs
|
||||
|
||||
use crate::extractor::{TextItem, TextLine, group_into_lines};
|
||||
use crate::extractor::{group_into_lines, TextItem, TextLine};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Options for markdown conversion
|
||||
@@ -103,7 +103,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
|
||||
// Calculate font statistics
|
||||
let font_stats = calculate_font_stats(&lines);
|
||||
let base_size = options.base_font_size.unwrap_or(font_stats.most_common_size);
|
||||
let base_size = options
|
||||
.base_font_size
|
||||
.unwrap_or(font_stats.most_common_size);
|
||||
|
||||
let mut output = String::new();
|
||||
let mut current_page = 0u32;
|
||||
@@ -201,9 +203,7 @@ fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
|
||||
.map(|(size, _)| *size as f32 / 10.0)
|
||||
.unwrap_or(12.0);
|
||||
|
||||
FontStats {
|
||||
most_common_size,
|
||||
}
|
||||
FontStats { most_common_size }
|
||||
}
|
||||
|
||||
/// Detect header level from font size
|
||||
@@ -242,7 +242,7 @@ fn is_list_item(text: &str) -> bool {
|
||||
let first_chars: String = trimmed.chars().take(5).collect();
|
||||
if first_chars.contains(|c: char| c.is_ascii_digit()) {
|
||||
// Check for "1.", "1)", "10."
|
||||
if let Some(idx) = first_chars.find(|c: char| c == '.' || c == ')') {
|
||||
if let Some(idx) = first_chars.find(['.', ')']) {
|
||||
let prefix = &first_chars[..idx];
|
||||
if prefix.chars().all(|c| c.is_ascii_digit()) {
|
||||
return true;
|
||||
@@ -292,10 +292,24 @@ fn is_code_like(text: &str) -> bool {
|
||||
// Code patterns
|
||||
let code_patterns = [
|
||||
// Language keywords
|
||||
"import ", "export ", "from ", "const ", "let ", "var ", "function ",
|
||||
"class ", "def ", "pub fn ", "fn ", "async fn ", "impl ",
|
||||
"import ",
|
||||
"export ",
|
||||
"from ",
|
||||
"const ",
|
||||
"let ",
|
||||
"var ",
|
||||
"function ",
|
||||
"class ",
|
||||
"def ",
|
||||
"pub fn ",
|
||||
"fn ",
|
||||
"async fn ",
|
||||
"impl ",
|
||||
// Syntax patterns
|
||||
"=> ", "-> ", ":: ", ":= ",
|
||||
"=> ",
|
||||
"-> ",
|
||||
":: ",
|
||||
":= ",
|
||||
// Common code endings
|
||||
];
|
||||
|
||||
@@ -306,7 +320,8 @@ fn is_code_like(text: &str) -> bool {
|
||||
}
|
||||
|
||||
// Check for code-like syntax
|
||||
let special_chars: usize = trimmed.chars()
|
||||
let special_chars: usize = trimmed
|
||||
.chars()
|
||||
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
|
||||
.count();
|
||||
|
||||
@@ -326,9 +341,20 @@ fn is_code_like(text: &str) -> bool {
|
||||
fn is_monospace_font(font_name: &str) -> bool {
|
||||
let lower = font_name.to_lowercase();
|
||||
let patterns = [
|
||||
"courier", "consolas", "monaco", "menlo", "mono", "fixed",
|
||||
"terminal", "typewriter", "source code", "fira code",
|
||||
"jetbrains", "inconsolata", "dejavu sans mono", "liberation mono",
|
||||
"courier",
|
||||
"consolas",
|
||||
"monaco",
|
||||
"menlo",
|
||||
"mono",
|
||||
"fixed",
|
||||
"terminal",
|
||||
"typewriter",
|
||||
"source code",
|
||||
"fira code",
|
||||
"jetbrains",
|
||||
"inconsolata",
|
||||
"dejavu sans mono",
|
||||
"liberation mono",
|
||||
];
|
||||
|
||||
patterns.iter().any(|p| lower.contains(p))
|
||||
|
||||
+57
-28
@@ -1,11 +1,11 @@
|
||||
//! Integration tests for pdf-to-markdown library
|
||||
|
||||
use pdf_to_markdown::{
|
||||
detect_pdf_type, extract_text, extract_text_with_positions, to_markdown,
|
||||
MarkdownOptions, PdfType, TextItem,
|
||||
use pdf_inspector::detector::DetectionConfig;
|
||||
use pdf_inspector::extractor::{group_into_lines, TextLine};
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_text, extract_text_with_positions, to_markdown, MarkdownOptions,
|
||||
PdfType, TextItem,
|
||||
};
|
||||
use pdf_to_markdown::detector::DetectionConfig;
|
||||
use pdf_to_markdown::extractor::{group_into_lines, TextLine};
|
||||
|
||||
// Helper to create test TextItems
|
||||
fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
|
||||
@@ -21,7 +21,14 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
|
||||
}
|
||||
}
|
||||
|
||||
fn make_text_item_with_font(text: &str, x: f32, y: f32, font_size: f32, font: &str, page: u32) -> TextItem {
|
||||
fn make_text_item_with_font(
|
||||
text: &str,
|
||||
x: f32,
|
||||
y: f32,
|
||||
font_size: f32,
|
||||
font: &str,
|
||||
page: u32,
|
||||
) -> TextItem {
|
||||
TextItem {
|
||||
text: text.to_string(),
|
||||
x,
|
||||
@@ -347,7 +354,7 @@ fn test_to_markdown_whitespace_only_lines() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_empty() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
let items: Vec<TextItem> = vec![];
|
||||
let md = to_markdown_from_items(items, MarkdownOptions::default());
|
||||
assert!(md.is_empty());
|
||||
@@ -355,7 +362,7 @@ fn test_markdown_from_items_empty() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_single() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
let items = vec![make_text_item("Hello", 100.0, 700.0, 12.0, 1)];
|
||||
let md = to_markdown_from_items(items, MarkdownOptions::default());
|
||||
assert!(md.contains("Hello"));
|
||||
@@ -363,7 +370,7 @@ fn test_markdown_from_items_single() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_header_detection() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
make_text_item("Title", 100.0, 750.0, 24.0, 1), // Large font = H1
|
||||
@@ -378,10 +385,13 @@ fn test_markdown_from_items_header_detection() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_h2_detection() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
make_text_item("Subtitle", 100.0, 750.0, 18.0, 1), // 1.5x = H2
|
||||
make_text_item("Body text", 100.0, 700.0, 12.0, 1),
|
||||
make_text_item("Body text one", 100.0, 700.0, 12.0, 1),
|
||||
make_text_item("Body text two", 100.0, 680.0, 12.0, 1),
|
||||
make_text_item("Body text three", 100.0, 660.0, 12.0, 1),
|
||||
];
|
||||
let md = to_markdown_from_items(items, MarkdownOptions::default());
|
||||
assert!(md.contains("## Subtitle"));
|
||||
@@ -389,10 +399,15 @@ fn test_markdown_from_items_h2_detection() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_monospace_code() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
let items = vec![
|
||||
make_text_item_with_font("let x = 5", 100.0, 700.0, 12.0, "Courier", 1),
|
||||
];
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
let items = vec![make_text_item_with_font(
|
||||
"let x = 5",
|
||||
100.0,
|
||||
700.0,
|
||||
12.0,
|
||||
"Courier",
|
||||
1,
|
||||
)];
|
||||
let md = to_markdown_from_items(items, MarkdownOptions::default());
|
||||
assert!(md.contains("```"));
|
||||
assert!(md.contains("let x = 5"));
|
||||
@@ -400,7 +415,7 @@ fn test_markdown_from_items_monospace_code() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_items_page_breaks() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
let items = vec![
|
||||
make_text_item("Page 1", 100.0, 700.0, 12.0, 1),
|
||||
make_text_item("Page 2", 100.0, 700.0, 12.0, 2),
|
||||
@@ -415,7 +430,7 @@ fn test_markdown_from_items_page_breaks() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_lines_empty() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_lines;
|
||||
use pdf_inspector::markdown::to_markdown_from_lines;
|
||||
let lines: Vec<TextLine> = vec![];
|
||||
let md = to_markdown_from_lines(lines, MarkdownOptions::default());
|
||||
assert!(md.is_empty());
|
||||
@@ -423,7 +438,7 @@ fn test_markdown_from_lines_empty() {
|
||||
|
||||
#[test]
|
||||
fn test_markdown_from_lines_basic() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_lines;
|
||||
use pdf_inspector::markdown::to_markdown_from_lines;
|
||||
let lines = vec![
|
||||
TextLine {
|
||||
items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)],
|
||||
@@ -557,18 +572,32 @@ fn test_non_code_text() {
|
||||
|
||||
#[test]
|
||||
fn test_monospace_font_names() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// Font names that contain the patterns in is_monospace_font
|
||||
let monospace_fonts = [
|
||||
"Courier", "Consolas", "Monaco", "Menlo",
|
||||
"Fira Code", "JetBrains Mono", "Inconsolata",
|
||||
"DejaVu Sans Mono", "Liberation Mono", "Fixed", "Terminal",
|
||||
"Courier",
|
||||
"Consolas",
|
||||
"Monaco",
|
||||
"Menlo",
|
||||
"Fira Code",
|
||||
"JetBrains Mono",
|
||||
"Inconsolata",
|
||||
"DejaVu Sans Mono",
|
||||
"Liberation Mono",
|
||||
"Fixed",
|
||||
"Terminal",
|
||||
];
|
||||
|
||||
for font in &monospace_fonts {
|
||||
let items = vec![make_text_item_with_font("code", 100.0, 700.0, 12.0, font, 1)];
|
||||
let items = vec![make_text_item_with_font(
|
||||
"code", 100.0, 700.0, 12.0, font, 1,
|
||||
)];
|
||||
let md = to_markdown_from_items(items, MarkdownOptions::default());
|
||||
assert!(md.contains("```"), "Font not detected as monospace: {}", font);
|
||||
assert!(
|
||||
md.contains("```"),
|
||||
"Font not detected as monospace: {}",
|
||||
font
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -578,7 +607,7 @@ fn test_monospace_font_names() {
|
||||
|
||||
#[test]
|
||||
fn test_header_level_h1() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// 24.0 / 12.0 = 2.0x = H1
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
@@ -593,7 +622,7 @@ fn test_header_level_h1() {
|
||||
|
||||
#[test]
|
||||
fn test_header_level_h2() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// 18.0 / 12.0 = 1.5x = H2
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
@@ -608,7 +637,7 @@ fn test_header_level_h2() {
|
||||
|
||||
#[test]
|
||||
fn test_header_level_h3() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// 15.0 / 12.0 = 1.25x = H3
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
@@ -623,7 +652,7 @@ fn test_header_level_h3() {
|
||||
|
||||
#[test]
|
||||
fn test_header_level_h4() {
|
||||
use pdf_to_markdown::markdown::to_markdown_from_items;
|
||||
use pdf_inspector::markdown::to_markdown_from_items;
|
||||
// 13.5 / 12.0 = 1.125x = H4 (>= 1.1)
|
||||
// Need multiple body items to establish base font size
|
||||
let items = vec![
|
||||
|
||||
Reference in New Issue
Block a user