mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ci, plus new package

Browse Source
This commit is contained in:
Abimael Martell
2026-02-06 21:39:21 -08:00
parent ee09e96cc4
commit 0ef7b2adfc
10 changed files with 279 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bin/detect_pdf.rs
+9 -3
View File
@@ -1,6 +1,6 @@
//! CLI tool for detecting PDF type (text-based vs scanned)
use pdf_to_markdown::{detect_pdf_type, PdfType};
use pdf_inspector::{detect_pdf_type, PdfType};
use std::env;
use std::process;
use std::time::Instant;
@@ -36,7 +36,11 @@ fn main() {
result.pages_sampled,
result.pages_with_text,
result.confidence,
result.title.as_ref().map(|t| format!("\"{}\"", t.replace('"', "\\\""))).unwrap_or_else(|| "null".to_string()),
result
.title
.as_ref()
.map(|t| format!("\"{}\"", t.replace('"', "\\\"")))
.unwrap_or_else(|| "null".to_string()),
elapsed.as_millis()
);
} else {
@@ -77,7 +81,9 @@ fn main() {
println!("Recommendation: Use OCR for best results");
}
PdfType::Mixed => {
println!("Recommendation: Try text extraction first, use OCR for image pages");
println!(
"Recommendation: Try text extraction first, use OCR for image pages"
);
}
}
}
src/bin/pdf2md.rs
+6 -2
View File
@@ -1,6 +1,6 @@
//! CLI tool for PDF to Markdown conversion
use pdf_to_markdown::{process_pdf, PdfType};
use pdf_inspector::{process_pdf, PdfType};
use std::env;
use std::fs;
use std::process;
@@ -32,7 +32,11 @@ fn main() {
let md_escaped = result
.markdown
.as_ref()
.map(|m| m.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n"))
.map(|m| {
m.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
})
.unwrap_or_default();
println!(
src/detector.rs
+6 -11
View File
@@ -263,18 +263,13 @@ fn scan_content_for_text_operators(content: &[u8]) -> (u32, bool) {
}
}
// Look for BT (Begin Text) as additional confirmation
if b == b'B' && i + 1 < content.len() && content[i + 1] == b'T' {
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
// BT found - text block marker
}
}
// Look for 'Do' operator (XObject/image placement)
if b == b'D' && i + 1 < content.len() && content[i + 1] == b'o' {
if i + 2 >= content.len() || content[i + 2].is_ascii_whitespace() {
has_images = true;
}
if b == b'D'
&& i + 1 < content.len()
&& content[i + 1] == b'o'
&& (i + 2 >= content.len() || content[i + 2].is_ascii_whitespace())
{
has_images = true;
}
i += 1;
src/extractor.rs
+22 -10
View File
@@ -37,7 +37,11 @@ pub struct TextLine {
impl TextLine {
pub fn text(&self) -> String {
self.items.iter().map(|i| i.text.as_str()).collect::<Vec<_>>().join(" ")
self.items
.iter()
.map(|i| i.text.as_str())
.collect::<Vec<_>>()
.join(" ")
}
}
@@ -101,11 +105,11 @@ fn extract_page_text_items(
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
// Get content
let content_data = doc.get_page_content(page_id)
let content_data = doc
.get_page_content(page_id)
.map_err(|e| PdfError::Parse(e.to_string()))?;
let content = Content::decode(&content_data)
.map_err(|e| PdfError::Parse(e.to_string()))?;
let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?;
// Text state tracking
let mut current_font = String::new();
@@ -153,7 +157,8 @@ fn extract_page_text_items(
// Set text matrix
if op.operands.len() >= 6 {
for (i, operand) in op.operands.iter().take(6).enumerate() {
text_matrix[i] = get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 });
text_matrix[i] =
get_number(operand).unwrap_or(if i == 0 || i == 3 { 1.0 } else { 0.0 });
}
line_matrix = text_matrix;
}
@@ -166,7 +171,9 @@ fn extract_page_text_items(
"Tj" => {
// Show text string
if in_text_block && !op.operands.is_empty() {
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font) {
if let Some(text) =
extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font)
{
if !text.trim().is_empty() {
items.push(TextItem {
text,
@@ -188,7 +195,9 @@ fn extract_page_text_items(
if let Ok(array) = op.operands[0].as_array() {
let mut combined_text = String::new();
for item in array {
if let Some(text) = extract_text_from_operand(item, doc, &fonts, &current_font) {
if let Some(text) =
extract_text_from_operand(item, doc, &fonts, &current_font)
{
combined_text.push_str(&text);
}
}
@@ -212,7 +221,9 @@ fn extract_page_text_items(
line_matrix[5] -= current_font_size * 1.2;
text_matrix = line_matrix;
if !op.operands.is_empty() {
if let Some(text) = extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font) {
if let Some(text) =
extract_text_from_operand(&op.operands[0], doc, &fonts, &current_font)
{
if !text.trim().is_empty() {
items.push(TextItem {
text,
@@ -239,7 +250,7 @@ fn extract_page_text_items(
fn get_number(obj: &Object) -> Option<f32> {
match obj {
Object::Integer(i) => Some(*i as f32),
Object::Real(r) => Some(*r as f32),
Object::Real(r) => Some(*r),
_ => None,
}
}
@@ -286,7 +297,8 @@ pub fn group_into_lines(items: Vec<TextItem>) -> Vec<TextLine> {
// Sort by page, then by Y (descending for PDF coords), then by X
let mut sorted = items;
sorted.sort_by(|a, b| {
a.page.cmp(&b.page)
a.page
.cmp(&b.page)
.then(b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal))
.then(a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal))
});
src/lib.rs
+13 -11
View File
@@ -69,7 +69,9 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
PdfType::Mixed => {
// Try to extract what we can
let text = extract_text(&path).ok();
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
let markdown = text
.as_ref()
.map(|t| to_markdown(t, MarkdownOptions::default()));
PdfProcessResult {
pdf_type: PdfType::Mixed,
@@ -105,18 +107,18 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Scanned | PdfType::ImageBased => {
PdfProcessResult {
pdf_type: detection.pdf_type,
text: None,
markdown: None,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
}
}
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
pdf_type: detection.pdf_type,
text: None,
markdown: None,
page_count: detection.page_count,
processing_time_ms: start.elapsed().as_millis() as u64,
},
PdfType::Mixed => {
let text = extractor::extract_text_mem(buffer).ok();
let markdown = text.as_ref().map(|t| to_markdown(t, MarkdownOptions::default()));
let markdown = text
.as_ref()
.map(|t| to_markdown(t, MarkdownOptions::default()));
PdfProcessResult {
pdf_type: PdfType::Mixed,
src/markdown.rs
+39 -13
View File
@@ -6,7 +6,7 @@
//! - Code blocks (monospace fonts, indentation)
//! - Paragraphs
use crate::extractor::{TextItem, TextLine, group_into_lines};
use crate::extractor::{group_into_lines, TextItem, TextLine};
use std::collections::HashMap;
/// Options for markdown conversion
@@ -103,7 +103,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
// Calculate font statistics
let font_stats = calculate_font_stats(&lines);
let base_size = options.base_font_size.unwrap_or(font_stats.most_common_size);
let base_size = options
.base_font_size
.unwrap_or(font_stats.most_common_size);
let mut output = String::new();
let mut current_page = 0u32;
@@ -201,9 +203,7 @@ fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
.map(|(size, _)| *size as f32 / 10.0)
.unwrap_or(12.0);
FontStats {
most_common_size,
}
FontStats { most_common_size }
}
/// Detect header level from font size
@@ -242,7 +242,7 @@ fn is_list_item(text: &str) -> bool {
let first_chars: String = trimmed.chars().take(5).collect();
if first_chars.contains(|c: char| c.is_ascii_digit()) {
// Check for "1.", "1)", "10."
if let Some(idx) = first_chars.find(|c: char| c == '.' || c == ')') {
if let Some(idx) = first_chars.find(['.', ')']) {
let prefix = &first_chars[..idx];
if prefix.chars().all(|c| c.is_ascii_digit()) {
return true;
@@ -292,10 +292,24 @@ fn is_code_like(text: &str) -> bool {
// Code patterns
let code_patterns = [
// Language keywords
"import ", "export ", "from ", "const ", "let ", "var ", "function ",
"class ", "def ", "pub fn ", "fn ", "async fn ", "impl ",
"import ",
"export ",
"from ",
"const ",
"let ",
"var ",
"function ",
"class ",
"def ",
"pub fn ",
"fn ",
"async fn ",
"impl ",
// Syntax patterns
"=> ", "-> ", ":: ", ":= ",
"=> ",
"-> ",
":: ",
":= ",
// Common code endings
];
@@ -306,7 +320,8 @@ fn is_code_like(text: &str) -> bool {
}
// Check for code-like syntax
let special_chars: usize = trimmed.chars()
let special_chars: usize = trimmed
.chars()
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
.count();
@@ -326,9 +341,20 @@ fn is_code_like(text: &str) -> bool {
fn is_monospace_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
let patterns = [
"courier", "consolas", "monaco", "menlo", "mono", "fixed",
"terminal", "typewriter", "source code", "fira code",
"jetbrains", "inconsolata", "dejavu sans mono", "liberation mono",
"courier",
"consolas",
"monaco",
"menlo",
"mono",
"fixed",
"terminal",
"typewriter",
"source code",
"fira code",
"jetbrains",
"inconsolata",
"dejavu sans mono",
"liberation mono",
];
patterns.iter().any(|p| lower.contains(p))