mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): Add --pages and --select-pages flags, fix CMap matching and whitespace tracking

Browse Source 
- Add --pages flag to insert <!-- Page N --> markers between pages
- Add --select-pages flag to process only specific pages (e.g. 1,3,5-10)
- Wire MarkdownOptions and page filter through process_pdf_with_config
- Fix fuzzy CMap matching that caused Cyrillic substitution on Latin text
- Fix whitespace Tj items not advancing text matrix (broke gap detection)
- Tighten single-char fragment join threshold from 0.25 to 0.20
- Gitignore debug/diagnostic binaries and remove tracked ones

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-17 08:58:58 -08:00
co-authored by Claude Opus 4.6
parent 9a98fbc9ec
commit 350bbc0086
12 changed files with 265 additions and 537 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+7
View File
@@ -30,3 +30,10 @@ scripts/
# Test output
test_output/
# Debug/diagnostic binaries
src/bin/debug_*.rs
src/bin/dump_*.rs
src/bin/profile_*.rs
src/bin/detection_report.rs
REPORT.md
Cargo.toml
+12
View File
@@ -50,3 +50,15 @@ path = "src/bin/debug_ygaps.rs"
[[bin]]
name = "debug_pages"
path = "src/bin/debug_pages.rs"
[[bin]]
name = "debug_fonts"
path = "src/bin/debug_fonts.rs"
[[bin]]
name = "dump_ops"
path = "src/bin/dump_ops.rs"
[[bin]]
name = "debug_spaces"
path = "src/bin/debug_spaces.rs"
src/bin/debug_ligatures.rs
-180
View File
@@ -1,180 +0,0 @@
use pdf_inspector::extract_text_with_positions;
use pdf_inspector::tounicode::FontCMaps;
fn main() {
let path = std::env::args()
.nth(1)
.expect("Usage: debug_ligatures <pdf>");
// Load PDF and extract CMaps
let pdf_bytes = std::fs::read(&path).unwrap();
let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes);
println!("=== Font CMaps ===");
if font_cmaps.by_name.is_empty() && font_cmaps.by_obj_num.is_empty() {
println!(" (none found)");
}
for (name, cmap) in &font_cmaps.by_name {
println!(
" font={:30} code_byte_length={} char_map_entries={} ranges={}",
name,
cmap.code_byte_length,
cmap.char_map.len(),
cmap.ranges.len()
);
}
// Load with lopdf to inspect font Differences arrays
let doc = lopdf::Document::load_mem(&pdf_bytes).unwrap();
let pages = doc.get_pages();
println!("\n=== Font Encoding Differences ===");
for (page_num, &page_id) in pages.iter() {
println!("--- Page {} ---", page_num);
let fonts = match doc.get_page_fonts(page_id) {
Ok(f) => f,
Err(_) => continue,
};
for (font_name_bytes, font_dict) in &fonts {
let font_name = String::from_utf8_lossy(font_name_bytes).to_string();
// Check for Encoding
if let Ok(encoding_obj) = font_dict.get(b"Encoding") {
let enc_dict = match encoding_obj {
lopdf::Object::Dictionary(d) => Some(d.clone()),
lopdf::Object::Reference(r) => doc.get_dictionary(*r).ok().cloned(),
lopdf::Object::Name(name) => {
println!(
" font={}: Encoding={}",
font_name,
String::from_utf8_lossy(name)
);
None
}
_ => None,
};
if let Some(enc_dict) = enc_dict {
// Check BaseEncoding
if let Ok(lopdf::Object::Name(name)) = enc_dict.get(b"BaseEncoding") {
println!(
" font={}: BaseEncoding={}",
font_name,
String::from_utf8_lossy(name)
);
}
// Dump Differences
if let Ok(diff_obj) = enc_dict.get(b"Differences") {
let diff_array = match diff_obj {
lopdf::Object::Array(arr) => Some(arr.clone()),
lopdf::Object::Reference(r) => {
if let Ok(lopdf::Object::Array(arr)) = doc.get_object(*r) {
Some(arr.clone())
} else {
None
}
}
_ => None,
};
if let Some(diff_array) = diff_array {
let mut current_code: u8 = 0;
let mut entries = Vec::new();
let mut total_glyphs = 0;
for item in &diff_array {
match item {
lopdf::Object::Integer(n) => {
current_code = *n as u8;
}
lopdf::Object::Name(name) => {
let glyph = String::from_utf8_lossy(name).to_string();
entries.push((current_code, glyph));
current_code = current_code.wrapping_add(1);
total_glyphs += 1;
}
_ => {}
}
}
println!(
" font={}: Differences has {} glyph entries",
font_name, total_glyphs
);
// Show ligature entries specifically
for (code, glyph) in &entries {
if glyph == "fi"
|| glyph == "fl"
|| glyph == "ffi"
|| glyph == "ffl"
{
println!(
" code=0x{:02X} ({:3}) glyph={:?} (LIGATURE)",
code, code, glyph
);
}
}
// Check coverage: does it have standard ASCII letters?
let has_a = entries.iter().any(|(_, g)| g == "a");
let has_space = entries.iter().any(|(_, g)| g == "space");
let has_period = entries.iter().any(|(_, g)| g == "period");
println!(
" has 'a': {}, has 'space': {}, has 'period': {}",
has_a, has_space, has_period
);
// Show first 10 and last 5 entries
println!(" First 10 entries:");
for (code, glyph) in entries.iter().take(10) {
println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph);
}
if entries.len() > 15 {
println!(" ...");
println!(" Last 5 entries:");
for (code, glyph) in entries
.iter()
.rev()
.take(5)
.collect::<Vec<_>>()
.iter()
.rev()
{
println!(" 0x{:02X} ({:3}) -> {:?}", code, code, glyph);
}
}
}
}
}
} else {
println!(" font={}: no Encoding", font_name);
}
}
}
// Now extract text and look for ligatures
let items = extract_text_with_positions(&path).unwrap();
println!("\n=== Items containing fi or fl (first 10) ===");
let mut count = 0;
for item in items.iter() {
if item.text.contains('\u{FB01}') || item.text.contains('\u{FB02}') {
println!(
" page={} font={} text={:?}",
item.page, item.font, item.text
);
count += 1;
if count >= 10 {
break;
}
}
}
let total_lig = items
.iter()
.filter(|i| i.text.contains('\u{FB01}') || i.text.contains('\u{FB02}'))
.count();
println!(" Total items with ligatures: {}", total_lig);
}
src/bin/debug_order.rs
-46
View File
@@ -1,46 +0,0 @@
use pdf_inspector::extract_text_with_positions;
fn main() {
let items = extract_text_with_positions("samples/tables/doc.pdf").unwrap();
// Find items containing "Description" or section numbers
println!("Items containing section markers:");
for item in items.iter().filter(|i| i.page == 1) {
if item.text.contains("Description")
|| item.text.starts_with("3 ")
|| item.text.starts_with("3 ")
{
println!(" x={:6.1} y={:6.1} \"{}\"", item.x, item.y, item.text);
}
}
// Look at the Y range for right column
let right_col: Vec<_> = items
.iter()
.filter(|i| i.page == 1 && i.x > 300.0 && i.x < 400.0)
.collect();
let y_min = right_col.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
let y_max = right_col
.iter()
.map(|i| i.y)
.fold(f32::NEG_INFINITY, f32::max);
println!(
"\nRight column (x=300-400) Y range: {:.1} to {:.1}",
y_min, y_max
);
// Show items near Y=675 (where "3 Description" appears)
println!("\nItems near Y=675 (±10):");
for item in items
.iter()
.filter(|i| i.page == 1 && (i.y - 675.0).abs() < 10.0)
{
println!(
" x={:6.1} y={:6.1} \"{}\"",
item.x,
item.y,
&item.text[..item.text.len().min(50)]
);
}
}
src/bin/debug_pages.rs
-31
View File
@@ -1,31 +0,0 @@
use pdf_inspector::extract_text_with_positions;
use std::env;
fn main() {
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: debug_pages <pdf_path> [max_page | min-max]");
std::process::exit(1);
}
let range = args.get(2).map(|s| s.as_str()).unwrap_or("1-3");
let (min_page, max_page) = if let Some((a, b)) = range.split_once('-') {
(a.parse().unwrap_or(1), b.parse().unwrap_or(3))
} else {
(1, range.parse().unwrap_or(3))
};
let items = extract_text_with_positions(&args[1]).expect("Failed to extract");
for page in min_page..=max_page {
let page_items: Vec<_> = items.iter().filter(|i| i.page == page).collect();
println!("=== PAGE {} ({} items) ===", page, page_items.len());
for item in &page_items {
println!(
" x={:7.1} y={:7.1} w={:7.1} fs={:5.1} text={:?}",
item.x, item.y, item.width, item.font_size, item.text
);
}
println!();
}
}
src/bin/debug_ygaps.rs
-199
View File
@@ -1,199 +0,0 @@
//! Debug tool: Print Y positions and gaps between consecutive lines
//!
//! Usage: debug_ygaps <pdf_file> [page_number]
//!
//! Shows text lines grouped by page with Y coordinates, gaps from previous line,
//! font sizes, and whether each gap would be treated as a paragraph break.
use pdf_inspector::extract_text_with_positions;
use pdf_inspector::extractor::{group_into_lines, TextLine};
use std::env;
use std::process;
fn main() {
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: {} <pdf_file> [page_number]", args[0]);
eprintln!();
eprintln!("Prints Y positions and gaps between consecutive text lines.");
eprintln!("If page_number is given, only that page is shown.");
process::exit(1);
}
let pdf_path = &args[1];
let filter_page: Option<u32> = args.get(2).and_then(|s| s.parse().ok());
let items = match extract_text_with_positions(pdf_path) {
Ok(items) => items,
Err(e) => {
eprintln!("Error extracting text: {}", e);
process::exit(1);
}
};
if items.is_empty() {
eprintln!("No text items found in PDF.");
process::exit(0);
}
// Compute base font size (most common font size >= 9pt)
let base_size = {
let mut size_counts: std::collections::HashMap<i32, usize> =
std::collections::HashMap::new();
for item in &items {
if item.font_size >= 9.0 {
let key = (item.font_size * 10.0) as i32;
*size_counts.entry(key).or_insert(0) += 1;
}
}
size_counts
.into_iter()
.max_by_key(|&(_, count)| count)
.map(|(size_key, _)| size_key as f32 / 10.0)
.unwrap_or(10.0)
};
eprintln!("Base font size: {:.1}pt", base_size);
eprintln!(
"Paragraph break threshold: y_gap > {:.1} (base * 1.8)",
base_size * 1.8
);
eprintln!();
// Group into lines
let lines = group_into_lines(items);
// Get unique pages
let mut pages: Vec<u32> = lines.iter().map(|l| l.page).collect();
pages.sort();
pages.dedup();
for page in pages {
if let Some(fp) = filter_page {
if page != fp {
continue;
}
}
let page_lines: Vec<&TextLine> = lines.iter().filter(|l| l.page == page).collect();
println!("===== PAGE {} ({} lines) =====", page, page_lines.len());
println!(
"{:>8} {:>8} {:>8} {:>6} {:>5} Text (first 80 chars)",
"Y", "Gap", "GapRatio", "Font", "Bold"
);
println!("{}", "-".repeat(120));
let mut prev_y: Option<f32> = None;
for line in &page_lines {
let font_size = line.items.first().map(|i| i.font_size).unwrap_or(0.0);
let is_bold = line.items.first().map(|i| i.is_bold).unwrap_or(false);
let text = line.text();
let display_text: String = text.chars().take(80).collect();
let (gap_str, ratio_str, marker) = if let Some(py) = prev_y {
let gap = py - line.y;
let ratio = gap / base_size;
let is_para = gap > base_size * 1.8;
let marker = if is_para { " <<PARA>>" } else { "" };
(
format!("{:8.1}", gap),
format!("{:8.2}", ratio),
marker.to_string(),
)
} else {
(
" ---".to_string(),
" ---".to_string(),
String::new(),
)
};
println!(
"{:8.1} {} {} {:6.1} {:>5} {}{}",
line.y,
gap_str,
ratio_str,
font_size,
if is_bold { "B" } else { "" },
display_text,
marker
);
prev_y = Some(line.y);
}
println!();
// Summary statistics for this page
let mut gaps: Vec<f32> = Vec::new();
let mut prev_y: Option<f32> = None;
for line in &page_lines {
if let Some(py) = prev_y {
let gap = py - line.y;
if gap > 0.0 && gap < 200.0 {
gaps.push(gap);
}
}
prev_y = Some(line.y);
}
if !gaps.is_empty() {
gaps.sort_by(|a, b| a.partial_cmp(b).unwrap());
let min = gaps.first().unwrap();
let max = gaps.last().unwrap();
let median = gaps[gaps.len() / 2];
let mean: f32 = gaps.iter().sum::<f32>() / gaps.len() as f32;
println!(" Gap statistics for page {}:", page);
println!(" Count: {}", gaps.len());
println!(" Min: {:6.1} (ratio: {:.2})", min, min / base_size);
println!(" Max: {:6.1} (ratio: {:.2})", max, max / base_size);
println!(
" Median: {:6.1} (ratio: {:.2})",
median,
median / base_size
);
println!(" Mean: {:6.1} (ratio: {:.2})", mean, mean / base_size);
// Histogram of gap ratios
println!();
println!(" Gap ratio histogram (gap / base_size):");
let buckets: Vec<f32> = vec![
0.0,
0.5,
1.0,
1.2,
1.5,
1.8,
2.0,
2.5,
3.0,
5.0,
10.0,
f32::INFINITY,
];
for i in 0..buckets.len() - 1 {
let count = gaps
.iter()
.filter(|&&g| {
let r = g / base_size;
r >= buckets[i] && r < buckets[i + 1]
})
.count();
if count > 0 {
let label = if buckets[i + 1] == f32::INFINITY {
format!("{:4.1}+ ", buckets[i])
} else {
format!("{:4.1}-{:<4.1}", buckets[i], buckets[i + 1])
};
let bar: String = "#".repeat(count.min(60));
println!(" {} | {:3} {}", label, count, bar);
}
}
println!();
}
}
}
src/bin/pdf2md.rs
+70 -2
View File
@@ -1,10 +1,47 @@
//! CLI tool for PDF to Markdown conversion
use pdf_inspector::{process_pdf, PdfType};
use pdf_inspector::{process_pdf_with_config_pages, DetectionConfig, MarkdownOptions, PdfType};
use std::collections::HashSet;
use std::env;
use std::fs;
use std::process;
/// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
let mut pages = HashSet::new();
for part in spec.split(',') {
let part = part.trim();
if let Some((start, end)) = part.split_once('-') {
let start: u32 = start
.trim()
.parse()
.map_err(|_| format!("invalid page number: {}", start.trim()))?;
let end: u32 = end
.trim()
.parse()
.map_err(|_| format!("invalid page number: {}", end.trim()))?;
if start == 0 || end == 0 {
return Err("page numbers are 1-indexed".to_string());
}
if start > end {
return Err(format!("invalid range: {}-{}", start, end));
}
for p in start..=end {
pages.insert(p);
}
} else {
let p: u32 = part
.parse()
.map_err(|_| format!("invalid page number: {}", part))?;
if p == 0 {
return Err("page numbers are 1-indexed".to_string());
}
pages.insert(p);
}
}
Ok(pages)
}
fn main() {
let args: Vec<String> = env::args().collect();
@@ -19,18 +56,49 @@ fn main() {
eprintln!("Options:");
eprintln!(" --json Output result as JSON");
eprintln!(" --raw Output only markdown (no headers)");
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
process::exit(1);
}
let pdf_path = &args[1];
let json_output = args.iter().any(|a| a == "--json");
let raw_output = args.iter().any(|a| a == "--raw");
let page_numbers = args.iter().any(|a| a == "--pages");
// Parse --select-pages value
let page_filter = args
.iter()
.position(|a| a == "--select-pages")
.map(|i| {
args.get(i + 1)
.unwrap_or_else(|| {
eprintln!("Error: --select-pages requires a value (e.g. 1,3,5-10)");
process::exit(1);
})
.as_str()
})
.map(|spec| {
parse_page_spec(spec).unwrap_or_else(|e| {
eprintln!("Error: invalid --select-pages value: {}", e);
process::exit(1);
})
});
let output_file = args
.get(2)
.filter(|a| !a.starts_with("--"))
.map(|s| s.as_str());
match process_pdf(pdf_path) {
let mut md_options = MarkdownOptions::default();
md_options.include_page_numbers = page_numbers;
match process_pdf_with_config_pages(
pdf_path,
DetectionConfig::default(),
md_options,
page_filter.as_ref(),
) {
Ok(result) => {
if json_output {
let md_escaped = result
src/extractor.rs
+99 -14
View File
@@ -6,7 +6,7 @@ use crate::glyph_names::glyph_to_char;
use crate::tounicode::FontCMaps;
use crate::PdfError;
use lopdf::{Document, Encoding, Object, ObjectId};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::Path;
/// Font encoding map: maps byte codes to Unicode characters
@@ -747,10 +747,10 @@ fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool {
}
// Single-character fragment joined to a multi-character item: use a
// more generous threshold to rejoin split words like "b" + "illion"
// or "C" + "ultural".
// moderately generous threshold to rejoin split words like "b" + "illion"
// or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words.
if (prev_chars == 1) != (curr_chars == 1) {
return gap < font_size * 0.25;
return gap < font_size * 0.20;
}
// Both single-char: per-glyph positioning. For numeric characters
@@ -874,34 +874,70 @@ fn extract_text_from_doc(doc: &Document) -> Result<String, PdfError> {
/// Extract text with position information from PDF file
pub fn extract_text_with_positions<P: AsRef<Path>>(path: P) -> Result<Vec<TextItem>, PdfError> {
extract_text_with_positions_pages(path, None)
}
/// Extract text with positions from a file, limited to specific pages.
///
/// `page_filter` is an optional set of 1-indexed page numbers to process.
/// When `None`, all pages are processed.
pub fn extract_text_with_positions_pages<P: AsRef<Path>>(
path: P,
page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> {
// Read the raw PDF bytes for ToUnicode extraction
let pdf_bytes = std::fs::read(path.as_ref())?;
crate::validate_pdf_bytes(&pdf_bytes)?;
let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes);
let doc = Document::load_mem(&pdf_bytes)?;
extract_positioned_text_from_doc(&doc, &font_cmaps)
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
}
/// Extract text with positions from memory buffer
pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result<Vec<TextItem>, PdfError> {
extract_text_with_positions_mem_pages(buffer, None)
}
/// Extract text with positions from memory buffer, limited to specific pages.
pub fn extract_text_with_positions_mem_pages(
buffer: &[u8],
page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> {
crate::validate_pdf_bytes(buffer)?;
// Extract ToUnicode CMaps from raw PDF bytes
let font_cmaps = FontCMaps::from_pdf_bytes(buffer);
let doc = Document::load_mem(buffer)?;
extract_positioned_text_from_doc(&doc, &font_cmaps)
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
}
/// Extract positioned text from loaded document
fn extract_positioned_text_from_doc(
doc: &Document,
font_cmaps: &FontCMaps,
page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> {
// If raw byte scanning found no CMaps, populate from the document model.
// This handles PDFs with compressed object streams where raw scanning fails.
let mut font_cmaps_owned;
let font_cmaps = if font_cmaps.by_obj_num.is_empty() {
font_cmaps_owned = font_cmaps.clone();
populate_cmaps_from_doc(doc, &mut font_cmaps_owned);
&font_cmaps_owned
} else {
font_cmaps
};
let pages = doc.get_pages();
let mut all_items = Vec::new();
for (page_num, &page_id) in pages.iter() {
if let Some(filter) = page_filter {
if !filter.contains(page_num) {
continue;
}
}
let items = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
all_items.extend(items);
@@ -913,6 +949,54 @@ fn extract_positioned_text_from_doc(
Ok(all_items)
}
/// Populate FontCMaps from the lopdf document model for ToUnicode streams
/// that weren't found by raw byte scanning (e.g. in compressed object streams).
fn populate_cmaps_from_doc(doc: &Document, font_cmaps: &mut FontCMaps) {
use crate::tounicode::ToUnicodeCMap;
for (_page_num, &page_id) in doc.get_pages().iter() {
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
for (font_name, font_dict) in &fonts {
if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") {
if let Ok(obj_ref) = tounicode_ref.as_reference() {
let obj_num = obj_ref.0;
if font_cmaps.by_obj_num.contains_key(&obj_num) {
continue;
}
// Try to get the stream content via lopdf
if let Ok(stream) = doc.get_object(obj_ref) {
if let Ok(stream) = stream.as_stream() {
if let Ok(data) = stream.decompressed_content() {
if let Some(cmap) = ToUnicodeCMap::parse(&data) {
let resource_name =
String::from_utf8_lossy(font_name).to_string();
let base_name = font_dict
.get(b"BaseFont")
.ok()
.and_then(|o| o.as_name().ok())
.map(|n| String::from_utf8_lossy(n).to_string());
// Store by object number
font_cmaps.by_obj_num.insert(obj_num, cmap.clone());
// Store by resource name
font_cmaps
.by_name
.insert(resource_name.clone(), cmap.clone());
if let Some(base) = base_name {
let unique_key = format!("{}_{}", base, obj_num);
font_cmaps.by_name.insert(unique_key, cmap.clone());
font_cmaps.by_name.insert(base, cmap);
}
}
}
}
}
}
}
}
}
}
/// Multiply two 2D transformation matrices
/// Matrix format: [a, b, c, d, e, f] representing:
/// | a b 0 |
@@ -1146,9 +1230,7 @@ fn extract_page_text_items(
&font_encodings,
&encoding_cache,
) {
if !text.trim().is_empty() {
let rendered_size =
effective_font_size(current_font_size, &text_matrix);
let rendered_size = effective_font_size(current_font_size, &text_matrix);
let combined = multiply_matrices(&text_matrix, &ctm);
let (x, y) = (combined[4], combined[5]);
let width = if let Some(font_info) = font_widths.get(&current_font) {
@@ -1160,14 +1242,16 @@ fn extract_page_text_items(
);
text_matrix[4] += w_ts * text_matrix[0];
text_matrix[5] += w_ts * text_matrix[1];
(w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]))
.abs()
(w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])).abs()
} else {
0.0
}
} else {
0.0
};
// Only create text item for non-whitespace; whitespace
// still advances the text matrix above so gap detection works
if !text.trim().is_empty() {
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
@@ -1659,9 +1743,7 @@ fn extract_form_xobject_text(
&font_encodings,
&encoding_cache,
) {
if !text.trim().is_empty() {
let rendered_size =
effective_font_size(current_font_size, &text_matrix);
let rendered_size = effective_font_size(current_font_size, &text_matrix);
let combined = multiply_matrices(&text_matrix, parent_ctm);
let (x, y) = (combined[4], combined[5]);
let width = if let Some(font_info) = font_widths.get(&current_font) {
@@ -1683,6 +1765,9 @@ fn extract_form_xobject_text(
} else {
0.0
};
// Only create text item for non-whitespace; whitespace
// still advances the text matrix above so gap detection works
if !text.trim().is_empty() {
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
src/lib.rs
+26 -9
View File
@@ -16,7 +16,9 @@ pub use detector::{
detect_pdf_type, detect_pdf_type_mem, detect_pdf_type_mem_with_config,
detect_pdf_type_with_config, DetectionConfig, PdfType, PdfTypeResult, ScanStrategy,
};
pub use extractor::{extract_text, extract_text_with_positions, TextItem};
pub use extractor::{
extract_text, extract_text_with_positions, extract_text_with_positions_pages, TextItem,
};
pub use markdown::{to_markdown, to_markdown_from_items, MarkdownOptions};
use std::path::Path;
@@ -112,10 +114,24 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
Ok(result)
}
/// Process a PDF file with custom detection configuration
/// Process a PDF file with custom detection and markdown configuration
pub fn process_pdf_with_config<P: AsRef<Path>>(
path: P,
config: DetectionConfig,
markdown_options: MarkdownOptions,
) -> Result<PdfProcessResult, PdfError> {
process_pdf_with_config_pages(path, config, markdown_options, None)
}
/// Process a PDF file with custom configuration and optional page filter.
///
/// `page_filter` limits extraction to the given 1-indexed page numbers.
/// When `None`, all pages are processed.
pub fn process_pdf_with_config_pages<P: AsRef<Path>>(
path: P,
config: DetectionConfig,
markdown_options: MarkdownOptions,
page_filter: Option<&std::collections::HashSet<u32>>,
) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
@@ -130,8 +146,8 @@ pub fn process_pdf_with_config<P: AsRef<Path>>(
let result = match pdf_type {
PdfType::TextBased => {
let items = extract_text_with_positions(&path)?;
let markdown = to_markdown_from_items(items, MarkdownOptions::default());
let items = extract_text_with_positions_pages(&path, page_filter)?;
let markdown = to_markdown_from_items(items, markdown_options);
PdfProcessResult {
pdf_type,
@@ -155,8 +171,8 @@ pub fn process_pdf_with_config<P: AsRef<Path>>(
confidence,
},
PdfType::Mixed => {
let items = extract_text_with_positions(&path).ok();
let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default()));
let items = extract_text_with_positions_pages(&path, page_filter).ok();
let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone()));
PdfProcessResult {
pdf_type,
@@ -235,10 +251,11 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
Ok(result)
}
/// Process PDF from memory buffer with custom detection configuration
/// Process PDF from memory buffer with custom detection and markdown configuration
pub fn process_pdf_mem_with_config(
buffer: &[u8],
config: DetectionConfig,
markdown_options: MarkdownOptions,
) -> Result<PdfProcessResult, PdfError> {
let start = std::time::Instant::now();
@@ -254,7 +271,7 @@ pub fn process_pdf_mem_with_config(
let result = match pdf_type {
PdfType::TextBased => {
let items = extractor::extract_text_with_positions_mem(buffer)?;
let markdown = to_markdown_from_items(items, MarkdownOptions::default());
let markdown = to_markdown_from_items(items, markdown_options);
PdfProcessResult {
pdf_type,
@@ -279,7 +296,7 @@ pub fn process_pdf_mem_with_config(
},
PdfType::Mixed => {
let items = extractor::extract_text_with_positions_mem(buffer).ok();
let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default()));
let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone()));
PdfProcessResult {
pdf_type,
src/markdown.rs
+11
View File
@@ -36,6 +36,8 @@ pub struct MarkdownOptions {
pub include_images: bool,
/// Include extracted hyperlinks
pub include_links: bool,
/// Insert page break markers (<!-- Page N -->) between pages
pub include_page_numbers: bool,
}
impl Default for MarkdownOptions {
@@ -52,6 +54,7 @@ impl Default for MarkdownOptions {
detect_italic: true,
include_images: true,
include_links: true,
include_page_numbers: false,
}
}
}
@@ -518,6 +521,10 @@ fn to_markdown_from_lines_with_tables_and_images(
current_page = line.page;
prev_y = f32::MAX;
if options.include_page_numbers {
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
}
}
// Check if we should insert a table before this line
@@ -757,6 +764,10 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
prev_y = f32::MAX;
in_list = false;
last_list_x = None;
if options.include_page_numbers {
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
}
}
// Paragraph break (large Y gap relative to document's typical line spacing)
src/tounicode.rs
+3 -20
View File
@@ -535,7 +535,7 @@ pub fn extract_tounicode_cmaps(pdf_bytes: &[u8]) -> HashMap<u32, ToUnicodeCMap>
}
/// Collection of ToUnicode CMaps indexed by font name
#[derive(Debug, Default)]
#[derive(Debug, Default, Clone)]
pub struct FontCMaps {
/// Map of font name (e.g., "FNotoSans0") to ToUnicodeCMap
pub by_name: HashMap<String, ToUnicodeCMap>,
@@ -605,26 +605,9 @@ impl FontCMaps {
}
}
/// Get a CMap for a font name
/// Get a CMap for a font name (exact match only)
pub fn get(&self, font_name: &str) -> Option<&ToUnicodeCMap> {
// Try exact match first
if let Some(cmap) = self.by_name.get(font_name) {
return Some(cmap);
}
// Try without leading 'F' if present (resource names sometimes differ)
// but only if the stripped name is long enough to avoid false matches
// (e.g., "F1" → "1" would match too many things)
let stripped = font_name.strip_prefix('F').unwrap_or(font_name);
if stripped.len() >= 4 {
for (name, cmap) in &self.by_name {
if name.contains(stripped) || stripped.contains(name.as_str()) {
return Some(cmap);
}
}
}
None
self.by_name.get(font_name)
}
/// Get a CMap by ToUnicode object number
tests/integration_tests.rs
+1
View File
@@ -265,6 +265,7 @@ fn test_markdown_options_custom() {
detect_italic: false,
include_images: false,
include_links: false,
include_page_numbers: false,
};
assert!(!opts.detect_headers);
assert!(opts.detect_lists);