Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7467f842a5 | ||
|
|
740488d875 | ||
|
|
0b894b7d3f | ||
|
|
ebc2f05bcd |
@@ -26,16 +26,16 @@ Evaluated on the [opendataloader-bench](https://github.com/opendataloader-projec
|
||||
|
||||
| Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
|
||||
|---|---|---|---|---|---|
|
||||
| pdf-inspector | 0.83 | 0.88 | 0.66 | 0.74 | 4s |
|
||||
| pdf-inspector | 0.78 | 0.87 | 0.59 | 0.57 | 4s |
|
||||
| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
|
||||
| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
|
||||
| markitdown | 0.58 | 0.88 | 0.00 | 0.00 | 8s |
|
||||
|
||||
For context, engines that use OCR/ML (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus — pdf-inspector reaches the low end of that range without any OCR, in 4 seconds.
|
||||
For context, engines that use OCR/ML (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus.
|
||||
|
||||
**Where we do well:** Speed (fastest of all engines), the best table detection of any engine shown, and heading detection now on par with opendataloader. Overall lands within 0.01 of opendataloader at roughly 2.5× the speed.
|
||||
**Where we do well:** Speed (fastest of all engines), reading order, table detection vs other direct-text tools.
|
||||
|
||||
**Where we lag:** Reading order still trails opendataloader slightly, and table structure trails OCR-based engines that can see visual layout.
|
||||
**Where we lag:** Heading detection trails opendataloader — many PDFs use bold text at body font size for headings, or headings that are only slightly larger than body text. Table detection trails OCR-based engines that can see visual table structure.
|
||||
|
||||
## Quick start
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.10.1",
|
||||
"version": "1.10.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -208,7 +208,6 @@ fn main() {
|
||||
eprintln!(" --raw Output only markdown (no headers)");
|
||||
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
|
||||
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
|
||||
eprintln!(" --password PW Password for an encrypted PDF");
|
||||
eprintln!(" --detect-only Only detect PDF type (no extraction)");
|
||||
eprintln!(" --analyze Detect + extract + layout analysis (no markdown)");
|
||||
process::exit(1);
|
||||
@@ -222,16 +221,6 @@ fn main() {
|
||||
let detect_only = args.iter().any(|a| a == "--detect-only");
|
||||
let analyze = args.iter().any(|a| a == "--analyze");
|
||||
|
||||
// Parse --password value
|
||||
let password = args.iter().position(|a| a == "--password").map(|i| {
|
||||
args.get(i + 1)
|
||||
.unwrap_or_else(|| {
|
||||
eprintln!("Error: --password requires a value");
|
||||
process::exit(1);
|
||||
})
|
||||
.clone()
|
||||
});
|
||||
|
||||
// Parse --select-pages value
|
||||
let page_filter = args
|
||||
.iter()
|
||||
@@ -280,7 +269,6 @@ fn main() {
|
||||
if let Some(pages) = page_filter {
|
||||
options.page_filter = Some(pages);
|
||||
}
|
||||
options.password = password;
|
||||
|
||||
match process_pdf_with_options(pdf_path, options) {
|
||||
Ok(result) => {
|
||||
|
||||
+2
-245
@@ -9,7 +9,7 @@ mod links;
|
||||
pub(crate) mod underline;
|
||||
mod xobjects;
|
||||
|
||||
use crate::text_utils::{is_cjk_char, is_rtl_text};
|
||||
use crate::text_utils::is_rtl_text;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
|
||||
use crate::PdfError;
|
||||
@@ -527,136 +527,6 @@ fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
|
||||
saw_backtrack
|
||||
}
|
||||
|
||||
/// Detect a tracked (letter-spaced) run of single-glyph items and derive its
|
||||
/// run-local space floor.
|
||||
///
|
||||
/// Display type set with tracking renders one glyph per show op; the merge
|
||||
/// loop's fixed thresholds (0.08-0.13 em) then read every letter gap as a
|
||||
/// word boundary and emit "H O W" instead of "HOW". Within such a run the
|
||||
/// gaps carry the real signal: letter gaps cluster tightly just above the
|
||||
/// fixed threshold, word gaps sit clearly higher. Returns (run_end_index,
|
||||
/// space_floor) when the run starting at `start` is tracked — spaces are
|
||||
/// then inserted only at gaps above the floor (infinity = single word).
|
||||
/// Normal text (multi-char items, or single-char runs with sub-threshold
|
||||
/// gaps) returns None and keeps the existing behavior.
|
||||
/// Han/Kana scripts write without inter-word spaces. Hangul (Korean) DOES
|
||||
/// space between words and deliberately stays out of this set — a Korean
|
||||
/// tracked run keeps normal word-boundary handling.
|
||||
fn is_spaceless_cjk(c: char) -> bool {
|
||||
matches!(c,
|
||||
'\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation
|
||||
| '\u{3040}'..='\u{309F}' // Hiragana
|
||||
| '\u{30A0}'..='\u{30FF}' // Katakana
|
||||
| '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs
|
||||
| '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs
|
||||
| '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms
|
||||
)
|
||||
}
|
||||
|
||||
fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize, f32)> {
|
||||
const MIN_GAPS: usize = 4;
|
||||
let first = group[start];
|
||||
if first.text.trim().chars().count() != 1 {
|
||||
return None;
|
||||
}
|
||||
let fs = first.font_size;
|
||||
if fs <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Walk the run under the SAME break conditions as the merge loop
|
||||
// (size band, style equality, mergeable gap) so indices stay aligned.
|
||||
let mut gaps: Vec<f32> = Vec::new();
|
||||
let mut end_x = first.x + effective_merge_width(first);
|
||||
let mut end = start;
|
||||
for (offset, next) in group[start + 1..].iter().enumerate() {
|
||||
if next.text.trim().chars().count() != 1 {
|
||||
break;
|
||||
}
|
||||
if (next.font_size - fs).abs() > fs * 0.20 {
|
||||
break;
|
||||
}
|
||||
if next.is_bold != first.is_bold
|
||||
|| next.is_italic != first.is_italic
|
||||
|| next.is_underline != first.is_underline
|
||||
|| next.is_strikeout != first.is_strikeout
|
||||
{
|
||||
break;
|
||||
}
|
||||
let gap = next.x - end_x;
|
||||
if gap > fs * 0.5 || gap < -fs * 0.5 {
|
||||
break;
|
||||
}
|
||||
gaps.push(gap / fs);
|
||||
end_x = next.x + effective_merge_width(next);
|
||||
end = start + 1 + offset;
|
||||
}
|
||||
if gaps.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Tracked signature: the run's TYPICAL gap clears the fixed space
|
||||
// threshold (0.08) — the merge loop would break almost every letter
|
||||
// pair into "words". Short runs (2-3 gaps: "H O W") demand a stricter
|
||||
// shape — clearly wide, uniform, ALL-CAPS — because a genuine spaced
|
||||
// sequence of single letters ("x y z" variables) has the same gap
|
||||
// count; display tracking is a caps convention.
|
||||
let mut sorted = gaps.clone();
|
||||
sorted.sort_by(|a, b| a.total_cmp(b));
|
||||
let median = sorted[sorted.len() / 2];
|
||||
// Typographic convention gate, both tiers: display tracking is an
|
||||
// all-caps convention, and Han/Kana never space between glyphs. Mixed-
|
||||
// or lowercase Latin runs keep their boundaries because geometry alone
|
||||
// cannot distinguish spaced singles ("A b c d e") from a tracked
|
||||
// title-case word ("B u f f a l o").
|
||||
let run_chars = || {
|
||||
group[start..=end]
|
||||
.iter()
|
||||
.flat_map(|it| it.text.trim().chars())
|
||||
};
|
||||
let spaceless_cjk = run_chars().all(|c| is_spaceless_cjk(c) || !c.is_alphanumeric())
|
||||
&& run_chars().any(is_spaceless_cjk);
|
||||
let all_caps = run_chars().all(|c| c.is_uppercase() || is_cjk_char(c) || !c.is_alphabetic());
|
||||
if !(spaceless_cjk || all_caps) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if gaps.len() >= MIN_GAPS {
|
||||
if median <= 0.075 {
|
||||
return None;
|
||||
}
|
||||
} else {
|
||||
let uniform = sorted[sorted.len() - 1] <= sorted[0].max(0.01) * 1.4;
|
||||
if median < 0.09 || !uniform {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
// Han/Kana: no inter-glyph spaces, period — a nonuniform gap
|
||||
// distribution (punctuation spacing, justification) must not
|
||||
// manufacture word boundaries.
|
||||
if spaceless_cjk {
|
||||
return Some((end, f32::INFINITY));
|
||||
}
|
||||
|
||||
// Word gaps, if present, form a second mode above the letter-gap
|
||||
// cluster: split at the largest relative jump. Unimodal → one word.
|
||||
let mut best_jump = 1.0f32;
|
||||
let mut floor = f32::INFINITY;
|
||||
for pair in sorted.windows(2) {
|
||||
let (lo, hi) = (pair[0].max(0.01), pair[1].max(0.01));
|
||||
let jump = hi / lo;
|
||||
if jump > best_jump {
|
||||
best_jump = jump;
|
||||
floor = (lo + hi) / 2.0;
|
||||
}
|
||||
}
|
||||
if best_jump < 1.4 {
|
||||
floor = f32::INFINITY;
|
||||
}
|
||||
Some((end, floor * fs))
|
||||
}
|
||||
|
||||
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if items.is_empty() {
|
||||
return items;
|
||||
@@ -704,14 +574,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
let mut text = first.text.clone();
|
||||
let mut end_x = first.x + effective_merge_width(first);
|
||||
|
||||
// Tracked display text: run-local space floor overrides the
|
||||
// fixed thresholds for this run's junctions (see helper).
|
||||
let tracked = if *preserve_stream_order {
|
||||
None
|
||||
} else {
|
||||
tracked_run_space_floor(group, i)
|
||||
};
|
||||
|
||||
let mut j = i + 1;
|
||||
while j < group.len() {
|
||||
let next = group[j];
|
||||
@@ -766,11 +628,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
let needs_bullet_space = *preserve_stream_order
|
||||
&& is_standalone_bullet_text(&text)
|
||||
&& !next.text.trim().is_empty();
|
||||
let effective_threshold = match tracked {
|
||||
Some((run_end, floor)) if j <= run_end => floor,
|
||||
_ => threshold,
|
||||
};
|
||||
if needs_bullet_space || gap > effective_threshold {
|
||||
if needs_bullet_space || gap > threshold {
|
||||
text.push(' ');
|
||||
}
|
||||
text.push_str(&next.text);
|
||||
@@ -936,107 +794,6 @@ mod tests {
|
||||
use crate::types::{ItemType, PdfLine, TextLine};
|
||||
use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
|
||||
|
||||
/// Glyph-per-item run at `fs`=12 with the given inter-glyph gap (pt).
|
||||
fn glyph_run(chars: &str, start_x: f32, glyph_w: f32, gap: f32) -> Vec<TextItem> {
|
||||
let mut x = start_x;
|
||||
let mut out = Vec::new();
|
||||
for c in chars.chars() {
|
||||
out.push(make_merge_item(&c.to_string(), x, glyph_w));
|
||||
x += glyph_w + gap;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_caps_run_collapses_to_word() {
|
||||
// Display tracking: every letter gap (0.19 em) clears the fixed
|
||||
// space threshold — without the run-local floor this reads "H O W".
|
||||
let items = glyph_run("HOW", 100.0, 10.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "HOW");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_run_keeps_word_gaps_bimodal() {
|
||||
// Letters at 0.19 em, word gaps at 0.42 em (below the 0.5 em item
|
||||
// break): the split must land between the modes. Needs >=4 gaps to
|
||||
// enter the bimodal tier — short runs use the strict uniform gate.
|
||||
let mut items = glyph_run("ITISOK", 100.0, 8.0, 2.3);
|
||||
for i in 2..6 {
|
||||
items[i].x += 2.8; // word gap at T|I
|
||||
}
|
||||
for i in 4..6 {
|
||||
items[i].x += 2.8; // word gap at S|O
|
||||
}
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "IT IS OK");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lowercase_spaced_singles_stay_words() {
|
||||
// "x y z" variables: same gap shape but lowercase — the short-run
|
||||
// caps requirement keeps genuine spaced singles apart.
|
||||
let items = glyph_run("xyz", 100.0, 6.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "x y z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kerned_singles_unaffected() {
|
||||
// Tiny kerning gaps never triggered spaces before and still don't.
|
||||
let items = glyph_run("WORD", 100.0, 8.0, 0.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "WORD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_lowercase_spaced_singles_keep_boundaries() {
|
||||
// Review: a 5+ single-letter lowercase list has the tracked gap
|
||||
// shape at any length — the convention gate must protect it in
|
||||
// the >=4-gap tier too.
|
||||
let items = glyph_run("abcde", 100.0, 6.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "a b c d e");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn han_run_with_nonuniform_gaps_never_gains_spaces() {
|
||||
// Review: a bimodal gap distribution (justification, punctuation
|
||||
// spacing) must not manufacture word boundaries in Han text.
|
||||
let mut items = glyph_run("北京时事快报", 100.0, 12.0, 1.4);
|
||||
for item in items.iter_mut().skip(3) {
|
||||
item.x += 3.0; // wide gap after the third glyph
|
||||
}
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "北京时事快报");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uppercase_leading_spaced_singles_keep_boundaries() {
|
||||
// "A b c d e" is indistinguishable from a title-case tracked word
|
||||
// without reliable tracking metadata, so preserve its boundaries.
|
||||
let items = glyph_run("Abcde", 100.0, 7.0, 2.3);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "A b c d e");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cjk_glyph_run_collapses_without_spaces() {
|
||||
// CJK sets one glyph per item with loose gaps; CJK uses no spaces,
|
||||
// and the non-alphabetic run passes the caps gate.
|
||||
let items = glyph_run("北京时事", 100.0, 12.0, 1.4);
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "北京时事");
|
||||
}
|
||||
|
||||
fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.into(),
|
||||
|
||||
+9
-69
@@ -125,7 +125,7 @@ pub struct PdfProcessResult {
|
||||
/// .mode(ProcessMode::Analyze)
|
||||
/// .pages([1, 3, 5]);
|
||||
/// ```
|
||||
#[derive(Clone)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PdfOptions {
|
||||
/// How far the pipeline should run (default: [`ProcessMode::Full`]).
|
||||
pub mode: ProcessMode,
|
||||
@@ -135,23 +135,6 @@ pub struct PdfOptions {
|
||||
pub markdown: MarkdownOptions,
|
||||
/// Optional set of 1-indexed pages to process. `None` = all pages.
|
||||
pub page_filter: Option<HashSet<u32>>,
|
||||
/// Password for decrypting an encrypted PDF. `None` falls back to the
|
||||
/// empty password (owner-only encryption).
|
||||
pub password: Option<String>,
|
||||
}
|
||||
|
||||
// Manual `Debug` so the password is never leaked through debug logging or a
|
||||
// panic that formats the options; it renders as `Some("[REDACTED]")`.
|
||||
impl std::fmt::Debug for PdfOptions {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PdfOptions")
|
||||
.field("mode", &self.mode)
|
||||
.field("detection", &self.detection)
|
||||
.field("markdown", &self.markdown)
|
||||
.field("page_filter", &self.page_filter)
|
||||
.field("password", &self.password.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PdfOptions {
|
||||
@@ -161,7 +144,6 @@ impl Default for PdfOptions {
|
||||
detection: DetectionConfig::default(),
|
||||
markdown: MarkdownOptions::default(),
|
||||
page_filter: None,
|
||||
password: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -203,12 +185,6 @@ impl PdfOptions {
|
||||
self.page_filter = Some(pages.into_iter().collect());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the password used to decrypt an encrypted PDF.
|
||||
pub fn password(mut self, password: impl Into<String>) -> Self {
|
||||
self.password = Some(password.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
@@ -241,8 +217,7 @@ pub fn process_pdf_with_options<P: AsRef<Path>>(
|
||||
validate_pdf_file(&path)?;
|
||||
|
||||
// Load the document once — shared by detection AND extraction.
|
||||
let (doc, page_count) =
|
||||
load_document_from_path_with_password(&path, options.password.as_deref())?;
|
||||
let (doc, page_count) = load_document_from_path(&path)?;
|
||||
|
||||
process_document(doc, page_count, options, start)
|
||||
}
|
||||
@@ -267,8 +242,7 @@ pub fn process_pdf_mem_with_options(
|
||||
let start = std::time::Instant::now();
|
||||
validate_pdf_bytes(buffer)?;
|
||||
|
||||
let (doc, page_count) =
|
||||
load_document_from_mem_with_password(buffer, options.password.as_deref())?;
|
||||
let (doc, page_count) = load_document_from_mem(buffer)?;
|
||||
|
||||
process_document(doc, page_count, options, start)
|
||||
}
|
||||
@@ -3293,40 +3267,24 @@ fn tsr_region_contains_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
/// page count from it directly to avoid the metadata-only round-trip.
|
||||
pub(crate) fn load_document_from_path<P: AsRef<Path>>(
|
||||
path: P,
|
||||
) -> Result<(Document, u32), PdfError> {
|
||||
load_document_from_path_with_password(path, None)
|
||||
}
|
||||
|
||||
/// Load a PDF file, decrypting with `password` if the file is encrypted.
|
||||
pub(crate) fn load_document_from_path_with_password<P: AsRef<Path>>(
|
||||
path: P,
|
||||
password: Option<&str>,
|
||||
) -> Result<(Document, u32), PdfError> {
|
||||
let buffer = std::fs::read(&path)?;
|
||||
load_document_from_mem_with_password(&buffer, password)
|
||||
load_document_from_mem(&buffer)
|
||||
}
|
||||
|
||||
/// Load a PDF from a memory buffer.
|
||||
pub(crate) fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
|
||||
load_document_from_mem_with_password(buffer, None)
|
||||
}
|
||||
|
||||
/// Load a PDF from a memory buffer, decrypting with `password` if encrypted.
|
||||
pub(crate) fn load_document_from_mem_with_password(
|
||||
buffer: &[u8],
|
||||
password: Option<&str>,
|
||||
) -> Result<(Document, u32), PdfError> {
|
||||
// Fix malformed struct element names before parsing. Some PDF generators
|
||||
// write bare names (/S Code) instead of proper PDF names (/S /Code), which
|
||||
// causes lopdf to silently drop the entire object.
|
||||
let fixed = structure_tree::fix_bare_struct_names(buffer);
|
||||
let buf = fixed.as_ref();
|
||||
|
||||
let doc = match load_document_bytes(buf, password) {
|
||||
let doc = match load_document_bytes(buf) {
|
||||
Ok(doc) => doc,
|
||||
Err(first_err) => {
|
||||
for repaired in repair_pdf_container_candidates(buf) {
|
||||
match load_document_bytes(&repaired, password) {
|
||||
match load_document_bytes(&repaired) {
|
||||
Ok(doc) => {
|
||||
log::debug!("loaded PDF after repairing malformed container bytes");
|
||||
let page_count = doc.get_pages().len() as u32;
|
||||
@@ -3346,31 +3304,13 @@ pub(crate) fn load_document_from_mem_with_password(
|
||||
Ok((doc, page_count))
|
||||
}
|
||||
|
||||
fn load_document_bytes(buf: &[u8], password: Option<&str>) -> Result<Document, lopdf::Error> {
|
||||
fn load_document_bytes(buf: &[u8]) -> Result<Document, lopdf::Error> {
|
||||
match Document::load_mem(buf) {
|
||||
// Some encrypted PDFs load structurally but leave their streams
|
||||
// encrypted (`is_encrypted()` stays true); reading them yields garbage
|
||||
// until we re-load with a password. Others fail load_mem outright with
|
||||
// an encryption error. Handle both by re-loading with the password.
|
||||
Ok(doc) if doc.is_encrypted() => decrypt_document_bytes(buf, password),
|
||||
Ok(doc) => Ok(doc),
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => decrypt_document_bytes(buf, password),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-load an encrypted PDF, decrypting with `password`. Falls back to the
|
||||
/// empty password (owner-only encryption, the common "protected" case) when a
|
||||
/// non-empty password was supplied but rejected.
|
||||
fn decrypt_document_bytes(buf: &[u8], password: Option<&str>) -> Result<Document, lopdf::Error> {
|
||||
let pw = password.unwrap_or("");
|
||||
match Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(pw)) {
|
||||
Ok(doc) => Ok(doc),
|
||||
Err(inner) if !pw.is_empty() => {
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))
|
||||
.map_err(|_| inner)
|
||||
}
|
||||
Err(inner) => Err(inner),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -174,21 +174,6 @@ pub(crate) fn is_toc_marker_heading(text: &str) -> bool {
|
||||
pub(crate) fn is_heading_fragment(text: &str) -> bool {
|
||||
let t = text.trim_end();
|
||||
|
||||
// A lowercase-initial one-or-two-word "heading" is a mid-sentence
|
||||
// fragment beside display math ("or inversely", "and therefore") —
|
||||
// real headings that short start uppercase. Measured as spurious
|
||||
// headings on academic docs (fire-pdf ENG-5029 / opendataloader MHS).
|
||||
{
|
||||
let words: Vec<&str> = t.split_whitespace().collect();
|
||||
if words.len() <= 2 {
|
||||
if let Some(first_alpha) = t.chars().find(|c| c.is_alphabetic()) {
|
||||
if first_alpha.is_lowercase() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_equation_number(s: &str) -> bool {
|
||||
s.strip_prefix('(')
|
||||
.and_then(|r| r.strip_suffix(')'))
|
||||
@@ -475,10 +460,6 @@ mod tests {
|
||||
#[test]
|
||||
fn heading_fragments() {
|
||||
// Equation lead-ins: colon ending + inline equation reference
|
||||
assert!(is_heading_fragment("or inversely"));
|
||||
assert!(is_heading_fragment("and therefore"));
|
||||
assert!(!is_heading_fragment("Introduction"));
|
||||
assert!(!is_heading_fragment("iPhone Sales Strategy Overview")); // 4 words, exempt
|
||||
assert!(is_heading_fragment("Rearranging Equation (8) gives:"));
|
||||
// Display-equation neighbours ending in an equation number
|
||||
assert!(is_heading_fragment("S = kB ln W, (2)"));
|
||||
|
||||
BIN
Binary file not shown.
@@ -3606,45 +3606,3 @@ fn test_markdown_options_default_has_include_images_false() {
|
||||
let opts = MarkdownOptions::default();
|
||||
assert!(!opts.include_images);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_pdf_decrypts_with_correct_password() {
|
||||
let path = "tests/fixtures/encrypted-secret123.pdf";
|
||||
|
||||
// No password: the file is encrypted and can't be read.
|
||||
let no_pw = process_pdf_with_options(path, PdfOptions::new());
|
||||
assert!(
|
||||
matches!(no_pw, Err(PdfError::Encrypted)),
|
||||
"expected Encrypted without a password, got {no_pw:?}"
|
||||
);
|
||||
|
||||
// Wrong password: still rejected.
|
||||
let wrong = process_pdf_with_options(path, PdfOptions::new().password("wrong"));
|
||||
assert!(
|
||||
matches!(wrong, Err(PdfError::Encrypted)),
|
||||
"expected Encrypted with a wrong password, got {wrong:?}"
|
||||
);
|
||||
|
||||
// Correct password: decrypts and extracts real content.
|
||||
let ok = process_pdf_with_options(path, PdfOptions::new().password("secret123"))
|
||||
.expect("correct password should decrypt");
|
||||
let md = ok.markdown.unwrap_or_default();
|
||||
// Assert a stable fixture token so a garbled-but-long extraction (the
|
||||
// encrypted-stream regression this guards) still fails the test.
|
||||
assert!(
|
||||
md.contains("Procurement"),
|
||||
"decrypted markdown should contain the fixture's real text, got {} chars",
|
||||
md.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pdf_options_debug_redacts_password() {
|
||||
let opts = PdfOptions::new().password("secret123");
|
||||
let dbg = format!("{opts:?}");
|
||||
assert!(
|
||||
!dbg.contains("secret123"),
|
||||
"password leaked in Debug: {dbg}"
|
||||
);
|
||||
assert!(dbg.contains("REDACTED"), "expected redaction marker: {dbg}");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user