Compare commits

..
Author SHA1 Message Date
Cursor AgentandAbimael Martell bfb33caa37 fix(glyph_names): handle non-ASCII input in uniXXXX glyph name parsing
Use str::get instead of a byte-length check plus slice when parsing the
uniXXXX glyph-name form. The byte-length guard only proved the index was
in bounds, not on a UTF-8 char boundary, so a glyph name containing
non-ASCII bytes could cause a slice on a non-boundary index. Switch to a
checked slice that folds into the existing Option flow, and add tests.

Co-authored-by: Abimael Martell <abimaelmartell@users.noreply.github.com>
2026-08-09 07:04:43 +00:00
15 changed files with 121 additions and 1337 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "pdf-inspector"
version = "0.1.8"
version = "0.1.7"
edition = "2021"
autobins = false
authors = ["Firecrawl Team"]
+3 -4
View File
@@ -5,15 +5,14 @@
If you believe you've found a security vulnerability in pdf-inspector, please
report it privately so we can fix it before public disclosure.
**Preferred:** Submit through Firecrawl's Bugcrowd vulnerability disclosure
program at <https://bugcrowd.com/engagements/firecrawl-vdp-ess>. Please include:
**Preferred:** Email **help@firecrawl.dev** with:
- A description of the issue and its impact
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
- The version or commit hash of pdf-inspector you tested against
**Alternative:** If you'd rather not use Bugcrowd, email
**help@firecrawl.dev** with the same details.
**Alternative:** Use GitHub's private vulnerability reporting under the
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
We'll acknowledge your report in a timely manner and keep you updated on
remediation progress. Please do not open a public GitHub issue for security
+2 -2
View File
@@ -851,7 +851,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "pdf-inspector"
version = "0.1.8"
version = "0.1.7"
dependencies = [
"env_logger",
"include_dir",
@@ -867,7 +867,7 @@ dependencies = [
[[package]]
name = "pdf-inspector-napi"
version = "0.2.3"
version = "0.2.2"
dependencies = [
"napi",
"napi-build",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "pdf-inspector-napi"
version = "0.2.3"
version = "0.2.2"
edition = "2021"
[lib]
-16
View File
@@ -83,22 +83,6 @@ for (const region of result[0].regions) {
}
```
### Async variants
`processPdf`, `classifyPdf`, and `extractPagesMarkdown` are synchronous and parse on the calling thread — in Node, that's the event loop. For a one-off call in a script that's fine, but in a server a large document can hold the loop for tens to hundreds of milliseconds.
`processPdfAsync`, `classifyPdfAsync`, and `extractPagesMarkdownAsync` take the same arguments and produce the same results, but run the parse on the libuv thread pool and return a promise, keeping the event loop free. The input buffer is copied before the call returns, so it's safe to reuse or mutate immediately:
```typescript
import { classifyPdfAsync, extractPagesMarkdownAsync } from '@firecrawl/pdf-inspector'
const classification = await classifyPdfAsync(pdf)
if (classification.pdfType === 'TextBased') {
const { pages } = await extractPagesMarkdownAsync(pdf)
// ...
}
```
## Types
```typescript
+6 -6
View File
@@ -8,12 +8,12 @@
"@napi-rs/cli": "^3.4.1",
},
"optionalDependencies": {
"@firecrawl/pdf-inspector-darwin-arm64": "1.13.0",
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.13.0",
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.13.0",
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.13.0",
"@firecrawl/pdf-inspector-linux-x64-musl": "1.13.0",
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.13.0",
"@firecrawl/pdf-inspector-darwin-arm64": "1.12.0",
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.12.0",
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.12.0",
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.12.0",
"@firecrawl/pdf-inspector-linux-x64-musl": "1.12.0",
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.12.0",
},
},
},
+7 -7
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.13.0",
"version": "1.12.0",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
@@ -52,11 +52,11 @@
"@napi-rs/cli": "^3.4.1"
},
"optionalDependencies": {
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.13.0",
"@firecrawl/pdf-inspector-linux-x64-musl": "1.13.0",
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.13.0",
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.13.0",
"@firecrawl/pdf-inspector-darwin-arm64": "1.13.0",
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.13.0"
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.12.0",
"@firecrawl/pdf-inspector-linux-x64-musl": "1.12.0",
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.12.0",
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.12.0",
"@firecrawl/pdf-inspector-darwin-arm64": "1.12.0",
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.12.0"
}
}
+41 -182
View File
@@ -153,7 +153,9 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
}
}
fn to_napi_page_ocr_reasons(reasons: Vec<pdf_inspector::PageOcrReasons>) -> Vec<PageOcrReasons> {
fn to_napi_page_ocr_reasons(
reasons: Vec<pdf_inspector::PageOcrReasons>,
) -> Vec<PageOcrReasons> {
reasons
.into_iter()
.map(|reason| PageOcrReasons {
@@ -200,31 +202,6 @@ where
}
}
// ---------------------------------------------------------------------------
// Shared implementations (single body behind sync and async entry points)
// ---------------------------------------------------------------------------
fn process_pdf_impl(bytes: &[u8], pages: Option<Vec<u32>>) -> Result<PdfResult> {
let mut opts = pdf_inspector::PdfOptions::new();
if let Some(p) = pages {
opts = opts.pages(p);
}
let result = pdf_inspector::process_pdf_mem_with_options(bytes, opts)
.map_err(|e| to_napi_err(e, "process_pdf"))?;
Ok(to_napi_result(result))
}
fn classify_pdf_impl(bytes: &[u8]) -> Result<PdfClassification> {
let result =
pdf_inspector::classify_pdf_mem(bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
Ok(PdfClassification {
pdf_type: convert_pdf_type(result.pdf_type),
page_count: result.page_count,
pages_needing_ocr: result.pages_needing_ocr,
confidence: result.confidence as f64,
})
}
// ---------------------------------------------------------------------------
// Public NAPI API
// ---------------------------------------------------------------------------
@@ -233,7 +210,15 @@ fn classify_pdf_impl(bytes: &[u8]) -> Result<PdfClassification> {
#[napi]
pub fn process_pdf(buffer: Buffer, pages: Option<Vec<u32>>) -> Result<PdfResult> {
let bytes: Vec<u8> = buffer.to_vec();
catch_panic("process_pdf", move || process_pdf_impl(&bytes, pages))
catch_panic("process_pdf", move || {
let mut opts = pdf_inspector::PdfOptions::new();
if let Some(p) = pages {
opts = opts.pages(p);
}
let result = pdf_inspector::process_pdf_mem_with_options(&bytes, opts)
.map_err(|e| to_napi_err(e, "process_pdf"))?;
Ok(to_napi_result(result))
})
}
/// Fast detection only — no text extraction or markdown.
@@ -253,7 +238,16 @@ pub fn detect_pdf(buffer: Buffer) -> Result<PdfResult> {
#[napi]
pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
let bytes: Vec<u8> = buffer.to_vec();
catch_panic("classify_pdf", move || classify_pdf_impl(&bytes))
catch_panic("classify_pdf", move || {
let result =
pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
Ok(PdfClassification {
pdf_type: convert_pdf_type(result.pdf_type),
page_count: result.page_count,
pages_needing_ocr: result.pages_needing_ocr,
confidence: result.confidence as f64,
})
})
}
/// Extract plain text from a PDF Buffer.
@@ -639,32 +633,25 @@ pub fn extract_pages_markdown(
) -> Result<PagesExtractionResult> {
let bytes: Vec<u8> = buffer.to_vec();
catch_panic("extract_pages_markdown", move || {
extract_pages_markdown_impl(&bytes, pages.as_deref())
})
}
fn extract_pages_markdown_impl(
bytes: &[u8],
pages: Option<&[u32]>,
) -> Result<PagesExtractionResult> {
let result = pdf_inspector::extract_pages_markdown_mem(bytes, pages)
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
Ok(PagesExtractionResult {
pages: result
.pages
.into_iter()
.map(|r| PageMarkdownResult {
page: r.page,
markdown: r.markdown,
needs_ocr: r.needs_ocr,
ocr_reason: r.ocr_reason,
})
.collect(),
pages_with_tables: result.pages_with_tables,
pages_with_columns: result.pages_with_columns,
pages_needing_ocr: result.pages_needing_ocr,
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
is_complex: result.is_complex,
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
Ok(PagesExtractionResult {
pages: result
.pages
.into_iter()
.map(|r| PageMarkdownResult {
page: r.page,
markdown: r.markdown,
needs_ocr: r.needs_ocr,
ocr_reason: r.ocr_reason,
})
.collect(),
pages_with_tables: result.pages_with_tables,
pages_with_columns: result.pages_with_columns,
pages_needing_ocr: result.pages_needing_ocr,
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
is_complex: result.is_complex,
})
})
}
@@ -705,131 +692,3 @@ fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<Pa
})
.collect()
}
// ---------------------------------------------------------------------------
// Async variants (libuv thread pool via AsyncTask)
//
// The synchronous exports above parse on the calling thread, which in Node is
// the event loop. These `*Async` variants run the same shared implementations
// on the libuv thread pool and hand JavaScript a promise, so servers under
// concurrent load keep answering requests while a document parses. The sync
// exports keep their names, signatures, and behaviour.
//
// Each factory copies the input Buffer to an owned `Vec<u8>` on the calling
// (JS) thread — deliberately. JS execution is single-threaded, so no JS code
// can mutate the buffer while the synchronous part of the call copies it.
// Holding the napi `Buffer` and reading it from the worker instead would be
// zero-copy, but a caller mutating the buffer before the promise settles
// would then race the worker's reads — undefined behavior, not a recoverable
// error (a known napi-rs soundness hazard with cross-thread Buffer access).
// The copy is a one-time memcpy, negligible next to the parse it unblocks.
// ---------------------------------------------------------------------------
pub struct ProcessPdfTask {
bytes: Vec<u8>,
pages: Option<Vec<u32>>,
}
impl Task for ProcessPdfTask {
type Output = PdfResult;
type JsValue = PdfResult;
fn compute(&mut self) -> Result<Self::Output> {
let bytes = std::mem::take(&mut self.bytes);
let pages = self.pages.take();
// AssertUnwindSafe: `bytes`/`pages` are moved into the closure and
// dropped on unwind — no shared state can be observed broken.
catch_panic(
"process_pdf",
panic::AssertUnwindSafe(move || process_pdf_impl(&bytes, pages)),
)
}
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(output)
}
}
/// Async variant of [`processPdf`]: same result, but the parse runs on the
/// libuv thread pool instead of the event loop and the call returns a
/// promise. The buffer is copied before the call returns, so it may be
/// reused or mutated immediately.
// ts_return_type is required: napi-rs emits `Promise<unknown>` for
// `AsyncTask<T>` returns without it.
#[napi(ts_return_type = "Promise<PdfResult>")]
pub fn process_pdf_async(buffer: Buffer, pages: Option<Vec<u32>>) -> AsyncTask<ProcessPdfTask> {
AsyncTask::new(ProcessPdfTask {
bytes: buffer.to_vec(),
pages,
})
}
pub struct ClassifyPdfTask {
bytes: Vec<u8>,
}
impl Task for ClassifyPdfTask {
type Output = PdfClassification;
type JsValue = PdfClassification;
fn compute(&mut self) -> Result<Self::Output> {
let bytes = std::mem::take(&mut self.bytes);
catch_panic(
"classify_pdf",
panic::AssertUnwindSafe(move || classify_pdf_impl(&bytes)),
)
}
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(output)
}
}
/// Async variant of [`classifyPdf`]: same result, but the classification runs
/// on the libuv thread pool instead of the event loop and the call returns a
/// promise. The buffer is copied before the call returns, so it may be
/// reused or mutated immediately.
#[napi(ts_return_type = "Promise<PdfClassification>")]
pub fn classify_pdf_async(buffer: Buffer) -> AsyncTask<ClassifyPdfTask> {
AsyncTask::new(ClassifyPdfTask {
bytes: buffer.to_vec(),
})
}
pub struct ExtractPagesMarkdownTask {
bytes: Vec<u8>,
pages: Option<Vec<u32>>,
}
impl Task for ExtractPagesMarkdownTask {
type Output = PagesExtractionResult;
type JsValue = PagesExtractionResult;
fn compute(&mut self) -> Result<Self::Output> {
let bytes = std::mem::take(&mut self.bytes);
let pages = self.pages.take();
catch_panic(
"extract_pages_markdown",
panic::AssertUnwindSafe(move || extract_pages_markdown_impl(&bytes, pages.as_deref())),
)
}
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(output)
}
}
/// Async variant of [`extractPagesMarkdown`]: same result, but the extraction
/// runs on the libuv thread pool instead of the event loop and the call
/// returns a promise. The buffer is copied before the call returns, so it
/// may be reused or mutated immediately.
#[napi(ts_return_type = "Promise<PagesExtractionResult>")]
pub fn extract_pages_markdown_async(
buffer: Buffer,
pages: Option<Vec<u32>>,
) -> AsyncTask<ExtractPagesMarkdownTask> {
AsyncTask::new(ExtractPagesMarkdownTask {
bytes: buffer.to_vec(),
pages,
})
}
-68
View File
@@ -2,16 +2,13 @@ import { readFileSync } from 'fs';
import { strict as assert } from 'assert';
import {
processPdf,
processPdfAsync,
detectPdf,
classifyPdf,
classifyPdfAsync,
extractText,
extractTextWithPositions,
extractTextInRegions,
detectVectorGridInRegion,
extractPagesMarkdown,
extractPagesMarkdownAsync,
} from './index.js';
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
@@ -127,75 +124,10 @@ assert.equal(picked.pages[0].page, 2);
assert.equal(picked.pages[1].page, 0);
console.log(' extractPagesMarkdown with pages: OK');
// --- Async variants ---
console.log('Testing async variants...');
// processPdfAsync returns a promise and matches the sync result
const asyncResultPromise = processPdfAsync(fixture);
assert.ok(asyncResultPromise instanceof Promise);
const asyncResult = await asyncResultPromise;
assert.equal(asyncResult.pdfType, result.pdfType);
assert.equal(asyncResult.pageCount, result.pageCount);
assert.equal(asyncResult.markdown, result.markdown);
console.log(' processPdfAsync: OK');
// processPdfAsync with pages
const asyncResult2 = await processPdfAsync(fixture, [1]);
assert.equal(asyncResult2.markdown, result2.markdown);
console.log(' processPdfAsync with pages: OK');
// classifyPdfAsync matches the sync result
const asyncClassified = await classifyPdfAsync(fixture);
assert.equal(asyncClassified.pdfType, classified.pdfType);
assert.equal(asyncClassified.pageCount, classified.pageCount);
assert.equal(asyncClassified.confidence, classified.confidence);
assert.deepEqual(asyncClassified.pagesNeedingOcr, classified.pagesNeedingOcr);
console.log(' classifyPdfAsync: OK');
// extractPagesMarkdownAsync matches the sync result
const asyncAllPages = await extractPagesMarkdownAsync(fixture);
assert.equal(asyncAllPages.pages.length, allPages.pages.length);
assert.deepEqual(
asyncAllPages.pages.map(p => p.markdown),
allPages.pages.map(p => p.markdown),
);
assert.equal(asyncAllPages.isComplex, allPages.isComplex);
console.log(' extractPagesMarkdownAsync: OK');
// selected pages preserve caller order
const asyncPicked = await extractPagesMarkdownAsync(fixture, [2, 0]);
assert.equal(asyncPicked.pages.length, 2);
assert.equal(asyncPicked.pages[0].page, 2);
assert.equal(asyncPicked.pages[1].page, 0);
console.log(' extractPagesMarkdownAsync with pages: OK');
// input buffer is copied at call time: mutating it immediately after the
// call must not affect the in-flight parse
const scratch = Buffer.from(fixture);
const inFlight = processPdfAsync(scratch);
scratch.fill(0);
const fromMutated = await inFlight;
assert.equal(fromMutated.markdown, result.markdown);
console.log(' processPdfAsync input copied at call time: OK');
// concurrent async calls all settle
const [c1, c2, c3] = await Promise.all([
processPdfAsync(fixture),
classifyPdfAsync(fixture),
extractPagesMarkdownAsync(fixture),
]);
assert.equal(c1.pdfType, 'TextBased');
assert.equal(c2.pdfType, 'TextBased');
assert.equal(c3.pages.length, 3);
console.log(' concurrent async calls: OK');
// --- Error handling ---
console.log('Testing error handling...');
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
assert.throws(() => classifyPdf(Buffer.from('')), /classify_pdf/);
await assert.rejects(processPdfAsync(Buffer.from('not a pdf')), /process_pdf/);
await assert.rejects(classifyPdfAsync(Buffer.from('')), /classify_pdf/);
await assert.rejects(extractPagesMarkdownAsync(Buffer.from('')), /extract_pages_markdown/);
console.log(' error handling: OK');
console.log('\nAll NAPI tests passed!');
+1 -1
View File
@@ -6,7 +6,7 @@ build-backend = "maturin"
name = "pdf-inspector"
# Bump this to publish to PyPI — CI publishes automatically when the version
# changes on main (same flow as napi/package.json for npm).
version = "0.2.7"
version = "0.2.6"
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
readme = "docs/python.md"
license = { text = "MIT" }
+17 -327
View File
@@ -41,110 +41,15 @@ pub(crate) fn detect_columns(
}
debug!("page {}: detect_columns: {} items", page, page_items.len());
// The width of one ordinary page, used three ways below: as the largest
// credible width for a single text run, as the size of empty gap that marks
// content as detached, and as the span past which those checks run at all.
// This is a heuristic, not a format rule: PDF 2.0 sets no page-size limit,
// and since PDF 1.6 `UserUnit` scales a page's physical size independently
// of its coordinates. 14_400 units (200in at the default 1/72in unit) is
// the traditional Acrobat architectural limit, which makes it a reasonable
// "wider than any ordinary page" mark in coordinate space.
const MAX_PAGE_EXTENT: f32 = 14_400.0;
// A detached cluster is only dropped if it also holds a small minority of
// the items, so a genuine two-part layout keeps its full bounds even when
// the halves are far apart.
const MAX_TRIM_FRACTION: f32 = 0.10;
// Position and width of each item, skipping only non-finite geometry.
let finite_span = |i: &&TextItem| -> Option<(f32, f32)> {
let (left, width) = (i.x, effective_width(i));
(left.is_finite() && (left + width).is_finite()).then_some((left, width))
};
let (min_left, max_right, total) = page_items.iter().filter_map(finite_span).fold(
(f32::INFINITY, f32::NEG_INFINITY, 0usize),
|(lo, hi, n), (left, width)| (lo.min(left), hi.max(left + width), n + 1),
);
// No item had usable geometry, so there is no layout to report.
if total == 0 {
return vec![];
}
// Every threshold below (gutter margins, spanning-item width, the XY-cut
// margin) is a fraction of the page width, so a far item can set the scale
// for the whole page and shrink the effective detection window to a
// rounding error — real gutters then fall inside the margin band and a
// genuine multi-column page collapses to one region.
//
// Anything inside one page extent is ordinary, so the common case keeps the
// plain bounds and skips the work below entirely.
let (x_min, x_max) = if max_right - min_left <= MAX_PAGE_EXTENT {
(min_left, max_right)
} else {
// Discarding content needs positive evidence that it is not part of the
// layout, because a count-based rule alone cannot tell a stray from a
// sparse far sidebar. The evidence is geometric: positions are grouped
// into clusters separated by more than a whole page of continuous
// emptiness. Real content, however sparse, does not leave a void that
// large; a malformed coordinate sits alone beyond one.
let mut spans: Vec<(f32, f32)> = page_items.iter().filter_map(finite_span).collect();
spans.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut core: Option<std::ops::Range<usize>> = None;
let mut start = 0usize;
for i in 1..=spans.len() {
if i < spans.len() && spans[i].0 - spans[i - 1].0 <= MAX_PAGE_EXTENT {
continue;
}
if core.as_ref().is_none_or(|best| i - start > best.len()) {
core = Some(start..i);
}
start = i;
}
let mut core = core.unwrap_or(0..spans.len());
// Only drop the detached clusters when they are a small minority, so a
// genuine two-part layout keeps its full bounds.
let dropped = spans.len() - core.len();
if dropped as f32 > spans.len() as f32 * MAX_TRIM_FRACTION {
core = 0..spans.len();
}
let core = &spans[core];
// Positions cannot be inflated by a bogus width, so the spread of the
// content is a sound scale for judging one. A run much wider than the
// page's own content is a malformed width — the test is relative, so a
// genuinely large page keeps its genuinely long runs.
let (lo, widest_left) = (core[0].0, core[core.len() - 1].0);
let max_run_width = (widest_left - lo) + MAX_PAGE_EXTENT;
let hi = core
.iter()
.filter(|&&(_, width)| width <= max_run_width)
.map(|&(left, width)| left + width)
.fold(widest_left, f32::max);
if lo != min_left || hi != max_right {
debug!(
"page {page}: bounds {min_left}..{max_right} exceed one page; \
dropped {dropped}/{} detached item(s), using {lo}..{hi}",
spans.len()
);
}
(lo, hi)
};
// Hard ceiling on the histogram size, independent of the trimming above:
// the bounds are attacker-influenced, so an unclamped
// `page_width / BIN_WIDTH` lets a crafted PDF force an arbitrarily large
// `vec![0u32; num_bins]` allocation. 65_536 bins covers ~128k points at
// BIN_WIDTH 2.0 — roughly 9x the largest legal page — so this never binds
// on a real layout. Kept as a bound that does not depend on the outlier
// heuristic staying correct.
const MAX_BINS: usize = 65_536;
// Find page bounds
let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
let x_max = page_items
.iter()
.map(|i| i.x + effective_width(i))
.fold(f32::NEG_INFINITY, f32::max);
let page_width = x_max - x_min;
if !page_width.is_finite() || page_width < 200.0 {
if page_width < 200.0 {
return vec![ColumnRegion { x_min, x_max }];
}
@@ -152,20 +57,13 @@ pub(crate) fn detect_columns(
return vec![ColumnRegion { x_min, x_max }];
}
// Widen the bins rather than dropping the tail of the page. Clamping the
// count alone would leave anything past MAX_BINS * BIN_WIDTH outside the
// histogram, folded into the last bin, which places gutters at the wrong
// coordinates. Scaling keeps full coverage under the same allocation
// ceiling; only the resolution degrades, and only beyond ~131k points.
let bin_width = BIN_WIDTH.max(page_width / MAX_BINS as f32);
// Build occupancy histogram.
// Exclude items wider than 60% of page width — these are spanning items
// (titles, full-width paragraphs) that would fill the gutter and prevent
// detection of partial-page column layouts (e.g. two-column abstracts on
// a page that also has single-column introduction text).
let wide_threshold = page_width * 0.6;
let num_bins = ((page_width / bin_width).ceil() as usize).clamp(1, MAX_BINS);
let num_bins = ((page_width / BIN_WIDTH).ceil() as usize).max(1);
let mut histogram = vec![0u32; num_bins];
for item in &page_items {
@@ -173,8 +71,8 @@ pub(crate) fn detect_columns(
if w > wide_threshold {
continue;
}
let left = ((item.x - x_min) / bin_width).floor() as usize;
let right = (((item.x + w) - x_min) / bin_width).ceil() as usize;
let left = ((item.x - x_min) / BIN_WIDTH).floor() as usize;
let right = (((item.x + w) - x_min) / BIN_WIDTH).ceil() as usize;
let left = left.min(num_bins);
let right = right.min(num_bins);
for count in histogram.iter_mut().take(right).skip(left) {
@@ -211,12 +109,12 @@ pub(crate) fn detect_columns(
let valleys: Vec<(usize, usize)> = valleys
.into_iter()
.filter(|&(start, end)| {
let width_pts = (end - start) as f32 * bin_width;
let width_pts = (end - start) as f32 * BIN_WIDTH;
if width_pts < MIN_GUTTER_WIDTH {
return false;
}
// Valley center must not be within 5% of page edges
let center_pts = ((start + end) as f32 / 2.0) * bin_width;
let center_pts = ((start + end) as f32 / 2.0) * BIN_WIDTH;
center_pts > margin_threshold && center_pts < (page_width - margin_threshold)
})
.collect();
@@ -234,7 +132,7 @@ pub(crate) fn detect_columns(
&histogram,
num_bins,
x_min,
bin_width,
BIN_WIDTH,
page_width,
margin_threshold,
);
@@ -243,7 +141,7 @@ pub(crate) fn detect_columns(
&rel_valleys,
&page_items,
x_min,
bin_width,
BIN_WIDTH,
x_max,
MIN_ITEMS_PER_COLUMN,
MIN_VERTICAL_SPAN_RATIO,
@@ -284,7 +182,7 @@ pub(crate) fn detect_columns(
&valleys,
&page_items,
x_min,
bin_width,
BIN_WIDTH,
x_max,
MIN_ITEMS_PER_COLUMN,
MIN_VERTICAL_SPAN_RATIO,
@@ -298,7 +196,7 @@ pub(crate) fn detect_columns(
&valleys,
&page_items,
x_min,
bin_width,
BIN_WIDTH,
x_max,
MIN_ITEMS_PER_COLUMN,
MIN_VERTICAL_SPAN_RATIO,
@@ -1929,7 +1827,7 @@ fn split_column_stragglers(lines: Vec<TextLine>) -> (Vec<TextLine>, Vec<TextLine
.unwrap();
let (cs, ce) = segments[core_seg];
let mut core = Vec::with_capacity(ce.saturating_sub(cs));
let mut core = Vec::with_capacity(ce - cs);
let mut stragglers = Vec::new();
for (i, line) in lines.into_iter().enumerate() {
if i >= cs && i < ce {
@@ -2635,214 +2533,6 @@ mod tests {
);
}
#[test]
fn extreme_far_coordinate_does_not_allocate_unboundedly() {
// A crafted PDF can place a text run at an arbitrary coordinate via the
// text matrix. The derived page width must not drive an unbounded
// histogram allocation (previously `page_width / BIN_WIDTH` bins with no
// upper bound would try to reserve terabytes and abort the process).
let mut items = Vec::new();
for i in 0..24 {
items.push(make_item(1, i as f32 * 10.0, 700.0 - i as f32 * 5.0, "A"));
}
// Item placed 1e12 points away — 5e11 bins if left unclamped.
items.push(make_item(1, 1e12, 700.0, "Z"));
// Must return without aborting; content is preserved as a single region.
let cols = detect_columns(&items, 1, false);
assert!(!cols.is_empty());
}
#[test]
fn non_finite_coordinates_never_leak_into_region_bounds() {
// An inf/NaN coordinate must not escape as a column boundary: callers
// treat these as page/column edges.
for bad_x in [f32::INFINITY, f32::NEG_INFINITY, f32::NAN] {
let mut items = Vec::new();
for i in 0..24 {
items.push(make_item(1, i as f32 * 10.0, 700.0 - i as f32 * 5.0, "A"));
}
items.push(make_item(1, bad_x, 700.0, "Z"));
for col in detect_columns(&items, 1, false) {
assert!(
col.x_min.is_finite() && col.x_max.is_finite(),
"bad_x {bad_x} leaked bounds {}..{}",
col.x_min,
col.x_max
);
}
}
}
#[test]
fn all_non_finite_coordinates_yield_no_columns() {
let items: Vec<TextItem> = (0..24)
.map(|i| make_item(1, f32::NAN, 700.0 - i as f32 * 5.0, "A"))
.collect();
assert!(detect_columns(&items, 1, false).is_empty());
}
#[test]
fn one_bad_item_does_not_disable_column_detection() {
// A single stray item should not collapse a clean two-column page to
// one region. Every gutter threshold is a fraction of the page width,
// so an untrimmed outlier pushes real gutters inside the rejected
// margin band. A malformed *width* at an ordinary position poisons the
// bounds just as a malformed position does.
for (label, bad_x, bad_width) in [
("nan position", f32::NAN, 0.0),
("inf position", f32::INFINITY, 0.0),
("far position", 50_000.0, 0.0),
("very far position", 1e12, 0.0),
("huge width", 100.0, 1e12),
("inf width", 100.0, f32::INFINITY),
] {
let mut items = Vec::new();
items.extend(fill_zone(1, 30.0, 280.0, 750.0, 50.0));
items.extend(fill_zone(1, 320.0, 570.0, 750.0, 50.0));
let mut bad = make_item(1, bad_x, 400.0, "Z");
bad.width = bad_width;
items.push(bad);
let cols = detect_columns(&items, 1, false);
assert_eq!(
cols.len(),
2,
"{label}: expected 2 columns, got {}",
cols.len()
);
for col in &cols {
assert!(
col.x_max - col.x_min <= MAX_PAGE_EXTENT_FOR_TEST,
"{label}: region {}..{} exceeds one page",
col.x_min,
col.x_max
);
}
}
}
/// Mirrors `MAX_PAGE_EXTENT` in `detect_columns`.
const MAX_PAGE_EXTENT_FOR_TEST: f32 = 14_400.0;
#[test]
fn very_wide_page_keeps_full_histogram_coverage() {
// Beyond MAX_BINS * BIN_WIDTH (~131k points) the bins must widen rather
// than stop covering the page. Three zones: the first gutter is inside
// the old coverage limit, the second is past it. Because the first
// gutter is found, the XY-cut fallback never runs, so a truncated
// histogram silently reports two columns instead of three.
let mut items = Vec::new();
items.extend(fill_zone(1, 0.0, 60_000.0, 750.0, 700.0));
items.extend(fill_zone(1, 70_000.0, 140_000.0, 750.0, 700.0));
items.extend(fill_zone(1, 160_000.0, 200_000.0, 750.0, 700.0));
let cols = detect_columns(&items, 1, false);
assert_eq!(
cols.len(),
3,
"Expected 3 columns across a 200k-wide page, got {}",
cols.len()
);
assert!(
(140_000.0..=160_000.0).contains(&cols[1].x_max),
"second gutter at {}, expected inside the real 140k..160k gap",
cols[1].x_max
);
}
#[test]
fn large_page_with_legitimately_long_runs_is_kept() {
// On a very large page, individual runs can exceed one ordinary page's
// width. They are real content, so they must not be judged malformed:
// the page keeps its columns and its full right edge.
let mut items = Vec::new();
for row in 0..30 {
let y = 750.0 - row as f32 * 14.0;
let mut left = make_item(1, 0.0, y, "Left run");
left.width = 20_000.0;
let mut right = make_item(1, 25_000.0, y, "Right run");
right.width = 20_000.0;
items.extend([left, right]);
}
let cols = detect_columns(&items, 1, false);
assert!(
!cols.is_empty(),
"a page of long-but-valid runs must still report a layout"
);
let right_edge = cols
.iter()
.map(|c| c.x_max)
.fold(f32::NEG_INFINITY, f32::max);
assert!(
right_edge > 44_000.0,
"long runs were treated as malformed: right edge {right_edge}, expected ~45_000"
);
}
#[test]
fn sparse_far_sidebar_on_a_large_page_is_kept() {
// A large-format page with a thin, sparsely-populated sidebar far from
// the main block. The sidebar is a small minority of the items, so an
// item-count rule alone would discard it — but nothing about its
// geometry says it is invalid, so its bounds must survive.
let mut items = Vec::new();
items.extend(fill_zone(1, 0.0, 12_000.0, 750.0, 500.0));
for i in 0..12 {
items.push(make_item(1, 24_000.0, 750.0 - i as f32 * 14.0, "Sidebar"));
}
let cols = detect_columns(&items, 1, false);
let right_edge = cols
.iter()
.map(|c| c.x_max)
.fold(f32::NEG_INFINITY, f32::max);
assert!(
right_edge > 24_000.0,
"sidebar was trimmed away: right edge {right_edge}, expected >24_000"
);
}
#[test]
fn genuinely_wide_layout_keeps_its_true_bounds() {
// A large-format page whose content really is spread beyond one
// ordinary page must not be trimmed to the median cluster: its far
// items are the majority, not strays.
let mut items = Vec::new();
items.extend(fill_zone(1, 100.0, 20_000.0, 750.0, 600.0));
items.extend(fill_zone(1, 22_000.0, 40_000.0, 750.0, 600.0));
let cols = detect_columns(&items, 1, false);
let widest = cols
.iter()
.map(|c| c.x_max)
.fold(f32::NEG_INFINITY, f32::max);
assert!(
widest > 35_000.0,
"wide layout was trimmed: right edge {widest}, expected ~40_000"
);
}
#[test]
fn oversized_but_legal_page_is_not_trimmed() {
// A wide-format page well inside the 14_400pt spec limit must keep its
// real bounds — outlier trimming is only for spans beyond a legal page.
let mut items = Vec::new();
items.extend(fill_zone(1, 100.0, 4_000.0, 750.0, 400.0));
items.extend(fill_zone(1, 4_400.0, 8_000.0, 750.0, 400.0));
let cols = detect_columns(&items, 1, false);
assert_eq!(cols.len(), 2, "Expected 2 columns, got {}", cols.len());
assert!(
cols[1].x_max > 7_000.0,
"right column should keep its true extent, got {}",
cols[1].x_max
);
}
#[test]
fn two_column_regression_guard() {
// Standard 2-column layout with clear gutter at center
+38 -700
View File
@@ -9,7 +9,7 @@
use log::debug;
use lopdf::{Document, Object, ObjectId};
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
// ─── Standard structure types ────────────────────────────────────────
@@ -252,28 +252,10 @@ impl StructTree {
let role_map = parse_role_map(doc, struct_root);
debug!("structure tree: {} role map entries", role_map.len());
// Seed the cycle guard with the struct-root's own object id so a `/K`
// that points back at the root is treated as a cycle, and bound total
// node materialization with a global budget.
let mut walk = StructWalk::new();
if let Ok(root_id) = struct_root_obj.as_reference() {
walk.active.insert(root_id);
}
// Parse child elements from /K
let children = parse_kids(doc, struct_root, &role_map, None, 0, &mut walk);
let children = parse_kids(doc, struct_root, &role_map, None, 0);
debug!("structure tree: {} top-level elements", children.len());
if walk.truncated {
log::warn!(
"structure tree parsing was truncated (node budget of \
{MAX_STRUCT_NODES} or traversal budget of {MAX_STRUCT_WORK} \
reached, a `/K` reference cycle, or the max nesting depth of \
{MAX_DEPTH}); tagged roles/tables may be incomplete (likely a \
very large or malformed tagged PDF)"
);
}
if children.is_empty() {
return None;
}
@@ -497,123 +479,6 @@ fn parse_role_map(doc: &Document, struct_root: &lopdf::Dictionary) -> HashMap<St
/// malformed PDFs).
const MAX_DEPTH: usize = 64;
/// Global cap on the number of structure-tree nodes materialized in a single
/// parse. Real tagged trees are far smaller; a crafted PDF can alias one struct
/// element into its own `/K` (e.g. `/K [n 0 R n 0 R]`) so the tree branches
/// exponentially (2^depth) before the depth cap is reached, exhausting memory.
/// This budget bounds total work and allocation regardless of tree shape.
const MAX_STRUCT_NODES: usize = 500_000;
/// Cap on the number of `/K` items *examined* during a single parse, regardless
/// of whether they materialize anything. Bounds CPU for crafted wide `/K` arrays
/// of non-materializing entries (unsupported value types, `/OBJR` dicts, cycle
/// back-edges) that would otherwise be scanned in full without ever touching the
/// node budget. Kept well above the node budget so it never truncates content
/// that already fits within `MAX_STRUCT_NODES`.
const MAX_STRUCT_WORK: usize = 2_000_000;
/// Traversal state shared across the recursive structure-tree parse.
///
/// `budget` is a global allowance charged once per materialized item — each
/// struct-element node and each marked-content reference — so total work is
/// bounded even for aliased/DAG-shaped `/K` graphs of distinct objects or a
/// single element with a very wide `/K` array. `active` holds the object IDs
/// currently on the depth-first path so a struct element that references itself
/// (or an ancestor) is not expanded into an unbounded/exponential subtree.
/// `budget` bounds *materialization* (nodes + content refs). `work` separately
/// bounds *traversal* — every `/K` item examined is charged against it, even
/// ones that materialize nothing (unsupported values, `/OBJR`, cycle back-edges)
/// — so a wide malformed array cannot force an unbounded scan, and those skipped
/// items don't drain the materialization budget and truncate real content.
/// `truncated` records whether any parse work was skipped — the budget was
/// exhausted, a `/K` reference cycle was broken, or the depth cap was hit — so
/// the caller can log it once rather than per skipped item. `stalled` is set
/// when an atomic multi-unit reservation could not fit in the remaining budget;
/// it makes [`exhausted`](Self::exhausted) report done so a wide `/K` array is
/// not scanned to the end once no further leaf can be materialized.
struct StructWalk {
budget: usize,
work: usize,
active: HashSet<ObjectId>,
truncated: bool,
stalled: bool,
}
impl StructWalk {
fn new() -> Self {
Self {
budget: MAX_STRUCT_NODES,
work: MAX_STRUCT_WORK,
active: HashSet::new(),
truncated: false,
stalled: false,
}
}
/// Charge one unit of traversal work for an examined `/K` item, whether or
/// not it materializes anything. Returns `false` (flagging truncation) once
/// the traversal budget is spent, so an enclosing loop stops instead of
/// scanning the rest of a wide array of non-materializing entries.
fn spend_work(&mut self) -> bool {
if self.work == 0 {
self.truncated = true;
return false;
}
self.work -= 1;
true
}
/// Record that some parse work was skipped for a non-budget reason (a `/K`
/// reference cycle or the depth cap), so the one-shot truncation warning
/// also covers malformed/over-deep trees, not just budget exhaustion.
fn note_skipped(&mut self) {
self.truncated = true;
}
/// Charge one unit against the budget for a materialized item (a struct
/// element node or a marked-content reference). Returns `false` — without
/// underflowing — once the budget is exhausted, so callers skip the item.
fn charge(&mut self) -> bool {
if self.budget == 0 {
self.truncated = true;
return false;
}
self.budget -= 1;
true
}
/// Atomically charge `n` units for a single item that materializes several
/// budget-counted parts at once (a leaf wrapper node *plus* its content
/// reference). Charges nothing when fewer than `n` units remain — so a
/// partial reservation never wastes capacity — and marks the walk `stalled`
/// so the enclosing loop stops instead of scanning the rest of a wide `/K`
/// array that can no longer fit any leaf.
fn charge_n(&mut self, n: usize) -> bool {
if self.budget < n {
self.truncated = true;
self.stalled = true;
return false;
}
self.budget -= n;
true
}
/// Whether traversal should stop: the budget is spent, or a multi-unit
/// reservation could not fit (`stalled`) so no further leaf will materialize.
/// Use this at the guards that break/return to skip remaining items; it
/// records that truncation occurred (a guard only fires while an item is
/// still pending), so callers that drop work without going through
/// [`charge`](Self::charge) still flag the truncation for logging.
fn exhausted(&mut self) -> bool {
if self.budget == 0 || self.stalled {
self.truncated = true;
true
} else {
false
}
}
}
/// Parse child elements from a `/K` entry.
fn parse_kids(
doc: &Document,
@@ -621,13 +486,8 @@ fn parse_kids(
role_map: &HashMap<String, String>,
inherited_page: Option<ObjectId>,
depth: usize,
walk: &mut StructWalk,
) -> Vec<StructElement> {
if depth >= MAX_DEPTH {
walk.note_skipped();
return Vec::new();
}
if walk.exhausted() {
return Vec::new();
}
@@ -638,59 +498,22 @@ fn parse_kids(
// /Pg on this element (inherited by children)
let page_id = get_page_ref(doc, dict).or(inherited_page);
let mut children = Vec::new();
match k_obj {
Object::Array(arr) => {
let mut children = Vec::new();
for item in arr {
if walk.exhausted() || !walk.spend_work() {
break;
}
process_kid_item(doc, item, role_map, page_id, depth, &mut children, walk);
let resolved = resolve_obj(doc, item);
parse_kid(doc, resolved, role_map, page_id, depth, &mut children);
}
children
}
other => {
process_kid_item(doc, other, role_map, page_id, depth, &mut children, walk);
let resolved = resolve_obj(doc, other);
let mut children = Vec::new();
parse_kid(doc, resolved, role_map, page_id, depth, &mut children);
children
}
}
children
}
/// Resolve one `/K` array item (following at most one level of indirection),
/// guarding against reference cycles and the global node budget, then dispatch
/// it via [`parse_kid`].
fn process_kid_item(
doc: &Document,
item: &Object,
role_map: &HashMap<String, String>,
inherited_page: Option<ObjectId>,
depth: usize,
out: &mut Vec<StructElement>,
walk: &mut StructWalk,
) {
if walk.exhausted() {
return;
}
if depth >= MAX_DEPTH {
walk.note_skipped();
return;
}
// If this child is an indirect reference, track its id on the active path so
// a self/ancestor reference is not expanded into an exponential subtree.
let ref_id = match item {
Object::Reference(id) => Some(*id),
_ => None,
};
if let Some(id) = ref_id {
if !walk.active.insert(id) {
walk.note_skipped();
return; // cycle: this object is already on the current path
}
}
let resolved = resolve_obj(doc, item);
parse_kid(doc, resolved, role_map, inherited_page, depth, out, walk);
if let Some(id) = ref_id {
walk.active.remove(&id);
}
}
/// Parse a single child (either a struct element dict or an MCID integer).
@@ -701,16 +524,10 @@ fn parse_kid(
inherited_page: Option<ObjectId>,
depth: usize,
out: &mut Vec<StructElement>,
walk: &mut StructWalk,
) {
match obj {
// Direct MCID integer — create a leaf wrapper
Object::Integer(mcid) => {
// A wrapper node plus its content reference — two items — reserved
// atomically so we never consume one unit without emitting both.
if !walk.charge_n(2) {
return;
}
// This is a bare MCID at the struct-element level.
// We attach it to the parent element, so we create a wrapper struct element.
// Actually, bare MCIDs inside /K are content refs for the parent,
@@ -729,11 +546,11 @@ fn parse_kid(
});
}
Object::Dictionary(d) => {
parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out, walk);
parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out);
}
Object::Stream(s) => {
// Some PDFs wrap struct elements in streams (rare)
parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out, walk);
parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out);
}
_ => {}
}
@@ -748,23 +565,13 @@ fn parse_struct_element_dict(
inherited_page: Option<ObjectId>,
depth: usize,
out: &mut Vec<StructElement>,
walk: &mut StructWalk,
) {
if depth >= MAX_DEPTH {
walk.note_skipped();
return;
}
// A marked-content reference dict materializes a wrapper node + one content
// reference (two items). Reserve both atomically *before* the node charge so
// we never consume a unit without emitting the reference — which would also
// deny that unit to a later element that would have fit. This matches the
// bare-MCID path.
// Check if this is a marked-content reference dict (has /Type /MCR)
if is_mcr_dict(dict) {
if let Ok(Object::Integer(mcid)) = dict.get(b"MCID") {
if !walk.charge_n(2) {
return;
}
let page_id = get_page_ref(doc, dict).or(inherited_page);
out.push(StructElement {
role: StructRole::Span,
@@ -781,15 +588,12 @@ fn parse_struct_element_dict(
return;
}
// Skip object-reference dicts (`/Type /OBJR`) — they materialize no node, so
// recognize and return *before* charging the budget (otherwise a document
// full of OBJRs would drain the shared budget and truncate real content).
// Check if this is an object reference dict (has /Type /OBJR) — skip these
if is_objr_dict(dict) {
return;
}
// It's a struct element — parse its /S (structure type). A dict without a
// valid /S also materializes nothing, so validate before charging.
// It's a struct element — parse its /S (structure type)
let role_name = match dict.get(b"S") {
Ok(s_obj) => {
let resolved = resolve_obj(doc, s_obj);
@@ -801,12 +605,6 @@ fn parse_struct_element_dict(
Err(_) => return,
};
// Charge the node only now that we know it will materialize (bounds
// aliased/DAG-shaped `/K` graphs the per-path cycle guard alone cannot stop).
if !walk.charge() {
return;
}
let role = StructRole::from_name_with_role_map(&role_name, role_map);
let page_id = get_page_ref(doc, dict).or(inherited_page);
@@ -823,73 +621,51 @@ fn parse_struct_element_dict(
let k_resolved = resolve_obj(doc, k_obj);
match k_resolved {
Object::Integer(mcid) => {
if walk.charge() {
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id,
});
}
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id,
});
}
Object::Array(arr) => {
for item in arr {
if walk.exhausted() || !walk.spend_work() {
break;
}
// Only content-ref items (bare MCIDs / MCR dicts) are charged
// here — those are the unbounded allocations. Structural
// children are charged once at their own node entry in the
// recursive call, so charging them here too would double-count
// and drain the budget ~2× faster than the per-node semantics.
let ref_id = match item {
Object::Reference(id) => Some(*id),
_ => None,
};
let resolved = resolve_obj(doc, item);
match resolved {
Object::Integer(mcid) => {
if walk.charge() {
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id,
});
}
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id,
});
}
Object::Dictionary(d) => {
if is_mcr_dict(d) {
if let Ok(Object::Integer(mcid)) = d.get(b"MCID") {
if walk.charge() {
let pg = get_page_ref(doc, d).or(page_id);
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id: pg,
});
}
let pg = get_page_ref(doc, d).or(page_id);
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id: pg,
});
}
} else if is_objr_dict(d) {
// Skip object references
} else {
recurse_struct_child(
parse_struct_element_dict(
doc,
ref_id,
d,
role_map,
page_id,
depth,
depth + 1,
&mut children,
walk,
);
}
}
Object::Stream(s) => {
recurse_struct_child(
parse_struct_element_dict(
doc,
ref_id,
&s.dict,
role_map,
page_id,
depth,
depth + 1,
&mut children,
walk,
);
}
_ => {}
@@ -899,29 +675,14 @@ fn parse_struct_element_dict(
Object::Dictionary(d) => {
if is_mcr_dict(d) {
if let Ok(Object::Integer(mcid)) = d.get(b"MCID") {
if walk.charge() {
let pg = get_page_ref(doc, d).or(page_id);
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id: pg,
});
}
let pg = get_page_ref(doc, d).or(page_id);
content_refs.push(MarkedContentRef {
mcid: *mcid,
page_id: pg,
});
}
} else {
let ref_id = match k_obj {
Object::Reference(id) => Some(*id),
_ => None,
};
recurse_struct_child(
doc,
ref_id,
d,
role_map,
page_id,
depth,
&mut children,
walk,
);
parse_struct_element_dict(doc, d, role_map, page_id, depth + 1, &mut children);
}
}
_ => {}
@@ -938,37 +699,6 @@ fn parse_struct_element_dict(
});
}
/// Recurse into a child struct-element dictionary, guarding against reference
/// cycles (via the active-path object-id set) and the global node budget.
///
/// `ref_id` is the object id of the child when it was reached through an
/// indirect reference (`None` for an inline dictionary, which cannot alias).
#[allow(clippy::too_many_arguments)]
fn recurse_struct_child(
doc: &Document,
ref_id: Option<ObjectId>,
dict: &lopdf::Dictionary,
role_map: &HashMap<String, String>,
inherited_page: Option<ObjectId>,
depth: usize,
out: &mut Vec<StructElement>,
walk: &mut StructWalk,
) {
if walk.exhausted() {
return;
}
if let Some(id) = ref_id {
if !walk.active.insert(id) {
walk.note_skipped();
return; // cycle: this object is already on the current path
}
}
parse_struct_element_dict(doc, dict, role_map, inherited_page, depth + 1, out, walk);
if let Some(id) = ref_id {
walk.active.remove(&id);
}
}
/// Check if dict has `/Type /MCR`.
fn is_mcr_dict(dict: &lopdf::Dictionary) -> bool {
dict.get(b"Type")
@@ -1171,7 +901,6 @@ fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use lopdf::dictionary;
#[test]
fn non_heading_content_roles() {
@@ -1502,395 +1231,4 @@ mod tests {
let role_map = tree.mcid_to_roles(&page_ids);
assert!(!role_map.is_empty(), "Should have MCID→role mappings");
}
fn count_nodes(elems: &[StructElement]) -> usize {
elems.iter().map(|e| 1 + count_nodes(&e.children)).sum()
}
/// Wrap already-created struct elements under a `/StructTreeRoot` and
/// `/Catalog`, returning a document ready for [`StructTree::from_doc`].
/// `root_kid` is the top-level element the root's `/K` points at.
fn finalize_tagged_doc(mut doc: Document, root_kid: ObjectId) -> Document {
let root_id = doc.add_object(dictionary! {
"Type" => "StructTreeRoot",
"K" => vec![Object::Reference(root_kid)],
});
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"StructTreeRoot" => Object::Reference(root_id),
});
doc.trailer.set("Root", Object::Reference(catalog_id));
doc
}
#[test]
fn struct_tree_self_alias_kids_terminates() {
// A struct element that lists itself twice in `/K` (`/K [n 0 R n 0 R]`)
// must not expand into an exponential tree.
let mut doc = Document::new();
let elem = doc.new_object_id();
doc.set_object(
elem,
dictionary! {
"Type" => "StructElem",
"S" => "Div",
"K" => vec![Object::Reference(elem), Object::Reference(elem)],
},
);
let doc = finalize_tagged_doc(doc, elem);
let tree = StructTree::from_doc(&doc).expect("tree should parse");
let n = count_nodes(&tree.children);
assert!(
n < 10,
"self-alias must not explode; materialized {n} nodes"
);
}
#[test]
fn struct_tree_mutual_alias_kids_terminates() {
// A → B → A cycle via `/K` must terminate.
let mut doc = Document::new();
let a = doc.new_object_id();
let b = doc.new_object_id();
doc.set_object(
a,
dictionary! {
"Type" => "StructElem",
"S" => "Div",
"K" => vec![Object::Reference(b), Object::Reference(b)],
},
);
doc.set_object(
b,
dictionary! {
"Type" => "StructElem",
"S" => "Div",
"K" => vec![Object::Reference(a), Object::Reference(a)],
},
);
let doc = finalize_tagged_doc(doc, a);
let tree = StructTree::from_doc(&doc).expect("tree should parse");
let n = count_nodes(&tree.children);
assert!(
n < 100,
"mutual alias must terminate small; materialized {n} nodes"
);
}
#[test]
fn struct_tree_aliased_dag_respects_node_budget() {
// Distinct elements, each aliased twice in the next level's `/K`, form a
// DAG that would expand to 2^depth nodes (the per-path cycle guard does
// not catch this since every id is on the path only once). The global
// node budget must cap total materialization.
let mut doc = Document::new();
let levels = 22; // 2^22 ≈ 4.2M unbounded, well past the budget
let ids: Vec<ObjectId> = (0..=levels).map(|_| doc.new_object_id()).collect();
for i in 0..levels {
doc.set_object(
ids[i],
dictionary! {
"Type" => "StructElem",
"S" => "Div",
"K" => vec![Object::Reference(ids[i + 1]), Object::Reference(ids[i + 1])],
},
);
}
doc.set_object(
ids[levels],
dictionary! { "Type" => "StructElem", "S" => "P" },
);
let root_id = doc.add_object(dictionary! {
"Type" => "StructTreeRoot",
"K" => vec![Object::Reference(ids[0])],
});
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"StructTreeRoot" => Object::Reference(root_id),
});
doc.trailer.set("Root", Object::Reference(catalog_id));
let tree = StructTree::from_doc(&doc).expect("tree should parse");
let n = count_nodes(&tree.children);
assert!(
n <= MAX_STRUCT_NODES,
"node count {n} exceeded budget {MAX_STRUCT_NODES}"
);
}
#[test]
fn struct_tree_wide_mcid_array_respects_budget() {
// A single struct element with a `/K` array of bare MCIDs wider than the
// budget must not allocate `content_refs` without bound — each array item
// is charged, so materialized marked-content refs stay within the budget.
let mut doc = Document::new();
let elem = doc.new_object_id();
let kids: Vec<Object> = (0..(MAX_STRUCT_NODES as i64 + 100))
.map(Object::Integer)
.collect();
doc.set_object(
elem,
dictionary! {
"Type" => "StructElem",
"S" => "P",
"K" => kids,
},
);
let doc = finalize_tagged_doc(doc, elem);
let tree = StructTree::from_doc(&doc).expect("tree should parse");
assert!(
tree.mcid_count() <= MAX_STRUCT_NODES,
"content_refs unbounded: {} > {MAX_STRUCT_NODES}",
tree.mcid_count()
);
}
#[test]
fn budget_charge_flags_truncation_once_exhausted() {
let mut walk = StructWalk::new();
walk.budget = 1;
assert!(walk.charge(), "should spend the last unit");
assert!(!walk.truncated, "not truncated while budget remained");
assert!(!walk.charge(), "budget exhausted");
assert!(walk.truncated, "exhaustion must set the truncation flag");
// Stays exhausted/flagged on subsequent calls.
assert!(!walk.charge());
assert!(walk.truncated);
}
#[test]
fn exhausted_flags_truncation_after_budget_spent_by_charge() {
// The dominant truncation path: the budget is driven to 0 by a
// successful `charge()` (which does not set the flag), and remaining
// items are then dropped by an `exhausted()` guard — which must flag it.
let mut walk = StructWalk::new();
walk.budget = 1;
assert!(walk.charge());
assert!(
!walk.truncated,
"spending the last unit is not truncation yet"
);
assert!(walk.exhausted(), "budget is now spent");
assert!(
walk.truncated,
"the guard that skips work must flag truncation"
);
}
#[test]
fn wide_kids_array_flags_truncation_via_parser() {
// Reproduce the reviewer's scenario through the real parser: a `/K`
// array wider than the budget drives the budget to 0 via `charge()`,
// then the loop guard drops the rest — the truncation flag must be set
// (so `from_doc` logs it) rather than staying silently false.
let mut doc = Document::new();
let elem = doc.new_object_id();
let kids: Vec<Object> = (0..20i64).map(Object::Integer).collect();
doc.set_object(
elem,
dictionary! { "Type" => "StructElem", "S" => "P", "K" => kids },
);
let dict = doc.get_dictionary(elem).unwrap().clone();
let mut walk = StructWalk::new();
walk.budget = 5; // smaller than the 20-item `/K` array
let role_map = HashMap::new();
let mut out = Vec::new();
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
assert!(
walk.truncated,
"a `/K` array wider than the budget must flag truncation"
);
}
#[test]
fn cycle_skip_flags_truncation() {
// A `/K` reference cycle is dropped rather than expanded; that skip must
// still flag truncation so the one-shot warning fires for malformed
// trees, not only for budget exhaustion.
let mut doc = Document::new();
let elem = doc.new_object_id();
doc.set_object(
elem,
dictionary! {
"Type" => "StructElem",
"S" => "Div",
"K" => vec![Object::Reference(elem), Object::Reference(elem)],
},
);
let dict = doc.get_dictionary(elem).unwrap().clone();
let mut walk = StructWalk::new();
walk.active.insert(elem); // simulate `elem` already on the DFS path
let role_map = HashMap::new();
let mut out = Vec::new();
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
assert!(
walk.truncated,
"a cycle-skipped `/K` child must flag truncation"
);
}
#[test]
fn bare_mcid_charges_node_and_reference() {
// A bare MCID `/K` child becomes a wrapper node carrying one content
// reference — two materialized items — so it must charge two budget
// units, not one.
let doc = Document::new();
let obj = Object::Integer(7);
let role_map = HashMap::new();
let mut out = Vec::new();
let mut walk = StructWalk::new();
let before = walk.budget;
parse_kid(&doc, &obj, &role_map, None, 0, &mut out, &mut walk);
assert_eq!(
out.len(),
1,
"bare MCID should materialize one wrapper node"
);
assert_eq!(
before - walk.budget,
2,
"bare MCID must charge for both the node and its content reference"
);
}
#[test]
fn mcr_dict_charges_node_and_reference() {
// A top-level MCR `/K` dict materializes the same wrapper node + content
// reference as a bare MCID, so it must charge the same two budget units
// (not one), keeping the per-item budgeting uniform.
let doc = Document::new();
let obj = Object::Dictionary(dictionary! { "Type" => "MCR", "MCID" => 3 });
let role_map = HashMap::new();
let mut out = Vec::new();
let mut walk = StructWalk::new();
let before = walk.budget;
parse_kid(&doc, &obj, &role_map, None, 0, &mut out, &mut walk);
assert_eq!(out.len(), 1, "MCR dict should materialize one wrapper node");
assert_eq!(
before - walk.budget,
2,
"MCR dict must charge for both the node and its content reference"
);
}
#[test]
fn leaf_wrappers_reserve_both_units_atomically() {
// With only one unit left, a two-item leaf wrapper (bare MCID or MCR
// dict) must consume nothing and flag truncation, leaving the unit for a
// later single-item element instead of half-charging.
let doc = Document::new();
let role_map = HashMap::new();
// Bare MCID via parse_kid.
let mut walk = StructWalk::new();
walk.budget = 1;
let mut out = Vec::new();
parse_kid(
&doc,
&Object::Integer(5),
&role_map,
None,
0,
&mut out,
&mut walk,
);
assert!(out.is_empty(), "bare MCID must not partially materialize");
assert_eq!(walk.budget, 1, "the leftover unit must be preserved");
assert!(walk.truncated);
// MCR dict via parse_struct_element_dict.
let mcr = dictionary! { "Type" => "MCR", "MCID" => 1 };
let mut walk = StructWalk::new();
walk.budget = 1;
let mut out = Vec::new();
parse_struct_element_dict(&doc, &mcr, &role_map, None, 0, &mut out, &mut walk);
assert!(out.is_empty(), "MCR dict must not partially materialize");
assert_eq!(walk.budget, 1, "the leftover unit must be preserved");
assert!(walk.truncated);
}
#[test]
fn insufficient_reservation_stops_the_scan() {
// A one-unit budget is not "exhausted" for a one-unit item, but once a
// two-unit leaf reservation fails, the walk is stalled so enclosing `/K`
// loops stop instead of scanning the rest of a wide array.
let mut walk = StructWalk::new();
walk.budget = 1;
assert!(
!walk.exhausted(),
"one unit left must still allow a one-unit item"
);
assert!(!walk.charge_n(2), "cannot reserve two units from one");
assert!(
walk.exhausted(),
"an insufficient reservation must stop the loop"
);
assert!(walk.truncated);
}
#[test]
fn work_budget_bounds_examined_items() {
let mut walk = StructWalk::new();
walk.work = 2;
assert!(walk.spend_work());
assert!(walk.spend_work());
assert!(!walk.spend_work(), "traversal budget exhausted");
assert!(walk.truncated);
}
#[test]
fn wide_unsupported_kids_stop_at_work_budget() {
// A wide `/K` array of unsupported values (nulls) materializes nothing;
// it must stop at the traversal budget instead of scanning every entry.
let mut doc = Document::new();
let elem = doc.new_object_id();
let kids: Vec<Object> = (0..1000).map(|_| Object::Null).collect();
doc.set_object(
elem,
dictionary! { "Type" => "StructElem", "S" => "P", "K" => kids },
);
let dict = doc.get_dictionary(elem).unwrap().clone();
let mut walk = StructWalk::new();
walk.work = 10; // far smaller than the 1000-entry array
let role_map = HashMap::new();
let mut out = Vec::new();
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
assert!(
walk.truncated,
"a wide unsupported `/K` array must hit the work budget"
);
}
#[test]
fn non_materializing_dicts_do_not_charge_node_budget() {
let doc = Document::new();
let role_map = HashMap::new();
// OBJR dict: materializes no node, so it must not spend the node budget.
let objr = dictionary! { "Type" => "OBJR" };
let mut walk = StructWalk::new();
let before = walk.budget;
let mut out = Vec::new();
parse_struct_element_dict(&doc, &objr, &role_map, None, 0, &mut out, &mut walk);
assert!(out.is_empty());
assert_eq!(walk.budget, before, "OBJR must not spend the node budget");
// A struct dict without a valid /S also materializes nothing.
let no_s = dictionary! { "Type" => "StructElem" };
let mut walk = StructWalk::new();
let before = walk.budget;
let mut out = Vec::new();
parse_struct_element_dict(&doc, &no_s, &role_map, None, 0, &mut out, &mut walk);
assert!(out.is_empty());
assert_eq!(
walk.budget, before,
"a dict without /S must not spend the node budget"
);
}
}
+1 -19
View File
@@ -594,7 +594,7 @@ fn hex_to_unicode_string(hex: &str) -> Option<String> {
let bytes: Option<Vec<u8>> = (0..hex.len())
.step_by(2)
.map(|i| u8::from_str_radix(hex.get(i..i + 2)?, 16).ok())
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).ok())
.collect();
let bytes = bytes?;
@@ -2606,24 +2606,6 @@ endcmap
assert_eq!(cmap.lookup(0x0025), Some("B".to_string()));
}
#[test]
fn test_hex_to_unicode_non_ascii_no_panic() {
// A destination containing a multi-byte char makes the byte length even
// while a byte offset can land inside a char. Slicing must not panic;
// it should be rejected gracefully.
assert_eq!(hex_to_unicode_string("XéY"), None);
assert_eq!(hex_to_unicode_string("\u{fffd}0"), None);
}
#[test]
fn test_parse_bfchar_non_ascii_destination_no_panic() {
// Crafted /ToUnicode CMap: a non-hex, non-ASCII destination previously
// triggered a char-boundary panic in hex_to_unicode_string.
let cmap_content = "beginbfchar <0041> <XéY> endbfchar";
// Must not panic; the malformed entry is simply skipped.
let _ = ToUnicodeCMap::parse(cmap_content.as_bytes());
}
#[test]
fn test_parse_bfchar_1byte() {
// This is the pattern that caused the CJK bug: codespace is <0000><FFFF>
+2 -2
View File
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "pdf-inspector"
version = "0.1.8"
version = "0.1.7"
dependencies = [
"env_logger",
"include_dir",
@@ -740,7 +740,7 @@ dependencies = [
[[package]]
name = "pdf-inspector-wasm"
version = "0.1.4"
version = "0.1.3"
dependencies = [
"console_error_panic_hook",
"js-sys",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "pdf-inspector-wasm"
version = "0.1.4"
version = "0.1.3"
edition = "2021"
authors = ["Firecrawl Team"]
description = "Browser WebAssembly bindings for pdf-inspector"