Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3fe9ccee07 | ||
|
|
36dd5fa426 | ||
|
|
fabec0aec3 | ||
|
|
1f28c00a13 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.7"
|
||||
version = "0.1.8"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
|
||||
Generated
+2
-2
@@ -851,7 +851,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.7"
|
||||
version = "0.1.8"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -867,7 +867,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -83,6 +83,22 @@ for (const region of result[0].regions) {
|
||||
}
|
||||
```
|
||||
|
||||
### Async variants
|
||||
|
||||
`processPdf`, `classifyPdf`, and `extractPagesMarkdown` are synchronous and parse on the calling thread — in Node, that's the event loop. For a one-off call in a script that's fine, but in a server a large document can hold the loop for tens to hundreds of milliseconds.
|
||||
|
||||
`processPdfAsync`, `classifyPdfAsync`, and `extractPagesMarkdownAsync` take the same arguments and produce the same results, but run the parse on the libuv thread pool and return a promise, keeping the event loop free. The input buffer is copied before the call returns, so it's safe to reuse or mutate immediately:
|
||||
|
||||
```typescript
|
||||
import { classifyPdfAsync, extractPagesMarkdownAsync } from '@firecrawl/pdf-inspector'
|
||||
|
||||
const classification = await classifyPdfAsync(pdf)
|
||||
if (classification.pdfType === 'TextBased') {
|
||||
const { pages } = await extractPagesMarkdownAsync(pdf)
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## Types
|
||||
|
||||
```typescript
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.13.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.12.0",
|
||||
"version": "1.13.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -52,11 +52,11 @@
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.12.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.12.0"
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.13.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.13.0"
|
||||
}
|
||||
}
|
||||
|
||||
+182
-41
@@ -153,9 +153,7 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_napi_page_ocr_reasons(
|
||||
reasons: Vec<pdf_inspector::PageOcrReasons>,
|
||||
) -> Vec<PageOcrReasons> {
|
||||
fn to_napi_page_ocr_reasons(reasons: Vec<pdf_inspector::PageOcrReasons>) -> Vec<PageOcrReasons> {
|
||||
reasons
|
||||
.into_iter()
|
||||
.map(|reason| PageOcrReasons {
|
||||
@@ -202,6 +200,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared implementations (single body behind sync and async entry points)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn process_pdf_impl(bytes: &[u8], pages: Option<Vec<u32>>) -> Result<PdfResult> {
|
||||
let mut opts = pdf_inspector::PdfOptions::new();
|
||||
if let Some(p) = pages {
|
||||
opts = opts.pages(p);
|
||||
}
|
||||
let result = pdf_inspector::process_pdf_mem_with_options(bytes, opts)
|
||||
.map_err(|e| to_napi_err(e, "process_pdf"))?;
|
||||
Ok(to_napi_result(result))
|
||||
}
|
||||
|
||||
fn classify_pdf_impl(bytes: &[u8]) -> Result<PdfClassification> {
|
||||
let result =
|
||||
pdf_inspector::classify_pdf_mem(bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: convert_pdf_type(result.pdf_type),
|
||||
page_count: result.page_count,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
confidence: result.confidence as f64,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public NAPI API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -210,15 +233,7 @@ where
|
||||
#[napi]
|
||||
pub fn process_pdf(buffer: Buffer, pages: Option<Vec<u32>>) -> Result<PdfResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("process_pdf", move || {
|
||||
let mut opts = pdf_inspector::PdfOptions::new();
|
||||
if let Some(p) = pages {
|
||||
opts = opts.pages(p);
|
||||
}
|
||||
let result = pdf_inspector::process_pdf_mem_with_options(&bytes, opts)
|
||||
.map_err(|e| to_napi_err(e, "process_pdf"))?;
|
||||
Ok(to_napi_result(result))
|
||||
})
|
||||
catch_panic("process_pdf", move || process_pdf_impl(&bytes, pages))
|
||||
}
|
||||
|
||||
/// Fast detection only — no text extraction or markdown.
|
||||
@@ -238,16 +253,7 @@ pub fn detect_pdf(buffer: Buffer) -> Result<PdfResult> {
|
||||
#[napi]
|
||||
pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("classify_pdf", move || {
|
||||
let result =
|
||||
pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: convert_pdf_type(result.pdf_type),
|
||||
page_count: result.page_count,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
confidence: result.confidence as f64,
|
||||
})
|
||||
})
|
||||
catch_panic("classify_pdf", move || classify_pdf_impl(&bytes))
|
||||
}
|
||||
|
||||
/// Extract plain text from a PDF Buffer.
|
||||
@@ -633,25 +639,32 @@ pub fn extract_pages_markdown(
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|r| PageMarkdownResult {
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
extract_pages_markdown_impl(&bytes, pages.as_deref())
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_pages_markdown_impl(
|
||||
bytes: &[u8],
|
||||
pages: Option<&[u32]>,
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(bytes, pages)
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|r| PageMarkdownResult {
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -692,3 +705,131 @@ fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<Pa
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Async variants (libuv thread pool via AsyncTask)
|
||||
//
|
||||
// The synchronous exports above parse on the calling thread, which in Node is
|
||||
// the event loop. These `*Async` variants run the same shared implementations
|
||||
// on the libuv thread pool and hand JavaScript a promise, so servers under
|
||||
// concurrent load keep answering requests while a document parses. The sync
|
||||
// exports keep their names, signatures, and behaviour.
|
||||
//
|
||||
// Each factory copies the input Buffer to an owned `Vec<u8>` on the calling
|
||||
// (JS) thread — deliberately. JS execution is single-threaded, so no JS code
|
||||
// can mutate the buffer while the synchronous part of the call copies it.
|
||||
// Holding the napi `Buffer` and reading it from the worker instead would be
|
||||
// zero-copy, but a caller mutating the buffer before the promise settles
|
||||
// would then race the worker's reads — undefined behavior, not a recoverable
|
||||
// error (a known napi-rs soundness hazard with cross-thread Buffer access).
|
||||
// The copy is a one-time memcpy, negligible next to the parse it unblocks.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct ProcessPdfTask {
|
||||
bytes: Vec<u8>,
|
||||
pages: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
impl Task for ProcessPdfTask {
|
||||
type Output = PdfResult;
|
||||
type JsValue = PdfResult;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let pages = self.pages.take();
|
||||
// AssertUnwindSafe: `bytes`/`pages` are moved into the closure and
|
||||
// dropped on unwind — no shared state can be observed broken.
|
||||
catch_panic(
|
||||
"process_pdf",
|
||||
panic::AssertUnwindSafe(move || process_pdf_impl(&bytes, pages)),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`processPdf`]: same result, but the parse runs on the
|
||||
/// libuv thread pool instead of the event loop and the call returns a
|
||||
/// promise. The buffer is copied before the call returns, so it may be
|
||||
/// reused or mutated immediately.
|
||||
// ts_return_type is required: napi-rs emits `Promise<unknown>` for
|
||||
// `AsyncTask<T>` returns without it.
|
||||
#[napi(ts_return_type = "Promise<PdfResult>")]
|
||||
pub fn process_pdf_async(buffer: Buffer, pages: Option<Vec<u32>>) -> AsyncTask<ProcessPdfTask> {
|
||||
AsyncTask::new(ProcessPdfTask {
|
||||
bytes: buffer.to_vec(),
|
||||
pages,
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ClassifyPdfTask {
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Task for ClassifyPdfTask {
|
||||
type Output = PdfClassification;
|
||||
type JsValue = PdfClassification;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
catch_panic(
|
||||
"classify_pdf",
|
||||
panic::AssertUnwindSafe(move || classify_pdf_impl(&bytes)),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`classifyPdf`]: same result, but the classification runs
|
||||
/// on the libuv thread pool instead of the event loop and the call returns a
|
||||
/// promise. The buffer is copied before the call returns, so it may be
|
||||
/// reused or mutated immediately.
|
||||
#[napi(ts_return_type = "Promise<PdfClassification>")]
|
||||
pub fn classify_pdf_async(buffer: Buffer) -> AsyncTask<ClassifyPdfTask> {
|
||||
AsyncTask::new(ClassifyPdfTask {
|
||||
bytes: buffer.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ExtractPagesMarkdownTask {
|
||||
bytes: Vec<u8>,
|
||||
pages: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
impl Task for ExtractPagesMarkdownTask {
|
||||
type Output = PagesExtractionResult;
|
||||
type JsValue = PagesExtractionResult;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let pages = self.pages.take();
|
||||
catch_panic(
|
||||
"extract_pages_markdown",
|
||||
panic::AssertUnwindSafe(move || extract_pages_markdown_impl(&bytes, pages.as_deref())),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`extractPagesMarkdown`]: same result, but the extraction
|
||||
/// runs on the libuv thread pool instead of the event loop and the call
|
||||
/// returns a promise. The buffer is copied before the call returns, so it
|
||||
/// may be reused or mutated immediately.
|
||||
#[napi(ts_return_type = "Promise<PagesExtractionResult>")]
|
||||
pub fn extract_pages_markdown_async(
|
||||
buffer: Buffer,
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> AsyncTask<ExtractPagesMarkdownTask> {
|
||||
AsyncTask::new(ExtractPagesMarkdownTask {
|
||||
bytes: buffer.to_vec(),
|
||||
pages,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,13 +2,16 @@ import { readFileSync } from 'fs';
|
||||
import { strict as assert } from 'assert';
|
||||
import {
|
||||
processPdf,
|
||||
processPdfAsync,
|
||||
detectPdf,
|
||||
classifyPdf,
|
||||
classifyPdfAsync,
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
detectVectorGridInRegion,
|
||||
extractPagesMarkdown,
|
||||
extractPagesMarkdownAsync,
|
||||
} from './index.js';
|
||||
|
||||
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
|
||||
@@ -124,10 +127,75 @@ assert.equal(picked.pages[0].page, 2);
|
||||
assert.equal(picked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdown with pages: OK');
|
||||
|
||||
// --- Async variants ---
|
||||
console.log('Testing async variants...');
|
||||
|
||||
// processPdfAsync returns a promise and matches the sync result
|
||||
const asyncResultPromise = processPdfAsync(fixture);
|
||||
assert.ok(asyncResultPromise instanceof Promise);
|
||||
const asyncResult = await asyncResultPromise;
|
||||
assert.equal(asyncResult.pdfType, result.pdfType);
|
||||
assert.equal(asyncResult.pageCount, result.pageCount);
|
||||
assert.equal(asyncResult.markdown, result.markdown);
|
||||
console.log(' processPdfAsync: OK');
|
||||
|
||||
// processPdfAsync with pages
|
||||
const asyncResult2 = await processPdfAsync(fixture, [1]);
|
||||
assert.equal(asyncResult2.markdown, result2.markdown);
|
||||
console.log(' processPdfAsync with pages: OK');
|
||||
|
||||
// classifyPdfAsync matches the sync result
|
||||
const asyncClassified = await classifyPdfAsync(fixture);
|
||||
assert.equal(asyncClassified.pdfType, classified.pdfType);
|
||||
assert.equal(asyncClassified.pageCount, classified.pageCount);
|
||||
assert.equal(asyncClassified.confidence, classified.confidence);
|
||||
assert.deepEqual(asyncClassified.pagesNeedingOcr, classified.pagesNeedingOcr);
|
||||
console.log(' classifyPdfAsync: OK');
|
||||
|
||||
// extractPagesMarkdownAsync matches the sync result
|
||||
const asyncAllPages = await extractPagesMarkdownAsync(fixture);
|
||||
assert.equal(asyncAllPages.pages.length, allPages.pages.length);
|
||||
assert.deepEqual(
|
||||
asyncAllPages.pages.map(p => p.markdown),
|
||||
allPages.pages.map(p => p.markdown),
|
||||
);
|
||||
assert.equal(asyncAllPages.isComplex, allPages.isComplex);
|
||||
console.log(' extractPagesMarkdownAsync: OK');
|
||||
|
||||
// selected pages preserve caller order
|
||||
const asyncPicked = await extractPagesMarkdownAsync(fixture, [2, 0]);
|
||||
assert.equal(asyncPicked.pages.length, 2);
|
||||
assert.equal(asyncPicked.pages[0].page, 2);
|
||||
assert.equal(asyncPicked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdownAsync with pages: OK');
|
||||
|
||||
// input buffer is copied at call time: mutating it immediately after the
|
||||
// call must not affect the in-flight parse
|
||||
const scratch = Buffer.from(fixture);
|
||||
const inFlight = processPdfAsync(scratch);
|
||||
scratch.fill(0);
|
||||
const fromMutated = await inFlight;
|
||||
assert.equal(fromMutated.markdown, result.markdown);
|
||||
console.log(' processPdfAsync input copied at call time: OK');
|
||||
|
||||
// concurrent async calls all settle
|
||||
const [c1, c2, c3] = await Promise.all([
|
||||
processPdfAsync(fixture),
|
||||
classifyPdfAsync(fixture),
|
||||
extractPagesMarkdownAsync(fixture),
|
||||
]);
|
||||
assert.equal(c1.pdfType, 'TextBased');
|
||||
assert.equal(c2.pdfType, 'TextBased');
|
||||
assert.equal(c3.pages.length, 3);
|
||||
console.log(' concurrent async calls: OK');
|
||||
|
||||
// --- Error handling ---
|
||||
console.log('Testing error handling...');
|
||||
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
|
||||
assert.throws(() => classifyPdf(Buffer.from('')), /classify_pdf/);
|
||||
await assert.rejects(processPdfAsync(Buffer.from('not a pdf')), /process_pdf/);
|
||||
await assert.rejects(classifyPdfAsync(Buffer.from('')), /classify_pdf/);
|
||||
await assert.rejects(extractPagesMarkdownAsync(Buffer.from('')), /extract_pages_markdown/);
|
||||
console.log(' error handling: OK');
|
||||
|
||||
console.log('\nAll NAPI tests passed!');
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
||||
name = "pdf-inspector"
|
||||
# Bump this to publish to PyPI — CI publishes automatically when the version
|
||||
# changes on main (same flow as napi/package.json for npm).
|
||||
version = "0.2.6"
|
||||
version = "0.2.7"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
+327
-17
@@ -41,15 +41,110 @@ pub(crate) fn detect_columns(
|
||||
}
|
||||
debug!("page {}: detect_columns: {} items", page, page_items.len());
|
||||
|
||||
// Find page bounds
|
||||
let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
|
||||
let x_max = page_items
|
||||
.iter()
|
||||
.map(|i| i.x + effective_width(i))
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
// The width of one ordinary page, used three ways below: as the largest
|
||||
// credible width for a single text run, as the size of empty gap that marks
|
||||
// content as detached, and as the span past which those checks run at all.
|
||||
// This is a heuristic, not a format rule: PDF 2.0 sets no page-size limit,
|
||||
// and since PDF 1.6 `UserUnit` scales a page's physical size independently
|
||||
// of its coordinates. 14_400 units (200in at the default 1/72in unit) is
|
||||
// the traditional Acrobat architectural limit, which makes it a reasonable
|
||||
// "wider than any ordinary page" mark in coordinate space.
|
||||
const MAX_PAGE_EXTENT: f32 = 14_400.0;
|
||||
// A detached cluster is only dropped if it also holds a small minority of
|
||||
// the items, so a genuine two-part layout keeps its full bounds even when
|
||||
// the halves are far apart.
|
||||
const MAX_TRIM_FRACTION: f32 = 0.10;
|
||||
|
||||
// Position and width of each item, skipping only non-finite geometry.
|
||||
let finite_span = |i: &&TextItem| -> Option<(f32, f32)> {
|
||||
let (left, width) = (i.x, effective_width(i));
|
||||
(left.is_finite() && (left + width).is_finite()).then_some((left, width))
|
||||
};
|
||||
|
||||
let (min_left, max_right, total) = page_items.iter().filter_map(finite_span).fold(
|
||||
(f32::INFINITY, f32::NEG_INFINITY, 0usize),
|
||||
|(lo, hi, n), (left, width)| (lo.min(left), hi.max(left + width), n + 1),
|
||||
);
|
||||
|
||||
// No item had usable geometry, so there is no layout to report.
|
||||
if total == 0 {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
// Every threshold below (gutter margins, spanning-item width, the XY-cut
|
||||
// margin) is a fraction of the page width, so a far item can set the scale
|
||||
// for the whole page and shrink the effective detection window to a
|
||||
// rounding error — real gutters then fall inside the margin band and a
|
||||
// genuine multi-column page collapses to one region.
|
||||
//
|
||||
// Anything inside one page extent is ordinary, so the common case keeps the
|
||||
// plain bounds and skips the work below entirely.
|
||||
let (x_min, x_max) = if max_right - min_left <= MAX_PAGE_EXTENT {
|
||||
(min_left, max_right)
|
||||
} else {
|
||||
// Discarding content needs positive evidence that it is not part of the
|
||||
// layout, because a count-based rule alone cannot tell a stray from a
|
||||
// sparse far sidebar. The evidence is geometric: positions are grouped
|
||||
// into clusters separated by more than a whole page of continuous
|
||||
// emptiness. Real content, however sparse, does not leave a void that
|
||||
// large; a malformed coordinate sits alone beyond one.
|
||||
let mut spans: Vec<(f32, f32)> = page_items.iter().filter_map(finite_span).collect();
|
||||
spans.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
|
||||
let mut core: Option<std::ops::Range<usize>> = None;
|
||||
let mut start = 0usize;
|
||||
for i in 1..=spans.len() {
|
||||
if i < spans.len() && spans[i].0 - spans[i - 1].0 <= MAX_PAGE_EXTENT {
|
||||
continue;
|
||||
}
|
||||
if core.as_ref().is_none_or(|best| i - start > best.len()) {
|
||||
core = Some(start..i);
|
||||
}
|
||||
start = i;
|
||||
}
|
||||
let mut core = core.unwrap_or(0..spans.len());
|
||||
|
||||
// Only drop the detached clusters when they are a small minority, so a
|
||||
// genuine two-part layout keeps its full bounds.
|
||||
let dropped = spans.len() - core.len();
|
||||
if dropped as f32 > spans.len() as f32 * MAX_TRIM_FRACTION {
|
||||
core = 0..spans.len();
|
||||
}
|
||||
let core = &spans[core];
|
||||
|
||||
// Positions cannot be inflated by a bogus width, so the spread of the
|
||||
// content is a sound scale for judging one. A run much wider than the
|
||||
// page's own content is a malformed width — the test is relative, so a
|
||||
// genuinely large page keeps its genuinely long runs.
|
||||
let (lo, widest_left) = (core[0].0, core[core.len() - 1].0);
|
||||
let max_run_width = (widest_left - lo) + MAX_PAGE_EXTENT;
|
||||
let hi = core
|
||||
.iter()
|
||||
.filter(|&&(_, width)| width <= max_run_width)
|
||||
.map(|&(left, width)| left + width)
|
||||
.fold(widest_left, f32::max);
|
||||
|
||||
if lo != min_left || hi != max_right {
|
||||
debug!(
|
||||
"page {page}: bounds {min_left}..{max_right} exceed one page; \
|
||||
dropped {dropped}/{} detached item(s), using {lo}..{hi}",
|
||||
spans.len()
|
||||
);
|
||||
}
|
||||
(lo, hi)
|
||||
};
|
||||
|
||||
// Hard ceiling on the histogram size, independent of the trimming above:
|
||||
// the bounds are attacker-influenced, so an unclamped
|
||||
// `page_width / BIN_WIDTH` lets a crafted PDF force an arbitrarily large
|
||||
// `vec![0u32; num_bins]` allocation. 65_536 bins covers ~128k points at
|
||||
// BIN_WIDTH 2.0 — roughly 9x the largest legal page — so this never binds
|
||||
// on a real layout. Kept as a bound that does not depend on the outlier
|
||||
// heuristic staying correct.
|
||||
const MAX_BINS: usize = 65_536;
|
||||
|
||||
let page_width = x_max - x_min;
|
||||
if page_width < 200.0 {
|
||||
if !page_width.is_finite() || page_width < 200.0 {
|
||||
return vec![ColumnRegion { x_min, x_max }];
|
||||
}
|
||||
|
||||
@@ -57,13 +152,20 @@ pub(crate) fn detect_columns(
|
||||
return vec![ColumnRegion { x_min, x_max }];
|
||||
}
|
||||
|
||||
// Widen the bins rather than dropping the tail of the page. Clamping the
|
||||
// count alone would leave anything past MAX_BINS * BIN_WIDTH outside the
|
||||
// histogram, folded into the last bin, which places gutters at the wrong
|
||||
// coordinates. Scaling keeps full coverage under the same allocation
|
||||
// ceiling; only the resolution degrades, and only beyond ~131k points.
|
||||
let bin_width = BIN_WIDTH.max(page_width / MAX_BINS as f32);
|
||||
|
||||
// Build occupancy histogram.
|
||||
// Exclude items wider than 60% of page width — these are spanning items
|
||||
// (titles, full-width paragraphs) that would fill the gutter and prevent
|
||||
// detection of partial-page column layouts (e.g. two-column abstracts on
|
||||
// a page that also has single-column introduction text).
|
||||
let wide_threshold = page_width * 0.6;
|
||||
let num_bins = ((page_width / BIN_WIDTH).ceil() as usize).max(1);
|
||||
let num_bins = ((page_width / bin_width).ceil() as usize).clamp(1, MAX_BINS);
|
||||
let mut histogram = vec![0u32; num_bins];
|
||||
|
||||
for item in &page_items {
|
||||
@@ -71,8 +173,8 @@ pub(crate) fn detect_columns(
|
||||
if w > wide_threshold {
|
||||
continue;
|
||||
}
|
||||
let left = ((item.x - x_min) / BIN_WIDTH).floor() as usize;
|
||||
let right = (((item.x + w) - x_min) / BIN_WIDTH).ceil() as usize;
|
||||
let left = ((item.x - x_min) / bin_width).floor() as usize;
|
||||
let right = (((item.x + w) - x_min) / bin_width).ceil() as usize;
|
||||
let left = left.min(num_bins);
|
||||
let right = right.min(num_bins);
|
||||
for count in histogram.iter_mut().take(right).skip(left) {
|
||||
@@ -109,12 +211,12 @@ pub(crate) fn detect_columns(
|
||||
let valleys: Vec<(usize, usize)> = valleys
|
||||
.into_iter()
|
||||
.filter(|&(start, end)| {
|
||||
let width_pts = (end - start) as f32 * BIN_WIDTH;
|
||||
let width_pts = (end - start) as f32 * bin_width;
|
||||
if width_pts < MIN_GUTTER_WIDTH {
|
||||
return false;
|
||||
}
|
||||
// Valley center must not be within 5% of page edges
|
||||
let center_pts = ((start + end) as f32 / 2.0) * BIN_WIDTH;
|
||||
let center_pts = ((start + end) as f32 / 2.0) * bin_width;
|
||||
center_pts > margin_threshold && center_pts < (page_width - margin_threshold)
|
||||
})
|
||||
.collect();
|
||||
@@ -132,7 +234,7 @@ pub(crate) fn detect_columns(
|
||||
&histogram,
|
||||
num_bins,
|
||||
x_min,
|
||||
BIN_WIDTH,
|
||||
bin_width,
|
||||
page_width,
|
||||
margin_threshold,
|
||||
);
|
||||
@@ -141,7 +243,7 @@ pub(crate) fn detect_columns(
|
||||
&rel_valleys,
|
||||
&page_items,
|
||||
x_min,
|
||||
BIN_WIDTH,
|
||||
bin_width,
|
||||
x_max,
|
||||
MIN_ITEMS_PER_COLUMN,
|
||||
MIN_VERTICAL_SPAN_RATIO,
|
||||
@@ -182,7 +284,7 @@ pub(crate) fn detect_columns(
|
||||
&valleys,
|
||||
&page_items,
|
||||
x_min,
|
||||
BIN_WIDTH,
|
||||
bin_width,
|
||||
x_max,
|
||||
MIN_ITEMS_PER_COLUMN,
|
||||
MIN_VERTICAL_SPAN_RATIO,
|
||||
@@ -196,7 +298,7 @@ pub(crate) fn detect_columns(
|
||||
&valleys,
|
||||
&page_items,
|
||||
x_min,
|
||||
BIN_WIDTH,
|
||||
bin_width,
|
||||
x_max,
|
||||
MIN_ITEMS_PER_COLUMN,
|
||||
MIN_VERTICAL_SPAN_RATIO,
|
||||
@@ -1827,7 +1929,7 @@ fn split_column_stragglers(lines: Vec<TextLine>) -> (Vec<TextLine>, Vec<TextLine
|
||||
.unwrap();
|
||||
|
||||
let (cs, ce) = segments[core_seg];
|
||||
let mut core = Vec::with_capacity(ce - cs);
|
||||
let mut core = Vec::with_capacity(ce.saturating_sub(cs));
|
||||
let mut stragglers = Vec::new();
|
||||
for (i, line) in lines.into_iter().enumerate() {
|
||||
if i >= cs && i < ce {
|
||||
@@ -2533,6 +2635,214 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extreme_far_coordinate_does_not_allocate_unboundedly() {
|
||||
// A crafted PDF can place a text run at an arbitrary coordinate via the
|
||||
// text matrix. The derived page width must not drive an unbounded
|
||||
// histogram allocation (previously `page_width / BIN_WIDTH` bins with no
|
||||
// upper bound would try to reserve terabytes and abort the process).
|
||||
let mut items = Vec::new();
|
||||
for i in 0..24 {
|
||||
items.push(make_item(1, i as f32 * 10.0, 700.0 - i as f32 * 5.0, "A"));
|
||||
}
|
||||
// Item placed 1e12 points away — 5e11 bins if left unclamped.
|
||||
items.push(make_item(1, 1e12, 700.0, "Z"));
|
||||
|
||||
// Must return without aborting; content is preserved as a single region.
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert!(!cols.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_finite_coordinates_never_leak_into_region_bounds() {
|
||||
// An inf/NaN coordinate must not escape as a column boundary: callers
|
||||
// treat these as page/column edges.
|
||||
for bad_x in [f32::INFINITY, f32::NEG_INFINITY, f32::NAN] {
|
||||
let mut items = Vec::new();
|
||||
for i in 0..24 {
|
||||
items.push(make_item(1, i as f32 * 10.0, 700.0 - i as f32 * 5.0, "A"));
|
||||
}
|
||||
items.push(make_item(1, bad_x, 700.0, "Z"));
|
||||
|
||||
for col in detect_columns(&items, 1, false) {
|
||||
assert!(
|
||||
col.x_min.is_finite() && col.x_max.is_finite(),
|
||||
"bad_x {bad_x} leaked bounds {}..{}",
|
||||
col.x_min,
|
||||
col.x_max
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_non_finite_coordinates_yield_no_columns() {
|
||||
let items: Vec<TextItem> = (0..24)
|
||||
.map(|i| make_item(1, f32::NAN, 700.0 - i as f32 * 5.0, "A"))
|
||||
.collect();
|
||||
|
||||
assert!(detect_columns(&items, 1, false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_bad_item_does_not_disable_column_detection() {
|
||||
// A single stray item should not collapse a clean two-column page to
|
||||
// one region. Every gutter threshold is a fraction of the page width,
|
||||
// so an untrimmed outlier pushes real gutters inside the rejected
|
||||
// margin band. A malformed *width* at an ordinary position poisons the
|
||||
// bounds just as a malformed position does.
|
||||
for (label, bad_x, bad_width) in [
|
||||
("nan position", f32::NAN, 0.0),
|
||||
("inf position", f32::INFINITY, 0.0),
|
||||
("far position", 50_000.0, 0.0),
|
||||
("very far position", 1e12, 0.0),
|
||||
("huge width", 100.0, 1e12),
|
||||
("inf width", 100.0, f32::INFINITY),
|
||||
] {
|
||||
let mut items = Vec::new();
|
||||
items.extend(fill_zone(1, 30.0, 280.0, 750.0, 50.0));
|
||||
items.extend(fill_zone(1, 320.0, 570.0, 750.0, 50.0));
|
||||
let mut bad = make_item(1, bad_x, 400.0, "Z");
|
||||
bad.width = bad_width;
|
||||
items.push(bad);
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(
|
||||
cols.len(),
|
||||
2,
|
||||
"{label}: expected 2 columns, got {}",
|
||||
cols.len()
|
||||
);
|
||||
for col in &cols {
|
||||
assert!(
|
||||
col.x_max - col.x_min <= MAX_PAGE_EXTENT_FOR_TEST,
|
||||
"{label}: region {}..{} exceeds one page",
|
||||
col.x_min,
|
||||
col.x_max
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors `MAX_PAGE_EXTENT` in `detect_columns`.
|
||||
const MAX_PAGE_EXTENT_FOR_TEST: f32 = 14_400.0;
|
||||
|
||||
#[test]
|
||||
fn very_wide_page_keeps_full_histogram_coverage() {
|
||||
// Beyond MAX_BINS * BIN_WIDTH (~131k points) the bins must widen rather
|
||||
// than stop covering the page. Three zones: the first gutter is inside
|
||||
// the old coverage limit, the second is past it. Because the first
|
||||
// gutter is found, the XY-cut fallback never runs, so a truncated
|
||||
// histogram silently reports two columns instead of three.
|
||||
let mut items = Vec::new();
|
||||
items.extend(fill_zone(1, 0.0, 60_000.0, 750.0, 700.0));
|
||||
items.extend(fill_zone(1, 70_000.0, 140_000.0, 750.0, 700.0));
|
||||
items.extend(fill_zone(1, 160_000.0, 200_000.0, 750.0, 700.0));
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(
|
||||
cols.len(),
|
||||
3,
|
||||
"Expected 3 columns across a 200k-wide page, got {}",
|
||||
cols.len()
|
||||
);
|
||||
assert!(
|
||||
(140_000.0..=160_000.0).contains(&cols[1].x_max),
|
||||
"second gutter at {}, expected inside the real 140k..160k gap",
|
||||
cols[1].x_max
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_page_with_legitimately_long_runs_is_kept() {
|
||||
// On a very large page, individual runs can exceed one ordinary page's
|
||||
// width. They are real content, so they must not be judged malformed:
|
||||
// the page keeps its columns and its full right edge.
|
||||
let mut items = Vec::new();
|
||||
for row in 0..30 {
|
||||
let y = 750.0 - row as f32 * 14.0;
|
||||
let mut left = make_item(1, 0.0, y, "Left run");
|
||||
left.width = 20_000.0;
|
||||
let mut right = make_item(1, 25_000.0, y, "Right run");
|
||||
right.width = 20_000.0;
|
||||
items.extend([left, right]);
|
||||
}
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert!(
|
||||
!cols.is_empty(),
|
||||
"a page of long-but-valid runs must still report a layout"
|
||||
);
|
||||
let right_edge = cols
|
||||
.iter()
|
||||
.map(|c| c.x_max)
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
assert!(
|
||||
right_edge > 44_000.0,
|
||||
"long runs were treated as malformed: right edge {right_edge}, expected ~45_000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_far_sidebar_on_a_large_page_is_kept() {
|
||||
// A large-format page with a thin, sparsely-populated sidebar far from
|
||||
// the main block. The sidebar is a small minority of the items, so an
|
||||
// item-count rule alone would discard it — but nothing about its
|
||||
// geometry says it is invalid, so its bounds must survive.
|
||||
let mut items = Vec::new();
|
||||
items.extend(fill_zone(1, 0.0, 12_000.0, 750.0, 500.0));
|
||||
for i in 0..12 {
|
||||
items.push(make_item(1, 24_000.0, 750.0 - i as f32 * 14.0, "Sidebar"));
|
||||
}
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
let right_edge = cols
|
||||
.iter()
|
||||
.map(|c| c.x_max)
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
assert!(
|
||||
right_edge > 24_000.0,
|
||||
"sidebar was trimmed away: right edge {right_edge}, expected >24_000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn genuinely_wide_layout_keeps_its_true_bounds() {
|
||||
// A large-format page whose content really is spread beyond one
|
||||
// ordinary page must not be trimmed to the median cluster: its far
|
||||
// items are the majority, not strays.
|
||||
let mut items = Vec::new();
|
||||
items.extend(fill_zone(1, 100.0, 20_000.0, 750.0, 600.0));
|
||||
items.extend(fill_zone(1, 22_000.0, 40_000.0, 750.0, 600.0));
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
let widest = cols
|
||||
.iter()
|
||||
.map(|c| c.x_max)
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
assert!(
|
||||
widest > 35_000.0,
|
||||
"wide layout was trimmed: right edge {widest}, expected ~40_000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_but_legal_page_is_not_trimmed() {
|
||||
// A wide-format page well inside the 14_400pt spec limit must keep its
|
||||
// real bounds — outlier trimming is only for spans beyond a legal page.
|
||||
let mut items = Vec::new();
|
||||
items.extend(fill_zone(1, 100.0, 4_000.0, 750.0, 400.0));
|
||||
items.extend(fill_zone(1, 4_400.0, 8_000.0, 750.0, 400.0));
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(cols.len(), 2, "Expected 2 columns, got {}", cols.len());
|
||||
assert!(
|
||||
cols[1].x_max > 7_000.0,
|
||||
"right column should keep its true extent, got {}",
|
||||
cols[1].x_max
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_column_regression_guard() {
|
||||
// Standard 2-column layout with clear gutter at center
|
||||
|
||||
+700
-38
@@ -9,7 +9,7 @@
|
||||
use log::debug;
|
||||
use lopdf::{Document, Object, ObjectId};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
// ─── Standard structure types ────────────────────────────────────────
|
||||
|
||||
@@ -252,10 +252,28 @@ impl StructTree {
|
||||
let role_map = parse_role_map(doc, struct_root);
|
||||
debug!("structure tree: {} role map entries", role_map.len());
|
||||
|
||||
// Seed the cycle guard with the struct-root's own object id so a `/K`
|
||||
// that points back at the root is treated as a cycle, and bound total
|
||||
// node materialization with a global budget.
|
||||
let mut walk = StructWalk::new();
|
||||
if let Ok(root_id) = struct_root_obj.as_reference() {
|
||||
walk.active.insert(root_id);
|
||||
}
|
||||
|
||||
// Parse child elements from /K
|
||||
let children = parse_kids(doc, struct_root, &role_map, None, 0);
|
||||
let children = parse_kids(doc, struct_root, &role_map, None, 0, &mut walk);
|
||||
debug!("structure tree: {} top-level elements", children.len());
|
||||
|
||||
if walk.truncated {
|
||||
log::warn!(
|
||||
"structure tree parsing was truncated (node budget of \
|
||||
{MAX_STRUCT_NODES} or traversal budget of {MAX_STRUCT_WORK} \
|
||||
reached, a `/K` reference cycle, or the max nesting depth of \
|
||||
{MAX_DEPTH}); tagged roles/tables may be incomplete (likely a \
|
||||
very large or malformed tagged PDF)"
|
||||
);
|
||||
}
|
||||
|
||||
if children.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -479,6 +497,123 @@ fn parse_role_map(doc: &Document, struct_root: &lopdf::Dictionary) -> HashMap<St
|
||||
/// malformed PDFs).
|
||||
const MAX_DEPTH: usize = 64;
|
||||
|
||||
/// Global cap on the number of structure-tree nodes materialized in a single
|
||||
/// parse. Real tagged trees are far smaller; a crafted PDF can alias one struct
|
||||
/// element into its own `/K` (e.g. `/K [n 0 R n 0 R]`) so the tree branches
|
||||
/// exponentially (2^depth) before the depth cap is reached, exhausting memory.
|
||||
/// This budget bounds total work and allocation regardless of tree shape.
|
||||
const MAX_STRUCT_NODES: usize = 500_000;
|
||||
|
||||
/// Cap on the number of `/K` items *examined* during a single parse, regardless
|
||||
/// of whether they materialize anything. Bounds CPU for crafted wide `/K` arrays
|
||||
/// of non-materializing entries (unsupported value types, `/OBJR` dicts, cycle
|
||||
/// back-edges) that would otherwise be scanned in full without ever touching the
|
||||
/// node budget. Kept well above the node budget so it never truncates content
|
||||
/// that already fits within `MAX_STRUCT_NODES`.
|
||||
const MAX_STRUCT_WORK: usize = 2_000_000;
|
||||
|
||||
/// Traversal state shared across the recursive structure-tree parse.
|
||||
///
|
||||
/// `budget` is a global allowance charged once per materialized item — each
|
||||
/// struct-element node and each marked-content reference — so total work is
|
||||
/// bounded even for aliased/DAG-shaped `/K` graphs of distinct objects or a
|
||||
/// single element with a very wide `/K` array. `active` holds the object IDs
|
||||
/// currently on the depth-first path so a struct element that references itself
|
||||
/// (or an ancestor) is not expanded into an unbounded/exponential subtree.
|
||||
/// `budget` bounds *materialization* (nodes + content refs). `work` separately
|
||||
/// bounds *traversal* — every `/K` item examined is charged against it, even
|
||||
/// ones that materialize nothing (unsupported values, `/OBJR`, cycle back-edges)
|
||||
/// — so a wide malformed array cannot force an unbounded scan, and those skipped
|
||||
/// items don't drain the materialization budget and truncate real content.
|
||||
/// `truncated` records whether any parse work was skipped — the budget was
|
||||
/// exhausted, a `/K` reference cycle was broken, or the depth cap was hit — so
|
||||
/// the caller can log it once rather than per skipped item. `stalled` is set
|
||||
/// when an atomic multi-unit reservation could not fit in the remaining budget;
|
||||
/// it makes [`exhausted`](Self::exhausted) report done so a wide `/K` array is
|
||||
/// not scanned to the end once no further leaf can be materialized.
|
||||
struct StructWalk {
|
||||
budget: usize,
|
||||
work: usize,
|
||||
active: HashSet<ObjectId>,
|
||||
truncated: bool,
|
||||
stalled: bool,
|
||||
}
|
||||
|
||||
impl StructWalk {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
budget: MAX_STRUCT_NODES,
|
||||
work: MAX_STRUCT_WORK,
|
||||
active: HashSet::new(),
|
||||
truncated: false,
|
||||
stalled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Charge one unit of traversal work for an examined `/K` item, whether or
|
||||
/// not it materializes anything. Returns `false` (flagging truncation) once
|
||||
/// the traversal budget is spent, so an enclosing loop stops instead of
|
||||
/// scanning the rest of a wide array of non-materializing entries.
|
||||
fn spend_work(&mut self) -> bool {
|
||||
if self.work == 0 {
|
||||
self.truncated = true;
|
||||
return false;
|
||||
}
|
||||
self.work -= 1;
|
||||
true
|
||||
}
|
||||
|
||||
/// Record that some parse work was skipped for a non-budget reason (a `/K`
|
||||
/// reference cycle or the depth cap), so the one-shot truncation warning
|
||||
/// also covers malformed/over-deep trees, not just budget exhaustion.
|
||||
fn note_skipped(&mut self) {
|
||||
self.truncated = true;
|
||||
}
|
||||
|
||||
/// Charge one unit against the budget for a materialized item (a struct
|
||||
/// element node or a marked-content reference). Returns `false` — without
|
||||
/// underflowing — once the budget is exhausted, so callers skip the item.
|
||||
fn charge(&mut self) -> bool {
|
||||
if self.budget == 0 {
|
||||
self.truncated = true;
|
||||
return false;
|
||||
}
|
||||
self.budget -= 1;
|
||||
true
|
||||
}
|
||||
|
||||
/// Atomically charge `n` units for a single item that materializes several
|
||||
/// budget-counted parts at once (a leaf wrapper node *plus* its content
|
||||
/// reference). Charges nothing when fewer than `n` units remain — so a
|
||||
/// partial reservation never wastes capacity — and marks the walk `stalled`
|
||||
/// so the enclosing loop stops instead of scanning the rest of a wide `/K`
|
||||
/// array that can no longer fit any leaf.
|
||||
fn charge_n(&mut self, n: usize) -> bool {
|
||||
if self.budget < n {
|
||||
self.truncated = true;
|
||||
self.stalled = true;
|
||||
return false;
|
||||
}
|
||||
self.budget -= n;
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether traversal should stop: the budget is spent, or a multi-unit
|
||||
/// reservation could not fit (`stalled`) so no further leaf will materialize.
|
||||
/// Use this at the guards that break/return to skip remaining items; it
|
||||
/// records that truncation occurred (a guard only fires while an item is
|
||||
/// still pending), so callers that drop work without going through
|
||||
/// [`charge`](Self::charge) still flag the truncation for logging.
|
||||
fn exhausted(&mut self) -> bool {
|
||||
if self.budget == 0 || self.stalled {
|
||||
self.truncated = true;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse child elements from a `/K` entry.
|
||||
fn parse_kids(
|
||||
doc: &Document,
|
||||
@@ -486,8 +621,13 @@ fn parse_kids(
|
||||
role_map: &HashMap<String, String>,
|
||||
inherited_page: Option<ObjectId>,
|
||||
depth: usize,
|
||||
walk: &mut StructWalk,
|
||||
) -> Vec<StructElement> {
|
||||
if depth >= MAX_DEPTH {
|
||||
walk.note_skipped();
|
||||
return Vec::new();
|
||||
}
|
||||
if walk.exhausted() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
@@ -498,22 +638,59 @@ fn parse_kids(
|
||||
// /Pg on this element (inherited by children)
|
||||
let page_id = get_page_ref(doc, dict).or(inherited_page);
|
||||
|
||||
let mut children = Vec::new();
|
||||
match k_obj {
|
||||
Object::Array(arr) => {
|
||||
let mut children = Vec::new();
|
||||
for item in arr {
|
||||
let resolved = resolve_obj(doc, item);
|
||||
parse_kid(doc, resolved, role_map, page_id, depth, &mut children);
|
||||
if walk.exhausted() || !walk.spend_work() {
|
||||
break;
|
||||
}
|
||||
process_kid_item(doc, item, role_map, page_id, depth, &mut children, walk);
|
||||
}
|
||||
children
|
||||
}
|
||||
other => {
|
||||
let resolved = resolve_obj(doc, other);
|
||||
let mut children = Vec::new();
|
||||
parse_kid(doc, resolved, role_map, page_id, depth, &mut children);
|
||||
children
|
||||
process_kid_item(doc, other, role_map, page_id, depth, &mut children, walk);
|
||||
}
|
||||
}
|
||||
children
|
||||
}
|
||||
|
||||
/// Resolve one `/K` array item (following at most one level of indirection),
|
||||
/// guarding against reference cycles and the global node budget, then dispatch
|
||||
/// it via [`parse_kid`].
|
||||
fn process_kid_item(
|
||||
doc: &Document,
|
||||
item: &Object,
|
||||
role_map: &HashMap<String, String>,
|
||||
inherited_page: Option<ObjectId>,
|
||||
depth: usize,
|
||||
out: &mut Vec<StructElement>,
|
||||
walk: &mut StructWalk,
|
||||
) {
|
||||
if walk.exhausted() {
|
||||
return;
|
||||
}
|
||||
if depth >= MAX_DEPTH {
|
||||
walk.note_skipped();
|
||||
return;
|
||||
}
|
||||
// If this child is an indirect reference, track its id on the active path so
|
||||
// a self/ancestor reference is not expanded into an exponential subtree.
|
||||
let ref_id = match item {
|
||||
Object::Reference(id) => Some(*id),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(id) = ref_id {
|
||||
if !walk.active.insert(id) {
|
||||
walk.note_skipped();
|
||||
return; // cycle: this object is already on the current path
|
||||
}
|
||||
}
|
||||
let resolved = resolve_obj(doc, item);
|
||||
parse_kid(doc, resolved, role_map, inherited_page, depth, out, walk);
|
||||
if let Some(id) = ref_id {
|
||||
walk.active.remove(&id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a single child (either a struct element dict or an MCID integer).
|
||||
@@ -524,10 +701,16 @@ fn parse_kid(
|
||||
inherited_page: Option<ObjectId>,
|
||||
depth: usize,
|
||||
out: &mut Vec<StructElement>,
|
||||
walk: &mut StructWalk,
|
||||
) {
|
||||
match obj {
|
||||
// Direct MCID integer — create a leaf wrapper
|
||||
Object::Integer(mcid) => {
|
||||
// A wrapper node plus its content reference — two items — reserved
|
||||
// atomically so we never consume one unit without emitting both.
|
||||
if !walk.charge_n(2) {
|
||||
return;
|
||||
}
|
||||
// This is a bare MCID at the struct-element level.
|
||||
// We attach it to the parent element, so we create a wrapper struct element.
|
||||
// Actually, bare MCIDs inside /K are content refs for the parent,
|
||||
@@ -546,11 +729,11 @@ fn parse_kid(
|
||||
});
|
||||
}
|
||||
Object::Dictionary(d) => {
|
||||
parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out);
|
||||
parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out, walk);
|
||||
}
|
||||
Object::Stream(s) => {
|
||||
// Some PDFs wrap struct elements in streams (rare)
|
||||
parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out);
|
||||
parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out, walk);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
@@ -565,13 +748,23 @@ fn parse_struct_element_dict(
|
||||
inherited_page: Option<ObjectId>,
|
||||
depth: usize,
|
||||
out: &mut Vec<StructElement>,
|
||||
walk: &mut StructWalk,
|
||||
) {
|
||||
if depth >= MAX_DEPTH {
|
||||
walk.note_skipped();
|
||||
return;
|
||||
}
|
||||
// Check if this is a marked-content reference dict (has /Type /MCR)
|
||||
|
||||
// A marked-content reference dict materializes a wrapper node + one content
|
||||
// reference (two items). Reserve both atomically *before* the node charge so
|
||||
// we never consume a unit without emitting the reference — which would also
|
||||
// deny that unit to a later element that would have fit. This matches the
|
||||
// bare-MCID path.
|
||||
if is_mcr_dict(dict) {
|
||||
if let Ok(Object::Integer(mcid)) = dict.get(b"MCID") {
|
||||
if !walk.charge_n(2) {
|
||||
return;
|
||||
}
|
||||
let page_id = get_page_ref(doc, dict).or(inherited_page);
|
||||
out.push(StructElement {
|
||||
role: StructRole::Span,
|
||||
@@ -588,12 +781,15 @@ fn parse_struct_element_dict(
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if this is an object reference dict (has /Type /OBJR) — skip these
|
||||
// Skip object-reference dicts (`/Type /OBJR`) — they materialize no node, so
|
||||
// recognize and return *before* charging the budget (otherwise a document
|
||||
// full of OBJRs would drain the shared budget and truncate real content).
|
||||
if is_objr_dict(dict) {
|
||||
return;
|
||||
}
|
||||
|
||||
// It's a struct element — parse its /S (structure type)
|
||||
// It's a struct element — parse its /S (structure type). A dict without a
|
||||
// valid /S also materializes nothing, so validate before charging.
|
||||
let role_name = match dict.get(b"S") {
|
||||
Ok(s_obj) => {
|
||||
let resolved = resolve_obj(doc, s_obj);
|
||||
@@ -605,6 +801,12 @@ fn parse_struct_element_dict(
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
// Charge the node only now that we know it will materialize (bounds
|
||||
// aliased/DAG-shaped `/K` graphs the per-path cycle guard alone cannot stop).
|
||||
if !walk.charge() {
|
||||
return;
|
||||
}
|
||||
|
||||
let role = StructRole::from_name_with_role_map(&role_name, role_map);
|
||||
let page_id = get_page_ref(doc, dict).or(inherited_page);
|
||||
|
||||
@@ -621,51 +823,73 @@ fn parse_struct_element_dict(
|
||||
let k_resolved = resolve_obj(doc, k_obj);
|
||||
match k_resolved {
|
||||
Object::Integer(mcid) => {
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id,
|
||||
});
|
||||
if walk.charge() {
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
Object::Array(arr) => {
|
||||
for item in arr {
|
||||
if walk.exhausted() || !walk.spend_work() {
|
||||
break;
|
||||
}
|
||||
// Only content-ref items (bare MCIDs / MCR dicts) are charged
|
||||
// here — those are the unbounded allocations. Structural
|
||||
// children are charged once at their own node entry in the
|
||||
// recursive call, so charging them here too would double-count
|
||||
// and drain the budget ~2× faster than the per-node semantics.
|
||||
let ref_id = match item {
|
||||
Object::Reference(id) => Some(*id),
|
||||
_ => None,
|
||||
};
|
||||
let resolved = resolve_obj(doc, item);
|
||||
match resolved {
|
||||
Object::Integer(mcid) => {
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id,
|
||||
});
|
||||
if walk.charge() {
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
Object::Dictionary(d) => {
|
||||
if is_mcr_dict(d) {
|
||||
if let Ok(Object::Integer(mcid)) = d.get(b"MCID") {
|
||||
let pg = get_page_ref(doc, d).or(page_id);
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id: pg,
|
||||
});
|
||||
if walk.charge() {
|
||||
let pg = get_page_ref(doc, d).or(page_id);
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id: pg,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if is_objr_dict(d) {
|
||||
// Skip object references
|
||||
} else {
|
||||
parse_struct_element_dict(
|
||||
recurse_struct_child(
|
||||
doc,
|
||||
ref_id,
|
||||
d,
|
||||
role_map,
|
||||
page_id,
|
||||
depth + 1,
|
||||
depth,
|
||||
&mut children,
|
||||
walk,
|
||||
);
|
||||
}
|
||||
}
|
||||
Object::Stream(s) => {
|
||||
parse_struct_element_dict(
|
||||
recurse_struct_child(
|
||||
doc,
|
||||
ref_id,
|
||||
&s.dict,
|
||||
role_map,
|
||||
page_id,
|
||||
depth + 1,
|
||||
depth,
|
||||
&mut children,
|
||||
walk,
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
@@ -675,14 +899,29 @@ fn parse_struct_element_dict(
|
||||
Object::Dictionary(d) => {
|
||||
if is_mcr_dict(d) {
|
||||
if let Ok(Object::Integer(mcid)) = d.get(b"MCID") {
|
||||
let pg = get_page_ref(doc, d).or(page_id);
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id: pg,
|
||||
});
|
||||
if walk.charge() {
|
||||
let pg = get_page_ref(doc, d).or(page_id);
|
||||
content_refs.push(MarkedContentRef {
|
||||
mcid: *mcid,
|
||||
page_id: pg,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
parse_struct_element_dict(doc, d, role_map, page_id, depth + 1, &mut children);
|
||||
let ref_id = match k_obj {
|
||||
Object::Reference(id) => Some(*id),
|
||||
_ => None,
|
||||
};
|
||||
recurse_struct_child(
|
||||
doc,
|
||||
ref_id,
|
||||
d,
|
||||
role_map,
|
||||
page_id,
|
||||
depth,
|
||||
&mut children,
|
||||
walk,
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
@@ -699,6 +938,37 @@ fn parse_struct_element_dict(
|
||||
});
|
||||
}
|
||||
|
||||
/// Recurse into a child struct-element dictionary, guarding against reference
|
||||
/// cycles (via the active-path object-id set) and the global node budget.
|
||||
///
|
||||
/// `ref_id` is the object id of the child when it was reached through an
|
||||
/// indirect reference (`None` for an inline dictionary, which cannot alias).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn recurse_struct_child(
|
||||
doc: &Document,
|
||||
ref_id: Option<ObjectId>,
|
||||
dict: &lopdf::Dictionary,
|
||||
role_map: &HashMap<String, String>,
|
||||
inherited_page: Option<ObjectId>,
|
||||
depth: usize,
|
||||
out: &mut Vec<StructElement>,
|
||||
walk: &mut StructWalk,
|
||||
) {
|
||||
if walk.exhausted() {
|
||||
return;
|
||||
}
|
||||
if let Some(id) = ref_id {
|
||||
if !walk.active.insert(id) {
|
||||
walk.note_skipped();
|
||||
return; // cycle: this object is already on the current path
|
||||
}
|
||||
}
|
||||
parse_struct_element_dict(doc, dict, role_map, inherited_page, depth + 1, out, walk);
|
||||
if let Some(id) = ref_id {
|
||||
walk.active.remove(&id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if dict has `/Type /MCR`.
|
||||
fn is_mcr_dict(dict: &lopdf::Dictionary) -> bool {
|
||||
dict.get(b"Type")
|
||||
@@ -901,6 +1171,7 @@ fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use lopdf::dictionary;
|
||||
|
||||
#[test]
|
||||
fn non_heading_content_roles() {
|
||||
@@ -1231,4 +1502,395 @@ mod tests {
|
||||
let role_map = tree.mcid_to_roles(&page_ids);
|
||||
assert!(!role_map.is_empty(), "Should have MCID→role mappings");
|
||||
}
|
||||
|
||||
fn count_nodes(elems: &[StructElement]) -> usize {
|
||||
elems.iter().map(|e| 1 + count_nodes(&e.children)).sum()
|
||||
}
|
||||
|
||||
/// Wrap already-created struct elements under a `/StructTreeRoot` and
|
||||
/// `/Catalog`, returning a document ready for [`StructTree::from_doc`].
|
||||
/// `root_kid` is the top-level element the root's `/K` points at.
|
||||
fn finalize_tagged_doc(mut doc: Document, root_kid: ObjectId) -> Document {
|
||||
let root_id = doc.add_object(dictionary! {
|
||||
"Type" => "StructTreeRoot",
|
||||
"K" => vec![Object::Reference(root_kid)],
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"StructTreeRoot" => Object::Reference(root_id),
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
doc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn struct_tree_self_alias_kids_terminates() {
|
||||
// A struct element that lists itself twice in `/K` (`/K [n 0 R n 0 R]`)
|
||||
// must not expand into an exponential tree.
|
||||
let mut doc = Document::new();
|
||||
let elem = doc.new_object_id();
|
||||
doc.set_object(
|
||||
elem,
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "Div",
|
||||
"K" => vec![Object::Reference(elem), Object::Reference(elem)],
|
||||
},
|
||||
);
|
||||
let doc = finalize_tagged_doc(doc, elem);
|
||||
|
||||
let tree = StructTree::from_doc(&doc).expect("tree should parse");
|
||||
let n = count_nodes(&tree.children);
|
||||
assert!(
|
||||
n < 10,
|
||||
"self-alias must not explode; materialized {n} nodes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn struct_tree_mutual_alias_kids_terminates() {
|
||||
// A → B → A cycle via `/K` must terminate.
|
||||
let mut doc = Document::new();
|
||||
let a = doc.new_object_id();
|
||||
let b = doc.new_object_id();
|
||||
doc.set_object(
|
||||
a,
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "Div",
|
||||
"K" => vec![Object::Reference(b), Object::Reference(b)],
|
||||
},
|
||||
);
|
||||
doc.set_object(
|
||||
b,
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "Div",
|
||||
"K" => vec![Object::Reference(a), Object::Reference(a)],
|
||||
},
|
||||
);
|
||||
let doc = finalize_tagged_doc(doc, a);
|
||||
|
||||
let tree = StructTree::from_doc(&doc).expect("tree should parse");
|
||||
let n = count_nodes(&tree.children);
|
||||
assert!(
|
||||
n < 100,
|
||||
"mutual alias must terminate small; materialized {n} nodes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn struct_tree_aliased_dag_respects_node_budget() {
|
||||
// Distinct elements, each aliased twice in the next level's `/K`, form a
|
||||
// DAG that would expand to 2^depth nodes (the per-path cycle guard does
|
||||
// not catch this since every id is on the path only once). The global
|
||||
// node budget must cap total materialization.
|
||||
let mut doc = Document::new();
|
||||
let levels = 22; // 2^22 ≈ 4.2M unbounded, well past the budget
|
||||
let ids: Vec<ObjectId> = (0..=levels).map(|_| doc.new_object_id()).collect();
|
||||
for i in 0..levels {
|
||||
doc.set_object(
|
||||
ids[i],
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "Div",
|
||||
"K" => vec![Object::Reference(ids[i + 1]), Object::Reference(ids[i + 1])],
|
||||
},
|
||||
);
|
||||
}
|
||||
doc.set_object(
|
||||
ids[levels],
|
||||
dictionary! { "Type" => "StructElem", "S" => "P" },
|
||||
);
|
||||
let root_id = doc.add_object(dictionary! {
|
||||
"Type" => "StructTreeRoot",
|
||||
"K" => vec![Object::Reference(ids[0])],
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"StructTreeRoot" => Object::Reference(root_id),
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let tree = StructTree::from_doc(&doc).expect("tree should parse");
|
||||
let n = count_nodes(&tree.children);
|
||||
assert!(
|
||||
n <= MAX_STRUCT_NODES,
|
||||
"node count {n} exceeded budget {MAX_STRUCT_NODES}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn struct_tree_wide_mcid_array_respects_budget() {
|
||||
// A single struct element with a `/K` array of bare MCIDs wider than the
|
||||
// budget must not allocate `content_refs` without bound — each array item
|
||||
// is charged, so materialized marked-content refs stay within the budget.
|
||||
let mut doc = Document::new();
|
||||
let elem = doc.new_object_id();
|
||||
let kids: Vec<Object> = (0..(MAX_STRUCT_NODES as i64 + 100))
|
||||
.map(Object::Integer)
|
||||
.collect();
|
||||
doc.set_object(
|
||||
elem,
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "P",
|
||||
"K" => kids,
|
||||
},
|
||||
);
|
||||
let doc = finalize_tagged_doc(doc, elem);
|
||||
|
||||
let tree = StructTree::from_doc(&doc).expect("tree should parse");
|
||||
assert!(
|
||||
tree.mcid_count() <= MAX_STRUCT_NODES,
|
||||
"content_refs unbounded: {} > {MAX_STRUCT_NODES}",
|
||||
tree.mcid_count()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_charge_flags_truncation_once_exhausted() {
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 1;
|
||||
assert!(walk.charge(), "should spend the last unit");
|
||||
assert!(!walk.truncated, "not truncated while budget remained");
|
||||
assert!(!walk.charge(), "budget exhausted");
|
||||
assert!(walk.truncated, "exhaustion must set the truncation flag");
|
||||
// Stays exhausted/flagged on subsequent calls.
|
||||
assert!(!walk.charge());
|
||||
assert!(walk.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_flags_truncation_after_budget_spent_by_charge() {
|
||||
// The dominant truncation path: the budget is driven to 0 by a
|
||||
// successful `charge()` (which does not set the flag), and remaining
|
||||
// items are then dropped by an `exhausted()` guard — which must flag it.
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 1;
|
||||
assert!(walk.charge());
|
||||
assert!(
|
||||
!walk.truncated,
|
||||
"spending the last unit is not truncation yet"
|
||||
);
|
||||
assert!(walk.exhausted(), "budget is now spent");
|
||||
assert!(
|
||||
walk.truncated,
|
||||
"the guard that skips work must flag truncation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wide_kids_array_flags_truncation_via_parser() {
|
||||
// Reproduce the reviewer's scenario through the real parser: a `/K`
|
||||
// array wider than the budget drives the budget to 0 via `charge()`,
|
||||
// then the loop guard drops the rest — the truncation flag must be set
|
||||
// (so `from_doc` logs it) rather than staying silently false.
|
||||
let mut doc = Document::new();
|
||||
let elem = doc.new_object_id();
|
||||
let kids: Vec<Object> = (0..20i64).map(Object::Integer).collect();
|
||||
doc.set_object(
|
||||
elem,
|
||||
dictionary! { "Type" => "StructElem", "S" => "P", "K" => kids },
|
||||
);
|
||||
let dict = doc.get_dictionary(elem).unwrap().clone();
|
||||
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 5; // smaller than the 20-item `/K` array
|
||||
let role_map = HashMap::new();
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(
|
||||
walk.truncated,
|
||||
"a `/K` array wider than the budget must flag truncation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cycle_skip_flags_truncation() {
|
||||
// A `/K` reference cycle is dropped rather than expanded; that skip must
|
||||
// still flag truncation so the one-shot warning fires for malformed
|
||||
// trees, not only for budget exhaustion.
|
||||
let mut doc = Document::new();
|
||||
let elem = doc.new_object_id();
|
||||
doc.set_object(
|
||||
elem,
|
||||
dictionary! {
|
||||
"Type" => "StructElem",
|
||||
"S" => "Div",
|
||||
"K" => vec![Object::Reference(elem), Object::Reference(elem)],
|
||||
},
|
||||
);
|
||||
let dict = doc.get_dictionary(elem).unwrap().clone();
|
||||
|
||||
let mut walk = StructWalk::new();
|
||||
walk.active.insert(elem); // simulate `elem` already on the DFS path
|
||||
let role_map = HashMap::new();
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(
|
||||
walk.truncated,
|
||||
"a cycle-skipped `/K` child must flag truncation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_mcid_charges_node_and_reference() {
|
||||
// A bare MCID `/K` child becomes a wrapper node carrying one content
|
||||
// reference — two materialized items — so it must charge two budget
|
||||
// units, not one.
|
||||
let doc = Document::new();
|
||||
let obj = Object::Integer(7);
|
||||
let role_map = HashMap::new();
|
||||
let mut out = Vec::new();
|
||||
let mut walk = StructWalk::new();
|
||||
let before = walk.budget;
|
||||
parse_kid(&doc, &obj, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert_eq!(
|
||||
out.len(),
|
||||
1,
|
||||
"bare MCID should materialize one wrapper node"
|
||||
);
|
||||
assert_eq!(
|
||||
before - walk.budget,
|
||||
2,
|
||||
"bare MCID must charge for both the node and its content reference"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mcr_dict_charges_node_and_reference() {
|
||||
// A top-level MCR `/K` dict materializes the same wrapper node + content
|
||||
// reference as a bare MCID, so it must charge the same two budget units
|
||||
// (not one), keeping the per-item budgeting uniform.
|
||||
let doc = Document::new();
|
||||
let obj = Object::Dictionary(dictionary! { "Type" => "MCR", "MCID" => 3 });
|
||||
let role_map = HashMap::new();
|
||||
let mut out = Vec::new();
|
||||
let mut walk = StructWalk::new();
|
||||
let before = walk.budget;
|
||||
parse_kid(&doc, &obj, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert_eq!(out.len(), 1, "MCR dict should materialize one wrapper node");
|
||||
assert_eq!(
|
||||
before - walk.budget,
|
||||
2,
|
||||
"MCR dict must charge for both the node and its content reference"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaf_wrappers_reserve_both_units_atomically() {
|
||||
// With only one unit left, a two-item leaf wrapper (bare MCID or MCR
|
||||
// dict) must consume nothing and flag truncation, leaving the unit for a
|
||||
// later single-item element instead of half-charging.
|
||||
let doc = Document::new();
|
||||
let role_map = HashMap::new();
|
||||
|
||||
// Bare MCID via parse_kid.
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 1;
|
||||
let mut out = Vec::new();
|
||||
parse_kid(
|
||||
&doc,
|
||||
&Object::Integer(5),
|
||||
&role_map,
|
||||
None,
|
||||
0,
|
||||
&mut out,
|
||||
&mut walk,
|
||||
);
|
||||
assert!(out.is_empty(), "bare MCID must not partially materialize");
|
||||
assert_eq!(walk.budget, 1, "the leftover unit must be preserved");
|
||||
assert!(walk.truncated);
|
||||
|
||||
// MCR dict via parse_struct_element_dict.
|
||||
let mcr = dictionary! { "Type" => "MCR", "MCID" => 1 };
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 1;
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &mcr, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(out.is_empty(), "MCR dict must not partially materialize");
|
||||
assert_eq!(walk.budget, 1, "the leftover unit must be preserved");
|
||||
assert!(walk.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insufficient_reservation_stops_the_scan() {
|
||||
// A one-unit budget is not "exhausted" for a one-unit item, but once a
|
||||
// two-unit leaf reservation fails, the walk is stalled so enclosing `/K`
|
||||
// loops stop instead of scanning the rest of a wide array.
|
||||
let mut walk = StructWalk::new();
|
||||
walk.budget = 1;
|
||||
assert!(
|
||||
!walk.exhausted(),
|
||||
"one unit left must still allow a one-unit item"
|
||||
);
|
||||
assert!(!walk.charge_n(2), "cannot reserve two units from one");
|
||||
assert!(
|
||||
walk.exhausted(),
|
||||
"an insufficient reservation must stop the loop"
|
||||
);
|
||||
assert!(walk.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn work_budget_bounds_examined_items() {
|
||||
let mut walk = StructWalk::new();
|
||||
walk.work = 2;
|
||||
assert!(walk.spend_work());
|
||||
assert!(walk.spend_work());
|
||||
assert!(!walk.spend_work(), "traversal budget exhausted");
|
||||
assert!(walk.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wide_unsupported_kids_stop_at_work_budget() {
|
||||
// A wide `/K` array of unsupported values (nulls) materializes nothing;
|
||||
// it must stop at the traversal budget instead of scanning every entry.
|
||||
let mut doc = Document::new();
|
||||
let elem = doc.new_object_id();
|
||||
let kids: Vec<Object> = (0..1000).map(|_| Object::Null).collect();
|
||||
doc.set_object(
|
||||
elem,
|
||||
dictionary! { "Type" => "StructElem", "S" => "P", "K" => kids },
|
||||
);
|
||||
let dict = doc.get_dictionary(elem).unwrap().clone();
|
||||
|
||||
let mut walk = StructWalk::new();
|
||||
walk.work = 10; // far smaller than the 1000-entry array
|
||||
let role_map = HashMap::new();
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &dict, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(
|
||||
walk.truncated,
|
||||
"a wide unsupported `/K` array must hit the work budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_materializing_dicts_do_not_charge_node_budget() {
|
||||
let doc = Document::new();
|
||||
let role_map = HashMap::new();
|
||||
|
||||
// OBJR dict: materializes no node, so it must not spend the node budget.
|
||||
let objr = dictionary! { "Type" => "OBJR" };
|
||||
let mut walk = StructWalk::new();
|
||||
let before = walk.budget;
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &objr, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(walk.budget, before, "OBJR must not spend the node budget");
|
||||
|
||||
// A struct dict without a valid /S also materializes nothing.
|
||||
let no_s = dictionary! { "Type" => "StructElem" };
|
||||
let mut walk = StructWalk::new();
|
||||
let before = walk.budget;
|
||||
let mut out = Vec::new();
|
||||
parse_struct_element_dict(&doc, &no_s, &role_map, None, 0, &mut out, &mut walk);
|
||||
assert!(out.is_empty());
|
||||
assert_eq!(
|
||||
walk.budget, before,
|
||||
"a dict without /S must not spend the node budget"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "0.1.7"
|
||||
version = "0.1.8"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "0.1.3"
|
||||
version = "0.1.4"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "0.1.3"
|
||||
version = "0.1.4"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Browser WebAssembly bindings for pdf-inspector"
|
||||
|
||||
Reference in New Issue
Block a user