Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
92e2d6ec8e | ||
|
|
74a633dfd0 | ||
|
|
c1957bf750 | ||
|
|
f4b8c9e854 | ||
|
|
69039f2728 | ||
|
|
3cca6446bd | ||
|
|
493fed498e | ||
|
|
436af97038 | ||
|
|
f731e1191c |
@@ -37,6 +37,7 @@ scripts/
|
||||
|
||||
# Test output
|
||||
test_output/
|
||||
.firecrawl/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
|
||||
+4
-3
@@ -5,14 +5,15 @@
|
||||
If you believe you've found a security vulnerability in pdf-inspector, please
|
||||
report it privately so we can fix it before public disclosure.
|
||||
|
||||
**Preferred:** Email **help@firecrawl.dev** with:
|
||||
**Preferred:** Submit through Firecrawl's Bugcrowd vulnerability disclosure
|
||||
program at <https://bugcrowd.com/engagements/firecrawl-vdp-ess>. Please include:
|
||||
|
||||
- A description of the issue and its impact
|
||||
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
|
||||
- The version or commit hash of pdf-inspector you tested against
|
||||
|
||||
**Alternative:** Use GitHub's private vulnerability reporting under the
|
||||
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
|
||||
**Alternative:** If you'd rather not use Bugcrowd, email
|
||||
**help@firecrawl.dev** with the same details.
|
||||
|
||||
We'll acknowledge your report in a timely manner and keep you updated on
|
||||
remediation progress. Please do not open a public GitHub issue for security
|
||||
|
||||
@@ -83,6 +83,22 @@ for (const region of result[0].regions) {
|
||||
}
|
||||
```
|
||||
|
||||
### Async variants
|
||||
|
||||
`processPdf`, `classifyPdf`, and `extractPagesMarkdown` are synchronous and parse on the calling thread — in Node, that's the event loop. For a one-off call in a script that's fine, but in a server a large document can hold the loop for tens to hundreds of milliseconds.
|
||||
|
||||
`processPdfAsync`, `classifyPdfAsync`, and `extractPagesMarkdownAsync` take the same arguments and produce the same results, but run the parse on the libuv thread pool and return a promise, keeping the event loop free. The input buffer is copied before the call returns, so it's safe to reuse or mutate immediately:
|
||||
|
||||
```typescript
|
||||
import { classifyPdfAsync, extractPagesMarkdownAsync } from '@firecrawl/pdf-inspector'
|
||||
|
||||
const classification = await classifyPdfAsync(pdf)
|
||||
if (classification.pdfType === 'TextBased') {
|
||||
const { pages } = await extractPagesMarkdownAsync(pdf)
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## Types
|
||||
|
||||
```typescript
|
||||
|
||||
+182
-41
@@ -153,9 +153,7 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_napi_page_ocr_reasons(
|
||||
reasons: Vec<pdf_inspector::PageOcrReasons>,
|
||||
) -> Vec<PageOcrReasons> {
|
||||
fn to_napi_page_ocr_reasons(reasons: Vec<pdf_inspector::PageOcrReasons>) -> Vec<PageOcrReasons> {
|
||||
reasons
|
||||
.into_iter()
|
||||
.map(|reason| PageOcrReasons {
|
||||
@@ -202,6 +200,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared implementations (single body behind sync and async entry points)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn process_pdf_impl(bytes: &[u8], pages: Option<Vec<u32>>) -> Result<PdfResult> {
|
||||
let mut opts = pdf_inspector::PdfOptions::new();
|
||||
if let Some(p) = pages {
|
||||
opts = opts.pages(p);
|
||||
}
|
||||
let result = pdf_inspector::process_pdf_mem_with_options(bytes, opts)
|
||||
.map_err(|e| to_napi_err(e, "process_pdf"))?;
|
||||
Ok(to_napi_result(result))
|
||||
}
|
||||
|
||||
fn classify_pdf_impl(bytes: &[u8]) -> Result<PdfClassification> {
|
||||
let result =
|
||||
pdf_inspector::classify_pdf_mem(bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: convert_pdf_type(result.pdf_type),
|
||||
page_count: result.page_count,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
confidence: result.confidence as f64,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public NAPI API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -210,15 +233,7 @@ where
|
||||
#[napi]
|
||||
pub fn process_pdf(buffer: Buffer, pages: Option<Vec<u32>>) -> Result<PdfResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("process_pdf", move || {
|
||||
let mut opts = pdf_inspector::PdfOptions::new();
|
||||
if let Some(p) = pages {
|
||||
opts = opts.pages(p);
|
||||
}
|
||||
let result = pdf_inspector::process_pdf_mem_with_options(&bytes, opts)
|
||||
.map_err(|e| to_napi_err(e, "process_pdf"))?;
|
||||
Ok(to_napi_result(result))
|
||||
})
|
||||
catch_panic("process_pdf", move || process_pdf_impl(&bytes, pages))
|
||||
}
|
||||
|
||||
/// Fast detection only — no text extraction or markdown.
|
||||
@@ -238,16 +253,7 @@ pub fn detect_pdf(buffer: Buffer) -> Result<PdfResult> {
|
||||
#[napi]
|
||||
pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("classify_pdf", move || {
|
||||
let result =
|
||||
pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: convert_pdf_type(result.pdf_type),
|
||||
page_count: result.page_count,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
confidence: result.confidence as f64,
|
||||
})
|
||||
})
|
||||
catch_panic("classify_pdf", move || classify_pdf_impl(&bytes))
|
||||
}
|
||||
|
||||
/// Extract plain text from a PDF Buffer.
|
||||
@@ -633,25 +639,32 @@ pub fn extract_pages_markdown(
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|r| PageMarkdownResult {
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
extract_pages_markdown_impl(&bytes, pages.as_deref())
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_pages_markdown_impl(
|
||||
bytes: &[u8],
|
||||
pages: Option<&[u32]>,
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(bytes, pages)
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|r| PageMarkdownResult {
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
ocr_reason: r.ocr_reason,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -692,3 +705,131 @@ fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<Pa
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Async variants (libuv thread pool via AsyncTask)
|
||||
//
|
||||
// The synchronous exports above parse on the calling thread, which in Node is
|
||||
// the event loop. These `*Async` variants run the same shared implementations
|
||||
// on the libuv thread pool and hand JavaScript a promise, so servers under
|
||||
// concurrent load keep answering requests while a document parses. The sync
|
||||
// exports keep their names, signatures, and behaviour.
|
||||
//
|
||||
// Each factory copies the input Buffer to an owned `Vec<u8>` on the calling
|
||||
// (JS) thread — deliberately. JS execution is single-threaded, so no JS code
|
||||
// can mutate the buffer while the synchronous part of the call copies it.
|
||||
// Holding the napi `Buffer` and reading it from the worker instead would be
|
||||
// zero-copy, but a caller mutating the buffer before the promise settles
|
||||
// would then race the worker's reads — undefined behavior, not a recoverable
|
||||
// error (a known napi-rs soundness hazard with cross-thread Buffer access).
|
||||
// The copy is a one-time memcpy, negligible next to the parse it unblocks.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct ProcessPdfTask {
|
||||
bytes: Vec<u8>,
|
||||
pages: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
impl Task for ProcessPdfTask {
|
||||
type Output = PdfResult;
|
||||
type JsValue = PdfResult;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let pages = self.pages.take();
|
||||
// AssertUnwindSafe: `bytes`/`pages` are moved into the closure and
|
||||
// dropped on unwind — no shared state can be observed broken.
|
||||
catch_panic(
|
||||
"process_pdf",
|
||||
panic::AssertUnwindSafe(move || process_pdf_impl(&bytes, pages)),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`processPdf`]: same result, but the parse runs on the
|
||||
/// libuv thread pool instead of the event loop and the call returns a
|
||||
/// promise. The buffer is copied before the call returns, so it may be
|
||||
/// reused or mutated immediately.
|
||||
// ts_return_type is required: napi-rs emits `Promise<unknown>` for
|
||||
// `AsyncTask<T>` returns without it.
|
||||
#[napi(ts_return_type = "Promise<PdfResult>")]
|
||||
pub fn process_pdf_async(buffer: Buffer, pages: Option<Vec<u32>>) -> AsyncTask<ProcessPdfTask> {
|
||||
AsyncTask::new(ProcessPdfTask {
|
||||
bytes: buffer.to_vec(),
|
||||
pages,
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ClassifyPdfTask {
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Task for ClassifyPdfTask {
|
||||
type Output = PdfClassification;
|
||||
type JsValue = PdfClassification;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
catch_panic(
|
||||
"classify_pdf",
|
||||
panic::AssertUnwindSafe(move || classify_pdf_impl(&bytes)),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`classifyPdf`]: same result, but the classification runs
|
||||
/// on the libuv thread pool instead of the event loop and the call returns a
|
||||
/// promise. The buffer is copied before the call returns, so it may be
|
||||
/// reused or mutated immediately.
|
||||
#[napi(ts_return_type = "Promise<PdfClassification>")]
|
||||
pub fn classify_pdf_async(buffer: Buffer) -> AsyncTask<ClassifyPdfTask> {
|
||||
AsyncTask::new(ClassifyPdfTask {
|
||||
bytes: buffer.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ExtractPagesMarkdownTask {
|
||||
bytes: Vec<u8>,
|
||||
pages: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
impl Task for ExtractPagesMarkdownTask {
|
||||
type Output = PagesExtractionResult;
|
||||
type JsValue = PagesExtractionResult;
|
||||
|
||||
fn compute(&mut self) -> Result<Self::Output> {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let pages = self.pages.take();
|
||||
catch_panic(
|
||||
"extract_pages_markdown",
|
||||
panic::AssertUnwindSafe(move || extract_pages_markdown_impl(&bytes, pages.as_deref())),
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
/// Async variant of [`extractPagesMarkdown`]: same result, but the extraction
|
||||
/// runs on the libuv thread pool instead of the event loop and the call
|
||||
/// returns a promise. The buffer is copied before the call returns, so it
|
||||
/// may be reused or mutated immediately.
|
||||
#[napi(ts_return_type = "Promise<PagesExtractionResult>")]
|
||||
pub fn extract_pages_markdown_async(
|
||||
buffer: Buffer,
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> AsyncTask<ExtractPagesMarkdownTask> {
|
||||
AsyncTask::new(ExtractPagesMarkdownTask {
|
||||
bytes: buffer.to_vec(),
|
||||
pages,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,13 +2,16 @@ import { readFileSync } from 'fs';
|
||||
import { strict as assert } from 'assert';
|
||||
import {
|
||||
processPdf,
|
||||
processPdfAsync,
|
||||
detectPdf,
|
||||
classifyPdf,
|
||||
classifyPdfAsync,
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
detectVectorGridInRegion,
|
||||
extractPagesMarkdown,
|
||||
extractPagesMarkdownAsync,
|
||||
} from './index.js';
|
||||
|
||||
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
|
||||
@@ -124,10 +127,75 @@ assert.equal(picked.pages[0].page, 2);
|
||||
assert.equal(picked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdown with pages: OK');
|
||||
|
||||
// --- Async variants ---
|
||||
console.log('Testing async variants...');
|
||||
|
||||
// processPdfAsync returns a promise and matches the sync result
|
||||
const asyncResultPromise = processPdfAsync(fixture);
|
||||
assert.ok(asyncResultPromise instanceof Promise);
|
||||
const asyncResult = await asyncResultPromise;
|
||||
assert.equal(asyncResult.pdfType, result.pdfType);
|
||||
assert.equal(asyncResult.pageCount, result.pageCount);
|
||||
assert.equal(asyncResult.markdown, result.markdown);
|
||||
console.log(' processPdfAsync: OK');
|
||||
|
||||
// processPdfAsync with pages
|
||||
const asyncResult2 = await processPdfAsync(fixture, [1]);
|
||||
assert.equal(asyncResult2.markdown, result2.markdown);
|
||||
console.log(' processPdfAsync with pages: OK');
|
||||
|
||||
// classifyPdfAsync matches the sync result
|
||||
const asyncClassified = await classifyPdfAsync(fixture);
|
||||
assert.equal(asyncClassified.pdfType, classified.pdfType);
|
||||
assert.equal(asyncClassified.pageCount, classified.pageCount);
|
||||
assert.equal(asyncClassified.confidence, classified.confidence);
|
||||
assert.deepEqual(asyncClassified.pagesNeedingOcr, classified.pagesNeedingOcr);
|
||||
console.log(' classifyPdfAsync: OK');
|
||||
|
||||
// extractPagesMarkdownAsync matches the sync result
|
||||
const asyncAllPages = await extractPagesMarkdownAsync(fixture);
|
||||
assert.equal(asyncAllPages.pages.length, allPages.pages.length);
|
||||
assert.deepEqual(
|
||||
asyncAllPages.pages.map(p => p.markdown),
|
||||
allPages.pages.map(p => p.markdown),
|
||||
);
|
||||
assert.equal(asyncAllPages.isComplex, allPages.isComplex);
|
||||
console.log(' extractPagesMarkdownAsync: OK');
|
||||
|
||||
// selected pages preserve caller order
|
||||
const asyncPicked = await extractPagesMarkdownAsync(fixture, [2, 0]);
|
||||
assert.equal(asyncPicked.pages.length, 2);
|
||||
assert.equal(asyncPicked.pages[0].page, 2);
|
||||
assert.equal(asyncPicked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdownAsync with pages: OK');
|
||||
|
||||
// input buffer is copied at call time: mutating it immediately after the
|
||||
// call must not affect the in-flight parse
|
||||
const scratch = Buffer.from(fixture);
|
||||
const inFlight = processPdfAsync(scratch);
|
||||
scratch.fill(0);
|
||||
const fromMutated = await inFlight;
|
||||
assert.equal(fromMutated.markdown, result.markdown);
|
||||
console.log(' processPdfAsync input copied at call time: OK');
|
||||
|
||||
// concurrent async calls all settle
|
||||
const [c1, c2, c3] = await Promise.all([
|
||||
processPdfAsync(fixture),
|
||||
classifyPdfAsync(fixture),
|
||||
extractPagesMarkdownAsync(fixture),
|
||||
]);
|
||||
assert.equal(c1.pdfType, 'TextBased');
|
||||
assert.equal(c2.pdfType, 'TextBased');
|
||||
assert.equal(c3.pages.length, 3);
|
||||
console.log(' concurrent async calls: OK');
|
||||
|
||||
// --- Error handling ---
|
||||
console.log('Testing error handling...');
|
||||
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
|
||||
assert.throws(() => classifyPdf(Buffer.from('')), /classify_pdf/);
|
||||
await assert.rejects(processPdfAsync(Buffer.from('not a pdf')), /process_pdf/);
|
||||
await assert.rejects(classifyPdfAsync(Buffer.from('')), /classify_pdf/);
|
||||
await assert.rejects(extractPagesMarkdownAsync(Buffer.from('')), /extract_pages_markdown/);
|
||||
console.log(' error handling: OK');
|
||||
|
||||
console.log('\nAll NAPI tests passed!');
|
||||
|
||||
+7
-7
@@ -858,22 +858,22 @@
|
||||
<p>Evaluated on the <a class="text-link" href="https://github.com/opendataloader-project/opendataloader-bench">opendataloader-bench</a> corpus of 200 PDFs. This comparison covers local engines without model-based PDF parsing, with OCR disabled. Higher scores are better.</p>
|
||||
</div>
|
||||
<div class="benchmark-card">
|
||||
<div class="benchmark-top"><span><strong>200 PDFs</strong> · OpenDataLoader benchmark</span><span>Apple M4 Pro · median of 3 runs</span></div>
|
||||
<div class="benchmark-top"><span><strong>200 PDFs</strong> · OpenDataLoader benchmark</span><span>Apple M4 Pro · median of 5 runs</span></div>
|
||||
<div class="table-scroll">
|
||||
<table aria-label="PDF extraction benchmark results">
|
||||
<thead>
|
||||
<tr><th>Engine</th><th>Overall</th><th>Reading order</th><th>Tables</th><th>Headings</th><th>Complete run</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="highlight"><td>pdf-inspector</td><td>0.875</td><td>0.915</td><td>0.814</td><td>0.788</td><td>2.8s</td></tr>
|
||||
<tr><td>LiteParse</td><td>0.870</td><td>0.908</td><td>0.693</td><td>0.811</td><td>13.9s</td></tr>
|
||||
<tr><td>OpenDataLoader</td><td>0.843</td><td>0.912</td><td>0.489</td><td>0.760</td><td>9.8s</td></tr>
|
||||
<tr><td>PyMuPDF4LLM</td><td>0.735</td><td>0.886</td><td>0.401</td><td>0.424</td><td>15.5s</td></tr>
|
||||
<tr><td>MarkItDown</td><td>0.583</td><td>0.879</td><td>0.000</td><td>0.000</td><td>6.7s</td></tr>
|
||||
<tr class="highlight"><td>pdf-inspector</td><td>0.875</td><td>0.915</td><td>0.814</td><td>0.788</td><td>0.470s</td></tr>
|
||||
<tr><td>LiteParse</td><td>0.873</td><td>0.913</td><td>0.693</td><td>0.811</td><td>0.750s</td></tr>
|
||||
<tr><td>OpenDataLoader</td><td>0.831</td><td>0.902</td><td>0.489</td><td>0.739</td><td>2.569s</td></tr>
|
||||
<tr><td>PyMuPDF4LLM</td><td>0.735</td><td>0.886</td><td>0.401</td><td>0.424</td><td>17.117s</td></tr>
|
||||
<tr><td>MarkItDown</td><td>0.589</td><td>0.844</td><td>0.273</td><td>0.000</td><td>16.165s</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="benchmark-note">Refreshed July 16, 2026. Scores use the benchmark’s NID, TEDS, and MHS evaluators.</div>
|
||||
<div class="benchmark-note">Refreshed July 31, 2026. Scores use the benchmark’s NID, TEDS, and MHS evaluators; speed is the median of five alternating or rotating complete corpus runs after an excluded warm-up. <a class="text-link" href="https://github.com/firecrawl/opendataloader-bench/tree/abi/pdf-parser-benchmark-results">Versions and raw artifacts</a>.</div>
|
||||
</div>
|
||||
<div class="best-fit">
|
||||
<strong>Best fit</strong>
|
||||
|
||||
+311
-5
@@ -2,11 +2,51 @@
|
||||
|
||||
use crate::types::{ItemType, TextItem};
|
||||
use lopdf::{Document, Object, ObjectId};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use super::fonts::{resolve_array, resolve_dict};
|
||||
use super::get_number;
|
||||
|
||||
/// Upper bound on the number of form-field nodes visited during a single
|
||||
/// `extract_form_fields` pass. A crafted PDF can chain thousands of distinct
|
||||
/// `/Kids` fields to blow the stack even without an outright reference cycle,
|
||||
/// so we cap total traversal work in addition to detecting cycles.
|
||||
const MAX_FORM_FIELD_NODES: usize = 100_000;
|
||||
|
||||
/// Upper bound on `/Kids` recursion depth. Real AcroForm hierarchies are only
|
||||
/// a few levels deep (fields → child fields → widgets); a crafted PDF can chain
|
||||
/// tens of thousands of distinct fields into a linear `/Kids` list that would
|
||||
/// overflow the stack via depth-first recursion long before the node budget is
|
||||
/// reached. This depth cap bounds the stack independently of total node count.
|
||||
const MAX_FORM_FIELD_DEPTH: usize = 100;
|
||||
|
||||
/// Traversal budget for the AcroForm field walk. Bounds both the number of
|
||||
/// distinct nodes visited *and* the total number of `/Fields`/`/Kids` entries
|
||||
/// examined.
|
||||
///
|
||||
/// Counting `visited` alone is not enough: invalid entries (non-references) and
|
||||
/// duplicate references never grow `visited`, so an oversized array full of them
|
||||
/// would iterate to completion no matter how large. Charging every examined
|
||||
/// entry against the same budget makes it a real cap on traversal work.
|
||||
pub(crate) struct FieldWalkBudget {
|
||||
visited: HashSet<ObjectId>,
|
||||
examined: usize,
|
||||
}
|
||||
|
||||
impl FieldWalkBudget {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
visited: HashSet::new(),
|
||||
examined: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// True once the budget is spent; callers must stop iterating and recursing.
|
||||
fn exhausted(&self) -> bool {
|
||||
self.visited.len() >= MAX_FORM_FIELD_NODES || self.examined >= MAX_FORM_FIELD_NODES
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> Vec<TextItem> {
|
||||
let mut links = Vec::new();
|
||||
|
||||
@@ -146,9 +186,12 @@ pub(crate) fn extract_form_fields(
|
||||
Err(_) => return items,
|
||||
};
|
||||
|
||||
// Borrow the array rather than cloning it: a crafted `/Fields` can be huge,
|
||||
// and cloning would pay an O(n) allocation/copy before the budget check
|
||||
// below can stop the work.
|
||||
let fields = match acroform.get(b"Fields") {
|
||||
Ok(obj) => match resolve_array(doc, obj) {
|
||||
Some(arr) => arr.clone(),
|
||||
Some(arr) => arr,
|
||||
None => return items,
|
||||
},
|
||||
Err(_) => return items,
|
||||
@@ -158,7 +201,19 @@ pub(crate) fn extract_form_fields(
|
||||
}
|
||||
let annotation_pages = annotation_page_map(doc, page_map);
|
||||
|
||||
for field_obj in &fields {
|
||||
// Bound the walk so a crafted PDF cannot send us into unbounded recursion
|
||||
// via a `/Kids` cycle, a deep chain, or an oversized array of invalid or
|
||||
// duplicate entries.
|
||||
let mut budget = FieldWalkBudget::new();
|
||||
|
||||
for field_obj in fields {
|
||||
// Stop once the budget is spent so a `/Fields` array wider than the
|
||||
// budget can't burn CPU iterating entries whose walk would no-op. Charge
|
||||
// every entry (including invalid ones) against the budget.
|
||||
if budget.exhausted() {
|
||||
break;
|
||||
}
|
||||
budget.examined += 1;
|
||||
if let Ok(field_ref) = field_obj.as_reference() {
|
||||
walk_form_fields(
|
||||
doc,
|
||||
@@ -168,6 +223,8 @@ pub(crate) fn extract_form_fields(
|
||||
page_map,
|
||||
&annotation_pages,
|
||||
&mut items,
|
||||
&mut budget,
|
||||
0,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -202,6 +259,7 @@ fn annotation_page_map(
|
||||
}
|
||||
|
||||
/// Recursively walk the form field tree, extracting leaf field values.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn walk_form_fields(
|
||||
doc: &Document,
|
||||
field_id: ObjectId,
|
||||
@@ -210,7 +268,22 @@ pub(crate) fn walk_form_fields(
|
||||
page_map: &HashMap<ObjectId, u32>,
|
||||
annotation_pages: &HashMap<ObjectId, u32>,
|
||||
items: &mut Vec<TextItem>,
|
||||
budget: &mut FieldWalkBudget,
|
||||
depth: usize,
|
||||
) {
|
||||
// Guard against `/Kids` cycles and pathologically large field trees.
|
||||
// Exceeding the depth cap means the chain is too deep to be a legitimate
|
||||
// form (and would overflow the stack); an exhausted budget means the tree is
|
||||
// too large. Both checks run *before* inserting so the visited set can never
|
||||
// grow past the budget.
|
||||
if depth > MAX_FORM_FIELD_DEPTH || budget.exhausted() {
|
||||
return;
|
||||
}
|
||||
// Revisiting an object ID means we hit a `/Kids` cycle.
|
||||
if !budget.visited.insert(field_id) {
|
||||
return;
|
||||
}
|
||||
|
||||
let field_dict = match doc.get_dictionary(field_id) {
|
||||
Ok(d) => d,
|
||||
Err(_) => return,
|
||||
@@ -241,9 +314,19 @@ pub(crate) fn walk_form_fields(
|
||||
|
||||
// Check for /Kids — if present, recurse into children
|
||||
if let Ok(kids_obj) = field_dict.get(b"Kids") {
|
||||
// Iterate the borrowed array directly — cloning a crafted, oversized
|
||||
// `/Kids` would allocate and copy every entry before the budget check
|
||||
// below could stop the work.
|
||||
if let Some(kids) = resolve_array(doc, kids_obj) {
|
||||
let kids = kids.clone();
|
||||
for kid in &kids {
|
||||
for kid in kids {
|
||||
// Stop once the budget is spent so a `/Kids` array wider than the
|
||||
// budget can't burn CPU iterating entries whose walk would no-op.
|
||||
// Charge every entry (including invalid/duplicate ones) against
|
||||
// the budget so this is a true traversal-work cap.
|
||||
if budget.exhausted() {
|
||||
break;
|
||||
}
|
||||
budget.examined += 1;
|
||||
if let Ok(kid_ref) = kid.as_reference() {
|
||||
walk_form_fields(
|
||||
doc,
|
||||
@@ -253,6 +336,8 @@ pub(crate) fn walk_form_fields(
|
||||
page_map,
|
||||
annotation_pages,
|
||||
items,
|
||||
budget,
|
||||
depth + 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -411,4 +496,225 @@ mod tests {
|
||||
assert_eq!(items[0].page, 2);
|
||||
assert_eq!(items[0].text, "customer: Alice");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kids_self_cycle_does_not_overflow_stack() {
|
||||
// A crafted AcroForm field that lists itself in `/Kids` must not send
|
||||
// the traversal into unbounded recursion.
|
||||
let mut doc = Document::new();
|
||||
let field_id = doc.new_object_id();
|
||||
doc.set_object(
|
||||
field_id,
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"T" => Object::string_literal("loop"),
|
||||
"Kids" => vec![Object::Reference(field_id)],
|
||||
},
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => vec![Object::Reference(field_id)],
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
// Completes (rather than overflowing the stack) and yields no items.
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
assert!(items.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kids_mutual_cycle_terminates() {
|
||||
// Two fields that reference each other via `/Kids` form a cycle that
|
||||
// must also terminate.
|
||||
let mut doc = Document::new();
|
||||
let field_a = doc.new_object_id();
|
||||
let field_b = doc.new_object_id();
|
||||
doc.set_object(
|
||||
field_a,
|
||||
dictionary! {
|
||||
"T" => Object::string_literal("a"),
|
||||
"Kids" => vec![Object::Reference(field_b)],
|
||||
},
|
||||
);
|
||||
doc.set_object(
|
||||
field_b,
|
||||
dictionary! {
|
||||
"T" => Object::string_literal("b"),
|
||||
"Kids" => vec![Object::Reference(field_a)],
|
||||
},
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => vec![Object::Reference(field_a)],
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
assert!(items.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_acyclic_kids_chain_does_not_overflow_stack() {
|
||||
// A long chain of *distinct* fields (no cycle) must also terminate:
|
||||
// the visited set alone would still recurse to the chain length, so
|
||||
// the depth cap is what prevents a stack overflow here.
|
||||
let mut doc = Document::new();
|
||||
let n = MAX_FORM_FIELD_DEPTH * 500;
|
||||
let ids: Vec<ObjectId> = (0..=n).map(|_| doc.new_object_id()).collect();
|
||||
for i in 0..n {
|
||||
doc.set_object(
|
||||
ids[i],
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"Kids" => vec![Object::Reference(ids[i + 1])],
|
||||
},
|
||||
);
|
||||
}
|
||||
// Leaf carries a value; it sits far below the depth cap so it is never
|
||||
// reached, proving traversal stops early rather than crashing.
|
||||
doc.set_object(
|
||||
ids[n],
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"T" => Object::string_literal("leaf"),
|
||||
"V" => Object::string_literal("x"),
|
||||
"Rect" => vec![10.into(), 20.into(), 110.into(), 40.into()],
|
||||
},
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => vec![Object::Reference(ids[0])],
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
assert!(items.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wide_tree_traversal_stops_at_node_budget() {
|
||||
// A single field with a `/Kids` array wider than the node budget must
|
||||
// stop traversal at the cap rather than growing `visited` (and the work)
|
||||
// without bound. Each processed leaf emits one item, so the item count
|
||||
// is bounded by the budget and reaches right up to it (a couple of
|
||||
// slots go to the root and the boundary node charged against the cap).
|
||||
let mut doc = Document::new();
|
||||
let fanout = MAX_FORM_FIELD_NODES + 50;
|
||||
let leaf_ids: Vec<ObjectId> = (0..fanout).map(|_| doc.new_object_id()).collect();
|
||||
for &leaf in &leaf_ids {
|
||||
doc.set_object(
|
||||
leaf,
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"V" => Object::string_literal("v"),
|
||||
"Rect" => vec![10.into(), 20.into(), 110.into(), 40.into()],
|
||||
},
|
||||
);
|
||||
}
|
||||
let kids: Vec<Object> = leaf_ids.iter().map(|&id| Object::Reference(id)).collect();
|
||||
let root_id = doc.add_object(dictionary! {
|
||||
"T" => Object::string_literal("root"),
|
||||
"Kids" => kids,
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => vec![Object::Reference(root_id)],
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
// Extraction stops at the budget: bounded above by the cap, and it gets
|
||||
// right up to it (allowing a small delta for the root/boundary nodes
|
||||
// charged against the budget).
|
||||
assert!(items.len() <= MAX_FORM_FIELD_NODES);
|
||||
assert!(items.len() >= MAX_FORM_FIELD_NODES - 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wide_top_level_fields_stop_at_node_budget() {
|
||||
// A top-level `/Fields` array wider than the budget must also stop at
|
||||
// the cap: the item count is bounded by the budget and reaches right up
|
||||
// to it.
|
||||
let mut doc = Document::new();
|
||||
let fanout = MAX_FORM_FIELD_NODES + 50;
|
||||
let leaf_ids: Vec<ObjectId> = (0..fanout).map(|_| doc.new_object_id()).collect();
|
||||
for &leaf in &leaf_ids {
|
||||
doc.set_object(
|
||||
leaf,
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"V" => Object::string_literal("v"),
|
||||
"Rect" => vec![10.into(), 20.into(), 110.into(), 40.into()],
|
||||
},
|
||||
);
|
||||
}
|
||||
let fields: Vec<Object> = leaf_ids.iter().map(|&id| Object::Reference(id)).collect();
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => fields,
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
assert!(items.len() <= MAX_FORM_FIELD_NODES);
|
||||
assert!(items.len() >= MAX_FORM_FIELD_NODES - 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_and_invalid_kids_entries_stop_at_budget() {
|
||||
// Duplicate references and non-reference junk never grow `visited`, so
|
||||
// without charging examined entries against the budget an oversized
|
||||
// array of them would iterate to completion. The walk must still
|
||||
// terminate and extract the single real leaf exactly once.
|
||||
let mut doc = Document::new();
|
||||
let leaf_id = doc.new_object_id();
|
||||
doc.set_object(
|
||||
leaf_id,
|
||||
dictionary! {
|
||||
"FT" => "Tx",
|
||||
"V" => Object::string_literal("v"),
|
||||
"Rect" => vec![10.into(), 20.into(), 110.into(), 40.into()],
|
||||
},
|
||||
);
|
||||
// A `/Kids` array far wider than the budget: half duplicate references
|
||||
// to the same leaf, half invalid (null) entries.
|
||||
let mut kids: Vec<Object> = Vec::new();
|
||||
for i in 0..(MAX_FORM_FIELD_NODES * 2) {
|
||||
if i % 2 == 0 {
|
||||
kids.push(Object::Reference(leaf_id));
|
||||
} else {
|
||||
kids.push(Object::Null);
|
||||
}
|
||||
}
|
||||
let root_id = doc.add_object(dictionary! {
|
||||
"T" => Object::string_literal("root"),
|
||||
"Kids" => kids,
|
||||
});
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"AcroForm" => dictionary! {
|
||||
"Fields" => vec![Object::Reference(root_id)],
|
||||
},
|
||||
});
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let page_map = HashMap::new();
|
||||
let items = extract_form_fields(&doc, &page_map);
|
||||
assert_eq!(items.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
+40
-3
@@ -4566,9 +4566,13 @@ pub fn glyph_to_char(name: &str) -> Option<char> {
|
||||
}
|
||||
}
|
||||
|
||||
// Try to parse uniXXXX format
|
||||
if name.starts_with("uni") && name.len() >= 7 {
|
||||
if let Ok(code) = u32::from_str_radix(&name[3..7], 16) {
|
||||
// Try to parse uniXXXX format.
|
||||
// Use `get` rather than a byte-length check + slice: `name` can contain
|
||||
// non-ASCII bytes (e.g. U+FFFD from lossy UTF-8 decoding of an attacker
|
||||
// controlled /Differences name), so byte index 7 may not be a char
|
||||
// boundary and `&name[3..7]` would panic.
|
||||
if let Some(hex) = name.strip_prefix("uni").and_then(|rest| rest.get(..4)) {
|
||||
if let Ok(code) = u32::from_str_radix(hex, 16) {
|
||||
// Strip PUA F000 offset: uniF0XX → U+00XX (Windows Symbol encoding convention)
|
||||
let code = if (0xF000..=0xF0FF).contains(&code) {
|
||||
code - 0xF000
|
||||
@@ -4588,3 +4592,36 @@ pub fn glyph_to_char(name: &str) -> Option<char> {
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn uni_hex_parsing() {
|
||||
assert_eq!(glyph_to_char("uni0041"), Some('A'));
|
||||
assert_eq!(glyph_to_char("uni00e9"), Some('\u{00e9}'));
|
||||
// PUA F0xx symbol-encoding offset is stripped.
|
||||
assert_eq!(glyph_to_char("uniF041"), Some('A'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn u_hex_parsing() {
|
||||
assert_eq!(glyph_to_char("u0041"), Some('A'));
|
||||
assert_eq!(glyph_to_char("u1F600"), Some('\u{1F600}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_ascii_uni_name_does_not_panic() {
|
||||
// A crafted /Differences name like `/uni#80#80#80#80` decodes via
|
||||
// from_utf8_lossy into "uni" followed by four U+FFFD replacements.
|
||||
// Byte index 7 lands mid-character, so a naive `&name[3..7]` slice
|
||||
// would panic. It must be handled gracefully instead.
|
||||
let crafted = format!("uni{0}{0}{0}{0}", '\u{FFFD}');
|
||||
assert_eq!(glyph_to_char(&crafted), None);
|
||||
|
||||
// Assorted non-ASCII bytes right after the "uni" prefix.
|
||||
assert_eq!(glyph_to_char("uni\u{FFFD}bc"), None);
|
||||
assert_eq!(glyph_to_char("uni\u{00e9}00"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +171,74 @@ pub(crate) fn is_toc_marker_heading(text: &str) -> bool {
|
||||
/// equation and absent from name-plus-number headings. A bare trailing colon
|
||||
/// is NOT a fragment signal either: real headings frequently end with colons
|
||||
/// ("Procedure:", "Steps for Using the Microscope:").
|
||||
/// True when the line opens with a section number ("3.", "2.1.4", "IV)").
|
||||
///
|
||||
/// Mirrors the acceptance of `heading::parse_numbering` rather than the
|
||||
/// stricter `convert::starts_with_section_number`, which deliberately
|
||||
/// requires two components because it bypasses isolation checks. Here a
|
||||
/// single "1." counts: numbering is independent evidence of a heading, and
|
||||
/// `heading.rs` applies its numbered-prefix allowance *after* consulting
|
||||
/// `is_heading_fragment`, so without this exemption a numbered
|
||||
/// sentence-case heading would be vetoed before that allowance can run.
|
||||
fn starts_with_numbering_prefix(t: &str) -> bool {
|
||||
let Some(first) = t.split_whitespace().next() else {
|
||||
return false;
|
||||
};
|
||||
let has_delimiter = first.ends_with(['.', ')', ':']);
|
||||
let token = first.trim_end_matches(['.', ')', ':']);
|
||||
if token.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let parts: Vec<&str> = token.split('.').collect();
|
||||
let decimal = parts
|
||||
.iter()
|
||||
.all(|p| !p.is_empty() && p.len() <= 3 && p.chars().all(|c| c.is_ascii_digit()));
|
||||
if decimal {
|
||||
// "1." / "2.1." carry a delimiter; "2.3 Title" is written without
|
||||
// one, so a multi-component number is accepted bare. A bare single
|
||||
// number ("3 apples") is not — that is ordinary prose.
|
||||
return has_delimiter || parts.len() >= 2;
|
||||
}
|
||||
// Roman numerals go through the heading parser's own grammar so the two
|
||||
// agree: uppercase I/V/X/L/C only, at most 8 characters. A looser rule
|
||||
// here would exempt markers the parser rejects — "iv)" or "d)" from an
|
||||
// alphabetical list — letting an ordinary list item bypass the veto and
|
||||
// reach heading promotion.
|
||||
//
|
||||
// A delimiter is also required: a bare leading "I" is the pronoun far
|
||||
// more often than a section number.
|
||||
has_delimiter && crate::markdown::heading::roman_value(token).is_some()
|
||||
}
|
||||
|
||||
/// True when the line reads as a title rather than a sentence: every
|
||||
/// content word (ignoring minor words) starts uppercase. Used to spare real
|
||||
/// headings from the dangling-verb veto — "Bond Yields" is a section title,
|
||||
/// "the method yields" is a stranded clause, and only the casing tells them
|
||||
/// apart.
|
||||
fn looks_title_case(t: &str) -> bool {
|
||||
const MINOR: &[&str] = &[
|
||||
"a", "an", "the", "of", "and", "or", "for", "to", "in", "on", "at", "by", "with", "from",
|
||||
"as", "is", "are", "that", "than", "into",
|
||||
];
|
||||
let mut content = 0usize;
|
||||
let mut capitalized = 0usize;
|
||||
for w in t.split_whitespace() {
|
||||
let cleaned: String = w.chars().filter(|c| c.is_alphabetic()).collect();
|
||||
if cleaned.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if MINOR.contains(&cleaned.to_lowercase().as_str()) {
|
||||
continue;
|
||||
}
|
||||
content += 1;
|
||||
if cleaned.chars().next().is_some_and(char::is_uppercase) {
|
||||
capitalized += 1;
|
||||
}
|
||||
}
|
||||
// A single content word ("Yields") is a title by default.
|
||||
content == 0 || capitalized == content
|
||||
}
|
||||
|
||||
pub(crate) fn is_heading_fragment(text: &str) -> bool {
|
||||
let t = text.trim_end();
|
||||
|
||||
@@ -244,9 +312,133 @@ pub(crate) fn is_heading_fragment(text: &str) -> bool {
|
||||
if t.ends_with(':') && t.split_whitespace().any(is_equation_number) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Dangling clause: a stranded sentence lead-in ends on a relational
|
||||
// verb with no terminal punctuation — "Note that the exact error equals"
|
||||
// left ahead of its formula when a phantom table dissolved.
|
||||
//
|
||||
// Gated on the line reading as prose rather than a title. Case is the
|
||||
// discriminator the trailing word alone cannot provide: a heading is
|
||||
// title case ("Bond Yields", "The Method Yields") while a stranded
|
||||
// lead-in is sentence case ("the method yields"). Without this gate the
|
||||
// veto eats real headings — "Bond Yields", "Crop Yields" and any wrapped
|
||||
// title-case heading the preprocessor failed to merge.
|
||||
if !t.ends_with(['.', '!', '?', ':', ';', ')', ']'])
|
||||
&& !looks_title_case(t)
|
||||
&& !starts_with_numbering_prefix(t)
|
||||
{
|
||||
if let Some(last) = t.split_whitespace().next_back() {
|
||||
let word: String = last
|
||||
.trim_matches(|c: char| !c.is_alphanumeric())
|
||||
.to_lowercase();
|
||||
// Relational verbs only, and only those with no common noun
|
||||
// sense. "yields" was dropped for exactly that reason: "Bond
|
||||
// Yields" is a real section title. Function words, copulas and
|
||||
// auxiliaries were measured and rejected outright — a heading
|
||||
// that wraps across lines ends on those, and suppressing them
|
||||
// destroyed real IRS Publication 17 headings.
|
||||
const DANGLING_TAIL: &[&str] =
|
||||
&["equals", "denotes", "implies", "satisfies", "signifies"];
|
||||
if DANGLING_TAIL.contains(&word.as_str()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod fragment_heading_tests {
|
||||
use super::is_heading_fragment;
|
||||
|
||||
#[test]
|
||||
fn dangling_tail_marks_stranded_clause() {
|
||||
// opendataloader 01030000000144: left behind when a phantom table
|
||||
// dissolved, ahead of its formula on the next line.
|
||||
assert!(is_heading_fragment("Note that the exact error equals"));
|
||||
assert!(is_heading_fragment("The remainder term satisfies"));
|
||||
assert!(is_heading_fragment("we conclude that the sum equals"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_headings_survive() {
|
||||
assert!(!is_heading_fragment("Introduction"));
|
||||
assert!(!is_heading_fragment("Error Analysis"));
|
||||
assert!(!is_heading_fragment("Materials and Methods"));
|
||||
assert!(!is_heading_fragment("Results"));
|
||||
assert!(!is_heading_fragment("3.2 Richardson Extrapolation"));
|
||||
assert!(!is_heading_fragment("Discussion and Conclusions"));
|
||||
// Terminal punctuation means the clause is complete.
|
||||
assert!(!is_heading_fragment("What is a Derivative?"));
|
||||
assert!(!is_heading_fragment("Procedure:"));
|
||||
assert!(!is_heading_fragment("Note that this is important."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn title_case_headings_ending_in_a_verb_survive() {
|
||||
// "yields" is also a plural noun; these are real section titles.
|
||||
assert!(!is_heading_fragment("Bond Yields"));
|
||||
assert!(!is_heading_fragment("Crop Yields"));
|
||||
assert!(!is_heading_fragment("Dividend Yields"));
|
||||
assert!(!is_heading_fragment("Yields"));
|
||||
// A wrapped title-case heading whose first line ends on a listed
|
||||
// verb must survive even if the preprocessor failed to merge it.
|
||||
assert!(!is_heading_fragment("The Theorem Implies"));
|
||||
assert!(!is_heading_fragment("What This Denotes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_sentence_case_headings_survive() {
|
||||
// heading.rs consults is_heading_fragment BEFORE applying its
|
||||
// numbered-prefix allowance, so the veto must not pre-empt it.
|
||||
assert!(!is_heading_fragment("1. What the model implies"));
|
||||
assert!(!is_heading_fragment("2.3 How the estimator satisfies"));
|
||||
assert!(!is_heading_fragment("IV) What this denotes"));
|
||||
// Without numbering the same wording is still a stranded clause.
|
||||
assert!(is_heading_fragment("What the model implies"));
|
||||
// A bare leading number or pronoun is prose, not numbering.
|
||||
assert!(is_heading_fragment("3 apples and what that implies"));
|
||||
assert!(is_heading_fragment("I think the model implies"));
|
||||
// Markers heading::parse_numbering rejects must not be exempted
|
||||
// either, or an ordinary list item bypasses the veto: lowercase
|
||||
// roman, alphabetical markers, and over-long tokens.
|
||||
assert!(is_heading_fragment("iv) the estimator satisfies"));
|
||||
assert!(is_heading_fragment("d) the value implies"));
|
||||
// Unsupported character (M is outside the parser's I/V/X/L/C set).
|
||||
assert!(is_heading_fragment("MMMM. the value implies"));
|
||||
// Over-long token: nine valid characters, so this exercises the
|
||||
// 8-character bound rather than the character set.
|
||||
assert!(is_heading_fragment("IIIIIIIII. the value implies"));
|
||||
// Eight is still within the bound and stays exempt.
|
||||
assert!(!is_heading_fragment("IIIIIIII. What this implies"));
|
||||
// Uppercase roman within the parser's grammar is still exempt.
|
||||
assert!(!is_heading_fragment("IV. What this denotes"));
|
||||
assert!(!is_heading_fragment("XII) What this implies"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrapped_headings_are_not_fragments() {
|
||||
// A heading that wraps across lines ends on a function word. These
|
||||
// are real headings from IRS Publication 17 and must survive.
|
||||
assert!(!is_heading_fragment("Casualty and"));
|
||||
assert!(!is_heading_fragment("Rule 10. You Must Be at"));
|
||||
assert!(!is_heading_fragment("Higher Standard Deduction for"));
|
||||
assert!(!is_heading_fragment("Qualifying Child of"));
|
||||
assert!(!is_heading_fragment("When Can I Withdraw or"));
|
||||
// Copulas and auxiliaries also end real wrapped headings.
|
||||
assert!(!is_heading_fragment("Rule 15. Your AGI Must Be"));
|
||||
assert!(!is_heading_fragment("What Medical Expenses Are"));
|
||||
assert!(!is_heading_fragment("Rule 13. You Must Have"));
|
||||
assert!(!is_heading_fragment("When Can a Roth IRA Be"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dangling_check_is_case_insensitive() {
|
||||
// All-caps is not sentence case, so the veto must not fire there.
|
||||
assert!(!is_heading_fragment("THE REMAINDER EQUALS"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the Y-gap threshold for paragraph break detection.
|
||||
///
|
||||
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
||||
|
||||
@@ -127,7 +127,9 @@ fn visual_style(line: &TextLine) -> Option<VisualStyle> {
|
||||
})
|
||||
}
|
||||
|
||||
fn roman_value(token: &str) -> Option<u32> {
|
||||
/// Shared with `analysis::starts_with_numbering_prefix` so the veto
|
||||
/// exemption and the heading parser agree on what a roman numeral is.
|
||||
pub(super) fn roman_value(token: &str) -> Option<u32> {
|
||||
if token.is_empty() || token.len() > 8 {
|
||||
return None;
|
||||
}
|
||||
|
||||
+308
-10
@@ -435,6 +435,72 @@ fn revised_table_cell_indices(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Index of candidate "body" items (larger-font attachment targets) sorted by
|
||||
/// Y, so script-attachment checks scan a narrow Y window instead of the whole
|
||||
/// page per candidate.
|
||||
struct ScriptBodyIndex<'a> {
|
||||
/// (y, item), sorted ascending by y
|
||||
by_y: Vec<(f32, &'a TextItem)>,
|
||||
/// widest vertical attachment window any body item can produce
|
||||
max_window: f32,
|
||||
}
|
||||
|
||||
impl<'a> ScriptBodyIndex<'a> {
|
||||
fn new(items: &'a [TextItem]) -> Self {
|
||||
// Smallest table-candidate font is 6pt, so any possible attachment
|
||||
// target is at least 6 x 1.2 pt.
|
||||
let mut by_y: Vec<(f32, &TextItem)> = items
|
||||
.iter()
|
||||
.filter(|i| i.font_size >= 6.0 * 1.2)
|
||||
.map(|i| (i.y, i))
|
||||
.collect();
|
||||
by_y.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
let max_window = by_y
|
||||
.iter()
|
||||
.map(|(_, i)| i.font_size * 0.8)
|
||||
.fold(0.0f32, f32::max);
|
||||
Self { by_y, max_window }
|
||||
}
|
||||
|
||||
/// True when a small-font item is horizontally attached to a larger-font
|
||||
/// item at a script baseline offset — a sub/superscript in running text
|
||||
/// or math (equation subscripts, footnote markers). Script attachments
|
||||
/// are not table cells; without this filter, display equations with
|
||||
/// sub/superscripts form phantom small-font table regions (e.g. TeX
|
||||
/// papers where log subscripts cluster with footnote lines into a fake
|
||||
/// 3-column table). A genuine baseline offset is required so same-line
|
||||
/// table neighbours (a small cell beside a larger label cell) are never
|
||||
/// classified as scripts.
|
||||
///
|
||||
/// `min_anchor_size` additionally constrains what counts as an
|
||||
/// attachment target: the small-font pass accepts any sufficiently
|
||||
/// larger item (0.0), while the body-font pass requires a heading-sized
|
||||
/// anchor so a body-size table cell beside a slightly larger label with
|
||||
/// baseline jitter is never treated as a script.
|
||||
fn is_script_attachment(&self, small: &TextItem, min_anchor_size: f32) -> bool {
|
||||
let attach_gap = small.font_size.max(4.0) * 0.6;
|
||||
let lo = self
|
||||
.by_y
|
||||
.partition_point(|(y, _)| *y < small.y - self.max_window);
|
||||
self.by_y[lo..]
|
||||
.iter()
|
||||
.take_while(|(y, _)| *y <= small.y + self.max_window)
|
||||
.any(|(_, body)| {
|
||||
let dy = (small.y - body.y).abs();
|
||||
body.font_size >= small.font_size * 1.2
|
||||
&& body.font_size >= min_anchor_size
|
||||
&& dy > body.font_size * 0.05
|
||||
&& dy <= body.font_size * 0.8
|
||||
&& {
|
||||
let gap_after_body = small.x - (body.x + body.width);
|
||||
let gap_before_body = body.x - (small.x + small.width);
|
||||
(-attach_gap..=attach_gap).contains(&gap_after_body)
|
||||
|| (-attach_gap..=attach_gap).contains(&gap_before_body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect tables in a set of text items from a single page
|
||||
pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bool) -> Vec<Table> {
|
||||
detect_tables_with_page_width(items, base_font_size, skip_body_font, content_width(items))
|
||||
@@ -483,6 +549,27 @@ pub(crate) fn detect_tables_with_page_width(
|
||||
// === Pass 1: Small-font tables (existing behavior) ===
|
||||
let table_font_threshold = base_font_size * 0.90;
|
||||
|
||||
// Mark sub/superscript attachments once per pass. They stay candidates —
|
||||
// the masks only remove them from region qualification and column/row
|
||||
// geometry.
|
||||
//
|
||||
// The two passes need different anchor thresholds. In the small-font pass
|
||||
// any sufficiently larger neighbour is a plausible base for a script. In
|
||||
// the body-font pass the candidates are themselves body-sized
|
||||
// (0.85..1.05x), so a merely "slightly larger" neighbour is usually a bold
|
||||
// label or an adjacent column header, not the base of a superscript —
|
||||
// treating it as one would strip real cells out of the geometry and lose
|
||||
// the table. Requiring a heading-sized anchor (>= 1.15x base) keeps the
|
||||
// body pass to genuine scripts hanging off headings.
|
||||
let script_index = ScriptBodyIndex::new(items);
|
||||
let script_flags: Vec<bool> = items
|
||||
.iter()
|
||||
.map(|item| script_index.is_script_attachment(item, 0.0))
|
||||
.collect();
|
||||
let body_script_flags: Vec<bool> = items
|
||||
.iter()
|
||||
.map(|item| script_index.is_script_attachment(item, base_font_size * 1.15))
|
||||
.collect();
|
||||
let table_candidates: Vec<(usize, &TextItem)> = items
|
||||
.iter()
|
||||
.enumerate()
|
||||
@@ -494,7 +581,14 @@ pub(crate) fn detect_tables_with_page_width(
|
||||
.collect();
|
||||
|
||||
if table_candidates.len() >= 6 {
|
||||
let regions = find_table_regions(&table_candidates);
|
||||
// Qualify regions from non-script items: a cluster of sub/superscripts
|
||||
// must not, on its own, mark out a table region.
|
||||
let region_evidence: Vec<(usize, &TextItem)> = table_candidates
|
||||
.iter()
|
||||
.filter(|(idx, _)| !script_flags[*idx])
|
||||
.cloned()
|
||||
.collect();
|
||||
let regions = find_table_regions(®ion_evidence);
|
||||
|
||||
for (y_min, y_max) in regions {
|
||||
let region_items: Vec<(usize, &TextItem)> = table_candidates
|
||||
@@ -508,7 +602,9 @@ pub(crate) fn detect_tables_with_page_width(
|
||||
}
|
||||
|
||||
if let Some(mut table) =
|
||||
detect_table_in_region(®ion_items, TableDetectionMode::SmallFont)
|
||||
detect_table_in_region(®ion_items, TableDetectionMode::SmallFont, &|i| {
|
||||
script_flags[i]
|
||||
})
|
||||
{
|
||||
// Try to recover body-font header row above the small-font table
|
||||
recover_header_row(&mut table, items, table_font_threshold);
|
||||
@@ -553,8 +649,20 @@ pub(crate) fn detect_tables_with_page_width(
|
||||
body_font_low,
|
||||
body_font_high,
|
||||
);
|
||||
// Scripts are NOT filtered out of the candidate set here, mirroring
|
||||
// the small-font pass: they must stay eligible for cell assignment so
|
||||
// a sub/superscript that belongs inside a table cell keeps its text.
|
||||
// The heading-anchored `body_script_flags` mask removes them from
|
||||
// geometry only.
|
||||
if body_candidates.len() >= 6 {
|
||||
let regions = find_table_regions_strict(&body_candidates);
|
||||
// Same reasoning as the small-font pass: scripts do not qualify
|
||||
// regions, but remain available for cell assignment within one.
|
||||
let region_evidence: Vec<(usize, &TextItem)> = body_candidates
|
||||
.iter()
|
||||
.filter(|(idx, _)| !body_script_flags[*idx])
|
||||
.cloned()
|
||||
.collect();
|
||||
let regions = find_table_regions_strict(®ion_evidence);
|
||||
log::debug!("body-font: {} strict regions found", regions.len());
|
||||
|
||||
for (y_min, y_max, _x_min, _x_max) in ®ions {
|
||||
@@ -580,7 +688,9 @@ pub(crate) fn detect_tables_with_page_width(
|
||||
}
|
||||
|
||||
if let Some(table) =
|
||||
detect_table_in_region(®ion_items, TableDetectionMode::BodyFont)
|
||||
detect_table_in_region(®ion_items, TableDetectionMode::BodyFont, &|i| {
|
||||
body_script_flags[i]
|
||||
})
|
||||
{
|
||||
tables.push(table);
|
||||
}
|
||||
@@ -808,10 +918,30 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32
|
||||
regions
|
||||
}
|
||||
|
||||
/// Detect a table within a specific region
|
||||
fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Option<Table> {
|
||||
// Find column boundaries
|
||||
let columns = find_column_boundaries(items, mode);
|
||||
/// Detect a table within a specific region.
|
||||
///
|
||||
/// `is_script` marks items that are sub/superscript attachments. Those are
|
||||
/// excluded from the *geometry* — they must not be able to create a column,
|
||||
/// which is how equation subscript clusters used to fabricate phantom grids —
|
||||
/// but they remain eligible for cell assignment, so legitimate cell content
|
||||
/// (exponents in an engineering-notation table, footnote markers) stays in
|
||||
/// the cell it belongs to instead of leaking out into the reading order.
|
||||
fn detect_table_in_region(
|
||||
items: &[(usize, &TextItem)],
|
||||
mode: TableDetectionMode,
|
||||
is_script: &dyn Fn(usize) -> bool,
|
||||
) -> Option<Table> {
|
||||
// Column geometry from non-script items only.
|
||||
let geometry_items: Vec<(usize, &TextItem)> = items
|
||||
.iter()
|
||||
.filter(|(idx, _)| !is_script(*idx))
|
||||
.cloned()
|
||||
.collect();
|
||||
// A region that is *entirely* scripts has no table structure at all.
|
||||
if geometry_items.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let columns = find_column_boundaries(&geometry_items, mode);
|
||||
let min_cols = 2;
|
||||
if columns.len() < min_cols || columns.len() > 25 {
|
||||
log::debug!(
|
||||
@@ -822,8 +952,8 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
return None;
|
||||
}
|
||||
|
||||
// Find row boundaries
|
||||
let rows = find_row_boundaries(items);
|
||||
// Find row boundaries (geometry items only, same reasoning)
|
||||
let rows = find_row_boundaries(&geometry_items);
|
||||
let min_rows = 2;
|
||||
if rows.len() < min_rows {
|
||||
log::debug!(
|
||||
@@ -842,6 +972,11 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
);
|
||||
|
||||
// Verify this looks like a table: multiple items should align to columns
|
||||
// Validate against ALL items, including scripts. Columns are derived from
|
||||
// non-script geometry so scripts cannot *create* a column, but excluding
|
||||
// them from validation too would let a region manufacture alignment: drop
|
||||
// the awkward items and whatever remains looks like a tidy grid. Block
|
||||
// diagrams did exactly that. Everything in the region must fit.
|
||||
let col_alignment = check_column_alignment(items, &columns, mode);
|
||||
let min_alignment = match mode {
|
||||
TableDetectionMode::SmallFont => 0.5,
|
||||
@@ -912,6 +1047,29 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
cells.push(row_cells);
|
||||
}
|
||||
|
||||
// Validation 0 (small-font pass only): reject tiny all-numeric
|
||||
// fragments. A <=2-row grid whose every cell is a bare 1-2 digit number
|
||||
// carries no tabular information — in practice these are
|
||||
// exponent/subscript clusters from display math that happen to align.
|
||||
// Body-font tables are not subject to this veto: their cells cannot be
|
||||
// script glyphs.
|
||||
if matches!(mode, TableDetectionMode::SmallFont) {
|
||||
let nonempty_cells: Vec<&String> =
|
||||
cells.iter().flatten().filter(|c| !c.is_empty()).collect();
|
||||
if rows.len() <= 2
|
||||
&& !nonempty_cells.is_empty()
|
||||
&& nonempty_cells
|
||||
.iter()
|
||||
.all(|c| c.len() <= 2 && c.chars().all(|ch| ch.is_ascii_digit()))
|
||||
{
|
||||
log::debug!(
|
||||
" validation 0 fail: tiny all-numeric fragment ({} cells)",
|
||||
nonempty_cells.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
// Validation 1: some rows should have content in first column.
|
||||
// Use a lower threshold (25%) for tables with wrapped cells where
|
||||
// continuation lines leave the first column empty.
|
||||
@@ -1977,6 +2135,146 @@ fn try_add_label_column(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
fn make_item(text: &str, x: f32, y: f32, font_size: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.to_string(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: font_size,
|
||||
font: "TestFont".to_string(),
|
||||
font_size,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_attachment_detects_subscript_after_body_text() {
|
||||
let body = make_item("log", 100.0, 500.0, 10.0, 15.0);
|
||||
let sub = make_item("10", 115.5, 497.0, 7.0, 7.0);
|
||||
let items = vec![body, sub.clone()];
|
||||
assert!(ScriptBodyIndex::new(&items).is_script_attachment(&sub, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_attachment_detects_superscript_footnote_marker() {
|
||||
let body = make_item("Hartley", 200.0, 500.0, 10.0, 35.0);
|
||||
let sup = make_item("2", 235.8, 504.0, 6.6, 3.5);
|
||||
let items = vec![body, sup.clone()];
|
||||
assert!(ScriptBodyIndex::new(&items).is_script_attachment(&sup, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_attachment_ignores_small_cell_far_from_body_text() {
|
||||
let body = make_item("Revenue", 100.0, 500.0, 10.0, 40.0);
|
||||
let cell = make_item("1,234", 180.0, 500.0, 7.0, 20.0);
|
||||
let items = vec![body, cell.clone()];
|
||||
assert!(!ScriptBodyIndex::new(&items).is_script_attachment(&cell, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_pass_anchor_spares_cells_beside_slightly_larger_labels() {
|
||||
// A body-font table cell (10pt) sitting beside a slightly larger,
|
||||
// NON-heading label (12.5pt) with a little baseline jitter. The
|
||||
// small-font pass treats any larger neighbour as a possible script
|
||||
// base, but the body pass must not: at body sizes a slightly larger
|
||||
// neighbour is a bold label or column header, and flagging the cell
|
||||
// would strip it out of the table geometry and lose the table.
|
||||
// Cell at the low end of the body band (0.85x base) beside a 10.5pt
|
||||
// label. 10.5 clears the inherent 1.2x-of-cell rule (10.2) but falls
|
||||
// below the body pass's heading anchor (11.5), which is exactly the
|
||||
// band where the two masks must disagree.
|
||||
let label = make_item("Revenue", 100.0, 500.0, 10.5, 40.0);
|
||||
let cell = make_item("1,234", 141.0, 496.5, 8.5, 22.0);
|
||||
let items = vec![label, cell.clone()];
|
||||
let index = ScriptBodyIndex::new(&items);
|
||||
let base = 10.0;
|
||||
assert!(
|
||||
index.is_script_attachment(&cell, 0.0),
|
||||
"small-font pass anchor should still see this as an attachment"
|
||||
);
|
||||
assert!(
|
||||
!index.is_script_attachment(&cell, base * 1.15),
|
||||
"body pass must not treat a cell beside a slightly larger label \
|
||||
as a script — that removes real cells from the geometry"
|
||||
);
|
||||
// A genuine heading-sized anchor still qualifies in the body pass.
|
||||
let heading = make_item("Section", 100.0, 500.0, 20.0, 60.0);
|
||||
let sup = make_item("3", 161.0, 508.0, 10.0, 5.0);
|
||||
let h_items = vec![heading, sup.clone()];
|
||||
assert!(
|
||||
ScriptBodyIndex::new(&h_items).is_script_attachment(&sup, base * 1.15),
|
||||
"script hanging off a heading must still be excluded in the body pass"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_attachment_ignores_same_baseline_neighbor_cell() {
|
||||
// A small cell beside a larger label on the SAME baseline is a table
|
||||
// layout, not a subscript — a genuine baseline offset is required.
|
||||
let label = make_item("Total", 100.0, 500.0, 10.0, 25.0);
|
||||
let cell = make_item("42", 127.0, 500.0, 7.5, 9.0);
|
||||
let items = vec![label, cell.clone()];
|
||||
assert!(!ScriptBodyIndex::new(&items).is_script_attachment(&cell, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_attachment_ignores_neighbor_on_different_line() {
|
||||
let body = make_item("Header", 100.0, 500.0, 10.0, 30.0);
|
||||
let cell = make_item("42", 131.0, 486.0, 7.0, 10.0);
|
||||
let items = vec![body, cell.clone()];
|
||||
assert!(!ScriptBodyIndex::new(&items).is_script_attachment(&cell, 0.0));
|
||||
}
|
||||
|
||||
/// Equation-subscript + footnote layout from Shannon entropy.pdf page 1,
|
||||
/// with real coordinates. Without the larger-font anchors the small items
|
||||
/// alone DO form a phantom table — proving the layout reaches detection —
|
||||
/// and adding the anchors must suppress it.
|
||||
fn shannon_page1_small_items() -> Vec<TextItem> {
|
||||
vec![
|
||||
make_item("2", 267.4, 133.9, 7.4, 3.7),
|
||||
make_item("10", 306.2, 133.9, 7.4, 7.4),
|
||||
make_item("10", 342.7, 133.9, 7.4, 7.4),
|
||||
make_item("10", 325.0, 118.9, 7.4, 7.4),
|
||||
make_item("Bell System Technical Journal,", 295.7, 101.9, 8.0, 95.0),
|
||||
make_item(
|
||||
"April 1924, p. 324; Certain Topics in",
|
||||
396.7,
|
||||
101.9,
|
||||
8.0,
|
||||
130.0,
|
||||
),
|
||||
make_item("v. 47, April 1928, p. 617.", 250.9, 92.5, 8.0, 90.0),
|
||||
make_item("Bell System Technical Journal,", 264.2, 82.6, 8.0, 95.0),
|
||||
make_item("July 1928, p. 535.", 364.3, 82.6, 8.0, 65.0),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equation_scripts_do_not_form_phantom_table() {
|
||||
let bare = shannon_page1_small_items();
|
||||
assert!(
|
||||
!detect_tables(&bare, 10.0, false).is_empty(),
|
||||
"test layout must form a phantom table when the filter cannot fire"
|
||||
);
|
||||
let mut items = shannon_page1_small_items();
|
||||
items.push(make_item("log", 253.0, 137.0, 10.0, 13.5));
|
||||
items.push(make_item("log", 291.5, 137.0, 10.0, 13.5));
|
||||
items.push(make_item("log", 328.0, 137.0, 10.0, 13.5));
|
||||
items.push(make_item("log", 310.3, 122.0, 10.0, 13.5));
|
||||
let tables = detect_tables(&items, 10.0, false);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"equation scripts + footnotes must not become a table: {tables:?}"
|
||||
);
|
||||
}
|
||||
use super::*;
|
||||
use crate::types::ItemType;
|
||||
|
||||
|
||||
+19
-1
@@ -594,7 +594,7 @@ fn hex_to_unicode_string(hex: &str) -> Option<String> {
|
||||
|
||||
let bytes: Option<Vec<u8>> = (0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).ok())
|
||||
.map(|i| u8::from_str_radix(hex.get(i..i + 2)?, 16).ok())
|
||||
.collect();
|
||||
let bytes = bytes?;
|
||||
|
||||
@@ -2606,6 +2606,24 @@ endcmap
|
||||
assert_eq!(cmap.lookup(0x0025), Some("B".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hex_to_unicode_non_ascii_no_panic() {
|
||||
// A destination containing a multi-byte char makes the byte length even
|
||||
// while a byte offset can land inside a char. Slicing must not panic;
|
||||
// it should be rejected gracefully.
|
||||
assert_eq!(hex_to_unicode_string("XéY"), None);
|
||||
assert_eq!(hex_to_unicode_string("\u{fffd}0"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bfchar_non_ascii_destination_no_panic() {
|
||||
// Crafted /ToUnicode CMap: a non-hex, non-ASCII destination previously
|
||||
// triggered a char-boundary panic in hex_to_unicode_string.
|
||||
let cmap_content = "beginbfchar <0041> <XéY> endbfchar";
|
||||
// Must not panic; the malformed entry is simply skipped.
|
||||
let _ = ToUnicodeCMap::parse(cmap_content.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bfchar_1byte() {
|
||||
// This is the pattern that caused the CJK bug: codespace is <0000><FFFF>
|
||||
|
||||
Reference in New Issue
Block a user