Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bee4f993b | ||
|
|
1719d24871 | ||
|
|
f114e79c8b | ||
|
|
544538b99f | ||
|
|
c8ba909407 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
|
||||
Generated
+2
-2
@@ -851,7 +851,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -867,7 +867,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.14.1",
|
||||
"version": "1.14.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -52,11 +52,11 @@
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.1"
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
||||
name = "pdf-inspector"
|
||||
# Keep package versions in sync with `python3 scripts/version.py <version>`.
|
||||
# CI publishes automatically when the synchronized change lands on main.
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
+1
-1
@@ -975,7 +975,7 @@ result = pdf_inspector.<span class="fn">process_pdf</span>(<span class="str">"do
|
||||
<script>
|
||||
(() => {
|
||||
const MAX_FILE_SIZE = 25 * 1024 * 1024;
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.1/pdf_inspector_wasm.js";
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.2/pdf_inspector_wasm.js";
|
||||
const input = document.querySelector("#pdf-input");
|
||||
const dropZone = document.querySelector("#drop-zone");
|
||||
const filePanel = document.querySelector("#demo-file");
|
||||
|
||||
+98
-24
@@ -1382,7 +1382,13 @@ fn scan_content_for_text_operators(
|
||||
let is_word_end =
|
||||
|pos: usize| -> bool { pos + 1 >= content.len() || content[pos + 1].is_ascii_whitespace() };
|
||||
|
||||
// Simple state machine to find operators
|
||||
// Simple state machine to find operators.
|
||||
// Each Tj/TJ/Tf lookback stops at the previous text/font operator so a
|
||||
// malformed `] TJ` (no `[`) cannot rescan the entire prefix — that was
|
||||
// quadratic in the number of operators.
|
||||
// `Tj`/`TJ` are only counted when the preceding token closes a string or
|
||||
// array (')', '>', ']'), so `Tj` inside `(Hello Tj World)` cannot pin the floor.
|
||||
let mut operand_floor = 0usize;
|
||||
let mut i = 0;
|
||||
while i < content.len() {
|
||||
let b = content[i];
|
||||
@@ -1392,14 +1398,15 @@ fn scan_content_for_text_operators(
|
||||
let next = content[i + 1];
|
||||
if next == b'j' || next == b'J' {
|
||||
// Verify it's an operator (followed by whitespace or newline)
|
||||
if i + 2 >= content.len()
|
||||
if (i + 2 >= content.len()
|
||||
|| content[i + 2].is_ascii_whitespace()
|
||||
|| content[i + 2] == b'\n'
|
||||
|| content[i + 2] == b'\r'
|
||||
|| content[i + 2] == b'\r')
|
||||
&& preceding_operand_closer(content, i, operand_floor)
|
||||
{
|
||||
text_ops += 1;
|
||||
// Scan backward for text string operand to collect unique chars
|
||||
collect_text_chars_before(content, i, unique_chars);
|
||||
collect_text_chars_before(content, i, unique_chars, operand_floor);
|
||||
operand_floor = i;
|
||||
}
|
||||
} else if next == b'f' {
|
||||
// Tf = set font operator
|
||||
@@ -1415,12 +1422,10 @@ fn scan_content_for_text_operators(
|
||||
|| content[i + 2] == b'<'
|
||||
|| content[i + 2] == b'/'
|
||||
{
|
||||
font_changes += 1;
|
||||
// Extract the font name operand preceding the size + Tf.
|
||||
// Pattern: /FontName <size> Tf
|
||||
// Scan backward past the size number and whitespace to find /Name.
|
||||
if let Some(name) = extract_font_name_before_tf(content, i) {
|
||||
if let Some(name) = extract_font_name_before_tf(content, i, operand_floor) {
|
||||
used_font_names.insert(name);
|
||||
font_changes += 1;
|
||||
operand_floor = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1466,6 +1471,20 @@ fn scan_content_for_text_operators(
|
||||
(text_ops, image_count, path_ops, font_changes)
|
||||
}
|
||||
|
||||
/// True when the token before `op_pos` (skipping whitespace, not crossing
|
||||
/// `floor`) is a string/array closer. Used so `Tj` inside `(Hello Tj World)`
|
||||
/// is not treated as an operator.
|
||||
fn preceding_operand_closer(content: &[u8], op_pos: usize, floor: usize) -> bool {
|
||||
let mut j = op_pos;
|
||||
while j > floor {
|
||||
j -= 1;
|
||||
if !content[j].is_ascii_whitespace() {
|
||||
return matches!(content[j], b')' | b'>' | b']');
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract the font name operand from content stream bytes preceding a Tf operator.
|
||||
///
|
||||
/// The Tf operator syntax is: `/FontName size Tf`
|
||||
@@ -1473,25 +1492,27 @@ fn scan_content_for_text_operators(
|
||||
/// whitespace to find the `/Name` token.
|
||||
///
|
||||
/// Returns the font name bytes (without the leading `/`), e.g. `b"F1"` for `/F1`.
|
||||
fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>> {
|
||||
/// `floor` is the start of the previous text/font operator (or 0); lookback
|
||||
/// must not cross it.
|
||||
fn extract_font_name_before_tf(content: &[u8], tf_pos: usize, floor: usize) -> Option<Vec<u8>> {
|
||||
// Scan backward past whitespace before "Tf"
|
||||
let mut j = tf_pos;
|
||||
while j > 0 && content[j - 1].is_ascii_whitespace() {
|
||||
while j > floor && content[j - 1].is_ascii_whitespace() {
|
||||
j -= 1;
|
||||
}
|
||||
// Scan backward past the size number (digits, '.', '-')
|
||||
while j > 0
|
||||
while j > floor
|
||||
&& (content[j - 1].is_ascii_digit() || content[j - 1] == b'.' || content[j - 1] == b'-')
|
||||
{
|
||||
j -= 1;
|
||||
}
|
||||
// Scan backward past whitespace between font name and size
|
||||
while j > 0 && content[j - 1].is_ascii_whitespace() {
|
||||
while j > floor && content[j - 1].is_ascii_whitespace() {
|
||||
j -= 1;
|
||||
}
|
||||
// Now j should point just after the font name. Scan backward to find '/'.
|
||||
let name_end = j;
|
||||
while j > 0 && content[j - 1] != b'/' {
|
||||
while j > floor && content[j - 1] != b'/' {
|
||||
// Font names consist of regular characters (not whitespace, not delimiters)
|
||||
if content[j - 1].is_ascii_whitespace() || content[j - 1] == b'(' || content[j - 1] == b')'
|
||||
{
|
||||
@@ -1499,7 +1520,7 @@ fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>>
|
||||
}
|
||||
j -= 1;
|
||||
}
|
||||
if j == 0 || content[j - 1] != b'/' {
|
||||
if j <= floor || content[j - 1] != b'/' {
|
||||
return None;
|
||||
}
|
||||
// j-1 is the '/', font name is content[j..name_end]
|
||||
@@ -1514,16 +1535,24 @@ fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>>
|
||||
/// and collect unique non-whitespace bytes from it.
|
||||
///
|
||||
/// Handles both literal strings `(...)` and hex strings `<...>`.
|
||||
fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut HashSet<u8>) {
|
||||
/// `floor` is the start of the previous text/font operator (or 0); lookback
|
||||
/// must not cross it, or a missing `[` before `TJ` rescans the whole prefix.
|
||||
fn collect_text_chars_before(
|
||||
content: &[u8],
|
||||
op_pos: usize,
|
||||
unique_chars: &mut HashSet<u8>,
|
||||
floor: usize,
|
||||
) {
|
||||
// Walk backward past whitespace to find the closing delimiter
|
||||
let mut j = op_pos;
|
||||
while j > 0 {
|
||||
while j > floor {
|
||||
j -= 1;
|
||||
if !content[j].is_ascii_whitespace() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if j == 0 {
|
||||
// All whitespace, or we landed on the previous operator token.
|
||||
if j == floor {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1533,7 +1562,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
// Literal string: scan backward for matching '('
|
||||
let mut depth = 1i32;
|
||||
let mut k = j;
|
||||
while k > 0 && depth > 0 {
|
||||
while k > floor && depth > 0 {
|
||||
k -= 1;
|
||||
match content[k] {
|
||||
b')' if k == 0 || content[k - 1] != b'\\' => depth += 1,
|
||||
@@ -1552,7 +1581,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
} else if closing == b'>' {
|
||||
// Hex string: scan backward for '<'
|
||||
let mut k = j;
|
||||
while k > 0 {
|
||||
while k > floor {
|
||||
k -= 1;
|
||||
if content[k] == b'<' {
|
||||
break;
|
||||
@@ -1582,7 +1611,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
} else if closing == b']' {
|
||||
// TJ array: scan backward for '[' and collect from all strings inside
|
||||
let mut k = j;
|
||||
while k > 0 {
|
||||
while k > floor {
|
||||
k -= 1;
|
||||
if content[k] == b'[' {
|
||||
break;
|
||||
@@ -2014,6 +2043,51 @@ mod tests {
|
||||
assert_eq!(imgs3, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_successive_tj_collects_each_operand() {
|
||||
// Lookback is floored at the previous Tj/TJ/Tf so later operators must
|
||||
// still see their own operands.
|
||||
let content = b"[(Hello)] TJ [(World)] TJ (More) Tj";
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, 3);
|
||||
for &ch in b"HeloWrdM" {
|
||||
assert!(uchars.contains(&ch), "missing char {}", ch as char);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_tj_inside_literal_is_not_an_operator() {
|
||||
// `Tj` followed by space inside a literal must not count as an operator
|
||||
// or pin the lookback floor; the real `Tj` still collects the string.
|
||||
let content = b"BT (Hello Tj World) Tj ET";
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, 1);
|
||||
for &ch in b"HeloTjWrd" {
|
||||
assert!(uchars.contains(&ch), "missing char {}", ch as char);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_malformed_tj_lookback_stays_linear() {
|
||||
// `] TJ` with no `[` used to walk the entire prefix for every operator
|
||||
// (quadratic). 30k repeats is enough that a prefix rescan would dominate
|
||||
// the test runtime; with the floor it is a single linear pass.
|
||||
let n = 30_000usize;
|
||||
let mut content = Vec::with_capacity(n * 5);
|
||||
for _ in 0..n {
|
||||
content.extend_from_slice(b"] TJ\n");
|
||||
}
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(&content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, n as u32);
|
||||
assert!(uchars.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_dominated_detection() {
|
||||
// Do operators are no longer counted as images by scan_content_for_text_operators.
|
||||
@@ -2772,14 +2846,14 @@ mod tests {
|
||||
fn test_extract_font_name_basic() {
|
||||
// Standard pattern: /F1 12 Tf
|
||||
let content = b"/F1 12 Tf";
|
||||
let name = extract_font_name_before_tf(content, 6); // 'T' is at index 6
|
||||
let name = extract_font_name_before_tf(content, 6, 0); // 'T' is at index 6
|
||||
assert_eq!(name, Some(b"F1".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_font_name_long_name() {
|
||||
let content = b"/ArialMT-Bold 9.5 Tf";
|
||||
let name = extract_font_name_before_tf(content, 18);
|
||||
let name = extract_font_name_before_tf(content, 18, 0);
|
||||
assert_eq!(name, Some(b"ArialMT-Bold".to_vec()));
|
||||
}
|
||||
|
||||
|
||||
+335
-15
@@ -1,6 +1,6 @@
|
||||
//! Rectangle-based table detection using union-find clustering.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
|
||||
use log::debug;
|
||||
|
||||
@@ -78,19 +78,111 @@ pub(crate) fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32),
|
||||
!(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom)
|
||||
}
|
||||
|
||||
fn grid_coord(value: f32, cell: f32) -> i32 {
|
||||
(value / cell).floor().clamp(-1_000_000.0, 1_000_000.0) as i32
|
||||
}
|
||||
|
||||
/// Inclusive grid range. `None` if the rect covers more cells than we will
|
||||
/// materialize — those rects are clustered via a bounded fallback.
|
||||
fn grid_span(lo: f32, hi: f32, cell: f32) -> Option<std::ops::RangeInclusive<i32>> {
|
||||
let a = grid_coord(lo.min(hi), cell);
|
||||
let b = grid_coord(lo.max(hi), cell);
|
||||
let span = b.saturating_sub(a);
|
||||
if span > 64 {
|
||||
return None;
|
||||
}
|
||||
Some(a..=b)
|
||||
}
|
||||
|
||||
fn union_bucket_pairs(
|
||||
uf: &mut UnionFind,
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
bucket: &[usize],
|
||||
tolerance: f32,
|
||||
) {
|
||||
let m = bucket.len();
|
||||
let mut pairs = 0usize;
|
||||
'cell: for a in 0..m {
|
||||
let i = bucket[a];
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
for &j in &bucket[a + 1..] {
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
break 'cell;
|
||||
}
|
||||
if uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn union_rect_against_bands(
|
||||
uf: &mut UnionFind,
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
i: usize,
|
||||
bands: &BTreeMap<i32, Vec<usize>>,
|
||||
lo: i32,
|
||||
hi: i32,
|
||||
tolerance: f32,
|
||||
) {
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
return;
|
||||
}
|
||||
let mut pairs = 0usize;
|
||||
let mut seen = HashSet::new();
|
||||
for (_, bucket) in bands.range(lo..=hi) {
|
||||
for &j in bucket {
|
||||
if !seen.insert(j) {
|
||||
continue;
|
||||
}
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
return;
|
||||
}
|
||||
if i == j || uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum component size for rect clustering. No real table has thousands
|
||||
/// of cell rects — once a component exceeds this, it is a vector drawing or
|
||||
/// page-spanning clipping path. We skip overlap checks for rects already in
|
||||
/// an oversized component, keeping the original O(n²) loop but making it
|
||||
/// effectively O(n) for pathological pages.
|
||||
/// an oversized component.
|
||||
const MAX_CLUSTER_RECTS: usize = 2000;
|
||||
|
||||
/// Pairwise-disjoint rects never merge, so a component-size cap does not
|
||||
/// stop an all-pairs loop. Rects are hashed into this many points of grid
|
||||
/// and compared only against others in the same cell.
|
||||
const CLUSTER_GRID_CELL: f32 = 64.0;
|
||||
|
||||
/// All-pairs AABB tests allowed inside one grid cell. A real table cell is
|
||||
/// tens of points wide, so a 64-pt cell holds a handful of neighbors — not
|
||||
/// thousands of stacked drawings.
|
||||
const MAX_CLUSTER_PAIRS_PER_CELL: usize = 16_384;
|
||||
|
||||
/// Cluster rects by spatial overlap using union-find.
|
||||
/// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned.
|
||||
///
|
||||
/// Skips overlap checks for rects whose component has already exceeded
|
||||
/// [`MAX_CLUSTER_RECTS`], so pages with tens of thousands of vector-drawing
|
||||
/// rects complete in milliseconds instead of minutes.
|
||||
/// Overlap tests run inside a uniform grid so far-apart rects are never
|
||||
/// compared, and each cell is pair-capped so a dense stack cannot go
|
||||
/// quadratic or starve an independent table in another cell.
|
||||
pub(crate) fn cluster_rects(
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
tolerance: f32,
|
||||
@@ -98,23 +190,144 @@ pub(crate) fn cluster_rects(
|
||||
) -> Vec<Vec<usize>> {
|
||||
let n = rects.len();
|
||||
let mut uf = UnionFind::new(n);
|
||||
let cell = CLUSTER_GRID_CELL.max(tolerance * 4.0);
|
||||
|
||||
for i in 0..n {
|
||||
// If rect i is already in an oversized component, no point comparing
|
||||
// it against further rects — the component won't be used for table
|
||||
// detection anyway.
|
||||
let mut grid: HashMap<(i32, i32), Vec<usize>> = HashMap::new();
|
||||
let mut large: Vec<usize> = Vec::new();
|
||||
for (idx, &(x, y, w, h)) in rects.iter().enumerate() {
|
||||
match (
|
||||
grid_span(x - tolerance, x + w + tolerance, cell),
|
||||
grid_span(y - tolerance, y + h + tolerance, cell),
|
||||
) {
|
||||
(Some(xs), Some(ys)) => {
|
||||
for gx in xs {
|
||||
for gy in ys.clone() {
|
||||
grid.entry((gx, gy)).or_default().push(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => large.push(idx),
|
||||
}
|
||||
}
|
||||
|
||||
let mut keys: Vec<_> = grid.keys().copied().collect();
|
||||
keys.sort_unstable();
|
||||
let mut keys_by_y: BTreeMap<i32, Vec<i32>> = BTreeMap::new();
|
||||
for &key in &keys {
|
||||
union_bucket_pairs(&mut uf, rects, &grid[&key], tolerance);
|
||||
keys_by_y.entry(key.1).or_default().push(key.0);
|
||||
}
|
||||
|
||||
// Oversized spans skip insert. Range-query occupied cells they cover so
|
||||
// later X-ranges are not starved and we do not scan unrelated rows.
|
||||
for &i in &large {
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
for j in (i + 1)..n {
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
// Check if the merged component just exceeded the cap —
|
||||
// if so, no need to test more pairs for rect i.
|
||||
let (x, y, w, h) = rects[i];
|
||||
let x_lo = grid_coord(x - tolerance, cell);
|
||||
let x_hi = grid_coord(x + w + tolerance, cell);
|
||||
let y_lo = grid_coord(y - tolerance, cell);
|
||||
let y_hi = grid_coord(y + h + tolerance, cell);
|
||||
for (&gy, gxs) in keys_by_y.range(y_lo..=y_hi) {
|
||||
let start = gxs.partition_point(|&gx| gx < x_lo);
|
||||
for &gx in &gxs[start..] {
|
||||
if gx > x_hi {
|
||||
break;
|
||||
}
|
||||
let bucket = &grid[&(gx, gy)];
|
||||
let mut pairs = 0usize;
|
||||
for &j in bucket {
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
break;
|
||||
}
|
||||
if uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Oversized-vs-oversized: band on the short axis so stacked or side-by-side
|
||||
// page-spanning rules stay linear. Wide vs tall pairs are matched by
|
||||
// querying the tall X-index; dual-oversized rects occupy every coarse-Y
|
||||
// cell they span.
|
||||
let mut large_x: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut large_y: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut large_coarse_y: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut wide: Vec<usize> = Vec::new();
|
||||
let mut dual: Vec<usize> = Vec::new();
|
||||
for &i in &large {
|
||||
let (x, y, w, h) = rects[i];
|
||||
let xs = grid_span(x - tolerance, x + w + tolerance, cell);
|
||||
let ys = grid_span(y - tolerance, y + h + tolerance, cell);
|
||||
match (xs, ys) {
|
||||
(Some(xs), _) => {
|
||||
for gx in xs {
|
||||
large_x.entry(gx).or_default().push(i);
|
||||
}
|
||||
}
|
||||
(_, Some(ys)) => {
|
||||
wide.push(i);
|
||||
for gy in ys {
|
||||
large_y.entry(gy).or_default().push(i);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
dual.push(i);
|
||||
let coarse = cell * 64.0;
|
||||
match grid_span(y - tolerance, y + h + tolerance, coarse) {
|
||||
Some(ys) => {
|
||||
for gy in ys {
|
||||
large_coarse_y.entry(gy).or_default().push(i);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
large_coarse_y.entry(i32::MIN).or_default().push(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for bands in [&large_x, &large_y, &large_coarse_y] {
|
||||
for bucket in bands.values() {
|
||||
union_bucket_pairs(&mut uf, rects, bucket, tolerance);
|
||||
}
|
||||
}
|
||||
// Cross-orientation is |wide|×|tall| if every wide rule spans the page.
|
||||
// Skip that pass when the product cannot be a table (a few rules).
|
||||
let tall_n = large
|
||||
.len()
|
||||
.saturating_sub(wide.len())
|
||||
.saturating_sub(dual.len());
|
||||
let cross_n =
|
||||
(wide.len() + dual.len()).saturating_mul(tall_n) + dual.len().saturating_mul(wide.len());
|
||||
if cross_n > 0 && cross_n <= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
for &i in wide.iter().chain(&dual) {
|
||||
let (x, _, w, _) = rects[i];
|
||||
let x_lo = grid_coord(x - tolerance, cell);
|
||||
let x_hi = grid_coord(x + w + tolerance, cell);
|
||||
union_rect_against_bands(&mut uf, rects, i, &large_x, x_lo, x_hi, tolerance);
|
||||
}
|
||||
for &i in &dual {
|
||||
let (_, y, _, h) = rects[i];
|
||||
let y_lo = grid_coord(y - tolerance, cell);
|
||||
let y_hi = grid_coord(y + h + tolerance, cell);
|
||||
union_rect_against_bands(&mut uf, rects, i, &large_y, y_lo, y_hi, tolerance);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3810,6 +4023,113 @@ mod tests {
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_overlapping_grid_still_clusters() {
|
||||
// Neighboring cells overlap; the grid must still union the whole table.
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..4 {
|
||||
for col in 0..4 {
|
||||
rects.push((col as f32 * 9.0, row as f32 * 9.0, 10.0, 10.0));
|
||||
}
|
||||
}
|
||||
let groups = cluster_rects(&rects, 0.0, 1);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_disjoint_stays_subquadratic() {
|
||||
// Pairwise-disjoint rects never merge, so a component-size cap does
|
||||
// not stop all-pairs overlap tests. Spread in X so they land in
|
||||
// different grid cells; 8k is enough that n² tests would dominate.
|
||||
let n = 8_000usize;
|
||||
let rects: Vec<(f32, f32, f32, f32)> =
|
||||
(0..n).map(|i| (i as f32 * 20.0, 0.0, 10.0, 10.0)).collect();
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert!(groups.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_stacked_disjoint_does_not_starve_later_table() {
|
||||
// Same X, spread in Y: a spatial grid must still union an overlapping
|
||||
// pair in another region of the page.
|
||||
let n = 8_000usize;
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> =
|
||||
(0..n).map(|i| (0.0, i as f32 * 20.0, 10.0, 10.0)).collect();
|
||||
rects.push((500.0, 0.0, 10.0, 10.0));
|
||||
rects.push((508.0, 0.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_oversized_span_still_unions() {
|
||||
// Wider than 64 grid cells; must still union the small overlapping rect.
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 10.0), (4900.0, 0.0, 10.0, 10.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 1);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_oversized_spans_all_get_a_pass() {
|
||||
// More than 32 huge rects: the last one must still union its overlap.
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = (0..40)
|
||||
.map(|i| (0.0, i as f32 * 20.0, 5000.0, 10.0))
|
||||
.collect();
|
||||
rects.push((4900.0, 39.0 * 20.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_oversized_not_starved_by_earlier_disjoint() {
|
||||
// 9k earlier disjoint drawings would exhaust an index-order cap of
|
||||
// 8,192 before the overlapping cell is visited.
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = (0..9_000)
|
||||
.map(|i| (10_000.0, i as f32 * 20.0, 10.0, 10.0))
|
||||
.collect();
|
||||
let wide = rects.len();
|
||||
rects.push((0.0, 0.0, 5000.0, 10.0));
|
||||
let target = rects.len();
|
||||
rects.push((4900.0, 0.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert!(
|
||||
groups
|
||||
.iter()
|
||||
.any(|g| g.contains(&wide) && g.contains(&target)),
|
||||
"wide rule and far-end cell must share a cluster"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_wide_and_tall_oversized_union() {
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 10.0), (0.0, 0.0, 10.0, 5000.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_dual_oversized_spans_coarse_y() {
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 5000.0), (0.0, 4500.0, 5000.0, 5000.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_wide_and_tall_stays_subquadratic() {
|
||||
let mut rects = Vec::with_capacity(4_000);
|
||||
for i in 0..2_000 {
|
||||
rects.push((0.0, i as f32 * 20.0, 5000.0, 10.0));
|
||||
rects.push((i as f32 * 20.0, 0.0, 10.0, 5000.0));
|
||||
}
|
||||
let _groups = cluster_rects(&rects, 0.0, 2);
|
||||
}
|
||||
|
||||
// --- snap_edges ---
|
||||
|
||||
#[test]
|
||||
|
||||
+64
-14
@@ -540,18 +540,14 @@ impl ToUnicodeCMap {
|
||||
|
||||
/// Remap a CMap that references pre-subsetting GIDs to sequential post-subsetting GIDs.
|
||||
/// Collects all source CIDs, sorts them, and reassigns to 1, 2, 3, ...
|
||||
///
|
||||
/// Range expansion stops after `MAX_CID_W_EXPANSION` CID visits, counting
|
||||
/// overwrites, so repeated full-width `bfrange`s cannot re-expand the
|
||||
/// 16-bit domain. Later overlapping ranges that would have introduced new
|
||||
/// CIDs after that many visits are truncated.
|
||||
pub fn remap_to_sequential(&self) -> ToUnicodeCMap {
|
||||
let mut cid_to_unicode: HashMap<u16, String> = HashMap::new();
|
||||
|
||||
// Expand ranges first
|
||||
for &(start, end, base) in &self.ranges {
|
||||
for cid in start..=end {
|
||||
let unicode_cp = base + (cid - start) as u32;
|
||||
if let Some(ch) = char::from_u32(unicode_cp) {
|
||||
cid_to_unicode.insert(cid, ch.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
expand_bfranges_for_remap(&self.ranges, &mut cid_to_unicode, MAX_CID_W_EXPANSION);
|
||||
|
||||
// char_map entries override range entries
|
||||
for (&cid, unicode) in &self.char_map {
|
||||
@@ -576,6 +572,33 @@ impl ToUnicodeCMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand `bfrange` entries into individual CID→Unicode inserts.
|
||||
/// Returns how many CIDs were visited. Counts overwrites so a repeated
|
||||
/// full-width range cannot keep working after `max_assignments`.
|
||||
fn expand_bfranges_for_remap(
|
||||
ranges: &[(u16, u16, u32)],
|
||||
cid_to_unicode: &mut HashMap<u16, String>,
|
||||
max_assignments: usize,
|
||||
) -> usize {
|
||||
let mut assigned = 0usize;
|
||||
'ranges: for &(start, end, base) in ranges {
|
||||
if start > end {
|
||||
continue;
|
||||
}
|
||||
for cid in start..=end {
|
||||
if assigned >= max_assignments {
|
||||
break 'ranges;
|
||||
}
|
||||
assigned += 1;
|
||||
let unicode_cp = base + (cid - start) as u32;
|
||||
if let Some(ch) = char::from_u32(unicode_cp) {
|
||||
cid_to_unicode.insert(cid, ch.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
assigned
|
||||
}
|
||||
|
||||
/// Parse a hex string to u16
|
||||
fn parse_hex_u16(hex: &str) -> Option<u16> {
|
||||
u16::from_str_radix(hex.trim(), 16).ok()
|
||||
@@ -1868,10 +1891,10 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap
|
||||
}
|
||||
|
||||
/// Shared 16-bit CID expansion cap (65,536).
|
||||
/// Encoding `begincidrange` and `/W` width assignment count every insert,
|
||||
/// including overwrites, so a repeated full-width range cannot keep working
|
||||
/// after the domain is filled. The `/W` unicode heuristic caps unique CIDs
|
||||
/// with the same number.
|
||||
/// Encoding `begincidrange`, `/W` width assignment, and ToUnicode sequential
|
||||
/// remap count every insert, including overwrites, so a repeated full-width
|
||||
/// range cannot keep working after the domain is filled. The `/W` unicode
|
||||
/// heuristic caps unique CIDs with the same number.
|
||||
pub(crate) const MAX_CID_W_EXPANSION: usize = 65_536;
|
||||
|
||||
/// Check if a CIDFont's /W (widths) array contains CID values that look like
|
||||
@@ -2924,6 +2947,33 @@ endbfrange
|
||||
assert!(remapped.ranges.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remap_to_sequential_repeated_full_bfranges_stay_bounded() {
|
||||
// 5,000 copies of `<0003> <ffff>` must stop after 65,536 CID visits,
|
||||
// not 5,000 × ~65,533 expansions.
|
||||
let ranges = vec![(3u16, 65535u16, 0x41u32); 5_000];
|
||||
let mut map = std::collections::HashMap::new();
|
||||
let assigned = expand_bfranges_for_remap(&ranges, &mut map, MAX_CID_W_EXPANSION);
|
||||
assert_eq!(assigned, MAX_CID_W_EXPANSION);
|
||||
assert!(map.len() <= MAX_CID_W_EXPANSION);
|
||||
|
||||
let mut body = String::new();
|
||||
let mut remaining = 5_000usize;
|
||||
while remaining > 0 {
|
||||
let n = remaining.min(100);
|
||||
body.push_str(&format!("{n} beginbfrange\n"));
|
||||
for _ in 0..n {
|
||||
body.push_str("<0003> <ffff> <0041>\n");
|
||||
}
|
||||
body.push_str("endbfrange\n");
|
||||
remaining -= n;
|
||||
}
|
||||
let data = format!("1 begincodespacerange\n<0000> <ffff>\nendcodespacerange\n{body}");
|
||||
let cmap = ToUnicodeCMap::parse(data.as_bytes()).unwrap();
|
||||
let remapped = cmap.remap_to_sequential();
|
||||
assert_eq!(remapped.lookup(1), Some("A".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_min_source_cid() {
|
||||
let cmap_content = r#"
|
||||
|
||||
Generated
+2
-2
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Browser WebAssembly bindings for pdf-inspector"
|
||||
|
||||
Reference in New Issue
Block a user