Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bee4f993b | ||
|
|
1719d24871 | ||
|
|
f114e79c8b | ||
|
|
544538b99f | ||
|
|
c8ba909407 | ||
|
|
076183e2e4 | ||
|
|
ec6e54afb8 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
|
||||
Generated
+2
-2
@@ -851,7 +851,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -867,7 +867,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.14.1",
|
||||
"version": "1.14.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -52,11 +52,11 @@
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.1",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.1"
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
||||
name = "pdf-inspector"
|
||||
# Keep package versions in sync with `python3 scripts/version.py <version>`.
|
||||
# CI publishes automatically when the synchronized change lands on main.
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
+1
-1
@@ -975,7 +975,7 @@ result = pdf_inspector.<span class="fn">process_pdf</span>(<span class="str">"do
|
||||
<script>
|
||||
(() => {
|
||||
const MAX_FILE_SIZE = 25 * 1024 * 1024;
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.1/pdf_inspector_wasm.js";
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.2/pdf_inspector_wasm.js";
|
||||
const input = document.querySelector("#pdf-input");
|
||||
const dropZone = document.querySelector("#drop-zone");
|
||||
const filePanel = document.querySelector("#demo-file");
|
||||
|
||||
+98
-24
@@ -1382,7 +1382,13 @@ fn scan_content_for_text_operators(
|
||||
let is_word_end =
|
||||
|pos: usize| -> bool { pos + 1 >= content.len() || content[pos + 1].is_ascii_whitespace() };
|
||||
|
||||
// Simple state machine to find operators
|
||||
// Simple state machine to find operators.
|
||||
// Each Tj/TJ/Tf lookback stops at the previous text/font operator so a
|
||||
// malformed `] TJ` (no `[`) cannot rescan the entire prefix — that was
|
||||
// quadratic in the number of operators.
|
||||
// `Tj`/`TJ` are only counted when the preceding token closes a string or
|
||||
// array (')', '>', ']'), so `Tj` inside `(Hello Tj World)` cannot pin the floor.
|
||||
let mut operand_floor = 0usize;
|
||||
let mut i = 0;
|
||||
while i < content.len() {
|
||||
let b = content[i];
|
||||
@@ -1392,14 +1398,15 @@ fn scan_content_for_text_operators(
|
||||
let next = content[i + 1];
|
||||
if next == b'j' || next == b'J' {
|
||||
// Verify it's an operator (followed by whitespace or newline)
|
||||
if i + 2 >= content.len()
|
||||
if (i + 2 >= content.len()
|
||||
|| content[i + 2].is_ascii_whitespace()
|
||||
|| content[i + 2] == b'\n'
|
||||
|| content[i + 2] == b'\r'
|
||||
|| content[i + 2] == b'\r')
|
||||
&& preceding_operand_closer(content, i, operand_floor)
|
||||
{
|
||||
text_ops += 1;
|
||||
// Scan backward for text string operand to collect unique chars
|
||||
collect_text_chars_before(content, i, unique_chars);
|
||||
collect_text_chars_before(content, i, unique_chars, operand_floor);
|
||||
operand_floor = i;
|
||||
}
|
||||
} else if next == b'f' {
|
||||
// Tf = set font operator
|
||||
@@ -1415,12 +1422,10 @@ fn scan_content_for_text_operators(
|
||||
|| content[i + 2] == b'<'
|
||||
|| content[i + 2] == b'/'
|
||||
{
|
||||
font_changes += 1;
|
||||
// Extract the font name operand preceding the size + Tf.
|
||||
// Pattern: /FontName <size> Tf
|
||||
// Scan backward past the size number and whitespace to find /Name.
|
||||
if let Some(name) = extract_font_name_before_tf(content, i) {
|
||||
if let Some(name) = extract_font_name_before_tf(content, i, operand_floor) {
|
||||
used_font_names.insert(name);
|
||||
font_changes += 1;
|
||||
operand_floor = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1466,6 +1471,20 @@ fn scan_content_for_text_operators(
|
||||
(text_ops, image_count, path_ops, font_changes)
|
||||
}
|
||||
|
||||
/// True when the token before `op_pos` (skipping whitespace, not crossing
|
||||
/// `floor`) is a string/array closer. Used so `Tj` inside `(Hello Tj World)`
|
||||
/// is not treated as an operator.
|
||||
fn preceding_operand_closer(content: &[u8], op_pos: usize, floor: usize) -> bool {
|
||||
let mut j = op_pos;
|
||||
while j > floor {
|
||||
j -= 1;
|
||||
if !content[j].is_ascii_whitespace() {
|
||||
return matches!(content[j], b')' | b'>' | b']');
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract the font name operand from content stream bytes preceding a Tf operator.
|
||||
///
|
||||
/// The Tf operator syntax is: `/FontName size Tf`
|
||||
@@ -1473,25 +1492,27 @@ fn scan_content_for_text_operators(
|
||||
/// whitespace to find the `/Name` token.
|
||||
///
|
||||
/// Returns the font name bytes (without the leading `/`), e.g. `b"F1"` for `/F1`.
|
||||
fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>> {
|
||||
/// `floor` is the start of the previous text/font operator (or 0); lookback
|
||||
/// must not cross it.
|
||||
fn extract_font_name_before_tf(content: &[u8], tf_pos: usize, floor: usize) -> Option<Vec<u8>> {
|
||||
// Scan backward past whitespace before "Tf"
|
||||
let mut j = tf_pos;
|
||||
while j > 0 && content[j - 1].is_ascii_whitespace() {
|
||||
while j > floor && content[j - 1].is_ascii_whitespace() {
|
||||
j -= 1;
|
||||
}
|
||||
// Scan backward past the size number (digits, '.', '-')
|
||||
while j > 0
|
||||
while j > floor
|
||||
&& (content[j - 1].is_ascii_digit() || content[j - 1] == b'.' || content[j - 1] == b'-')
|
||||
{
|
||||
j -= 1;
|
||||
}
|
||||
// Scan backward past whitespace between font name and size
|
||||
while j > 0 && content[j - 1].is_ascii_whitespace() {
|
||||
while j > floor && content[j - 1].is_ascii_whitespace() {
|
||||
j -= 1;
|
||||
}
|
||||
// Now j should point just after the font name. Scan backward to find '/'.
|
||||
let name_end = j;
|
||||
while j > 0 && content[j - 1] != b'/' {
|
||||
while j > floor && content[j - 1] != b'/' {
|
||||
// Font names consist of regular characters (not whitespace, not delimiters)
|
||||
if content[j - 1].is_ascii_whitespace() || content[j - 1] == b'(' || content[j - 1] == b')'
|
||||
{
|
||||
@@ -1499,7 +1520,7 @@ fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>>
|
||||
}
|
||||
j -= 1;
|
||||
}
|
||||
if j == 0 || content[j - 1] != b'/' {
|
||||
if j <= floor || content[j - 1] != b'/' {
|
||||
return None;
|
||||
}
|
||||
// j-1 is the '/', font name is content[j..name_end]
|
||||
@@ -1514,16 +1535,24 @@ fn extract_font_name_before_tf(content: &[u8], tf_pos: usize) -> Option<Vec<u8>>
|
||||
/// and collect unique non-whitespace bytes from it.
|
||||
///
|
||||
/// Handles both literal strings `(...)` and hex strings `<...>`.
|
||||
fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut HashSet<u8>) {
|
||||
/// `floor` is the start of the previous text/font operator (or 0); lookback
|
||||
/// must not cross it, or a missing `[` before `TJ` rescans the whole prefix.
|
||||
fn collect_text_chars_before(
|
||||
content: &[u8],
|
||||
op_pos: usize,
|
||||
unique_chars: &mut HashSet<u8>,
|
||||
floor: usize,
|
||||
) {
|
||||
// Walk backward past whitespace to find the closing delimiter
|
||||
let mut j = op_pos;
|
||||
while j > 0 {
|
||||
while j > floor {
|
||||
j -= 1;
|
||||
if !content[j].is_ascii_whitespace() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if j == 0 {
|
||||
// All whitespace, or we landed on the previous operator token.
|
||||
if j == floor {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1533,7 +1562,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
// Literal string: scan backward for matching '('
|
||||
let mut depth = 1i32;
|
||||
let mut k = j;
|
||||
while k > 0 && depth > 0 {
|
||||
while k > floor && depth > 0 {
|
||||
k -= 1;
|
||||
match content[k] {
|
||||
b')' if k == 0 || content[k - 1] != b'\\' => depth += 1,
|
||||
@@ -1552,7 +1581,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
} else if closing == b'>' {
|
||||
// Hex string: scan backward for '<'
|
||||
let mut k = j;
|
||||
while k > 0 {
|
||||
while k > floor {
|
||||
k -= 1;
|
||||
if content[k] == b'<' {
|
||||
break;
|
||||
@@ -1582,7 +1611,7 @@ fn collect_text_chars_before(content: &[u8], op_pos: usize, unique_chars: &mut H
|
||||
} else if closing == b']' {
|
||||
// TJ array: scan backward for '[' and collect from all strings inside
|
||||
let mut k = j;
|
||||
while k > 0 {
|
||||
while k > floor {
|
||||
k -= 1;
|
||||
if content[k] == b'[' {
|
||||
break;
|
||||
@@ -2014,6 +2043,51 @@ mod tests {
|
||||
assert_eq!(imgs3, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_successive_tj_collects_each_operand() {
|
||||
// Lookback is floored at the previous Tj/TJ/Tf so later operators must
|
||||
// still see their own operands.
|
||||
let content = b"[(Hello)] TJ [(World)] TJ (More) Tj";
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, 3);
|
||||
for &ch in b"HeloWrdM" {
|
||||
assert!(uchars.contains(&ch), "missing char {}", ch as char);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_tj_inside_literal_is_not_an_operator() {
|
||||
// `Tj` followed by space inside a literal must not count as an operator
|
||||
// or pin the lookback floor; the real `Tj` still collects the string.
|
||||
let content = b"BT (Hello Tj World) Tj ET";
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, 1);
|
||||
for &ch in b"HeloTjWrd" {
|
||||
assert!(uchars.contains(&ch), "missing char {}", ch as char);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_malformed_tj_lookback_stays_linear() {
|
||||
// `] TJ` with no `[` used to walk the entire prefix for every operator
|
||||
// (quadratic). 30k repeats is enough that a prefix rescan would dominate
|
||||
// the test runtime; with the floor it is a single linear pass.
|
||||
let n = 30_000usize;
|
||||
let mut content = Vec::with_capacity(n * 5);
|
||||
for _ in 0..n {
|
||||
content.extend_from_slice(b"] TJ\n");
|
||||
}
|
||||
let mut uchars = HashSet::new();
|
||||
let (ops, _, _, _) =
|
||||
scan_content_for_text_operators(&content, &mut uchars, &mut HashSet::new());
|
||||
assert_eq!(ops, n as u32);
|
||||
assert!(uchars.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_image_dominated_detection() {
|
||||
// Do operators are no longer counted as images by scan_content_for_text_operators.
|
||||
@@ -2772,14 +2846,14 @@ mod tests {
|
||||
fn test_extract_font_name_basic() {
|
||||
// Standard pattern: /F1 12 Tf
|
||||
let content = b"/F1 12 Tf";
|
||||
let name = extract_font_name_before_tf(content, 6); // 'T' is at index 6
|
||||
let name = extract_font_name_before_tf(content, 6, 0); // 'T' is at index 6
|
||||
assert_eq!(name, Some(b"F1".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_font_name_long_name() {
|
||||
let content = b"/ArialMT-Bold 9.5 Tf";
|
||||
let name = extract_font_name_before_tf(content, 18);
|
||||
let name = extract_font_name_before_tf(content, 18, 0);
|
||||
assert_eq!(name, Some(b"ArialMT-Bold".to_vec()));
|
||||
}
|
||||
|
||||
|
||||
+236
-3
@@ -890,6 +890,92 @@ fn tracked_run_space_floor(group: &[&TextItem], start: usize) -> Option<(usize,
|
||||
Some((end, floor * fs))
|
||||
}
|
||||
|
||||
/// Fractional font-size band within which `merge_text_items` treats two runs as
|
||||
/// the same size. Shared with `is_small_caps_continuation`, which exists only to
|
||||
/// rescue junctions this band would otherwise break.
|
||||
const MERGE_FONT_SIZE_BAND: f32 = 0.20;
|
||||
|
||||
/// Detect a small-caps continuation: typesetters render small caps as a
|
||||
/// full-size capital immediately followed by shrunken capitals in the same
|
||||
/// font (`(R) Tj` at 9.98pt, then `(OLANDO) Tj` at 6.74pt). Those runs are one
|
||||
/// word, but the font-size band in `merge_text_items` would split them,
|
||||
/// leaving "R" and "OLANDO" as separate items — which then read as separate
|
||||
/// table columns, since column boundaries cluster on item start positions.
|
||||
///
|
||||
/// Gated tightly so it cannot absorb the other reasons a smaller run follows a
|
||||
/// larger one:
|
||||
/// - runs the size band already accepts — excluded by requiring the junction
|
||||
/// to *cross* the band, so within-band pairs keep the normal word-spacing
|
||||
/// logic instead of having their space suppressed
|
||||
/// - superscripts / footnote markers — excluded by requiring an uppercase
|
||||
/// *letter* on both sides, so digits never qualify
|
||||
/// - drop caps — excluded because the body text that follows is mixed case
|
||||
/// - adjacent table cells or separate words — excluded by requiring the runs
|
||||
/// to be visually contiguous (essentially no gap)
|
||||
fn is_small_caps_continuation(
|
||||
text_so_far: &str,
|
||||
first: &TextItem,
|
||||
next: &TextItem,
|
||||
gap: f32,
|
||||
) -> bool {
|
||||
// Must shrink. Real small caps sit near 0.7-0.8 of the full cap height;
|
||||
// anything smaller is a superscript or a different run entirely.
|
||||
if first.font_size <= 0.0 || next.font_size >= first.font_size {
|
||||
return false;
|
||||
}
|
||||
// Only rescue junctions the size band would have broken. Within-band pairs
|
||||
// merge on their own, and suppressing their space would swallow real word
|
||||
// gaps between two similarly-sized uppercase words.
|
||||
if (next.font_size - first.font_size).abs() <= first.font_size * MERGE_FONT_SIZE_BAND {
|
||||
return false;
|
||||
}
|
||||
if next.font_size / first.font_size < 0.55 {
|
||||
return false;
|
||||
}
|
||||
// Visually contiguous: the capital and its small caps touch. A real word
|
||||
// space or a column gap disqualifies.
|
||||
if !(-first.font_size * 0.2..=first.font_size * 0.15).contains(&gap) {
|
||||
return false;
|
||||
}
|
||||
// The continuation must be all-uppercase letters (digits and lowercase
|
||||
// both disqualify), and must contain at least one letter.
|
||||
let mut saw_letter = false;
|
||||
for ch in next.text.chars() {
|
||||
if ch.is_alphabetic() {
|
||||
saw_letter = true;
|
||||
if !ch.is_uppercase() {
|
||||
return false;
|
||||
}
|
||||
} else if ch.is_numeric() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if !saw_letter {
|
||||
return false;
|
||||
}
|
||||
// What we are continuing must itself end in a capital. Check the actual
|
||||
// trailing character rather than skipping back to the nearest letter: after
|
||||
// "ANGELA M. MAZZARELLI1" the run to continue is the footnote marker, not
|
||||
// the "I" before it.
|
||||
let trimmed = text_so_far.trim_end();
|
||||
if trimmed.chars().last().is_some_and(|c| c.is_numeric()) {
|
||||
// One legitimate exception: an ordinal suffix set as a smaller run,
|
||||
// e.g. "JULY 4" + "TH". Only the four English suffixes qualify —
|
||||
// anything else after a digit is a footnote marker or numeric suffix.
|
||||
return matches!(trimmed_suffix(next), "TH" | "ST" | "ND" | "RD");
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.rev()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
/// The continuation run's text, trimmed — used to spot ordinal suffixes.
|
||||
fn trimmed_suffix(next: &TextItem) -> &str {
|
||||
next.text.trim()
|
||||
}
|
||||
|
||||
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if items.is_empty() {
|
||||
return items;
|
||||
@@ -948,8 +1034,16 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
let mut j = i + 1;
|
||||
while j < group.len() {
|
||||
let next = group[j];
|
||||
// Must be similar font size (within 20%)
|
||||
if (next.font_size - first.font_size).abs() > first.font_size * 0.20 {
|
||||
// A small-caps junction is mid-word: it both survives the
|
||||
// font-size band below and must never take a space.
|
||||
let small_caps_join =
|
||||
is_small_caps_continuation(&text, first, next, next.x - end_x);
|
||||
// Must be similar font size, except for genuine small-caps
|
||||
// runs, where the shrunken capitals are the same word as the
|
||||
// full-size initial (see helper).
|
||||
if (next.font_size - first.font_size).abs() > first.font_size * MERGE_FONT_SIZE_BAND
|
||||
&& !small_caps_join
|
||||
{
|
||||
break;
|
||||
}
|
||||
// Never merge across style boundaries: the merged item
|
||||
@@ -1003,7 +1097,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
Some((run_end, floor)) if j <= run_end => floor,
|
||||
_ => threshold,
|
||||
};
|
||||
if needs_bullet_space || gap > effective_threshold {
|
||||
if !small_caps_join && (needs_bullet_space || gap > effective_threshold) {
|
||||
text.push(' ');
|
||||
}
|
||||
text.push_str(&next.text);
|
||||
@@ -3007,6 +3101,145 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Small caps as typesetters emit them: a full-size capital at 9.98pt
|
||||
/// immediately followed by shrunken capitals at 6.74pt, touching.
|
||||
/// Modelled on `199AD3d.pdf` p.5 ("ROLANDO T. ACOSTA, P.J.").
|
||||
#[test]
|
||||
fn small_caps_run_merges_into_one_word() {
|
||||
let items = vec![
|
||||
make_item_fs("R", 144.36, 581.84, 7.20, 9.98),
|
||||
make_item_fs("OLANDO", 151.56, 581.84, 30.56, 6.74),
|
||||
make_item_fs("T. A", 185.45, 581.84, 17.58, 9.98),
|
||||
make_item_fs("COSTA", 203.94, 581.84, 23.15, 6.74),
|
||||
make_item_fs(", P.J.", 227.09, 581.84, 22.56, 9.98),
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1, "got {:?}", merged);
|
||||
assert_eq!(merged[0].text, "ROLANDO T. ACOSTA, P.J.");
|
||||
}
|
||||
|
||||
/// The full two-column row: both names must merge independently and the
|
||||
/// 72pt column gap between them must survive as an item boundary.
|
||||
#[test]
|
||||
fn small_caps_merge_does_not_swallow_a_second_column() {
|
||||
let items = vec![
|
||||
// Column 1: "ROLANDO T. ACOSTA, P.J." ending at x=249.65
|
||||
make_item_fs("R", 144.36, 581.84, 7.20, 9.98),
|
||||
make_item_fs("OLANDO", 151.56, 581.84, 30.56, 6.74),
|
||||
make_item_fs("T. A", 185.45, 581.84, 17.58, 9.98),
|
||||
make_item_fs("COSTA", 203.94, 581.84, 23.15, 6.74),
|
||||
make_item_fs(", P.J.", 227.09, 581.84, 22.56, 9.98),
|
||||
// Column 2 starts at x=321.96 — a 72pt gap.
|
||||
make_item_fs("A", 321.96, 581.84, 7.20, 9.98),
|
||||
make_item_fs("NIL", 329.17, 581.84, 12.72, 6.74),
|
||||
make_item_fs("C. S", 345.04, 581.84, 19.59, 9.98),
|
||||
make_item_fs("INGH", 364.62, 581.84, 19.08, 6.74),
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
let texts: Vec<&str> = merged.iter().map(|i| i.text.as_str()).collect();
|
||||
assert_eq!(
|
||||
texts,
|
||||
vec!["ROLANDO T. ACOSTA, P.J.", "ANIL C. SINGH"],
|
||||
"column gap should keep the two names apart"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_caps_merge_keeps_word_space_between_same_size_capitals() {
|
||||
// Two uppercase words at sizes the merge band already accepts (9.98 and
|
||||
// 9.0, a 10% drop) separated by a real word gap. The small-caps path
|
||||
// must not claim this junction and swallow the space.
|
||||
let items = vec![
|
||||
make_item_fs("SEE", 100.0, 500.0, 18.0, 9.98),
|
||||
make_item_fs("ALSO", 119.2, 500.0, 24.0, 9.0),
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1, "got {:?}", merged);
|
||||
assert_eq!(merged[0].text, "SEE ALSO");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_digit_is_not_a_capital_awaiting_small_caps() {
|
||||
// "...MAZZARELLI1" ends in a footnote marker; the backward search for an
|
||||
// uppercase letter must not skip the digit and glue the next run.
|
||||
assert!(!is_small_caps_continuation(
|
||||
"ANGELA M. MAZZARELLI1",
|
||||
&make_item_fs("ANGELA", 100.0, 500.0, 40.0, 9.98),
|
||||
&make_item_fs("SHULMAN", 140.0, 500.0, 30.0, 6.74),
|
||||
0.0,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinal_suffix_after_a_digit_still_merges() {
|
||||
// "TUESDAY, JULY 4" + "TH" is one word in the source; the digit guard
|
||||
// must not block the four English ordinal suffixes.
|
||||
for suffix in ["TH", "ST", "ND", "RD"] {
|
||||
assert!(
|
||||
is_small_caps_continuation(
|
||||
"TUESDAY, JULY 4",
|
||||
&make_item_fs("JULY", 100.0, 500.0, 30.0, 12.0),
|
||||
&make_item_fs(suffix, 130.0, 500.0, 8.0, 8.0),
|
||||
0.0,
|
||||
),
|
||||
"{suffix} should merge after a digit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superscript_footnote_marker_is_not_a_small_caps_continuation() {
|
||||
// A digit must never qualify — otherwise footnote markers get glued on
|
||||
// without the superscript handling.
|
||||
assert!(!is_small_caps_continuation(
|
||||
"MAZZARELLI",
|
||||
&make_item_fs("MAZZARELLI", 100.0, 500.0, 50.0, 9.98),
|
||||
&make_item_fs("1", 150.0, 503.0, 3.0, 6.74),
|
||||
0.0,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_cap_is_not_a_small_caps_continuation() {
|
||||
// Mixed-case body text after a large initial is a drop cap, not small
|
||||
// caps.
|
||||
assert!(!is_small_caps_continuation(
|
||||
"T",
|
||||
&make_item_fs("T", 100.0, 500.0, 20.0, 30.0),
|
||||
&make_item_fs("he court held", 120.0, 500.0, 60.0, 10.0),
|
||||
0.0,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn separate_word_is_not_a_small_caps_continuation() {
|
||||
// A real word space disqualifies even when both runs are uppercase.
|
||||
let first = make_item_fs("SEE", 100.0, 500.0, 20.0, 9.98);
|
||||
let next = make_item_fs("ALSO", 128.0, 500.0, 25.0, 6.74);
|
||||
assert!(!is_small_caps_continuation("SEE", &first, &next, 8.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lowercase_continuation_is_not_small_caps() {
|
||||
assert!(!is_small_caps_continuation(
|
||||
"SMALL",
|
||||
&make_item_fs("SMALL", 100.0, 500.0, 30.0, 9.98),
|
||||
&make_item_fs("caps", 130.0, 500.0, 20.0, 6.74),
|
||||
0.0,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_small_a_ratio_is_not_small_caps() {
|
||||
// 0.4 ratio is a superscript/sub-run, outside the small-caps band.
|
||||
assert!(!is_small_caps_continuation(
|
||||
"A",
|
||||
&make_item_fs("A", 100.0, 500.0, 7.0, 10.0),
|
||||
&make_item_fs("BC", 107.0, 500.0, 8.0, 4.0),
|
||||
0.0,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_subscript_items_chemical_formula() {
|
||||
// NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7
|
||||
|
||||
+335
-15
@@ -1,6 +1,6 @@
|
||||
//! Rectangle-based table detection using union-find clustering.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
|
||||
use log::debug;
|
||||
|
||||
@@ -78,19 +78,111 @@ pub(crate) fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32),
|
||||
!(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom)
|
||||
}
|
||||
|
||||
fn grid_coord(value: f32, cell: f32) -> i32 {
|
||||
(value / cell).floor().clamp(-1_000_000.0, 1_000_000.0) as i32
|
||||
}
|
||||
|
||||
/// Inclusive grid range. `None` if the rect covers more cells than we will
|
||||
/// materialize — those rects are clustered via a bounded fallback.
|
||||
fn grid_span(lo: f32, hi: f32, cell: f32) -> Option<std::ops::RangeInclusive<i32>> {
|
||||
let a = grid_coord(lo.min(hi), cell);
|
||||
let b = grid_coord(lo.max(hi), cell);
|
||||
let span = b.saturating_sub(a);
|
||||
if span > 64 {
|
||||
return None;
|
||||
}
|
||||
Some(a..=b)
|
||||
}
|
||||
|
||||
fn union_bucket_pairs(
|
||||
uf: &mut UnionFind,
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
bucket: &[usize],
|
||||
tolerance: f32,
|
||||
) {
|
||||
let m = bucket.len();
|
||||
let mut pairs = 0usize;
|
||||
'cell: for a in 0..m {
|
||||
let i = bucket[a];
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
for &j in &bucket[a + 1..] {
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
break 'cell;
|
||||
}
|
||||
if uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn union_rect_against_bands(
|
||||
uf: &mut UnionFind,
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
i: usize,
|
||||
bands: &BTreeMap<i32, Vec<usize>>,
|
||||
lo: i32,
|
||||
hi: i32,
|
||||
tolerance: f32,
|
||||
) {
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
return;
|
||||
}
|
||||
let mut pairs = 0usize;
|
||||
let mut seen = HashSet::new();
|
||||
for (_, bucket) in bands.range(lo..=hi) {
|
||||
for &j in bucket {
|
||||
if !seen.insert(j) {
|
||||
continue;
|
||||
}
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
return;
|
||||
}
|
||||
if i == j || uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum component size for rect clustering. No real table has thousands
|
||||
/// of cell rects — once a component exceeds this, it is a vector drawing or
|
||||
/// page-spanning clipping path. We skip overlap checks for rects already in
|
||||
/// an oversized component, keeping the original O(n²) loop but making it
|
||||
/// effectively O(n) for pathological pages.
|
||||
/// an oversized component.
|
||||
const MAX_CLUSTER_RECTS: usize = 2000;
|
||||
|
||||
/// Pairwise-disjoint rects never merge, so a component-size cap does not
|
||||
/// stop an all-pairs loop. Rects are hashed into this many points of grid
|
||||
/// and compared only against others in the same cell.
|
||||
const CLUSTER_GRID_CELL: f32 = 64.0;
|
||||
|
||||
/// All-pairs AABB tests allowed inside one grid cell. A real table cell is
|
||||
/// tens of points wide, so a 64-pt cell holds a handful of neighbors — not
|
||||
/// thousands of stacked drawings.
|
||||
const MAX_CLUSTER_PAIRS_PER_CELL: usize = 16_384;
|
||||
|
||||
/// Cluster rects by spatial overlap using union-find.
|
||||
/// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned.
|
||||
///
|
||||
/// Skips overlap checks for rects whose component has already exceeded
|
||||
/// [`MAX_CLUSTER_RECTS`], so pages with tens of thousands of vector-drawing
|
||||
/// rects complete in milliseconds instead of minutes.
|
||||
/// Overlap tests run inside a uniform grid so far-apart rects are never
|
||||
/// compared, and each cell is pair-capped so a dense stack cannot go
|
||||
/// quadratic or starve an independent table in another cell.
|
||||
pub(crate) fn cluster_rects(
|
||||
rects: &[(f32, f32, f32, f32)],
|
||||
tolerance: f32,
|
||||
@@ -98,23 +190,144 @@ pub(crate) fn cluster_rects(
|
||||
) -> Vec<Vec<usize>> {
|
||||
let n = rects.len();
|
||||
let mut uf = UnionFind::new(n);
|
||||
let cell = CLUSTER_GRID_CELL.max(tolerance * 4.0);
|
||||
|
||||
for i in 0..n {
|
||||
// If rect i is already in an oversized component, no point comparing
|
||||
// it against further rects — the component won't be used for table
|
||||
// detection anyway.
|
||||
let mut grid: HashMap<(i32, i32), Vec<usize>> = HashMap::new();
|
||||
let mut large: Vec<usize> = Vec::new();
|
||||
for (idx, &(x, y, w, h)) in rects.iter().enumerate() {
|
||||
match (
|
||||
grid_span(x - tolerance, x + w + tolerance, cell),
|
||||
grid_span(y - tolerance, y + h + tolerance, cell),
|
||||
) {
|
||||
(Some(xs), Some(ys)) => {
|
||||
for gx in xs {
|
||||
for gy in ys.clone() {
|
||||
grid.entry((gx, gy)).or_default().push(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => large.push(idx),
|
||||
}
|
||||
}
|
||||
|
||||
let mut keys: Vec<_> = grid.keys().copied().collect();
|
||||
keys.sort_unstable();
|
||||
let mut keys_by_y: BTreeMap<i32, Vec<i32>> = BTreeMap::new();
|
||||
for &key in &keys {
|
||||
union_bucket_pairs(&mut uf, rects, &grid[&key], tolerance);
|
||||
keys_by_y.entry(key.1).or_default().push(key.0);
|
||||
}
|
||||
|
||||
// Oversized spans skip insert. Range-query occupied cells they cover so
|
||||
// later X-ranges are not starved and we do not scan unrelated rows.
|
||||
for &i in &large {
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
for j in (i + 1)..n {
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
// Check if the merged component just exceeded the cap —
|
||||
// if so, no need to test more pairs for rect i.
|
||||
let (x, y, w, h) = rects[i];
|
||||
let x_lo = grid_coord(x - tolerance, cell);
|
||||
let x_hi = grid_coord(x + w + tolerance, cell);
|
||||
let y_lo = grid_coord(y - tolerance, cell);
|
||||
let y_hi = grid_coord(y + h + tolerance, cell);
|
||||
for (&gy, gxs) in keys_by_y.range(y_lo..=y_hi) {
|
||||
let start = gxs.partition_point(|&gx| gx < x_lo);
|
||||
for &gx in &gxs[start..] {
|
||||
if gx > x_hi {
|
||||
break;
|
||||
}
|
||||
let bucket = &grid[&(gx, gy)];
|
||||
let mut pairs = 0usize;
|
||||
for &j in bucket {
|
||||
if pairs >= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
break;
|
||||
}
|
||||
if uf.component_size(j) >= MAX_CLUSTER_RECTS {
|
||||
continue;
|
||||
}
|
||||
pairs += 1;
|
||||
if rects_overlap(&rects[i], &rects[j], tolerance) {
|
||||
uf.union(i, j);
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if uf.component_size(i) >= MAX_CLUSTER_RECTS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Oversized-vs-oversized: band on the short axis so stacked or side-by-side
|
||||
// page-spanning rules stay linear. Wide vs tall pairs are matched by
|
||||
// querying the tall X-index; dual-oversized rects occupy every coarse-Y
|
||||
// cell they span.
|
||||
let mut large_x: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut large_y: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut large_coarse_y: BTreeMap<i32, Vec<usize>> = BTreeMap::new();
|
||||
let mut wide: Vec<usize> = Vec::new();
|
||||
let mut dual: Vec<usize> = Vec::new();
|
||||
for &i in &large {
|
||||
let (x, y, w, h) = rects[i];
|
||||
let xs = grid_span(x - tolerance, x + w + tolerance, cell);
|
||||
let ys = grid_span(y - tolerance, y + h + tolerance, cell);
|
||||
match (xs, ys) {
|
||||
(Some(xs), _) => {
|
||||
for gx in xs {
|
||||
large_x.entry(gx).or_default().push(i);
|
||||
}
|
||||
}
|
||||
(_, Some(ys)) => {
|
||||
wide.push(i);
|
||||
for gy in ys {
|
||||
large_y.entry(gy).or_default().push(i);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
dual.push(i);
|
||||
let coarse = cell * 64.0;
|
||||
match grid_span(y - tolerance, y + h + tolerance, coarse) {
|
||||
Some(ys) => {
|
||||
for gy in ys {
|
||||
large_coarse_y.entry(gy).or_default().push(i);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
large_coarse_y.entry(i32::MIN).or_default().push(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for bands in [&large_x, &large_y, &large_coarse_y] {
|
||||
for bucket in bands.values() {
|
||||
union_bucket_pairs(&mut uf, rects, bucket, tolerance);
|
||||
}
|
||||
}
|
||||
// Cross-orientation is |wide|×|tall| if every wide rule spans the page.
|
||||
// Skip that pass when the product cannot be a table (a few rules).
|
||||
let tall_n = large
|
||||
.len()
|
||||
.saturating_sub(wide.len())
|
||||
.saturating_sub(dual.len());
|
||||
let cross_n =
|
||||
(wide.len() + dual.len()).saturating_mul(tall_n) + dual.len().saturating_mul(wide.len());
|
||||
if cross_n > 0 && cross_n <= MAX_CLUSTER_PAIRS_PER_CELL {
|
||||
for &i in wide.iter().chain(&dual) {
|
||||
let (x, _, w, _) = rects[i];
|
||||
let x_lo = grid_coord(x - tolerance, cell);
|
||||
let x_hi = grid_coord(x + w + tolerance, cell);
|
||||
union_rect_against_bands(&mut uf, rects, i, &large_x, x_lo, x_hi, tolerance);
|
||||
}
|
||||
for &i in &dual {
|
||||
let (_, y, _, h) = rects[i];
|
||||
let y_lo = grid_coord(y - tolerance, cell);
|
||||
let y_hi = grid_coord(y + h + tolerance, cell);
|
||||
union_rect_against_bands(&mut uf, rects, i, &large_y, y_lo, y_hi, tolerance);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3810,6 +4023,113 @@ mod tests {
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_overlapping_grid_still_clusters() {
|
||||
// Neighboring cells overlap; the grid must still union the whole table.
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..4 {
|
||||
for col in 0..4 {
|
||||
rects.push((col as f32 * 9.0, row as f32 * 9.0, 10.0, 10.0));
|
||||
}
|
||||
}
|
||||
let groups = cluster_rects(&rects, 0.0, 1);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_disjoint_stays_subquadratic() {
|
||||
// Pairwise-disjoint rects never merge, so a component-size cap does
|
||||
// not stop all-pairs overlap tests. Spread in X so they land in
|
||||
// different grid cells; 8k is enough that n² tests would dominate.
|
||||
let n = 8_000usize;
|
||||
let rects: Vec<(f32, f32, f32, f32)> =
|
||||
(0..n).map(|i| (i as f32 * 20.0, 0.0, 10.0, 10.0)).collect();
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert!(groups.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_stacked_disjoint_does_not_starve_later_table() {
|
||||
// Same X, spread in Y: a spatial grid must still union an overlapping
|
||||
// pair in another region of the page.
|
||||
let n = 8_000usize;
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> =
|
||||
(0..n).map(|i| (0.0, i as f32 * 20.0, 10.0, 10.0)).collect();
|
||||
rects.push((500.0, 0.0, 10.0, 10.0));
|
||||
rects.push((508.0, 0.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_oversized_span_still_unions() {
|
||||
// Wider than 64 grid cells; must still union the small overlapping rect.
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 10.0), (4900.0, 0.0, 10.0, 10.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 1);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_oversized_spans_all_get_a_pass() {
|
||||
// More than 32 huge rects: the last one must still union its overlap.
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = (0..40)
|
||||
.map(|i| (0.0, i as f32 * 20.0, 5000.0, 10.0))
|
||||
.collect();
|
||||
rects.push((4900.0, 39.0 * 20.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_oversized_not_starved_by_earlier_disjoint() {
|
||||
// 9k earlier disjoint drawings would exhaust an index-order cap of
|
||||
// 8,192 before the overlapping cell is visited.
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = (0..9_000)
|
||||
.map(|i| (10_000.0, i as f32 * 20.0, 10.0, 10.0))
|
||||
.collect();
|
||||
let wide = rects.len();
|
||||
rects.push((0.0, 0.0, 5000.0, 10.0));
|
||||
let target = rects.len();
|
||||
rects.push((4900.0, 0.0, 10.0, 10.0));
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert!(
|
||||
groups
|
||||
.iter()
|
||||
.any(|g| g.contains(&wide) && g.contains(&target)),
|
||||
"wide rule and far-end cell must share a cluster"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_wide_and_tall_oversized_union() {
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 10.0), (0.0, 0.0, 10.0, 5000.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_dual_oversized_spans_coarse_y() {
|
||||
let rects = vec![(0.0, 0.0, 5000.0, 5000.0), (0.0, 4500.0, 5000.0, 5000.0)];
|
||||
let groups = cluster_rects(&rects, 0.0, 2);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cluster_rects_many_wide_and_tall_stays_subquadratic() {
|
||||
let mut rects = Vec::with_capacity(4_000);
|
||||
for i in 0..2_000 {
|
||||
rects.push((0.0, i as f32 * 20.0, 5000.0, 10.0));
|
||||
rects.push((i as f32 * 20.0, 0.0, 10.0, 5000.0));
|
||||
}
|
||||
let _groups = cluster_rects(&rects, 0.0, 2);
|
||||
}
|
||||
|
||||
// --- snap_edges ---
|
||||
|
||||
#[test]
|
||||
|
||||
+141
-20
@@ -540,18 +540,14 @@ impl ToUnicodeCMap {
|
||||
|
||||
/// Remap a CMap that references pre-subsetting GIDs to sequential post-subsetting GIDs.
|
||||
/// Collects all source CIDs, sorts them, and reassigns to 1, 2, 3, ...
|
||||
///
|
||||
/// Range expansion stops after `MAX_CID_W_EXPANSION` CID visits, counting
|
||||
/// overwrites, so repeated full-width `bfrange`s cannot re-expand the
|
||||
/// 16-bit domain. Later overlapping ranges that would have introduced new
|
||||
/// CIDs after that many visits are truncated.
|
||||
pub fn remap_to_sequential(&self) -> ToUnicodeCMap {
|
||||
let mut cid_to_unicode: HashMap<u16, String> = HashMap::new();
|
||||
|
||||
// Expand ranges first
|
||||
for &(start, end, base) in &self.ranges {
|
||||
for cid in start..=end {
|
||||
let unicode_cp = base + (cid - start) as u32;
|
||||
if let Some(ch) = char::from_u32(unicode_cp) {
|
||||
cid_to_unicode.insert(cid, ch.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
expand_bfranges_for_remap(&self.ranges, &mut cid_to_unicode, MAX_CID_W_EXPANSION);
|
||||
|
||||
// char_map entries override range entries
|
||||
for (&cid, unicode) in &self.char_map {
|
||||
@@ -576,6 +572,33 @@ impl ToUnicodeCMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand `bfrange` entries into individual CID→Unicode inserts.
|
||||
/// Returns how many CIDs were visited. Counts overwrites so a repeated
|
||||
/// full-width range cannot keep working after `max_assignments`.
|
||||
fn expand_bfranges_for_remap(
|
||||
ranges: &[(u16, u16, u32)],
|
||||
cid_to_unicode: &mut HashMap<u16, String>,
|
||||
max_assignments: usize,
|
||||
) -> usize {
|
||||
let mut assigned = 0usize;
|
||||
'ranges: for &(start, end, base) in ranges {
|
||||
if start > end {
|
||||
continue;
|
||||
}
|
||||
for cid in start..=end {
|
||||
if assigned >= max_assignments {
|
||||
break 'ranges;
|
||||
}
|
||||
assigned += 1;
|
||||
let unicode_cp = base + (cid - start) as u32;
|
||||
if let Some(ch) = char::from_u32(unicode_cp) {
|
||||
cid_to_unicode.insert(cid, ch.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
assigned
|
||||
}
|
||||
|
||||
/// Parse a hex string to u16
|
||||
fn parse_hex_u16(hex: &str) -> Option<u16> {
|
||||
u16::from_str_radix(hex.trim(), 16).ok()
|
||||
@@ -1561,23 +1584,31 @@ fn parse_encoding_cmap_stream(data: &[u8]) -> Option<EncodingCMap> {
|
||||
}
|
||||
|
||||
let mut map = HashMap::new();
|
||||
let mut assigned = 0usize;
|
||||
let mut pos = 0;
|
||||
while let Some(start) = text[pos..].find("begincidchar") {
|
||||
let section_start = pos + start + "begincidchar".len();
|
||||
if let Some(end) = text[section_start..].find("endcidchar") {
|
||||
let section = &text[section_start..section_start + end];
|
||||
parse_cidchar_section(section, &mut map, &mut src_hex_lengths);
|
||||
if !parse_cidchar_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
|
||||
break;
|
||||
}
|
||||
pos = section_start + end;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
pos = 0;
|
||||
while let Some(start) = text[pos..].find("begincidrange") {
|
||||
while assigned < MAX_CID_W_EXPANSION {
|
||||
let Some(start) = text[pos..].find("begincidrange") else {
|
||||
break;
|
||||
};
|
||||
let section_start = pos + start + "begincidrange".len();
|
||||
if let Some(end) = text[section_start..].find("endcidrange") {
|
||||
let section = &text[section_start..section_start + end];
|
||||
parse_cidrange_section(section, &mut map, &mut src_hex_lengths);
|
||||
if !parse_cidrange_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
|
||||
break;
|
||||
}
|
||||
pos = section_start + end;
|
||||
} else {
|
||||
break;
|
||||
@@ -1612,7 +1643,8 @@ fn parse_cidchar_section(
|
||||
section: &str,
|
||||
map: &mut HashMap<u16, u16>,
|
||||
src_hex_lengths: &mut Vec<usize>,
|
||||
) {
|
||||
assigned: &mut usize,
|
||||
) -> bool {
|
||||
let mut chars = section.chars().peekable();
|
||||
loop {
|
||||
while chars.peek().is_some_and(|c| c.is_whitespace()) {
|
||||
@@ -1643,16 +1675,20 @@ fn parse_cidchar_section(
|
||||
}
|
||||
}
|
||||
if let (Some(code), Ok(cid)) = (parse_hex_u16(&src_hex), cid_str.parse::<u16>()) {
|
||||
map.insert(code, cid);
|
||||
if !assign_encoding_cid(map, code, cid, assigned) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn parse_cidrange_section(
|
||||
section: &str,
|
||||
map: &mut HashMap<u16, u16>,
|
||||
src_hex_lengths: &mut Vec<usize>,
|
||||
) {
|
||||
assigned: &mut usize,
|
||||
) -> bool {
|
||||
let mut chars = section.chars().peekable();
|
||||
loop {
|
||||
while chars.peek().is_some_and(|c| c.is_whitespace()) {
|
||||
@@ -1703,12 +1739,34 @@ fn parse_cidrange_section(
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if start > end {
|
||||
continue;
|
||||
}
|
||||
let mut cid = start_cid;
|
||||
for code in start..=end {
|
||||
map.insert(code, cid);
|
||||
if !assign_encoding_cid(map, code, cid, assigned) {
|
||||
return false;
|
||||
}
|
||||
cid = cid.saturating_add(1);
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn assign_encoding_cid(
|
||||
map: &mut HashMap<u16, u16>,
|
||||
code: u16,
|
||||
cid: u16,
|
||||
assigned: &mut usize,
|
||||
) -> bool {
|
||||
// Count overwrites: unique-key coverage alone would not stop a repeated
|
||||
// full-width range from re-inserting all 65,536 codes.
|
||||
if *assigned >= MAX_CID_W_EXPANSION {
|
||||
return false;
|
||||
}
|
||||
map.insert(code, cid);
|
||||
*assigned += 1;
|
||||
true
|
||||
}
|
||||
|
||||
fn parse_binary_cmap_encoding(data: &[u8]) -> Result<EncodingCMap, String> {
|
||||
@@ -1832,9 +1890,11 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap
|
||||
base
|
||||
}
|
||||
|
||||
/// Upper bound on CID `/W` range expansion. The CID domain is 16-bit, so more
|
||||
/// than 65,536 unique keys cannot exist; repeating full-width ranges must not
|
||||
/// re-expand the same domain.
|
||||
/// Shared 16-bit CID expansion cap (65,536).
|
||||
/// Encoding `begincidrange`, `/W` width assignment, and ToUnicode sequential
|
||||
/// remap count every insert, including overwrites, so a repeated full-width
|
||||
/// range cannot keep working after the domain is filled. The `/W` unicode
|
||||
/// heuristic caps unique CIDs with the same number.
|
||||
pub(crate) const MAX_CID_W_EXPANSION: usize = 65_536;
|
||||
|
||||
/// Check if a CIDFont's /W (widths) array contains CID values that look like
|
||||
@@ -2887,6 +2947,33 @@ endbfrange
|
||||
assert!(remapped.ranges.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remap_to_sequential_repeated_full_bfranges_stay_bounded() {
|
||||
// 5,000 copies of `<0003> <ffff>` must stop after 65,536 CID visits,
|
||||
// not 5,000 × ~65,533 expansions.
|
||||
let ranges = vec![(3u16, 65535u16, 0x41u32); 5_000];
|
||||
let mut map = std::collections::HashMap::new();
|
||||
let assigned = expand_bfranges_for_remap(&ranges, &mut map, MAX_CID_W_EXPANSION);
|
||||
assert_eq!(assigned, MAX_CID_W_EXPANSION);
|
||||
assert!(map.len() <= MAX_CID_W_EXPANSION);
|
||||
|
||||
let mut body = String::new();
|
||||
let mut remaining = 5_000usize;
|
||||
while remaining > 0 {
|
||||
let n = remaining.min(100);
|
||||
body.push_str(&format!("{n} beginbfrange\n"));
|
||||
for _ in 0..n {
|
||||
body.push_str("<0003> <ffff> <0041>\n");
|
||||
}
|
||||
body.push_str("endbfrange\n");
|
||||
remaining -= n;
|
||||
}
|
||||
let data = format!("1 begincodespacerange\n<0000> <ffff>\nendcodespacerange\n{body}");
|
||||
let cmap = ToUnicodeCMap::parse(data.as_bytes()).unwrap();
|
||||
let remapped = cmap.remap_to_sequential();
|
||||
assert_eq!(remapped.lookup(1), Some("A".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_min_source_cid() {
|
||||
let cmap_content = r#"
|
||||
@@ -3359,4 +3446,38 @@ endbfrange
|
||||
dict.set("W", Object::Array(w));
|
||||
assert!(cid_values_look_like_unicode(&dict));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding_cidrange_maps_a_normal_range() {
|
||||
let data = b"1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n\
|
||||
1 begincidrange\n<0041> <0043> 65\nendcidrange\n";
|
||||
let enc = parse_encoding_cmap_stream(data).unwrap();
|
||||
assert_eq!(enc.map.get(&0x41), Some(&65));
|
||||
assert_eq!(enc.map.get(&0x42), Some(&66));
|
||||
assert_eq!(enc.map.get(&0x43), Some(&67));
|
||||
assert_eq!(enc.map.len(), 3);
|
||||
assert_eq!(enc.code_byte_length, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding_cidrange_repeated_full_ranges_stay_bounded() {
|
||||
// 5,000 copies of `<0000> <ffff> 0` must not re-expand the 16-bit
|
||||
// domain on every declaration.
|
||||
let mut body = String::new();
|
||||
let mut remaining = 5_000usize;
|
||||
while remaining > 0 {
|
||||
let n = remaining.min(100);
|
||||
body.push_str(&format!("{n} begincidrange\n"));
|
||||
for _ in 0..n {
|
||||
body.push_str("<0000> <ffff> 0\n");
|
||||
}
|
||||
body.push_str("endcidrange\n");
|
||||
remaining -= n;
|
||||
}
|
||||
let data = format!("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n{body}");
|
||||
let enc = parse_encoding_cmap_stream(data.as_bytes()).unwrap();
|
||||
assert!(enc.map.len() <= MAX_CID_W_EXPANSION);
|
||||
assert_eq!(enc.map.get(&0), Some(&0));
|
||||
assert_eq!(enc.map.get(&65535), Some(&65535));
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.14.1"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Browser WebAssembly bindings for pdf-inspector"
|
||||
|
||||
Reference in New Issue
Block a user