Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
acebe6f7fa | ||
|
Abimael Martell
|
c248eeff8f |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.8.10",
|
||||
"version": "1.8.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { detectVectorGridInRegion } = require("./index.js");
|
||||
|
||||
const pdfPath =
|
||||
process.argv[2] ?? "/tmp/pdf_inspector_indent_fixtures/cis_edge_benchmark.pdf";
|
||||
const pdf = readFileSync(pdfPath);
|
||||
const dpi = Number(process.argv[3] ?? 200);
|
||||
|
||||
const crops = [
|
||||
{ pageIdx: 29, box: [0, 0, 612, 792], label: "page30-full" },
|
||||
{ pageIdx: 16, box: [0, 0, 612, 792], label: "page17-full" },
|
||||
{ pageIdx: 23, box: [0, 0, 612, 792], label: "page24-full" },
|
||||
];
|
||||
|
||||
for (const { pageIdx, box, label } of crops) {
|
||||
const result = detectVectorGridInRegion(pdf, pageIdx, box, dpi);
|
||||
if (!result) {
|
||||
console.log(`${label}: null`);
|
||||
continue;
|
||||
}
|
||||
const rows = result.structureTokens.filter((token) => token === "<tr>").length;
|
||||
const cols = rows > 0 ? result.cellBboxes.length / rows : 0;
|
||||
console.log(
|
||||
`${label}: cells=${result.cellBboxes.length} rows=${rows} cols=${cols}`,
|
||||
);
|
||||
}
|
||||
@@ -385,7 +385,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
@@ -534,7 +533,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
@@ -622,7 +620,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
if !text.trim().is_empty() {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
@@ -1015,17 +1012,9 @@ pub(crate) fn extract_page_text_items(
|
||||
// producing thousands of identical rects that yield a degenerate grid.
|
||||
// After dedup, if too few unique clip rects remain we fall through to
|
||||
// fill rects (explicitly drawn visible rectangles).
|
||||
//
|
||||
// When fill rects substantially outnumber clip rects, the clips are
|
||||
// typically section-level wrappers and the fills are the actual table
|
||||
// cell backgrounds (e.g. shaded-header tables drawn with `m`/`l`/`h`/`f*`
|
||||
// sequences). In that case, prefer fills.
|
||||
if rects.is_empty() {
|
||||
dedup_rects(&mut clip_rects);
|
||||
let prefer_fills = !fill_rects.is_empty() && fill_rects.len() >= clip_rects.len() * 3;
|
||||
if prefer_fills {
|
||||
rects = fill_rects;
|
||||
} else if clip_rects.len() >= 4 {
|
||||
if clip_rects.len() >= 4 {
|
||||
rects = clip_rects;
|
||||
} else if !fill_rects.is_empty() {
|
||||
rects = fill_rects;
|
||||
|
||||
+1
-122
@@ -720,11 +720,7 @@ pub(crate) fn extract_text_from_operand(
|
||||
font_encodings: &PageFontEncodings,
|
||||
encoding_cache: &HashMap<String, Encoding<'_>>,
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
font_widths: &PageFontWidths,
|
||||
) -> Option<String> {
|
||||
let is_type0_cid_font = font_widths
|
||||
.get(current_font)
|
||||
.is_some_and(|info| info.is_cid);
|
||||
let result = (|| -> Option<String> {
|
||||
if let Object::String(bytes, _) = obj {
|
||||
let mut decode_with_entry = |entry: &crate::tounicode::CMapEntry| -> Option<String> {
|
||||
@@ -966,31 +962,7 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
// canonical interpretation.
|
||||
// Latin-1 fallback
|
||||
Some(bytes.iter().map(|&b| b as char).collect())
|
||||
} else {
|
||||
None
|
||||
@@ -1241,97 +1213,4 @@ mod tests {
|
||||
let bad = "###!!!@@@$$$";
|
||||
assert!(score_text(good) > score_text(bad));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cid_font_with_unparseable_cmap_does_not_emit_latin1_mojibake() {
|
||||
// Type0/CID font (font_widths reports `is_cid=true`) where the
|
||||
// ToUnicode CMap couldn't be parsed (FontCMaps doesn't have the
|
||||
// obj_num). Bytes are a 2-byte CID stream containing high bytes
|
||||
// that aren't valid UTF-8 — exactly the case in the production
|
||||
// samples (Identity-H text where the ToUnicode CMap was missing
|
||||
// or malformed, scrape_id 019de78c-..., e.g. "Í Ù Z)¿").
|
||||
//
|
||||
// Without the guard, the function falls through to the byte-by-byte
|
||||
// Latin-1 fallback and produces "ÍÙ" (U+00CD U+00D9). The correct
|
||||
// behavior is to emit U+FFFD per CID so downstream
|
||||
// `detect_encoding_issues` flags the page for OCR.
|
||||
let bytes = vec![0xCD_u8, 0xD9, 0xCD, 0xD9];
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
let font_cmaps = FontCMaps::default();
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
font_tounicode_refs.insert("F0".to_string(), 999);
|
||||
let inline_cmaps = HashMap::new();
|
||||
let font_encodings: PageFontEncodings = HashMap::new();
|
||||
let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
|
||||
let mut decisions = CMapDecisionCache::new();
|
||||
let mut font_widths: PageFontWidths = HashMap::new();
|
||||
font_widths.insert("F0".to_string(), make_font_info(&[], 1000, true));
|
||||
|
||||
let result = extract_text_from_operand(
|
||||
&obj,
|
||||
"F0",
|
||||
None,
|
||||
&font_cmaps,
|
||||
&font_tounicode_refs,
|
||||
&inline_cmaps,
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut decisions,
|
||||
&font_widths,
|
||||
);
|
||||
|
||||
let text = result.expect("CID font fallback should still emit a marker");
|
||||
assert!(
|
||||
!text.contains('\u{00CD}') && !text.contains('\u{00D9}'),
|
||||
"CID font with unparseable CMap leaked Latin-1 mojibake: {text:?}"
|
||||
);
|
||||
assert!(
|
||||
text.contains('\u{FFFD}'),
|
||||
"CID font with unparseable CMap should emit U+FFFD so detect_encoding_issues fires: {text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_font_latin1_fallback_passes_high_bytes_through() {
|
||||
// A Type1/TrueType simple font (is_cid=false) with a `/ToUnicode`
|
||||
// reference but no usable CMap and no `/Differences` map.
|
||||
// Per-byte Latin-1 IS the canonical interpretation here — these
|
||||
// bytes are character codes, not CIDs. The CID guard must NOT
|
||||
// strip them. Reproduces the false positive that an earlier
|
||||
// version of the guard introduced for fonts in PDFs like
|
||||
// pdf-evals/Navigating-Artificial-Intelligence-..., where bytes
|
||||
// like 0xB6 are legitimate Latin-1 character codes.
|
||||
let bytes = vec![0x24_u8, 0x47, 0xB6, 0x56]; // "$G¶V"
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
let font_cmaps = FontCMaps::default();
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
font_tounicode_refs.insert("F1".to_string(), 999);
|
||||
let inline_cmaps = HashMap::new();
|
||||
let font_encodings: PageFontEncodings = HashMap::new();
|
||||
let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
|
||||
let mut decisions = CMapDecisionCache::new();
|
||||
let mut font_widths: PageFontWidths = HashMap::new();
|
||||
font_widths.insert("F1".to_string(), make_font_info(&[], 1000, false));
|
||||
|
||||
let text = extract_text_from_operand(
|
||||
&obj,
|
||||
"F1",
|
||||
None,
|
||||
&font_cmaps,
|
||||
&font_tounicode_refs,
|
||||
&inline_cmaps,
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut decisions,
|
||||
&font_widths,
|
||||
)
|
||||
.expect("simple font should round-trip Latin-1 bytes");
|
||||
assert_eq!(text, "$G\u{00B6}V");
|
||||
assert!(
|
||||
!text.contains('\u{FFFD}'),
|
||||
"simple font fallback must not stamp FFFD over legitimate bytes: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,7 +373,6 @@ fn extract_form_xobject_text_inner(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
@@ -518,7 +517,6 @@ fn extract_form_xobject_text_inner(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
|
||||
+91
-523
@@ -644,8 +644,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut rects_by_page: HashMap<u32, Vec<PdfRect>> = HashMap::new();
|
||||
let mut lines_by_page: HashMap<u32, Vec<PdfLine>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
@@ -658,7 +656,7 @@ pub fn extract_tables_in_regions_mem(
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
let ((mut items, rects, lines), has_gid, coords_rotated) =
|
||||
let ((mut items, _rects, _lines), has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
@@ -677,8 +675,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
rotated_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
rects_by_page.insert(*page_num, rects);
|
||||
lines_by_page.insert(*page_num, lines);
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
@@ -708,13 +704,15 @@ pub fn extract_tables_in_regions_mem(
|
||||
// content. This avoids rejecting clean tables just because an
|
||||
// unrelated decorative font on the same page is GID-encoded.
|
||||
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
let matched: Vec<TextItem> = match items {
|
||||
Some(items) => items
|
||||
.iter()
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect(),
|
||||
Some(items) => {
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
items
|
||||
.iter()
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
@@ -738,82 +736,42 @@ pub fn extract_tables_in_regions_mem(
|
||||
.unwrap_or(12.0)
|
||||
};
|
||||
|
||||
// Try rect-backed and line-backed vector-grid detectors first,
|
||||
// then fall back to the heuristic text-only detector. Each
|
||||
// candidate's markdown is quality-gated by the same
|
||||
// needs_ocr checks the heuristic-only path used: if a vector
|
||||
// detector produces a partial/garbled table, we ignore it and
|
||||
// try the next path rather than degrade the output.
|
||||
// needs_ocr fires on any of:
|
||||
// - garbage text (non-alphanumeric heavy)
|
||||
// - CID/Latin-1 mojibake
|
||||
// - encoding issues (U+FFFD, dollar-as-space)
|
||||
// - structural giveaways that the table is partial /
|
||||
// mis-detected (numeric "header", empty header cells,
|
||||
// duplicate header cells).
|
||||
// skip_body_font = false / layout_assisted = true because the
|
||||
// layout model already identified this region as a table.
|
||||
let region_rects: Vec<PdfRect> = rects_by_page
|
||||
.get(&page_1idx)
|
||||
.map(|rs| {
|
||||
rs.iter()
|
||||
.filter(|r| region_overlaps_rect(r, bounds))
|
||||
.cloned()
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let region_lines: Vec<PdfLine> = lines_by_page
|
||||
.get(&page_1idx)
|
||||
.map(|ls| {
|
||||
ls.iter()
|
||||
.filter(|l| region_overlaps_line(l, bounds))
|
||||
.cloned()
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
// Run heuristic table detection; skip_body_font = false since
|
||||
// the layout model already identified this region as a table.
|
||||
let detected = tables::detect_tables(&matched, base_font_size, false);
|
||||
|
||||
let evaluate = |t: &tables::Table| -> Option<String> {
|
||||
let md = tables::table_to_markdown(t);
|
||||
let trimmed = md.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if is_garbage_text(&md)
|
||||
|| is_cid_garbage(&md)
|
||||
|| detect_encoding_issues(&md)
|
||||
|| looks_like_partial_table_ex(&md, true)
|
||||
{
|
||||
None
|
||||
if let Some(table) = detected.into_iter().next() {
|
||||
let md = tables::table_to_markdown(&table);
|
||||
if md.trim().is_empty() {
|
||||
page_results.push(RegionText {
|
||||
text: String::new(),
|
||||
needs_ocr: true,
|
||||
});
|
||||
} else {
|
||||
Some(md)
|
||||
// needs_ocr fires on any of:
|
||||
// - garbage text (non-alphanumeric heavy)
|
||||
// - CID/Latin-1 mojibake
|
||||
// - encoding issues (U+FFFD, dollar-as-space)
|
||||
// - structural giveaways that the table is partial /
|
||||
// mis-detected (numeric "header", empty header cells,
|
||||
// duplicate header cells). Caught GLM-OCR-as-baseline
|
||||
// scoring 0 TEDS on real prod tables in eval.
|
||||
// Layout model already identified this region as a table,
|
||||
// so use relaxed partial-table checks (layout_assisted=true).
|
||||
let needs_ocr = is_garbage_text(&md)
|
||||
|| is_cid_garbage(&md)
|
||||
|| detect_encoding_issues(&md)
|
||||
|| looks_like_partial_table_ex(&md, true);
|
||||
page_results.push(RegionText {
|
||||
text: if needs_ocr { String::new() } else { md },
|
||||
needs_ocr,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let mut accepted_md: Option<String> = None;
|
||||
if !region_rects.is_empty() {
|
||||
let (rect_tables, _) =
|
||||
tables::detect_tables_from_rects(&matched, ®ion_rects, page_1idx);
|
||||
accepted_md = rect_tables.iter().find_map(&evaluate);
|
||||
}
|
||||
if accepted_md.is_none() && !region_lines.is_empty() {
|
||||
let line_tables =
|
||||
tables::detect_tables_from_lines(&matched, ®ion_lines, page_1idx);
|
||||
accepted_md = line_tables.iter().find_map(&evaluate);
|
||||
}
|
||||
if accepted_md.is_none() {
|
||||
let detected = tables::detect_tables(&matched, base_font_size, false);
|
||||
accepted_md = detected.iter().find_map(&evaluate);
|
||||
}
|
||||
|
||||
match accepted_md {
|
||||
Some(md) => page_results.push(RegionText {
|
||||
text: md,
|
||||
needs_ocr: false,
|
||||
}),
|
||||
None => page_results.push(RegionText {
|
||||
} else {
|
||||
page_results.push(RegionText {
|
||||
text: String::new(),
|
||||
needs_ocr: true,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1077,291 +1035,6 @@ mod vector_grid_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Helper: load a fixture PDF and run the rect-based table detector.
|
||||
fn detect_rect_tables_in_fixture_page(path: &str, page_num: u32) -> Vec<crate::tables::Table> {
|
||||
use crate::extractor::content_stream::extract_page_text_items;
|
||||
use crate::tables::detect_tables_from_rects;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use lopdf::Document;
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
|
||||
let buf = fs::read(path).unwrap();
|
||||
let doc = Document::load_mem(&buf).unwrap();
|
||||
let pages = doc.get_pages();
|
||||
let &page_id = pages.get(&page_num).unwrap();
|
||||
let needed: HashSet<u32> = HashSet::from([page_num]);
|
||||
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((items, rects, _lines), _has_gid, _rotated) =
|
||||
extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap();
|
||||
|
||||
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num);
|
||||
rect_tables
|
||||
}
|
||||
|
||||
fn detect_rect_tables_in_fixture(path: &str) -> Vec<crate::tables::Table> {
|
||||
detect_rect_tables_in_fixture_page(path, 1)
|
||||
}
|
||||
|
||||
/// Regression for the prose-in-a-frame failure mode introduced by the
|
||||
/// shaded-header detection lift (PR #76). The accessory_building permit
|
||||
/// form has a paragraph of legal text laid out in a 2-column justified
|
||||
/// block; the new fill-priority + dedup changes start producing rects
|
||||
/// for it, and the rect detector then admits a 10×2 fake table where
|
||||
/// every cell holds a sentence fragment ("I agree to comply...", "I",
|
||||
/// "It is the property owner's responsibility..."). This test asserts
|
||||
/// the detector REJECTS that fake table — only the real 5×3 form data
|
||||
/// table (TYPE / SIZE / SETBACKS) should survive. See pdf-evals PR #30
|
||||
/// for the original score regression that surfaced this.
|
||||
#[test]
|
||||
fn accessory_building_rejects_prose_in_frame() {
|
||||
let tables = detect_rect_tables_in_fixture(
|
||||
"tests/fixtures/accessory_building_permit_prose_frame.pdf",
|
||||
);
|
||||
// Real form data table (TYPE / SIZE / SETBACKS) must still be detected.
|
||||
let data_table = tables.iter().find(|t| t.columns.len() == 3);
|
||||
assert!(
|
||||
data_table.is_some(),
|
||||
"expected to keep the 5×3 TYPE/SIZE/SETBACKS data table; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
// Prose paragraph laid out in 2 cols must NOT be detected as a table.
|
||||
// If the rejection regresses, the 10×2 fake table reappears and
|
||||
// produces fragmented markdown like "I agree to comply..." | "I"
|
||||
// that fragments mid-sentence.
|
||||
let prose_table = tables.iter().find(|t| t.columns.len() == 2);
|
||||
assert!(
|
||||
prose_table.is_none(),
|
||||
"expected the 10×2 prose-in-frame block to be rejected; got rows×cols = {:?}",
|
||||
prose_table.map(|t| (t.rows.len(), t.columns.len()))
|
||||
);
|
||||
}
|
||||
|
||||
/// Wireless table regression: decorative/text-region rects may provide row
|
||||
/// bands, but without a real rect-derived column scaffold they must not be
|
||||
/// accepted as a vector grid.
|
||||
#[test]
|
||||
fn wireless_two_col_rejects_rect_grid() {
|
||||
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_two_col_no_rects.pdf");
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"expected no rect-detected tables for wireless content; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wireless_two_col_region_rejects_vector_grid() {
|
||||
let buf = std::fs::read("tests/fixtures/wireless_two_col_no_rects.pdf").unwrap();
|
||||
let crops = [
|
||||
[49.32_f32, 52.92, 558.72, 214.2],
|
||||
[49.32_f32, 288.72, 556.56, 378.0],
|
||||
[51.48_f32, 478.44, 558.36, 567.36],
|
||||
];
|
||||
for crop in crops {
|
||||
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
|
||||
assert!(
|
||||
detected.is_none(),
|
||||
"expected no vector grid for wireless crop {crop:?}; got {} cells",
|
||||
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Wireless dense table regression: text-position columns alone are not
|
||||
/// enough evidence for a rect-derived grid.
|
||||
#[test]
|
||||
fn wireless_dense_rejects_rect_grid() {
|
||||
let tables = detect_rect_tables_in_fixture("tests/fixtures/wireless_dense_no_rects.pdf");
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"expected no rect-detected tables for wireless content; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wireless_dense_region_rejects_vector_grid() {
|
||||
let buf = std::fs::read("tests/fixtures/wireless_dense_no_rects.pdf").unwrap();
|
||||
let crops = [
|
||||
[72.36_f32, 177.48, 243.72, 333.36],
|
||||
[72.0_f32, 390.24, 286.92, 417.6],
|
||||
];
|
||||
for crop in crops {
|
||||
let detected = crate::detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0).unwrap();
|
||||
assert!(
|
||||
detected.is_none(),
|
||||
"expected no vector grid for wireless crop {crop:?}; got {} cells",
|
||||
detected.map(|grid| grid.cell_bboxes.len()).unwrap_or(0)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiline_indent_cell_rect_grid_fixture_detects_table() {
|
||||
let tables = detect_rect_tables_in_fixture_page(
|
||||
"tests/fixtures/multiline_indent_cell_rect_grid.pdf",
|
||||
30,
|
||||
);
|
||||
let table = tables
|
||||
.iter()
|
||||
.max_by_key(|t| t.rows.len() * t.columns.len())
|
||||
.expect("expected a rect-detected table");
|
||||
assert_eq!(
|
||||
table.columns.len(),
|
||||
5,
|
||||
"expected the Controls Version / Control / IG table shape; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
assert!(
|
||||
table.rows.len() >= 3,
|
||||
"expected at least header plus data rows; got {}",
|
||||
table.rows.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiline_indent_cell_rect_grid_region_detects_vector_grid() {
|
||||
let buf = std::fs::read("tests/fixtures/multiline_indent_cell_rect_grid.pdf").unwrap();
|
||||
let detected =
|
||||
crate::detect_vector_grid_in_region_mem(&buf, 29, [0.0, 0.0, 612.0, 792.0], 200.0)
|
||||
.unwrap()
|
||||
.expect("expected vector grid for multiline indented description table");
|
||||
let rows = detected
|
||||
.structure_tokens
|
||||
.iter()
|
||||
.filter(|token| token.as_str() == "<tr>")
|
||||
.count();
|
||||
assert_eq!(detected.cell_bboxes.len() % rows, 0);
|
||||
assert_eq!(detected.cell_bboxes.len() / rows, 5);
|
||||
assert!(rows >= 3);
|
||||
assert!(!detected.cell_bboxes.is_empty());
|
||||
}
|
||||
|
||||
/// Regression for `greencomp_competence.pdf` — a 2-column "Area / Competence"
|
||||
/// glossary with a green-shaded header row and plain (line-drawn) body cells.
|
||||
/// Mirrors the production failure cohort #1 (Contractions glossary) and #6
|
||||
/// (BIO 350 course header): a few colored header rects sit in a horizontal
|
||||
/// strip while body rows are drawn with `m`/`l` operators, so the rect
|
||||
/// cluster has only 2 Y-edges and `try_build_grid` rejects.
|
||||
///
|
||||
/// IGNORED: lifting this shape required the exact-duplicate early-dedup
|
||||
/// (PR #76 first iteration), which had broad collateral damage on
|
||||
/// SEC 10-K TOCs and similar docs that draw rule-rects above + below
|
||||
/// section dividers (production diff: 0001104659-25-093871 lost its
|
||||
/// TOC structure, perf-graph data table, and qualifications matrix).
|
||||
/// Re-enable once a more surgical lift exists in `try_build_grid` or
|
||||
/// `snap_edges` that handles cell-border + inner-fill + text-bg rect
|
||||
/// triplets without page-wide dedup.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn greencomp_competence_two_cols() {
|
||||
let tables = detect_rect_tables_in_fixture("tests/fixtures/greencomp_competence.pdf");
|
||||
assert!(
|
||||
!tables.is_empty(),
|
||||
"expected at least one rect-detected table for shaded-header + plain-body shape"
|
||||
);
|
||||
let t = tables
|
||||
.iter()
|
||||
.max_by_key(|t| t.rows.len() * t.columns.len())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
t.columns.len(),
|
||||
2,
|
||||
"GreenComp competence is a 2-column table; got {}: {:?}",
|
||||
t.columns.len(),
|
||||
t.columns
|
||||
);
|
||||
assert!(
|
||||
t.rows.len() >= 6,
|
||||
"expected at least 6 rows of competences; got {}",
|
||||
t.rows.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for `upstage_key_functions.pdf` — a 4-column "Service Stage /
|
||||
/// Function Name / Explanation / Expected Benefit" table with a blue-shaded
|
||||
/// header band plus alternating row backgrounds. Mirrors production crops
|
||||
/// #2 (Parameter / Value with alternating blue rows) and #7 (Spanish XML
|
||||
/// schema with shaded header). Currently `pdf2md` returns zero markdown
|
||||
/// table rows.
|
||||
#[test]
|
||||
fn upstage_key_functions_four_cols() {
|
||||
let tables = detect_rect_tables_in_fixture("tests/fixtures/upstage_key_functions.pdf");
|
||||
assert!(
|
||||
!tables.is_empty(),
|
||||
"expected at least one rect-detected table for shaded-header + alt-row shape"
|
||||
);
|
||||
let t = tables
|
||||
.iter()
|
||||
.max_by_key(|t| t.rows.len() * t.columns.len())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
t.columns.len(),
|
||||
4,
|
||||
"Service Flow is a 4-column table; got {}: {:?}",
|
||||
t.columns.len(),
|
||||
t.columns
|
||||
);
|
||||
assert!(
|
||||
t.rows.len() >= 8,
|
||||
"expected at least 8 visible body rows; got {}",
|
||||
t.rows.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for `wired_header_data_misalign.pdf` — a single page from a
|
||||
/// parts catalog with a 4-column wire-bordered table (`Item | EAN | Nombre
|
||||
/// | Cant`). Column headers are centered/right-aligned inside their cells
|
||||
/// while data is left-aligned, so cluster_x_positions merges or drops
|
||||
/// columns and the cell-rect fallback used to assign text to the wrong
|
||||
/// columns (lost a column, fragmented neighbor cells). The fix prefers
|
||||
/// rect-border-derived column edges when they're well-distributed across
|
||||
/// the actual text items. This test asserts the detector keeps all 4
|
||||
/// columns and every column ends up populated.
|
||||
#[test]
|
||||
fn wired_header_data_misalign_keeps_all_columns() {
|
||||
let tables = detect_rect_tables_in_fixture("tests/fixtures/wired_header_data_misalign.pdf");
|
||||
let table = tables
|
||||
.iter()
|
||||
.find(|t| t.columns.len() == 4 && t.rows.len() >= 5)
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"expected a 4-column ≥5-row table; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
)
|
||||
});
|
||||
for c in 0..4 {
|
||||
let populated_rows = table
|
||||
.cells
|
||||
.iter()
|
||||
.filter(|row| !row[c].trim().is_empty())
|
||||
.count();
|
||||
assert!(
|
||||
populated_rows >= 2,
|
||||
"column {} only populated in {} rows; cells: {:?}",
|
||||
c,
|
||||
populated_rows,
|
||||
table.cells
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crop_px_bbox_is_plausible_bounds() {
|
||||
let crop = [10.0, 20.0, 110.0, 220.0];
|
||||
@@ -1883,32 +1556,26 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
// Stage 1: exclusive per-token assignment. Each PDF text item is
|
||||
// first split into whitespace-separated tokens with estimated x
|
||||
// positions (see `split_item_into_token_subitems`). For each token
|
||||
// we find the cell(s) whose (band-clamped) bbox satisfies the
|
||||
// strict membership rule (`tsr_region_contains_item`: center
|
||||
// inside OR >=60% overlap on both axes). If multiple cells
|
||||
// qualify, the closest-center wins. Tokens that don't land in any
|
||||
// cell are eligible for stage-2 orphan recovery.
|
||||
// Stage 1: exclusive per-item assignment. For each PDF text item,
|
||||
// find the cell(s) whose (band-clamped) bbox satisfies the strict
|
||||
// membership rule (`tsr_region_contains_item`: center inside OR
|
||||
// >=60% overlap on both axes). If multiple cells qualify, assign
|
||||
// the item to the cell whose center is geometrically closest. If
|
||||
// exactly one qualifies, assign to that. If none, the item is an
|
||||
// orphan and stage 2 below tries to recover it.
|
||||
//
|
||||
// Per-token (rather than per-item) routing is what prevents the
|
||||
// dense-grid collapse bug: a row rendered as one wide Tj
|
||||
// ("Marshall Islands 0.9 0.9 0.9") produces one TextItem whose
|
||||
// CENTER lies in only one cell, so per-item routing parks the
|
||||
// whole row in that single cell and leaves the rest of the row
|
||||
// empty. Per-token routing distributes the words to whichever
|
||||
// cells their (estimated) positions fall into. Single-token items
|
||||
// collapse to a one-element token list and behave exactly as
|
||||
// before.
|
||||
//
|
||||
// The token-level exclusivity (one token → one cell) still
|
||||
// prevents the cell-overlap bug where SLANet emits cells whose
|
||||
// y-extents overlap between rows. Closest-center disambiguation
|
||||
// routes each token to the correct row.
|
||||
// The exclusivity (one item → one cell) prevents the cell-overlap
|
||||
// bug where SLANet emits cells whose y-extents overlap between
|
||||
// rows: under the previous "for each cell, gather items" approach,
|
||||
// an item whose center fell in two cells' overlap got duplicated
|
||||
// into both. Closest-center disambiguation routes it to the
|
||||
// correct row.
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
let mut item_to_cell: std::collections::HashMap<usize, usize> =
|
||||
std::collections::HashMap::new();
|
||||
|
||||
// Pre-compute each cell's bounds + center (in PDF-pt-flipped space)
|
||||
// so we don't redo the work per token.
|
||||
// so we don't redo the work per item.
|
||||
let cell_meta: Vec<Option<(RegionBounds, f32, f32)>> = cells
|
||||
.iter()
|
||||
.map(|cell| {
|
||||
@@ -1923,53 +1590,44 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut per_cell_items: Vec<Vec<TextItem>> = vec![Vec::new(); cells.len()];
|
||||
// Tokens of an item that did NOT land in any cell during stage 1.
|
||||
// These are the orphan candidates handed to `tsr_assign_orphan_items`.
|
||||
// Using token-grain orphan candidates (rather than the original wide
|
||||
// item) lets stage 2 recover individual words that fell just outside
|
||||
// their cell's clamped band, without re-attributing already-claimed
|
||||
// tokens.
|
||||
let mut orphan_token_subitems: Vec<TextItem> = Vec::new();
|
||||
|
||||
for item in items.iter() {
|
||||
let token_subitems = split_item_into_token_subitems(item);
|
||||
for token_item in token_subitems {
|
||||
let token_w = text_utils::effective_width(&token_item);
|
||||
let token_cx = token_item.x + token_w * 0.5;
|
||||
let token_cy = token_item.y + token_item.height * 0.5;
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
for (cell_idx, meta) in cell_meta.iter().enumerate() {
|
||||
let Some((bounds, ccx, ccy)) = meta else {
|
||||
continue;
|
||||
};
|
||||
if !tsr_region_contains_item(&token_item, *bounds) {
|
||||
continue;
|
||||
}
|
||||
let dx = token_cx - ccx;
|
||||
let dy = token_cy - ccy;
|
||||
let dist_sq = dx * dx + dy * dy;
|
||||
if best.is_none_or(|(_, d)| dist_sq < d) {
|
||||
best = Some((cell_idx, dist_sq));
|
||||
}
|
||||
for (item_idx, item) in items.iter().enumerate() {
|
||||
let item_w = text_utils::effective_width(item);
|
||||
let item_cx = item.x + item_w * 0.5;
|
||||
let item_cy = item.y + item.height * 0.5;
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
for (cell_idx, meta) in cell_meta.iter().enumerate() {
|
||||
let Some((bounds, ccx, ccy)) = meta else {
|
||||
continue;
|
||||
};
|
||||
if !tsr_region_contains_item(item, *bounds) {
|
||||
continue;
|
||||
}
|
||||
if let Some((ci, _)) = best {
|
||||
per_cell_items[ci].push(token_item);
|
||||
} else {
|
||||
orphan_token_subitems.push(token_item);
|
||||
let dx = item_cx - ccx;
|
||||
let dy = item_cy - ccy;
|
||||
let dist_sq = dx * dx + dy * dy;
|
||||
if best.is_none_or(|(_, d)| dist_sq < d) {
|
||||
best = Some((cell_idx, dist_sq));
|
||||
}
|
||||
}
|
||||
if let Some((ci, _)) = best {
|
||||
claimed.insert(item_idx);
|
||||
item_to_cell.insert(item_idx, ci);
|
||||
}
|
||||
}
|
||||
|
||||
// Build per-cell text from the assigned tokens. Markdown cells must
|
||||
// Build per-cell text from the assigned items. Markdown cells must
|
||||
// be one line — collapse line breaks from the line-grouping pass.
|
||||
let mut per_cell_items: Vec<Vec<TextItem>> = vec![Vec::new(); cells.len()];
|
||||
for (&item_idx, &cell_idx) in &item_to_cell {
|
||||
per_cell_items[cell_idx].push(items[item_idx].clone());
|
||||
}
|
||||
for (cell_idx, matched) in per_cell_items.into_iter().enumerate() {
|
||||
cells[cell_idx].text = collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
.replace(['\n', '\r'], " ");
|
||||
}
|
||||
|
||||
// Stage 2: orphan assignment — tokens that didn't land in any cell
|
||||
// during stage 1 get assigned to their nearest *empty* cell,
|
||||
// Stage 2: orphan assignment — text items that didn't land in any
|
||||
// cell during stage 1 get assigned to their nearest *empty* cell,
|
||||
// clamped by a plausibility cap derived from cell geometry.
|
||||
//
|
||||
// This recovers two failure modes left by `normalize_cell_bands`:
|
||||
@@ -1981,13 +1639,7 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
// Empty-cell-only is the safety net: a cell already filled by stage 1
|
||||
// is never overwritten or augmented, so the cell-bleed case PR #62
|
||||
// closed cannot regress.
|
||||
tsr_assign_orphan_items(
|
||||
&orphan_token_subitems,
|
||||
&mut cells,
|
||||
&std::collections::HashSet::new(),
|
||||
page_h,
|
||||
coords,
|
||||
);
|
||||
tsr_assign_orphan_items(items, &mut cells, &claimed, page_h, coords);
|
||||
|
||||
results.push(cells);
|
||||
}
|
||||
@@ -1995,70 +1647,6 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Split a `TextItem` into one virtual sub-item per whitespace-separated
|
||||
/// token, with each token's `x` / `width` estimated from the original item's
|
||||
/// effective width and the token's character offset.
|
||||
///
|
||||
/// PDFs often render an entire row's content as a single Tj — e.g.
|
||||
/// "Marshall Islands 0.9 0.9 0.9" — producing one wide TextItem whose
|
||||
/// center sits in only one of the model-emitted cells. Per-item routing
|
||||
/// then parks the whole row in that one cell. Splitting on whitespace
|
||||
/// gives each word its own approximate position so per-cell routing can
|
||||
/// distribute the words to whichever cells their estimated centers fall
|
||||
/// into.
|
||||
///
|
||||
/// The character-width estimate is `effective_width / char_count`.
|
||||
/// `effective_width` returns the explicit `item.width` when known and
|
||||
/// otherwise falls back to `char_count * font_size * 0.5`. Either way the
|
||||
/// estimate is uniform across the item — fine for routing, since we only
|
||||
/// need to know which cell each token's center lands in, not its exact
|
||||
/// position. Single-token items collapse to a one-element vector
|
||||
/// equivalent to the input item, making this a no-op for the common case.
|
||||
fn split_item_into_token_subitems(item: &TextItem) -> Vec<TextItem> {
|
||||
let total_chars = item.text.chars().count();
|
||||
if total_chars == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let item_w = text_utils::effective_width(item);
|
||||
let char_w = item_w / total_chars as f32;
|
||||
|
||||
let mut tokens: Vec<TextItem> = Vec::new();
|
||||
let mut current_token = String::new();
|
||||
let mut current_start_idx: Option<usize> = None;
|
||||
|
||||
let push_token =
|
||||
|tokens: &mut Vec<TextItem>, text: String, start_idx: usize, end_idx: usize| {
|
||||
if text.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut sub = item.clone();
|
||||
sub.text = text;
|
||||
sub.x = item.x + start_idx as f32 * char_w;
|
||||
sub.width = (end_idx - start_idx) as f32 * char_w;
|
||||
tokens.push(sub);
|
||||
};
|
||||
|
||||
for (idx, ch) in item.text.chars().enumerate() {
|
||||
if ch.is_whitespace() {
|
||||
if let Some(start_idx) = current_start_idx.take() {
|
||||
let text = std::mem::take(&mut current_token);
|
||||
push_token(&mut tokens, text, start_idx, idx);
|
||||
}
|
||||
} else {
|
||||
if current_start_idx.is_none() {
|
||||
current_start_idx = Some(idx);
|
||||
}
|
||||
current_token.push(ch);
|
||||
}
|
||||
}
|
||||
if let Some(start_idx) = current_start_idx {
|
||||
let text = std::mem::take(&mut current_token);
|
||||
push_token(&mut tokens, text, start_idx, total_chars);
|
||||
}
|
||||
|
||||
tokens
|
||||
}
|
||||
|
||||
/// Compute plausibility caps for the orphan-assignment pass. Returns
|
||||
/// `(cap_x, cap_y)` — the maximum x/y distance from a text item's center
|
||||
/// to a candidate empty cell's bbox before the candidate is rejected.
|
||||
@@ -2564,15 +2152,13 @@ fn try_expand_multi_row_cells(
|
||||
/// * `phantom_empty_row` — a row whose every cell is empty, surrounded
|
||||
/// above and below by rows with content. SLANet sometimes emits an
|
||||
/// extra row that doesn't correspond to any visible PDF row.
|
||||
/// * `multi_row_in_cell` — at least one non-label `rowspan==1` cell
|
||||
/// encloses PDF text items that cluster into two distinct visual lines
|
||||
/// * `multi_row_in_cell` — at least one `rowspan==1` cell encloses
|
||||
/// PDF text items that cluster into two distinct visual lines
|
||||
/// separated by a whitespace gap larger than the line height. Cells
|
||||
/// declared as `rowspan>1` are excluded since they are *expected*
|
||||
/// to span multiple lines. First-row/first-column wraps are ignored
|
||||
/// unless the in-place row expansion has enough support to repair them,
|
||||
/// because those are often legitimate wrapped headers or row labels.
|
||||
/// SLANet's row under-detection on tightly-packed tables produces the
|
||||
/// rowspan==1-but-multi-line pattern (the FNBO failure mode).
|
||||
/// to span multiple lines. SLANet's row under-detection on
|
||||
/// tightly-packed tables produces the rowspan==1-but-multi-line
|
||||
/// pattern (the FNBO failure mode).
|
||||
fn detect_tsr_quality_issue(
|
||||
buffer: &[u8],
|
||||
input: &TsrTableInput,
|
||||
@@ -2630,8 +2216,6 @@ fn detect_tsr_quality_issue(
|
||||
};
|
||||
let expanded_cells =
|
||||
try_expand_multi_row_cells(cells, &items, page_h, coords, adaptive_threshold);
|
||||
let first_row = cells.iter().map(|cell| cell.row).min().unwrap_or(0);
|
||||
let first_col = cells.iter().map(|cell| cell.col).min().unwrap_or(0);
|
||||
|
||||
for cell in cells {
|
||||
// rowspan>1 cells are intentionally multi-line — skip them.
|
||||
@@ -2645,30 +2229,14 @@ fn detect_tsr_quality_issue(
|
||||
if cell_items.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
if cluster_tsr_cell_text_lines(cell_items).len() < 2 {
|
||||
continue;
|
||||
}
|
||||
if expanded_cells.is_some() {
|
||||
if cluster_tsr_cell_text_lines(cell_items).len() >= 2 {
|
||||
return Ok(Some(TsrQualityIssue::MultiRowInCell { expanded_cells }));
|
||||
}
|
||||
if !is_wrapped_tsr_label_cell(cell, first_row, first_col) {
|
||||
return Ok(Some(TsrQualityIssue::MultiRowInCell {
|
||||
expanded_cells: None,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn is_wrapped_tsr_label_cell(
|
||||
cell: &tables::StructuredCell,
|
||||
first_row: usize,
|
||||
first_col: usize,
|
||||
) -> bool {
|
||||
cell.is_header || cell.row == first_row || cell.col == first_col
|
||||
}
|
||||
|
||||
/// Auto-fallback variant of [`extract_tables_with_structure_mem`]:
|
||||
/// runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies (phantom rows, multi-row-in-cell text),
|
||||
|
||||
@@ -110,21 +110,15 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Reject page-spanning frames: a decorative outer border has just 4
|
||||
// edges (top/bottom/left/right). Real full-page tables — common in
|
||||
// governmental ledgers, financial reports, etc. — span the same A4 /
|
||||
// Letter dimensions but have many internal row/column rules. Only
|
||||
// reject when the line set looks like a bare frame, not a grid.
|
||||
// Reject page-spanning frames: if the grid covers >90% of a standard page
|
||||
// dimension in both axes, it's a border frame, not a table.
|
||||
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
||||
if table_width > 500.0 && table_height > 700.0 && horizontals.len() <= 4 && verticals.len() <= 4
|
||||
{
|
||||
if table_width > 500.0 && table_height > 700.0 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0}, {} h + {} v)",
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})",
|
||||
page,
|
||||
table_width,
|
||||
table_height,
|
||||
horizontals.len(),
|
||||
verticals.len()
|
||||
table_height
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -416,75 +410,6 @@ mod tests {
|
||||
assert!(tables.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_bare_frame_rejected() {
|
||||
// Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.
|
||||
// No internal structure → decorative border, not a table.
|
||||
let lines = vec![
|
||||
make_hline(20.0, 20.0, 575.0, 1), // top
|
||||
make_hline(820.0, 20.0, 575.0, 1), // bottom
|
||||
make_vline(20.0, 20.0, 820.0, 1), // left
|
||||
make_vline(575.0, 20.0, 820.0, 1), // right
|
||||
];
|
||||
let items = vec![
|
||||
make_item("title", 100.0, 100.0, 1),
|
||||
make_item("body", 100.0, 200.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"Page-sized 4-edge frame should be rejected as decoration"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_grid_with_internal_lines_accepted() {
|
||||
// Full-page table (governmental-ledger pattern): A4-sized grid
|
||||
// that previously hit the "page-spanning frame" early reject
|
||||
// before downstream validation could even look at it.
|
||||
// Verticals span the full table height so we isolate the
|
||||
// frame-vs-grid decision under test.
|
||||
let mut lines = Vec::new();
|
||||
// 13 horizontal rules: header + 12 row separators
|
||||
let h_ys = [
|
||||
22.5, 37.9, 95.5, 144.5, 184.9, 233.9, 291.7, 340.7, 415.8, 499.6, 574.7, 623.7, 698.8,
|
||||
];
|
||||
for &y in &h_ys {
|
||||
lines.push(make_hline(y, 22.6, 566.6, 1));
|
||||
}
|
||||
// 7 column dividers spanning full table height.
|
||||
let v_xs = [22.6, 66.3, 116.3, 186.6, 263.1, 493.5, 566.5];
|
||||
for &x in &v_xs {
|
||||
lines.push(make_vline(x, 22.5, 698.8, 1));
|
||||
}
|
||||
// Populate every cell so the capture-ratio + density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for r in 0..(h_ys.len() - 1) {
|
||||
let row_y = (h_ys[r] + h_ys[r + 1]) / 2.0;
|
||||
for c in 0..(v_xs.len() - 1) {
|
||||
let col_x = (v_xs[c] + v_xs[c + 1]) / 2.0;
|
||||
items.push(make_item("x", col_x, row_y, 1));
|
||||
}
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"Full-page table with internal grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 6,
|
||||
"expected ≥6 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert!(
|
||||
t.cells[0].len() >= 3,
|
||||
"expected ≥3 columns, got {}",
|
||||
t.cells[0].len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_rejected() {
|
||||
// Only 2 col edges (1 column) — not a table even with verticals
|
||||
|
||||
+30
-571
@@ -285,11 +285,7 @@ pub fn detect_tables_from_rects(
|
||||
//
|
||||
// Only remove when the container is a similarly-sized cell (height
|
||||
// ratio < 4×), NOT when the container is a table-wide background
|
||||
// that dwarfs the sub-rect. Origin-anchored page-background rects
|
||||
// also disqualify as containers — they normally exceed the 4× ratio,
|
||||
// but when the sub-rect is itself a tall table-frame the ratio can
|
||||
// fall under the gate, and dropping the frame collapses cluster
|
||||
// adjacency between adjacent column-cell groups.
|
||||
// that dwarfs the sub-rect.
|
||||
//
|
||||
// Skip this O(n²) dedup when there are too many rects — pages with
|
||||
// thousands of vector-drawing rects won't benefit from cell dedup.
|
||||
@@ -299,11 +295,9 @@ pub fn detect_tables_from_rects(
|
||||
page_rects.retain(|&(ax, ay, aw, ah)| {
|
||||
let tol = 2.0;
|
||||
!snapshot.iter().any(|&(bx, by, bw, bh)| {
|
||||
let container_is_page_bg = bx < 5.0 && by < 5.0;
|
||||
// b must strictly contain a (b is larger in area)
|
||||
bw * bh > aw * ah * 1.2
|
||||
&& bh < ah * 4.0 // container must be similarly sized, not a table background
|
||||
&& !container_is_page_bg
|
||||
&& bx <= ax + tol
|
||||
&& (bx + bw) >= (ax + aw) - tol
|
||||
&& by <= ay + tol
|
||||
@@ -1394,15 +1388,13 @@ fn detect_row_stripe_table(
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
// Allow longer cells for multi-column tables (descriptions in one column
|
||||
// are common). Narrow grids with giant cells are usually layout
|
||||
// backgrounds — but only when the row count is also small. A 4+-row
|
||||
// key/value table with one descriptive column reads as a real table
|
||||
// on every other gate, so don't reject it on cell length alone.
|
||||
// are common). Single-column or 2-column "tables" with giant cells are
|
||||
// almost always layout backgrounds.
|
||||
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
||||
if max_cell_len > max_allowed && non_empty_rows < 4 {
|
||||
if max_cell_len > max_allowed {
|
||||
debug!(
|
||||
" row-stripe rejected: max cell length {} > {} (layout background, {} rows)",
|
||||
max_cell_len, max_allowed, non_empty_rows
|
||||
" row-stripe rejected: max cell length {} > {} (layout background)",
|
||||
max_cell_len, max_allowed
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1634,59 +1626,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
}
|
||||
};
|
||||
|
||||
// For wired-grid tables whose header text is centered/right-aligned but
|
||||
// whose data is left-aligned, cluster_x_positions can drop the header-only
|
||||
// x-cluster in its singleton-filter pass and merge adjacent data clusters
|
||||
// when the gap is below threshold, losing a column. Rect borders are
|
||||
// ground truth in that case — but only when each rect column actually
|
||||
// holds text. Decorative or background rects (prose laid out in a frame,
|
||||
// cell-fill rects with extra borders) can produce more rect-derived
|
||||
// columns than the text supports; preferring rects there would split a
|
||||
// logical column into spurious sub-columns.
|
||||
let rect_cols_match_text = match (&rect_col_edges, &text_col_edges) {
|
||||
(Some(rect_edges), _) if rect_edges.len() >= 4 => {
|
||||
let num_rect_cols = rect_edges.len() - 1;
|
||||
let mut col_item_counts = vec![0usize; num_rect_cols];
|
||||
for (_, item) in &page_items {
|
||||
let cx = item.x + item.width / 2.0;
|
||||
for c in 0..num_rect_cols {
|
||||
if cx >= rect_edges[c] - 2.0 && cx <= rect_edges[c + 1] + 2.0 {
|
||||
col_item_counts[c] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Require every rect column to hold multiple text items. A rect
|
||||
// column with no (or only one) item is decorative or the rect grid
|
||||
// is detecting a spurious column the data does not need; in those
|
||||
// cases the old text-cluster preference is the safer fallback.
|
||||
col_item_counts.iter().all(|&n| n >= 2)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||
(Some(rect_edges), text_edges_opt) if rect_cols_match_text => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns (text clusters: {}; rect cols well-distributed)",
|
||||
rect_edges.len() - 1,
|
||||
text_edges_opt
|
||||
.as_ref()
|
||||
.map(|e| (e.len() - 1) as i32)
|
||||
.unwrap_or(-1)
|
||||
);
|
||||
(rect_edges, false)
|
||||
}
|
||||
let col_edges = match (rect_col_edges, text_col_edges) {
|
||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||
rect_edges.len() - 1,
|
||||
text_edges.len() - 1
|
||||
);
|
||||
(rect_edges, false)
|
||||
rect_edges
|
||||
}
|
||||
(_, Some(text_edges)) => (text_edges, true),
|
||||
(Some(rect_edges), None) => (rect_edges, false),
|
||||
(_, Some(text_edges)) => text_edges,
|
||||
(Some(rect_edges), None) => rect_edges,
|
||||
(None, None) => {
|
||||
debug!(
|
||||
" cell-rect rejected: only {} columns from text clustering",
|
||||
@@ -1711,25 +1661,12 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
page_items.len()
|
||||
);
|
||||
|
||||
let (mut cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
|
||||
let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
|
||||
|
||||
if item_indices.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut row_edges = row_edges;
|
||||
let (collapsed_cells, collapsed_row_edges, collapsed_rows) =
|
||||
collapse_multiline_description_rows(cells, row_edges, &col_edges);
|
||||
let has_wrapped_description_rows = collapsed_rows > 0;
|
||||
cells = collapsed_cells;
|
||||
row_edges = collapsed_row_edges;
|
||||
if collapsed_rows > 0 {
|
||||
debug!(
|
||||
" cell-rect collapsed {} wrapped description rows",
|
||||
collapsed_rows
|
||||
);
|
||||
}
|
||||
|
||||
// Validate: >=2 non-empty rows, >=25% density
|
||||
let non_empty_rows = cells
|
||||
.iter()
|
||||
@@ -1743,7 +1680,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
let num_rows = cells.len();
|
||||
let total_cells = (num_cols * num_rows) as f32;
|
||||
let non_empty_cells = cells
|
||||
.iter()
|
||||
@@ -1763,21 +1699,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject tables with paragraph-length cells — typically layout
|
||||
// backgrounds (sidebars, banners) where a single big rectangle
|
||||
// contains a wall of prose. Spare multi-row key/value tables where
|
||||
// the value column is a multi-bullet description: those pass every
|
||||
// other gate and shouldn't get killed on cell length alone.
|
||||
// Reject tables with paragraph-length cells (layout backgrounds, not tables)
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" cell-rect rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" cell-rect rejected: max cell length {} > 500",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1793,36 +1725,13 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
|
||||
// Reject "tables" that are actually prose in a framed region.
|
||||
// Columns here come from text X-position clustering; when prose wraps
|
||||
// inside a bounding-box rect (e.g. chat-transcript figures, two-column
|
||||
// legal-text blocks in forms) the word-boundary gaps cluster into
|
||||
// spurious columns, and the resulting cells hold sentence fragments
|
||||
// riddled with common English function words.
|
||||
//
|
||||
// Apply at any column count >= 2. The 2-col case is the bite — a
|
||||
// paragraph wrapped into 2 justified columns produces the same
|
||||
// surface signal as a real "label / value" table in the
|
||||
// well-distributed-cols check (both cols populated), so we need a
|
||||
// content-based signal to tell them apart.
|
||||
//
|
||||
// Layered checks combine after the 20%-of-cells prose-word
|
||||
// trigger fires:
|
||||
// (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
|
||||
// per non-empty cell (sentence fragments); real data tables
|
||||
// are typically <30 chars, occasionally up to ~55 for
|
||||
// descriptive 4-col tables. The 65-char threshold cleanly
|
||||
// separates them on observed fixtures (accessory_building
|
||||
// prose=74 chars, upstage data=53, greencomp=20). This
|
||||
// overrides the well-distributed relaxation — long cells
|
||||
// are the strongest prose signal even when both cols are
|
||||
// populated.
|
||||
// (b) Two-column text-only scaffold: when both columns were inferred
|
||||
// from text starts rather than rect edges, prose fragments can look
|
||||
// perfectly balanced. Require rect evidence for this relaxed shape.
|
||||
// (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
|
||||
// cells. Catches the prose-paragraph-as-many-cols shape
|
||||
// while admitting real "label / value / description /
|
||||
// benefit"-style tables.
|
||||
if num_cols >= 2 {
|
||||
// inside a bounding-box rect (e.g. chat-transcript figures) the
|
||||
// word-boundary gaps cluster into many spurious columns, and the
|
||||
// resulting cells hold sentence fragments riddled with common English
|
||||
// function words. Count cells with any such word and reject when
|
||||
// 20%+ of non-empty cells match — real tabular data (labels, units,
|
||||
// numbers) rarely contains these words.
|
||||
if num_cols >= 4 {
|
||||
const PROSE_WORDS: &[&str] = &[
|
||||
"a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on",
|
||||
"at", "with", "for", "by", "as", "and", "or", "but", "this", "that", "these", "those",
|
||||
@@ -1832,7 +1741,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
];
|
||||
let mut prose_cells = 0usize;
|
||||
let mut counted = 0usize;
|
||||
let mut total_chars = 0usize;
|
||||
for row in &cells {
|
||||
for cell in row {
|
||||
let t = cell.trim();
|
||||
@@ -1840,7 +1748,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
continue;
|
||||
}
|
||||
counted += 1;
|
||||
total_chars += t.chars().count();
|
||||
let lower = t.to_ascii_lowercase();
|
||||
let has_prose_word = lower
|
||||
.split(|c: char| !c.is_ascii_alphabetic() && c != '\'')
|
||||
@@ -1851,65 +1758,11 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
}
|
||||
}
|
||||
if counted > 0 && prose_cells * 5 >= counted {
|
||||
// (a) Long-cell content: overrides the well-distributed
|
||||
// relaxation. The 2-col prose-in-a-frame case populates
|
||||
// both cols (passes well-distributed) but every cell
|
||||
// holds a sentence fragment, so mean cell length is the
|
||||
// discriminator.
|
||||
const PROSE_MEAN_CHAR_THRESHOLD: usize = 65;
|
||||
let mean_chars = total_chars / counted;
|
||||
if mean_chars > PROSE_MEAN_CHAR_THRESHOLD && !has_wrapped_description_rows {
|
||||
debug!(
|
||||
" cell-rect rejected: prose-in-frame, mean non-empty cell {} chars > {} (prose words {}/{})",
|
||||
mean_chars, PROSE_MEAN_CHAR_THRESHOLD, prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
} else if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
|
||||
debug!(
|
||||
" cell-rect prose check relaxed: wrapped description rows, mean {} chars (prose words {}/{})",
|
||||
mean_chars, prose_cells, counted
|
||||
);
|
||||
}
|
||||
|
||||
// (b) Two text-derived columns are not enough vector evidence once
|
||||
// the content looks prose-like. Real 2-col rect tables still pass
|
||||
// when the column scaffold comes from drawn cell geometry.
|
||||
if columns_from_text && num_cols == 2 {
|
||||
debug!(
|
||||
" cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
|
||||
mean_chars, prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// (c) Well-distributed columns.
|
||||
let filled_cols = (0..num_cols)
|
||||
.filter(|&c| {
|
||||
cells
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
!row.get(c)
|
||||
.map(String::as_str)
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.is_empty()
|
||||
})
|
||||
.count()
|
||||
>= 2
|
||||
})
|
||||
.count();
|
||||
let well_distributed = filled_cols * 4 >= num_cols * 3;
|
||||
if !well_distributed {
|
||||
debug!(
|
||||
" cell-rect rejected: {}/{} cells contain prose function words — likely prose ({}/{} cols filled, mean {} chars)",
|
||||
prose_cells, counted, filled_cols, num_cols, mean_chars
|
||||
);
|
||||
return None;
|
||||
}
|
||||
debug!(
|
||||
" cell-rect prose check relaxed: {}/{} cols filled, mean {} chars — table-with-description-col",
|
||||
filled_cols, num_cols, mean_chars
|
||||
" cell-rect rejected: {}/{} cells contain prose function words — likely prose",
|
||||
prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1930,132 +1783,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Merge wrapped description-line bands back into their visual data rows.
|
||||
///
|
||||
/// Some Word/PDF exports draw enough rectangle geometry to prove a table exists
|
||||
/// but expose Y bands per wrapped text line instead of per cell row. In the
|
||||
/// common mapping-table shape, a narrow row-label column precedes one wide
|
||||
/// description column, and wrapped continuation bands have content only in that
|
||||
/// wide column. Merge only that high-confidence shape so framed prose still
|
||||
/// falls through the existing prose guards.
|
||||
fn collapse_multiline_description_rows(
|
||||
cells: Vec<Vec<String>>,
|
||||
row_edges: Vec<f32>,
|
||||
col_edges: &[f32],
|
||||
) -> (Vec<Vec<String>>, Vec<f32>, usize) {
|
||||
let num_rows = cells.len();
|
||||
let num_cols = col_edges.len().saturating_sub(1);
|
||||
if num_rows < 3 || num_cols < 3 || row_edges.len() != num_rows + 1 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let table_width = col_edges[num_cols] - col_edges[0];
|
||||
if table_width <= 0.0 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let Some((description_col, description_width)) = (0..num_cols)
|
||||
.map(|c| (c, col_edges[c + 1] - col_edges[c]))
|
||||
.max_by(|a, b| a.1.total_cmp(&b.1))
|
||||
else {
|
||||
return (cells, row_edges, 0);
|
||||
};
|
||||
|
||||
// Require a preceding row-label column. Without it (e.g. a prose frame
|
||||
// split into text-start columns), "one populated wide column" is not enough
|
||||
// evidence to find visual row starts safely.
|
||||
if description_col == 0 || description_width < table_width * 0.35 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let row_has_left_label = |row: &[String]| {
|
||||
row.iter()
|
||||
.take(description_col)
|
||||
.any(|cell| !cell.trim().is_empty())
|
||||
};
|
||||
let labeled_rows = cells.iter().filter(|row| row_has_left_label(row)).count();
|
||||
if labeled_rows < 2 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let mut merged_rows = 0usize;
|
||||
let mut wrapped_description_rows = 0usize;
|
||||
let mut new_cells: Vec<Vec<String>> = Vec::with_capacity(num_rows);
|
||||
let mut new_edges = Vec::with_capacity(row_edges.len());
|
||||
new_edges.push(row_edges[0]);
|
||||
|
||||
for (row_idx, row) in cells.into_iter().enumerate() {
|
||||
let desc_text = row
|
||||
.get(description_col)
|
||||
.map(String::as_str)
|
||||
.unwrap_or("")
|
||||
.trim();
|
||||
let left_label = row_has_left_label(&row);
|
||||
let non_desc_non_empty = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(col, cell)| *col != description_col && !cell.trim().is_empty())
|
||||
.count();
|
||||
|
||||
// Wrapped continuation bands contain only description-column text.
|
||||
// The preceding label/marker column is empty because the visual row's
|
||||
// label cell spans the whole wrapped block.
|
||||
let is_description_continuation = row_idx > 0
|
||||
&& !desc_text.is_empty()
|
||||
&& !left_label
|
||||
&& non_desc_non_empty == 0
|
||||
&& !new_cells.is_empty();
|
||||
|
||||
// Header cells are often split as "Controls" / "Version" in the first
|
||||
// column while the other header labels sit on the first band.
|
||||
let only_first_col = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.all(|(col, cell)| col == 0 || cell.trim().is_empty());
|
||||
let is_header_continuation = row_idx > 0
|
||||
&& only_first_col
|
||||
&& row
|
||||
.first()
|
||||
.is_some_and(|cell| !cell.trim().is_empty() && cell.chars().count() <= 24)
|
||||
&& !new_cells.is_empty()
|
||||
&& new_cells
|
||||
.last()
|
||||
.is_some_and(|prev| prev.iter().filter(|c| !c.trim().is_empty()).count() >= 2);
|
||||
|
||||
if is_description_continuation || is_header_continuation {
|
||||
if let Some(prev) = new_cells.last_mut() {
|
||||
for (col, cell) in row.iter().enumerate() {
|
||||
let text = cell.trim();
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !prev[col].trim().is_empty() {
|
||||
prev[col].push(' ');
|
||||
}
|
||||
prev[col].push_str(text);
|
||||
}
|
||||
}
|
||||
merged_rows += 1;
|
||||
if is_description_continuation {
|
||||
wrapped_description_rows += 1;
|
||||
}
|
||||
} else {
|
||||
if !new_cells.is_empty() {
|
||||
new_edges.push(row_edges[row_idx]);
|
||||
}
|
||||
new_cells.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
new_edges.push(*row_edges.last().unwrap());
|
||||
|
||||
if merged_rows == 0 || new_cells.len() < 2 || new_edges.len() != new_cells.len() + 1 {
|
||||
return (new_cells, row_edges, 0);
|
||||
}
|
||||
|
||||
(new_cells, new_edges, wrapped_description_rows)
|
||||
}
|
||||
|
||||
/// Detect a table by merging all cluster rects into one group.
|
||||
///
|
||||
/// This handles clip-path PDFs where each column's cell rects form a separate
|
||||
@@ -2191,20 +1918,18 @@ fn detect_merged_cluster_table(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject if any cell has excessive text — layout background rects
|
||||
// produce "cells" containing paragraphs, not short data-table values.
|
||||
// Multi-row key/value tables can legitimately have one column of
|
||||
// long descriptive text, so only reject narrow-row layouts here.
|
||||
// Reject if any cell has excessive text — layout background rects produce
|
||||
// "cells" containing paragraphs, not short data-table values.
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" merged-cluster rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" merged-cluster rejected: max cell length {} > 500 (layout background)",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2634,46 +2359,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_row_stripe_accepts_multi_row_key_value_long_cells() {
|
||||
// Multi-row 2-column key/value table where one value cell holds
|
||||
// a paragraph (>500 chars). The old `max_cell_len > 500` check
|
||||
// rejected this shape as a "layout background"; with the
|
||||
// multi-row guard, it should be accepted.
|
||||
let mut rects = Vec::new();
|
||||
let row_h = 25.0_f32;
|
||||
let y_top = 700.0_f32;
|
||||
for i in 0..8 {
|
||||
let y = y_top - (i as f32) * row_h;
|
||||
rects.push((40.0, y, 510.0, row_h));
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
for i in 0..8 {
|
||||
let row_center_y = y_top - (i as f32) * row_h + row_h / 2.0;
|
||||
// Left column: short label
|
||||
items.push(make_item(&format!("Field {}", i), 45.0, row_center_y, 10.0));
|
||||
// Right column: short value, except the last row which is a paragraph
|
||||
let value = if i == 7 {
|
||||
"X".repeat(800)
|
||||
} else {
|
||||
"value".to_string()
|
||||
};
|
||||
items.push(make_item(&value, 300.0, row_center_y, 10.0));
|
||||
}
|
||||
let result = detect_row_stripe_table(&items, &rects, 1);
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"multi-row key/value table with one long cell should be accepted"
|
||||
);
|
||||
let t = result.unwrap();
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 2, "expected 2 columns");
|
||||
}
|
||||
|
||||
// --- propagate_merged_cells ---
|
||||
|
||||
#[test]
|
||||
@@ -3288,232 +2973,6 @@ mod tests {
|
||||
// If tables were detected, that's also acceptable
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_derived_two_col_prose_is_not_cell_rect_table() {
|
||||
let page = 1;
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..8 {
|
||||
rects.push(PdfRect {
|
||||
x: 50.0,
|
||||
y: 100.0 + row as f32 * 20.0,
|
||||
width: 180.0,
|
||||
height: 18.0,
|
||||
page,
|
||||
});
|
||||
}
|
||||
|
||||
let mut items = Vec::new();
|
||||
let left = [
|
||||
"the annual plan was revised",
|
||||
"and the team noted changes",
|
||||
"this section explains limits",
|
||||
"with additional notes below",
|
||||
"the policy was reviewed",
|
||||
"and results are summarized",
|
||||
"this appendix describes scope",
|
||||
"with examples for reference",
|
||||
];
|
||||
let right = [
|
||||
"for each area in the review",
|
||||
"as part of the assessment",
|
||||
"that were applied in context",
|
||||
"to support the conclusion",
|
||||
"for use by the committee",
|
||||
"as shown in the narrative",
|
||||
"that remain under discussion",
|
||||
"to clarify the method",
|
||||
];
|
||||
for row in 0..8 {
|
||||
let y = 104.0 + row as f32 * 20.0;
|
||||
let mut left_item = make_item(left[row], 60.0, y, 9.0);
|
||||
left_item.width = 50.0;
|
||||
items.push(left_item);
|
||||
let mut right_item = make_item(right[row], 150.0, y, 9.0);
|
||||
right_item.width = 50.0;
|
||||
items.push(right_item);
|
||||
}
|
||||
|
||||
let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"text-derived two-column prose must not be accepted as a rect table; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiline_indented_description_rows_collapse_to_visual_rows() {
|
||||
let page = 1;
|
||||
let col_edges = [0.0, 60.0, 420.0, 460.0, 500.0, 540.0];
|
||||
let row_edges = [
|
||||
340.0, 320.0, 300.0, 270.0, 250.0, 230.0, 200.0, 180.0, 160.0,
|
||||
];
|
||||
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..row_edges.len() - 1 {
|
||||
let y_top = row_edges[row];
|
||||
let y_bot = row_edges[row + 1];
|
||||
for col in 0..col_edges.len() - 1 {
|
||||
rects.push((
|
||||
col_edges[col],
|
||||
y_bot,
|
||||
col_edges[col + 1] - col_edges[col],
|
||||
y_top - y_bot,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items = vec![
|
||||
make_item("Controls", 8.0, 330.0, 9.0),
|
||||
make_item("Control", 70.0, 330.0, 9.0),
|
||||
make_item("IG 1", 428.0, 330.0, 9.0),
|
||||
make_item("IG 2", 468.0, 330.0, 9.0),
|
||||
make_item("IG 3", 508.0, 330.0, 9.0),
|
||||
make_item("Version", 8.0, 310.0, 9.0),
|
||||
make_item("v8", 20.0, 285.0, 9.0),
|
||||
make_item(
|
||||
"4.5 Implement and Manage a Firewall on End-User Devices",
|
||||
70.0,
|
||||
285.0,
|
||||
9.0,
|
||||
),
|
||||
make_item("*", 438.0, 285.0, 9.0),
|
||||
make_item("*", 478.0, 285.0, 9.0),
|
||||
make_item("*", 518.0, 285.0, 9.0),
|
||||
make_item("v7", 20.0, 215.0, 9.0),
|
||||
make_item(
|
||||
"9.4 Apply Host-based Firewalls or Port-Filtering",
|
||||
70.0,
|
||||
215.0,
|
||||
9.0,
|
||||
),
|
||||
make_item("*", 478.0, 215.0, 9.0),
|
||||
make_item("*", 518.0, 215.0, 9.0),
|
||||
];
|
||||
items.push(make_item(
|
||||
"Implement and manage a host-based firewall or port-filtering tool",
|
||||
84.0,
|
||||
260.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"on end-user devices with a default-deny rule",
|
||||
84.0,
|
||||
240.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"Apply host-based firewalls or port filtering tools on end systems",
|
||||
84.0,
|
||||
190.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"and deny unauthorized network communication",
|
||||
84.0,
|
||||
170.0,
|
||||
8.0,
|
||||
));
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("expected multiline description table");
|
||||
assert_eq!(table.columns.len(), 5);
|
||||
assert_eq!(
|
||||
table.rows.len(),
|
||||
3,
|
||||
"wrapped lines should collapse to header plus two data rows"
|
||||
);
|
||||
assert_eq!(table.cells[0][0], "Controls Version");
|
||||
assert!(table.cells[1][1].contains("host-based firewall"));
|
||||
assert!(table.cells[1][1].contains("default-deny rule"));
|
||||
assert!(table.cells[2][1].contains("deny unauthorized"));
|
||||
}
|
||||
|
||||
/// Wire-bordered 4-column table whose header text is centered/right-aligned
|
||||
/// inside each cell while the data is left-aligned: cluster_x_positions
|
||||
/// merges adjacent columns (data Item→EAN gap is below threshold) and
|
||||
/// drops the header-only x-clusters in the filter pass, leaving only 3
|
||||
/// text-derived columns. Rect borders are 4 columns of ground truth.
|
||||
/// Before the fix the cell-rect path preferred text edges when they were
|
||||
/// the smaller set — losing a column. After the fix, 3+ rect columns
|
||||
/// always win.
|
||||
#[test]
|
||||
fn wired_header_data_misaligned_keeps_all_columns_from_rects() {
|
||||
let page = 1;
|
||||
// 4 cols: Item | EAN | Nombre | Cant
|
||||
let col_xs = [380.0_f32, 410.0, 470.0, 660.0, 700.0];
|
||||
// Header + 9 data rows at 15pt tall each (y descending).
|
||||
let row_ys: Vec<f32> = (0..=10).map(|r| 400.0 - 15.0 * r as f32).collect();
|
||||
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||
for r in 0..10 {
|
||||
let y_top = row_ys[r];
|
||||
let y_bot = row_ys[r + 1];
|
||||
for c in 0..4 {
|
||||
rects.push((col_xs[c], y_bot, col_xs[c + 1] - col_xs[c], y_top - y_bot));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items: Vec<TextItem> = Vec::new();
|
||||
// Header row (y ≈ 392.5): headers sit further to the right than data
|
||||
// because they are centered/right-aligned in the cells.
|
||||
items.push(make_item("Item", 389.0, 392.5, 9.0));
|
||||
items.push(make_item("EAN", 432.0, 392.5, 9.0));
|
||||
items.push(make_item("Nombre", 552.0, 392.5, 9.0));
|
||||
items.push(make_item("Cant", 672.0, 392.5, 9.0));
|
||||
|
||||
let names = [
|
||||
"Arnes Frontal",
|
||||
"Arnes Motor",
|
||||
"Arnes Piso",
|
||||
"Arnes Techo",
|
||||
"Arnes Puerta",
|
||||
"Arnes Tablero",
|
||||
"Arnes Trasero",
|
||||
"Arnes Lateral",
|
||||
"Arnes Sensor",
|
||||
];
|
||||
for r in 0..9 {
|
||||
let y = 377.5 - 15.0 * r as f32;
|
||||
items.push(make_item(&(r + 1).to_string(), 396.0, y, 9.0));
|
||||
items.push(make_item("7701023403016", 410.0, y, 9.0));
|
||||
items.push(make_item(names[r], 480.0, y, 9.0));
|
||||
items.push(make_item("1", 680.0, y, 9.0));
|
||||
}
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("wired 4-column table with header/data x-misalignment must detect");
|
||||
assert_eq!(
|
||||
table.columns.len(),
|
||||
4,
|
||||
"expected 4 columns from rect borders; cells: {:?}",
|
||||
table.cells
|
||||
);
|
||||
for c in 0..4 {
|
||||
let any_populated = table.cells.iter().any(|row| !row[c].trim().is_empty());
|
||||
assert!(
|
||||
any_populated,
|
||||
"column {} empty across all rows; cells: {:?}",
|
||||
c, table.cells
|
||||
);
|
||||
}
|
||||
// Header row populated in all 4 cells.
|
||||
let header = &table.cells[0];
|
||||
assert_eq!(header[0].trim(), "Item");
|
||||
assert_eq!(header[1].trim(), "EAN");
|
||||
assert_eq!(header[2].trim(), "Nombre");
|
||||
assert_eq!(header[3].trim(), "Cant");
|
||||
// First data row: Item="1", EAN, name, count="1" — no Item↔EAN merge.
|
||||
let data1 = &table.cells[1];
|
||||
assert_eq!(data1[0].trim(), "1");
|
||||
assert_eq!(data1[1].trim(), "7701023403016");
|
||||
assert!(data1[2].trim().contains("Arnes"));
|
||||
assert_eq!(data1[3].trim(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cluster_no_hint_without_items() {
|
||||
// Rects with no text items inside → no failed-cluster hint generated.
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+2
-448
@@ -1189,38 +1189,9 @@ fn test_firecrawl_tagged_pdf_struct_tree() {
|
||||
#[test]
|
||||
fn test_identity_h_no_tounicode_suppresses_garbage() {
|
||||
// shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no
|
||||
// usable ToUnicode CMap. The raw CID bytes (e.g. 0x08 0x37, 0x0E 0x0F)
|
||||
// contain non-ASCII high bytes and previously fell through to the
|
||||
// per-byte Latin-1 fallback, producing high-Latin-1 mojibake that
|
||||
// `is_cid_garbage` flagged. The Type0/CID guard in
|
||||
// `extract_text_from_operand` now emits one U+FFFD per CID instead of
|
||||
// mojibake; `detect_encoding_issues` trips on that and suppresses the
|
||||
// markdown / flags the page for OCR — so we still pass this test, but
|
||||
// via the deliberate marker path rather than by accident.
|
||||
// ToUnicode CMap. The raw CID values look like random Latin characters.
|
||||
// We should suppress the garbage and flag the page for OCR.
|
||||
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
|
||||
|
||||
// Pre-suppression check: the raw text items must contain the U+FFFD
|
||||
// markers that prove the Type0/CID fallback fired. This pins the
|
||||
// mechanism so a future regression that re-enables Latin-1 mojibake
|
||||
// would fail loudly here, not just silently change the suppression
|
||||
// chain to one that depends on `is_cid_garbage` + high-Latin-1 chars.
|
||||
let items = pdf_inspector::extractor::extract_text_with_positions_mem(&buf).unwrap();
|
||||
let combined: String = items.iter().map(|i| i.text.as_str()).collect();
|
||||
assert!(
|
||||
combined.contains('\u{FFFD}'),
|
||||
"Type0/CID font with unparseable ToUnicode CMap should emit U+FFFD per CID; \
|
||||
got {} chars: {:?}",
|
||||
combined.len(),
|
||||
&combined[..combined.len().min(100)]
|
||||
);
|
||||
assert!(
|
||||
!combined
|
||||
.chars()
|
||||
.any(|c| ('\u{0080}'..='\u{00FF}').contains(&c)),
|
||||
"Latin-1 mojibake (high bytes) must not leak from Type0/CID fallback; got: {:?}",
|
||||
&combined[..combined.len().min(100)]
|
||||
);
|
||||
|
||||
let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
|
||||
|
||||
// Page 1 should be flagged for OCR
|
||||
@@ -1648,33 +1619,6 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_in_regions_uses_line_grid() {
|
||||
// Stroked-grid table (m/l/S path operators forming a 2x2 grid).
|
||||
// The heuristic text-only detector handles the same cells already,
|
||||
// so this guards that the line-backed path doesn't regress: the
|
||||
// markdown still contains all four data cells.
|
||||
let buf = synthetic_vector_grid_pdf(false);
|
||||
let results =
|
||||
extract_tables_in_regions_mem(&buf, &[(0, vec![[40.0, 50.0, 220.0, 760.0]])]).unwrap();
|
||||
let region = &results[0].regions[0];
|
||||
assert!(
|
||||
!region.needs_ocr,
|
||||
"stroked-grid table should be extracted, got needs_ocr=true"
|
||||
);
|
||||
for tok in ["A1", "B1", "A2", "B2"] {
|
||||
assert!(
|
||||
region.text.contains(tok),
|
||||
"expected '{tok}' in output, got: {}",
|
||||
region.text
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
region.text.contains('|'),
|
||||
"expected pipe-delimited markdown"
|
||||
);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_mem tests (TSR-aware path)
|
||||
// =========================================================================
|
||||
@@ -2588,61 +2532,6 @@ fn test_auto_expands_under_counted_vector_grid_rows() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_keeps_wrapped_header_vector_grid_doc51() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = std::fs::read("tests/fixtures/government_positions_women.pdf").unwrap();
|
||||
let crop = [0.0, 0.0, 612.0, 792.0];
|
||||
let grid = detect_vector_grid_in_region_mem(&buf, 0, crop, 200.0)
|
||||
.unwrap()
|
||||
.expect("expected doc 51 vector grid");
|
||||
assert_eq!(
|
||||
grid.cell_bboxes.len(),
|
||||
36,
|
||||
"doc 51 should have a 9x4 vector grid"
|
||||
);
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: 200.0,
|
||||
structure_tokens: grid.structure_tokens,
|
||||
cell_bboxes: grid.cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
let r = &results[0];
|
||||
assert!(
|
||||
r.fallback_reason.is_none(),
|
||||
"wrapped header/label text should not trigger heuristic fallback: {:?}\n{}",
|
||||
r.fallback_reason,
|
||||
r.markdown
|
||||
);
|
||||
let md = &r.markdown;
|
||||
assert!(md.contains("Government Position"), "missing header: {md}");
|
||||
assert!(
|
||||
md.contains("Aquino Administration"),
|
||||
"missing Aquino header: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("Ramos Administration"),
|
||||
"missing Ramos header: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("City Municipal Councilor"),
|
||||
"row label was truncated: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("|Position||Administration"),
|
||||
"heuristic fallback split the header row: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_returns_empty_inputs() {
|
||||
use pdf_inspector::extract_tables_with_structure_auto_mem;
|
||||
@@ -3056,338 +2945,3 @@ fn test_extract_pages_markdown_path_none_returns_all_pages() {
|
||||
let result = extract_pages_markdown(path, None).unwrap();
|
||||
assert_eq!(result.pages.len() as u32, page_count);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PROBE: investigate dense-cell text-assignment failure mode (failure mode 2)
|
||||
// ============================================================================
|
||||
|
||||
fn synthetic_wide_row_pdf() -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let operations = vec![
|
||||
Operation::new("BT", vec![]),
|
||||
Operation::new("Tf", vec!["F1".into(), 10.into()]),
|
||||
Operation::new("Td", vec![20.into(), 700.into()]),
|
||||
// A single Tj that visually spans multiple cells. This mirrors PDFs
|
||||
// where a row's address/role/email columns are emitted as one literal
|
||||
// string with embedded spaces, producing one wide TextItem.
|
||||
Operation::new(
|
||||
"Tj",
|
||||
vec![Object::string_literal("Name JobTitle Email Phone")],
|
||||
),
|
||||
Operation::new("ET", vec![]),
|
||||
];
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 200.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_distributes_wide_item_across_cells() {
|
||||
use pdf_inspector::{extract_tables_with_structure_cells_mem, TsrTableInput};
|
||||
|
||||
// Reproduces failure mode 2: a row of multi-token text rendered as one
|
||||
// Tj produces a single wide TextItem that visually spans multiple cells.
|
||||
// The current first-match-by-center routing parks the entire item in
|
||||
// whichever cell holds the item's center, leaving the other cells empty.
|
||||
// See production samples in scrape_id 019de788-ff41-... where 10-column
|
||||
// grids ended up with row text packed into one cell.
|
||||
let buf = synthetic_wide_row_pdf();
|
||||
|
||||
// Helvetica 10pt with width=0 falls back to char_count*font_size*0.5.
|
||||
// "Name JobTitle Email Phone" is 25 chars → effective_width 125pt,
|
||||
// text starts at PDF (20, 700), top-down y=[90, 100], char_w≈5pt.
|
||||
// Tokens land at:
|
||||
// "Name" chars 0-3 center≈x=30
|
||||
// "JobTitle" chars 5-12 center≈x=65
|
||||
// "Email" chars 14-18 center≈x=100
|
||||
// "Phone" chars 20-24 center≈x=130
|
||||
let cell_bboxes = vec![
|
||||
poly(15.0, 88.0, 50.0, 102.0),
|
||||
poly(50.0, 88.0, 85.0, 102.0),
|
||||
poly(85.0, 88.0, 120.0, 102.0),
|
||||
poly(120.0, 88.0, 155.0, 102.0),
|
||||
];
|
||||
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
let cells_lists = extract_tables_with_structure_cells_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cells = &cells_lists[0];
|
||||
assert_eq!(cells.len(), 4);
|
||||
assert_eq!(
|
||||
cells[0].text, "Name",
|
||||
"cell 0 should hold 'Name', got {:?}",
|
||||
cells[0].text
|
||||
);
|
||||
assert_eq!(
|
||||
cells[1].text, "JobTitle",
|
||||
"cell 1 should hold 'JobTitle', got {:?}",
|
||||
cells[1].text
|
||||
);
|
||||
assert_eq!(
|
||||
cells[2].text, "Email",
|
||||
"cell 2 should hold 'Email', got {:?}",
|
||||
cells[2].text
|
||||
);
|
||||
assert_eq!(
|
||||
cells[3].text, "Phone",
|
||||
"cell 3 should hold 'Phone', got {:?}",
|
||||
cells[3].text
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PROPER TEST: synthetic Type0/Identity-H PDF with malformed ToUnicode CMap
|
||||
// ============================================================================
|
||||
//
|
||||
// Complements the existing real-PDF fixture `shinagawa_identity_h.pdf` by
|
||||
// building a minimal Type0 / Identity-H font in process. We control:
|
||||
// * the byte stream emitted by Tj (a 2-byte CID containing one high byte),
|
||||
// * the malformed ToUnicode contents (junk bytes that won't parse), and
|
||||
// * the DescendantFonts shape (just enough for `parse_type0_widths` to set
|
||||
// `is_cid=true`, which is what the new guard in `extract_text_from_operand`
|
||||
// keys off of).
|
||||
// No fixture file or external license to worry about.
|
||||
|
||||
fn synthetic_type0_broken_tounicode_pdf() -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let cid_font_id = doc.new_object_id();
|
||||
let descriptor_id = doc.new_object_id();
|
||||
let tounicode_id = doc.new_object_id();
|
||||
let cid_system_info_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
// Type0 font with Identity-H encoding and a broken ToUnicode reference.
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type0",
|
||||
"BaseFont" => "AAAAAA+SyntheticCID",
|
||||
"Encoding" => "Identity-H",
|
||||
"DescendantFonts" => vec![cid_font_id.into()],
|
||||
"ToUnicode" => tounicode_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
// CIDSystemInfo and a minimal CIDFontType2 descendant. parse_type0_widths
|
||||
// walks DescendantFonts → returns FontWidthInfo with is_cid=true. That's
|
||||
// the only thing the new Latin-1 guard needs to see.
|
||||
doc.objects.insert(
|
||||
cid_system_info_id,
|
||||
dictionary! {
|
||||
"Registry" => Object::string_literal("Adobe"),
|
||||
"Ordering" => Object::string_literal("Identity"),
|
||||
"Supplement" => 0,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
cid_font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "CIDFontType2",
|
||||
"BaseFont" => "AAAAAA+SyntheticCID",
|
||||
"CIDSystemInfo" => cid_system_info_id,
|
||||
"FontDescriptor" => descriptor_id,
|
||||
"DW" => 1000,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
descriptor_id,
|
||||
dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "AAAAAA+SyntheticCID",
|
||||
"Flags" => 4,
|
||||
"FontBBox" => vec![Object::Integer(-100), Object::Integer(-100), 1000.into(), 1000.into()],
|
||||
"ItalicAngle" => 0,
|
||||
"Ascent" => 800,
|
||||
"Descent" => Object::Integer(-200),
|
||||
"CapHeight" => 700,
|
||||
"StemV" => 80,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
// Intentionally malformed ToUnicode stream — just junk bytes. ToUnicode
|
||||
// CMap parsing will fail, so `font_cmaps.get_by_obj` returns None and
|
||||
// `has_cmap` stays false. The reference still exists in the font dict,
|
||||
// so `font_tounicode_refs` contains the entry — but the new guard now
|
||||
// routes off `is_cid` from font_widths instead, which is robust to a
|
||||
// failed CMap parse.
|
||||
doc.objects.insert(
|
||||
tounicode_id,
|
||||
Stream::new(dictionary! {}, b"this is not a valid CMap stream".to_vec()).into(),
|
||||
);
|
||||
|
||||
// Tj with a 2-byte CID stream containing a non-ASCII high byte.
|
||||
// Pre-fix this would have decoded as Latin-1 to "\u{00CD}\u{00D9}" ("ÍÙ").
|
||||
// Post-fix it should produce U+FFFD per CID.
|
||||
let cid_bytes = vec![0xCD_u8, 0xD9, 0xCD, 0xD9];
|
||||
let operations = vec![
|
||||
Operation::new("BT", vec![]),
|
||||
Operation::new("Tf", vec!["F0".into(), 12.into()]),
|
||||
Operation::new("Td", vec![50.into(), 100.into()]),
|
||||
Operation::new(
|
||||
"Tj",
|
||||
vec![Object::String(cid_bytes, lopdf::StringFormat::Hexadecimal)],
|
||||
),
|
||||
Operation::new("ET", vec![]),
|
||||
];
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 200.into(), 200.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F0" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_synthetic_type0_broken_tounicode_emits_fffd_not_latin1_mojibake() {
|
||||
let buf = synthetic_type0_broken_tounicode_pdf();
|
||||
|
||||
let items = pdf_inspector::extractor::extract_text_with_positions_mem(&buf).unwrap();
|
||||
let combined: String = items.iter().map(|i| i.text.as_str()).collect();
|
||||
|
||||
// Mojibake leak check: 2-byte CID 0xCDD9 must NOT come out as "ÍÙ"
|
||||
// (U+00CD U+00D9). That was the production scrape symptom.
|
||||
assert!(
|
||||
!combined.contains('\u{00CD}'),
|
||||
"Latin-1 mojibake leaked from Type0 font: {combined:?}"
|
||||
);
|
||||
assert!(
|
||||
!combined.contains('\u{00D9}'),
|
||||
"Latin-1 mojibake leaked from Type0 font: {combined:?}"
|
||||
);
|
||||
|
||||
// Marker presence: Type0/CID + non-ASCII bytes must produce U+FFFD so
|
||||
// `detect_encoding_issues` can flag the page for OCR downstream.
|
||||
assert!(
|
||||
combined.contains('\u{FFFD}'),
|
||||
"Type0 font with malformed ToUnicode CMap should emit U+FFFD per CID; got: {combined:?}"
|
||||
);
|
||||
|
||||
// End-to-end check: the page is correctly routed to OCR.
|
||||
let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
|
||||
assert!(
|
||||
result.pages_needing_ocr.contains(&1),
|
||||
"Type0 page with broken ToUnicode + non-ASCII bytes must be flagged for OCR; \
|
||||
pages_needing_ocr={:?}",
|
||||
result.pages_needing_ocr
|
||||
);
|
||||
}
|
||||
|
||||
@@ -54,14 +54,9 @@ company other than a life insurance company shall make a return on Form 1120PC.
|
||||
|
||||
annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return.
|
||||
|
||||
(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and
|
||||
(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company.
|
||||
(4) Exception for insurance companies filing their Federal income tax returns
|
||||
electronically. If an insurance company described in paragraph (c)(1), (c)(2), or
|
||||
|
||||
(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e).
|
||||
(5) Definition. For purposes of this section, the term annual statement means
|
||||
the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of
|
||||
||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and|
|
||||
|---|---|
|
||||
||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of|
|
||||
|
||||
Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement.
|
||||
|
||||
@@ -206,4 +201,3 @@ CFR part or section where Current OMB identified or described control No.
|
||||
Deputy Commissioner for Services and Enforcement.
|
||||
|
||||
Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user