Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0de11413ea |
@@ -20,7 +20,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --verbose
|
||||
@@ -51,9 +61,17 @@ jobs:
|
||||
components: clippy
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: clippy
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-clippy-
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy -- -D warnings
|
||||
@@ -71,9 +89,17 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: build
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-build-
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --verbose
|
||||
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you believe you've found a security vulnerability in pdf-inspector, please
|
||||
report it privately so we can fix it before public disclosure.
|
||||
|
||||
**Preferred:** Email **help@firecrawl.dev** with:
|
||||
|
||||
- A description of the issue and its impact
|
||||
- Steps to reproduce (a minimal PDF or input that triggers the bug is ideal)
|
||||
- The version or commit hash of pdf-inspector you tested against
|
||||
|
||||
**Alternative:** Use GitHub's private vulnerability reporting under the
|
||||
[Security tab](https://github.com/firecrawl/pdf-inspector/security/advisories/new).
|
||||
|
||||
We'll acknowledge your report in a timely manner and keep you updated on
|
||||
remediation progress. Please do not open a public GitHub issue for security
|
||||
bugs.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
- Memory-safety issues (panics, OOB reads, UB) reachable from a crafted PDF
|
||||
- Denial-of-service vectors (unbounded allocation, infinite loops) on
|
||||
reasonably-sized inputs
|
||||
- Bugs in the `pdf2md` / `detect-pdf` binaries or the `pdf-inspector` crate
|
||||
that affect downstream consumers
|
||||
|
||||
Out of scope:
|
||||
- Bugs in upstream dependencies (`lopdf`, etc.) — please report those upstream
|
||||
- Extraction quality issues (wrong text, missing tables) — open a regular
|
||||
GitHub issue instead
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.3",
|
||||
"version": "1.7.1",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { detectVectorGridInRegion } = require("./index.js");
|
||||
|
||||
const pdfPath =
|
||||
process.argv[2] ?? "/tmp/pdf_inspector_indent_fixtures/cis_edge_benchmark.pdf";
|
||||
const pdf = readFileSync(pdfPath);
|
||||
const dpi = Number(process.argv[3] ?? 200);
|
||||
|
||||
const crops = [
|
||||
{ pageIdx: 29, box: [0, 0, 612, 792], label: "page30-full" },
|
||||
{ pageIdx: 16, box: [0, 0, 612, 792], label: "page17-full" },
|
||||
{ pageIdx: 23, box: [0, 0, 612, 792], label: "page24-full" },
|
||||
];
|
||||
|
||||
for (const { pageIdx, box, label } of crops) {
|
||||
const result = detectVectorGridInRegion(pdf, pageIdx, box, dpi);
|
||||
if (!result) {
|
||||
console.log(`${label}: null`);
|
||||
continue;
|
||||
}
|
||||
const rows = result.structureTokens.filter((token) => token === "<tr>").length;
|
||||
const cols = rows > 0 ? result.cellBboxes.length / rows : 0;
|
||||
console.log(
|
||||
`${label}: cells=${result.cellBboxes.length} rows=${rows} cols=${cols}`,
|
||||
);
|
||||
}
|
||||
+7
-63
@@ -99,13 +99,6 @@ pub struct PageRegionTexts {
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// Vector-grid detection result compatible with `extractTablesWithStructure*`.
|
||||
#[napi(object)]
|
||||
pub struct VectorGridDetectionJs {
|
||||
pub structure_tokens: Vec<String>,
|
||||
pub cell_bboxes: Vec<Vec<f64>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -324,53 +317,6 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect a vector ruled-line / rectangle grid inside one page region.
|
||||
///
|
||||
/// Returns TSR-compatible structure tokens plus crop-pixel cell bboxes, or
|
||||
/// `null` when the region does not contain a valid vector grid.
|
||||
///
|
||||
/// `pageIdx` is 0-indexed. `regionPdfPtBbox` is `[x1,y1,x2,y2]` in PDF
|
||||
/// points with top-left origin. `renderDpi` is the DPI of the crop image that
|
||||
/// will consume the returned cell bboxes.
|
||||
#[napi]
|
||||
pub fn detect_vector_grid_in_region(
|
||||
buffer: Buffer,
|
||||
page_idx: u32,
|
||||
region_pdf_pt_bbox: Vec<f64>,
|
||||
render_dpi: f64,
|
||||
) -> Result<Option<VectorGridDetectionJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let region = if region_pdf_pt_bbox.len() == 4 {
|
||||
[
|
||||
region_pdf_pt_bbox[0] as f32,
|
||||
region_pdf_pt_bbox[1] as f32,
|
||||
region_pdf_pt_bbox[2] as f32,
|
||||
region_pdf_pt_bbox[3] as f32,
|
||||
]
|
||||
} else {
|
||||
[0.0, 0.0, 0.0, 0.0]
|
||||
};
|
||||
|
||||
catch_panic("detect_vector_grid_in_region", move || {
|
||||
let result = pdf_inspector::detect_vector_grid_in_region_mem(
|
||||
&bytes,
|
||||
page_idx,
|
||||
region,
|
||||
render_dpi as f32,
|
||||
)
|
||||
.map_err(|e| to_napi_err(e, "detect_vector_grid_in_region"))?;
|
||||
|
||||
Ok(result.map(|r| VectorGridDetectionJs {
|
||||
structure_tokens: r.structure_tokens,
|
||||
cell_bboxes: r
|
||||
.cell_bboxes
|
||||
.into_iter()
|
||||
.map(|bbox| bbox.into_iter().map(|v| v as f64).collect())
|
||||
.collect(),
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
/// One cropped table region plus its raw structure-recovery output, for
|
||||
/// `extractTablesWithStructure`.
|
||||
///
|
||||
@@ -482,10 +428,9 @@ pub fn extract_tables_with_structure_cells(
|
||||
/// `fallbackReason` is `null` when the TSR-hybrid path produced the
|
||||
/// markdown directly. When stage 1's quality check fires (the cells
|
||||
/// look like a SLANet detection pathology — phantom rows or multi-row
|
||||
/// content in a single cell), the auto path may expand the TSR cells
|
||||
/// in-place or run the heuristic table extractor on the same region.
|
||||
/// `fallbackReason` carries the diagnostic label (for example
|
||||
/// `"multi_row_in_cell_expanded"` or `"phantom_empty_row"`).
|
||||
/// content in a single cell), the heuristic table extractor is run on
|
||||
/// the same region instead, and `fallbackReason` carries the diagnostic
|
||||
/// label (`"phantom_empty_row"`, `"multi_row_in_cell"`).
|
||||
#[napi(object)]
|
||||
pub struct TableExtractionResultJs {
|
||||
pub markdown: String,
|
||||
@@ -495,14 +440,13 @@ pub struct TableExtractionResultJs {
|
||||
/// Auto-fallback variant of [`extractTablesWithStructure`].
|
||||
///
|
||||
/// Runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies, expands multi-row cells in-place when
|
||||
/// possible, and otherwise falls back to the heuristic
|
||||
/// `extractTablesInRegions` for inputs where the TSR path looks
|
||||
/// SLANet detection pathologies, and falls back to the heuristic
|
||||
/// `extractTablesInRegions` for any input where the TSR path looks
|
||||
/// compromised.
|
||||
///
|
||||
/// On clean inputs this returns identical markdown to
|
||||
/// `extractTablesWithStructure`; on flagged inputs `fallbackReason` is
|
||||
/// set to the recovery path that produced the result.
|
||||
/// `extractTablesWithStructure`; on flagged inputs the heuristic
|
||||
/// markdown replaces the TSR markdown and `fallbackReason` is set.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure_auto(
|
||||
buffer: Buffer,
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
detectVectorGridInRegion,
|
||||
extractPagesMarkdown,
|
||||
} from './index.js';
|
||||
|
||||
@@ -91,17 +90,6 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
|
||||
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
|
||||
console.log(' extractTextInRegions: OK');
|
||||
|
||||
// --- detectVectorGridInRegion ---
|
||||
console.log('Testing detectVectorGridInRegion...');
|
||||
const vectorGrid = detectVectorGridInRegion(fixture, 0, [0, 0, 600, 800], 72);
|
||||
assert.ok(vectorGrid === null || typeof vectorGrid === 'object');
|
||||
if (vectorGrid) {
|
||||
assert.ok(Array.isArray(vectorGrid.structureTokens));
|
||||
assert.ok(Array.isArray(vectorGrid.cellBboxes));
|
||||
assert.ok(vectorGrid.cellBboxes.every(bbox => Array.isArray(bbox) && bbox.length === 4));
|
||||
}
|
||||
console.log(' detectVectorGridInRegion: OK');
|
||||
|
||||
// --- extractPagesMarkdown ---
|
||||
console.log('Testing extractPagesMarkdown...');
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::fonts::{
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
/// Strip PDF comments (% to end of line) from content stream bytes.
|
||||
///
|
||||
@@ -385,7 +385,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
@@ -534,7 +533,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
@@ -622,7 +620,6 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
if !text.trim().is_empty() {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
@@ -664,29 +661,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if let Some(xobj_type) = xobjects.get(&xobj_name) {
|
||||
match xobj_type {
|
||||
XObjectType::Image => {
|
||||
// Emit a positional placeholder for the image
|
||||
// so downstream consumers (layout-aware
|
||||
// pipelines, figure-OCR routers) can locate
|
||||
// raster figures without parsing the PDF
|
||||
// again. The text field carries the
|
||||
// XObject resource name in the legacy
|
||||
// `[Image: Im0]` format that the markdown
|
||||
// emitter already recognizes.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
// Skip images — text extraction only
|
||||
}
|
||||
XObjectType::Form(form_id) => {
|
||||
// Extract text from Form XObject
|
||||
@@ -1037,17 +1012,9 @@ pub(crate) fn extract_page_text_items(
|
||||
// producing thousands of identical rects that yield a degenerate grid.
|
||||
// After dedup, if too few unique clip rects remain we fall through to
|
||||
// fill rects (explicitly drawn visible rectangles).
|
||||
//
|
||||
// When fill rects substantially outnumber clip rects, the clips are
|
||||
// typically section-level wrappers and the fills are the actual table
|
||||
// cell backgrounds (e.g. shaded-header tables drawn with `m`/`l`/`h`/`f*`
|
||||
// sequences). In that case, prefer fills.
|
||||
if rects.is_empty() {
|
||||
dedup_rects(&mut clip_rects);
|
||||
let prefer_fills = !fill_rects.is_empty() && fill_rects.len() >= clip_rects.len() * 3;
|
||||
if prefer_fills {
|
||||
rects = fill_rects;
|
||||
} else if clip_rects.len() >= 4 {
|
||||
if clip_rects.len() >= 4 {
|
||||
rects = clip_rects;
|
||||
} else if !fill_rects.is_empty() {
|
||||
rects = fill_rects;
|
||||
|
||||
+1
-122
@@ -720,11 +720,7 @@ pub(crate) fn extract_text_from_operand(
|
||||
font_encodings: &PageFontEncodings,
|
||||
encoding_cache: &HashMap<String, Encoding<'_>>,
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
font_widths: &PageFontWidths,
|
||||
) -> Option<String> {
|
||||
let is_type0_cid_font = font_widths
|
||||
.get(current_font)
|
||||
.is_some_and(|info| info.is_cid);
|
||||
let result = (|| -> Option<String> {
|
||||
if let Object::String(bytes, _) = obj {
|
||||
let mut decode_with_entry = |entry: &crate::tounicode::CMapEntry| -> Option<String> {
|
||||
@@ -966,31 +962,7 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
// canonical interpretation.
|
||||
// Latin-1 fallback
|
||||
Some(bytes.iter().map(|&b| b as char).collect())
|
||||
} else {
|
||||
None
|
||||
@@ -1241,97 +1213,4 @@ mod tests {
|
||||
let bad = "###!!!@@@$$$";
|
||||
assert!(score_text(good) > score_text(bad));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cid_font_with_unparseable_cmap_does_not_emit_latin1_mojibake() {
|
||||
// Type0/CID font (font_widths reports `is_cid=true`) where the
|
||||
// ToUnicode CMap couldn't be parsed (FontCMaps doesn't have the
|
||||
// obj_num). Bytes are a 2-byte CID stream containing high bytes
|
||||
// that aren't valid UTF-8 — exactly the case in the production
|
||||
// samples (Identity-H text where the ToUnicode CMap was missing
|
||||
// or malformed, scrape_id 019de78c-..., e.g. "Í Ù Z)¿").
|
||||
//
|
||||
// Without the guard, the function falls through to the byte-by-byte
|
||||
// Latin-1 fallback and produces "ÍÙ" (U+00CD U+00D9). The correct
|
||||
// behavior is to emit U+FFFD per CID so downstream
|
||||
// `detect_encoding_issues` flags the page for OCR.
|
||||
let bytes = vec![0xCD_u8, 0xD9, 0xCD, 0xD9];
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
let font_cmaps = FontCMaps::default();
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
font_tounicode_refs.insert("F0".to_string(), 999);
|
||||
let inline_cmaps = HashMap::new();
|
||||
let font_encodings: PageFontEncodings = HashMap::new();
|
||||
let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
|
||||
let mut decisions = CMapDecisionCache::new();
|
||||
let mut font_widths: PageFontWidths = HashMap::new();
|
||||
font_widths.insert("F0".to_string(), make_font_info(&[], 1000, true));
|
||||
|
||||
let result = extract_text_from_operand(
|
||||
&obj,
|
||||
"F0",
|
||||
None,
|
||||
&font_cmaps,
|
||||
&font_tounicode_refs,
|
||||
&inline_cmaps,
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut decisions,
|
||||
&font_widths,
|
||||
);
|
||||
|
||||
let text = result.expect("CID font fallback should still emit a marker");
|
||||
assert!(
|
||||
!text.contains('\u{00CD}') && !text.contains('\u{00D9}'),
|
||||
"CID font with unparseable CMap leaked Latin-1 mojibake: {text:?}"
|
||||
);
|
||||
assert!(
|
||||
text.contains('\u{FFFD}'),
|
||||
"CID font with unparseable CMap should emit U+FFFD so detect_encoding_issues fires: {text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_font_latin1_fallback_passes_high_bytes_through() {
|
||||
// A Type1/TrueType simple font (is_cid=false) with a `/ToUnicode`
|
||||
// reference but no usable CMap and no `/Differences` map.
|
||||
// Per-byte Latin-1 IS the canonical interpretation here — these
|
||||
// bytes are character codes, not CIDs. The CID guard must NOT
|
||||
// strip them. Reproduces the false positive that an earlier
|
||||
// version of the guard introduced for fonts in PDFs like
|
||||
// pdf-evals/Navigating-Artificial-Intelligence-..., where bytes
|
||||
// like 0xB6 are legitimate Latin-1 character codes.
|
||||
let bytes = vec![0x24_u8, 0x47, 0xB6, 0x56]; // "$G¶V"
|
||||
let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
|
||||
|
||||
let font_cmaps = FontCMaps::default();
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
font_tounicode_refs.insert("F1".to_string(), 999);
|
||||
let inline_cmaps = HashMap::new();
|
||||
let font_encodings: PageFontEncodings = HashMap::new();
|
||||
let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
|
||||
let mut decisions = CMapDecisionCache::new();
|
||||
let mut font_widths: PageFontWidths = HashMap::new();
|
||||
font_widths.insert("F1".to_string(), make_font_info(&[], 1000, false));
|
||||
|
||||
let text = extract_text_from_operand(
|
||||
&obj,
|
||||
"F1",
|
||||
None,
|
||||
&font_cmaps,
|
||||
&font_tounicode_refs,
|
||||
&inline_cmaps,
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
&mut decisions,
|
||||
&font_widths,
|
||||
)
|
||||
.expect("simple font should round-trip Latin-1 bytes");
|
||||
assert_eq!(text, "$G\u{00B6}V");
|
||||
assert!(
|
||||
!text.contains('\u{FFFD}'),
|
||||
"simple font fallback must not stamp FFFD over legitimate bytes: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,12 +29,8 @@ pub(crate) fn detect_columns(
|
||||
const MIN_ITEMS_PER_COLUMN: usize = 10;
|
||||
const NOISE_FRACTION: f32 = 0.15;
|
||||
|
||||
// Get items for this page. Strip Image placeholders — an image's left edge
|
||||
// would otherwise count toward the column projection profile.
|
||||
let page_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.page == page && crate::extractor::is_text_layout_item(i))
|
||||
.collect();
|
||||
// Get items for this page
|
||||
let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect();
|
||||
|
||||
if page_items.is_empty() {
|
||||
return vec![];
|
||||
|
||||
@@ -227,69 +227,6 @@ fn extract_positioned_text_impl(
|
||||
// Shared helpers (used by submodules via `super::`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Return true when this item should participate in text-layout
|
||||
/// heuristics (column detection, table grid detection, line grouping).
|
||||
///
|
||||
/// Image XObjects emit a positional placeholder via
|
||||
/// `extract_text_with_positions` (so layout-aware callers can crop +
|
||||
/// caption figures), but their bboxes don't carry text glyphs and would
|
||||
/// skew column/row clustering if they reached the heuristics. Hyperlinks
|
||||
/// and form fields *do* participate — the existing logic treats them as
|
||||
/// text-like and we keep that.
|
||||
pub(crate) fn is_text_layout_item(item: &crate::types::TextItem) -> bool {
|
||||
!matches!(item.item_type, crate::types::ItemType::Image)
|
||||
}
|
||||
|
||||
/// Map a (u, v) point in unit-square coordinates through the 6-element CTM
|
||||
/// to page-space. CTM format is `[a, b, c, d, e, f]` per
|
||||
/// [`multiply_matrices`].
|
||||
fn apply_ctm_point(ctm: &[f32; 6], u: f32, v: f32) -> (f32, f32) {
|
||||
(
|
||||
u * ctm[0] + v * ctm[2] + ctm[4],
|
||||
u * ctm[1] + v * ctm[3] + ctm[5],
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute the page-space axis-aligned bounding box of an Image XObject
|
||||
/// invoked under the given CTM.
|
||||
///
|
||||
/// Per the PDF spec, an image XObject is always rendered into a unit
|
||||
/// square `(0,0)–(1,1)` in its local coordinate system, and the `Do`
|
||||
/// operator applies the current CTM to position/scale/rotate that square
|
||||
/// onto the page. For the common axis-aligned case (no rotation/shear),
|
||||
/// the CTM reduces to `[w, 0, 0, h, x, y]` and the bbox is just
|
||||
/// `(x, y, w, h)`. For rotated/sheared images we transform all four
|
||||
/// corners and return their axis-aligned bbox so the caller always gets
|
||||
/// an upright rectangle.
|
||||
///
|
||||
/// Coordinates are PDF user space (origin at bottom-left, y-up). Width
|
||||
/// and height are non-negative.
|
||||
pub(crate) fn image_bbox_from_ctm(ctm: &[f32; 6]) -> (f32, f32, f32, f32) {
|
||||
let corners = [
|
||||
apply_ctm_point(ctm, 0.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 0.0),
|
||||
apply_ctm_point(ctm, 1.0, 1.0),
|
||||
apply_ctm_point(ctm, 0.0, 1.0),
|
||||
];
|
||||
let (mut x_min, mut x_max) = (corners[0].0, corners[0].0);
|
||||
let (mut y_min, mut y_max) = (corners[0].1, corners[0].1);
|
||||
for (cx, cy) in corners.iter().skip(1) {
|
||||
if *cx < x_min {
|
||||
x_min = *cx;
|
||||
}
|
||||
if *cx > x_max {
|
||||
x_max = *cx;
|
||||
}
|
||||
if *cy < y_min {
|
||||
y_min = *cy;
|
||||
}
|
||||
if *cy > y_max {
|
||||
y_max = *cy;
|
||||
}
|
||||
}
|
||||
(x_min, y_min, x_max - x_min, y_max - y_min)
|
||||
}
|
||||
|
||||
/// Multiply two 2D transformation matrices
|
||||
/// Matrix format: [a, b, c, d, e, f] representing:
|
||||
/// | a b 0 |
|
||||
|
||||
+13
-39
@@ -10,7 +10,7 @@ use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
use super::{get_number, multiply_matrices};
|
||||
|
||||
const MAX_FORM_XOBJECT_DEPTH: u8 = 5;
|
||||
|
||||
@@ -262,43 +262,19 @@ fn extract_form_xobject_text_inner(
|
||||
if !op.operands.is_empty() {
|
||||
if let Ok(name) = op.operands[0].as_name() {
|
||||
let xobj_name = String::from_utf8_lossy(name).to_string();
|
||||
match form_xobjects.get(&xobj_name) {
|
||||
Some(XObjectType::Form(nested_id)) => {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
if let Some(XObjectType::Form(nested_id)) = form_xobjects.get(&xobj_name) {
|
||||
if depth < MAX_FORM_XOBJECT_DEPTH {
|
||||
let nested_items = extract_form_xobject_text_inner(
|
||||
doc,
|
||||
*nested_id,
|
||||
page_num,
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
}
|
||||
Some(XObjectType::Image) => {
|
||||
// Mirror the top-level Image-XObject emission
|
||||
// in content_stream.rs so figures embedded
|
||||
// inside Form XObjects (common in print-to-PDF
|
||||
// workflows) aren't silently dropped.
|
||||
let (x, y, width, height) = image_bbox_from_ctm(&ctm);
|
||||
items.push(TextItem {
|
||||
text: format!("[Image: {}]", xobj_name),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
font: String::new(),
|
||||
font_size: 0.0,
|
||||
page: page_num,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -397,7 +373,6 @@ fn extract_form_xobject_text_inner(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
@@ -542,7 +517,6 @@ fn extract_form_xobject_text_inner(
|
||||
&font_encodings,
|
||||
&encoding_cache,
|
||||
cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
current_text.push_str(&text);
|
||||
}
|
||||
|
||||
+173
-2996
File diff suppressed because it is too large
Load Diff
+1
-10
@@ -422,16 +422,7 @@ impl Default for MarkdownOptions {
|
||||
fix_hyphenation: true,
|
||||
detect_bold: true,
|
||||
detect_italic: true,
|
||||
// `include_images: false` is intentional. The content-stream walker
|
||||
// now emits `ItemType::Image` `TextItem`s for every Image XObject
|
||||
// it encounters (see `extractor/content_stream.rs`). If we rendered
|
||||
// those into markdown by default, every existing caller would
|
||||
// suddenly see `` placeholders inserted
|
||||
// throughout their output — a silent regression for anyone who
|
||||
// upgrades. Image bboxes are still available via
|
||||
// `extract_text_with_positions` for callers (e.g. layout-aware
|
||||
// pipelines) that want to crop + caption figures themselves.
|
||||
include_images: false,
|
||||
include_images: true,
|
||||
include_links: true,
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: true,
|
||||
|
||||
+29
-261
@@ -4,74 +4,11 @@
|
||||
//! gridlines. Many IRS forms and government PDFs use these instead of
|
||||
//! `re` (rectangle) operators.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::tables::Table;
|
||||
use crate::types::{PdfLine, TextItem};
|
||||
|
||||
use super::detect_rects::{assign_items_to_grid, snap_edges};
|
||||
|
||||
/// Derive column edges from the x-endpoints of horizontal-rule
|
||||
/// segments when no vertical lines were drawn.
|
||||
///
|
||||
/// Catalog and archival-finding-aid tables are commonly drawn with
|
||||
/// per-row horizontal rules broken into N segments (one segment per
|
||||
/// cell), with no vertical dividers at all. The segment break points
|
||||
/// (e.g. `[50, 127], [127, 485], [485, 562]` per row) implicitly
|
||||
/// encode the column boundaries.
|
||||
///
|
||||
/// Returns column edges if ≥3 distinct x-positions each show up as a
|
||||
/// segment endpoint on ≥50% of the unique horizontal-line rows.
|
||||
/// Returns `None` otherwise — decorative rules with varying widths
|
||||
/// shouldn't be mistaken for a table.
|
||||
fn derive_columns_from_horizontal_segments(horizontals: &[(f32, f32, f32)]) -> Option<Vec<f32>> {
|
||||
if horizontals.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut endpoints: Vec<f32> = Vec::with_capacity(horizontals.len() * 2);
|
||||
for &(_, x_min, x_max) in horizontals {
|
||||
endpoints.push(x_min);
|
||||
endpoints.push(x_max);
|
||||
}
|
||||
let clusters = snap_edges(&endpoints, 5.0);
|
||||
if clusters.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Bucket y-values to count unique rows. Tolerance ~0.1pt (×10
|
||||
// rounding) tolerates the snap_edges 3pt clustering used later
|
||||
// for row edges.
|
||||
let unique_rows: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
if unique_rows.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let min_rows = (unique_rows.len() as f32 * 0.5).ceil() as usize;
|
||||
|
||||
let qualifying: Vec<f32> = clusters
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&cluster_x| {
|
||||
let rows_touched: HashSet<i32> = horizontals
|
||||
.iter()
|
||||
.filter(|&&(_, x_min, x_max)| {
|
||||
(x_min - cluster_x).abs() < 5.0 || (x_max - cluster_x).abs() < 5.0
|
||||
})
|
||||
.map(|&(y, _, _)| (y * 10.0).round() as i32)
|
||||
.collect();
|
||||
rows_touched.len() >= min_rows
|
||||
})
|
||||
.collect();
|
||||
|
||||
if qualifying.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
Some(qualifying)
|
||||
}
|
||||
|
||||
/// Detect tables from line segments on a given page.
|
||||
///
|
||||
/// Lines are classified as horizontal or vertical, snapped into grid edges,
|
||||
@@ -115,50 +52,25 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
// Diagonal lines are ignored
|
||||
}
|
||||
|
||||
if horizontals.len() < 3 {
|
||||
if horizontals.len() < 3 || verticals.len() < 2 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// If no/very-few vertical lines are drawn, try to derive column edges
|
||||
// from the x-endpoints of the horizontal-rule segments. Catalog and
|
||||
// archival-finding-aid layouts commonly draw each row's horizontal
|
||||
// rule as N segments (one per cell), with no vertical dividers at
|
||||
// all — the segment break points encode the column boundaries.
|
||||
let implicit_col_edges: Option<Vec<f32>> = if verticals.len() < 2 {
|
||||
derive_columns_from_horizontal_segments(&horizontals)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if verticals.len() < 2 && implicit_col_edges.is_none() {
|
||||
return Vec::new();
|
||||
}
|
||||
let cols_from_segments = implicit_col_edges.is_some();
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page){}",
|
||||
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
|
||||
page,
|
||||
horizontals.len(),
|
||||
verticals.len(),
|
||||
page_lines.len(),
|
||||
if cols_from_segments {
|
||||
" — columns from horizontal segments"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
page_lines.len()
|
||||
);
|
||||
|
||||
// Snap Y-values of horizontal lines → row edges
|
||||
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
|
||||
let row_edges = snap_edges(&h_ys, 3.0);
|
||||
|
||||
// Column edges from drawn verticals when present, else from the
|
||||
// horizontal-segment endpoints derived above.
|
||||
let col_edges = if let Some(c) = implicit_col_edges {
|
||||
c
|
||||
} else {
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
snap_edges(&v_xs, 3.0)
|
||||
};
|
||||
// Snap X-values of vertical lines → column edges
|
||||
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
|
||||
let col_edges = snap_edges(&v_xs, 3.0);
|
||||
|
||||
log::debug!(
|
||||
"detect_lines p{}: {} row edges, {} col edges after snap",
|
||||
@@ -198,21 +110,15 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Reject page-spanning frames: a decorative outer border has just 4
|
||||
// edges (top/bottom/left/right). Real full-page tables — common in
|
||||
// governmental ledgers, financial reports, etc. — span the same A4 /
|
||||
// Letter dimensions but have many internal row/column rules. Only
|
||||
// reject when the line set looks like a bare frame, not a grid.
|
||||
// Reject page-spanning frames: if the grid covers >90% of a standard page
|
||||
// dimension in both axes, it's a border frame, not a table.
|
||||
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
|
||||
if table_width > 500.0 && table_height > 700.0 && horizontals.len() <= 4 && verticals.len() <= 4
|
||||
{
|
||||
if table_width > 500.0 && table_height > 700.0 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0}, {} h + {} v)",
|
||||
"detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})",
|
||||
page,
|
||||
table_width,
|
||||
table_height,
|
||||
horizontals.len(),
|
||||
verticals.len()
|
||||
table_height
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -240,33 +146,24 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
|
||||
// Validate vertical lines: at least 2 should span a meaningful height.
|
||||
// Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
|
||||
// for tables with partial column separators. Skipped entirely when
|
||||
// columns came from horizontal-segment endpoints — there are no
|
||||
// vertical lines to validate against, and the segment-endpoint
|
||||
// consistency check in `derive_columns_from_horizontal_segments`
|
||||
// is the equivalent guard.
|
||||
let spanning_v = if cols_from_segments {
|
||||
0
|
||||
} else {
|
||||
let s = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let p = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if s < 2 && p < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
s,
|
||||
p
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
s
|
||||
};
|
||||
// for tables with partial column separators.
|
||||
let spanning_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
|
||||
.count();
|
||||
let partial_v = verticals
|
||||
.iter()
|
||||
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
|
||||
.count();
|
||||
if spanning_v < 2 && partial_v < 4 {
|
||||
log::debug!(
|
||||
"detect_lines p{}: rejected — {} spanning + {} partial V lines",
|
||||
page,
|
||||
spanning_v,
|
||||
partial_v
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Row edges need to be in descending order (top of page = higher Y first)
|
||||
let mut row_edges_desc = row_edges;
|
||||
@@ -513,135 +410,6 @@ mod tests {
|
||||
assert!(tables.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_only_implicit_columns_accepted() {
|
||||
// Catalog/finding-aid pattern: each row's horizontal rule is
|
||||
// drawn as 3 segments at consistent x-endpoints (50, 127, 485,
|
||||
// 562), with no vertical lines anywhere. The segment break
|
||||
// points must be inferred as column edges.
|
||||
let mut lines = Vec::new();
|
||||
// Slightly uneven row spacing so the chart-gridline rejector
|
||||
// (CV < 0.02) doesn't fire.
|
||||
let row_ys = [80.0_f32, 145.0, 215.0, 280.0, 350.0, 415.0, 485.0];
|
||||
for &y in &row_ys {
|
||||
lines.push(make_hline(y, 50.0, 127.0, 1));
|
||||
lines.push(make_hline(y, 127.0, 485.0, 1));
|
||||
lines.push(make_hline(y, 485.0, 562.0, 1));
|
||||
}
|
||||
// Populate every cell so capture / density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for w in row_ys.windows(2) {
|
||||
let row_y = (w[0] + w[1]) / 2.0;
|
||||
items.push(make_item("id", 80.0, row_y, 1));
|
||||
items.push(make_item("description here", 200.0, row_y, 1));
|
||||
items.push(make_item("date", 510.0, row_y, 1));
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"horizontal-segment-only grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 3, "expected 3 columns");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_horizontal_segments_with_inconsistent_endpoints_rejected() {
|
||||
// Decorative rules of varying widths shouldn't be detected as a
|
||||
// table — each line has its own x-endpoints, no consistent
|
||||
// column boundary survives the 50%-of-rows threshold.
|
||||
let lines = vec![
|
||||
make_hline(100.0, 50.0, 150.0, 1),
|
||||
make_hline(200.0, 50.0, 220.0, 1),
|
||||
make_hline(300.0, 50.0, 310.0, 1),
|
||||
make_hline(400.0, 50.0, 470.0, 1),
|
||||
];
|
||||
let items = vec![
|
||||
make_item("decorative", 100.0, 150.0, 1),
|
||||
make_item("text", 100.0, 250.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"varying-width decorative rules should not be detected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_bare_frame_rejected() {
|
||||
// Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.
|
||||
// No internal structure → decorative border, not a table.
|
||||
let lines = vec![
|
||||
make_hline(20.0, 20.0, 575.0, 1), // top
|
||||
make_hline(820.0, 20.0, 575.0, 1), // bottom
|
||||
make_vline(20.0, 20.0, 820.0, 1), // left
|
||||
make_vline(575.0, 20.0, 820.0, 1), // right
|
||||
];
|
||||
let items = vec![
|
||||
make_item("title", 100.0, 100.0, 1),
|
||||
make_item("body", 100.0, 200.0, 1),
|
||||
];
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"Page-sized 4-edge frame should be rejected as decoration"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_spanning_grid_with_internal_lines_accepted() {
|
||||
// Full-page table (governmental-ledger pattern): A4-sized grid
|
||||
// that previously hit the "page-spanning frame" early reject
|
||||
// before downstream validation could even look at it.
|
||||
// Verticals span the full table height so we isolate the
|
||||
// frame-vs-grid decision under test.
|
||||
let mut lines = Vec::new();
|
||||
// 13 horizontal rules: header + 12 row separators
|
||||
let h_ys = [
|
||||
22.5, 37.9, 95.5, 144.5, 184.9, 233.9, 291.7, 340.7, 415.8, 499.6, 574.7, 623.7, 698.8,
|
||||
];
|
||||
for &y in &h_ys {
|
||||
lines.push(make_hline(y, 22.6, 566.6, 1));
|
||||
}
|
||||
// 7 column dividers spanning full table height.
|
||||
let v_xs = [22.6, 66.3, 116.3, 186.6, 263.1, 493.5, 566.5];
|
||||
for &x in &v_xs {
|
||||
lines.push(make_vline(x, 22.5, 698.8, 1));
|
||||
}
|
||||
// Populate every cell so the capture-ratio + density checks pass.
|
||||
let mut items = Vec::new();
|
||||
for r in 0..(h_ys.len() - 1) {
|
||||
let row_y = (h_ys[r] + h_ys[r + 1]) / 2.0;
|
||||
for c in 0..(v_xs.len() - 1) {
|
||||
let col_x = (v_xs[c] + v_xs[c + 1]) / 2.0;
|
||||
items.push(make_item("x", col_x, row_y, 1));
|
||||
}
|
||||
}
|
||||
let tables = detect_tables_from_lines(&items, &lines, 1);
|
||||
assert_eq!(
|
||||
tables.len(),
|
||||
1,
|
||||
"Full-page table with internal grid should be accepted"
|
||||
);
|
||||
let t = &tables[0];
|
||||
assert!(
|
||||
t.cells.len() >= 6,
|
||||
"expected ≥6 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert!(
|
||||
t.cells[0].len() >= 3,
|
||||
"expected ≥3 columns, got {}",
|
||||
t.cells[0].len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_rejected() {
|
||||
// Only 2 col edges (1 column) — not a table even with verticals
|
||||
|
||||
+41
-635
@@ -231,15 +231,6 @@ pub fn detect_tables_from_rects(
|
||||
rects: &[PdfRect],
|
||||
page: u32,
|
||||
) -> (Vec<Table>, Vec<RectHintRegion>) {
|
||||
// Strip Image placeholders before column/row clustering — an image's bbox
|
||||
// would otherwise show up as a spurious column edge. See `is_text_layout_item`.
|
||||
let items_owned: Vec<TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| crate::extractor::is_text_layout_item(i))
|
||||
.cloned()
|
||||
.collect();
|
||||
let items = items_owned.as_slice();
|
||||
|
||||
// Filter rects on this page; normalize negative widths/heights; skip tiny rects.
|
||||
let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized
|
||||
for r in rects {
|
||||
@@ -294,11 +285,7 @@ pub fn detect_tables_from_rects(
|
||||
//
|
||||
// Only remove when the container is a similarly-sized cell (height
|
||||
// ratio < 4×), NOT when the container is a table-wide background
|
||||
// that dwarfs the sub-rect. Origin-anchored page-background rects
|
||||
// also disqualify as containers — they normally exceed the 4× ratio,
|
||||
// but when the sub-rect is itself a tall table-frame the ratio can
|
||||
// fall under the gate, and dropping the frame collapses cluster
|
||||
// adjacency between adjacent column-cell groups.
|
||||
// that dwarfs the sub-rect.
|
||||
//
|
||||
// Skip this O(n²) dedup when there are too many rects — pages with
|
||||
// thousands of vector-drawing rects won't benefit from cell dedup.
|
||||
@@ -308,11 +295,9 @@ pub fn detect_tables_from_rects(
|
||||
page_rects.retain(|&(ax, ay, aw, ah)| {
|
||||
let tol = 2.0;
|
||||
!snapshot.iter().any(|&(bx, by, bw, bh)| {
|
||||
let container_is_page_bg = bx < 5.0 && by < 5.0;
|
||||
// b must strictly contain a (b is larger in area)
|
||||
bw * bh > aw * ah * 1.2
|
||||
&& bh < ah * 4.0 // container must be similarly sized, not a table background
|
||||
&& !container_is_page_bg
|
||||
&& bx <= ax + tol
|
||||
&& (bx + bw) >= (ax + aw) - tol
|
||||
&& by <= ay + tol
|
||||
@@ -1403,15 +1388,13 @@ fn detect_row_stripe_table(
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
// Allow longer cells for multi-column tables (descriptions in one column
|
||||
// are common). Narrow grids with giant cells are usually layout
|
||||
// backgrounds — but only when the row count is also small. A 4+-row
|
||||
// key/value table with one descriptive column reads as a real table
|
||||
// on every other gate, so don't reject it on cell length alone.
|
||||
// are common). Single-column or 2-column "tables" with giant cells are
|
||||
// almost always layout backgrounds.
|
||||
let max_allowed = if num_cols >= 3 { 2000 } else { 500 };
|
||||
if max_cell_len > max_allowed && non_empty_rows < 4 {
|
||||
if max_cell_len > max_allowed {
|
||||
debug!(
|
||||
" row-stripe rejected: max cell length {} > {} (layout background, {} rows)",
|
||||
max_cell_len, max_allowed, non_empty_rows
|
||||
" row-stripe rejected: max cell length {} > {} (layout background)",
|
||||
max_cell_len, max_allowed
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1604,111 +1587,25 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Derive columns from text X-position clustering, but prefer rect
|
||||
// X-edges when they already provide a tighter scaffold. Some PDFs draw
|
||||
// only the row-index cells in the body plus a full header row; that is
|
||||
// not dense enough for `try_build_grid`, but the header rects still define
|
||||
// the real columns. Text starts inside wide cells can otherwise split the
|
||||
// table into spurious sub-columns.
|
||||
// Derive columns from text X-position clustering
|
||||
let columns = cluster_x_positions(&page_items, 15.0);
|
||||
let text_col_edges = if columns.len() >= 2 {
|
||||
let mut edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
|
||||
let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
|
||||
edges.push(min_x - 5.0);
|
||||
for pair in columns.windows(2) {
|
||||
edges.push((pair[0] + pair[1]) / 2.0);
|
||||
}
|
||||
let max_x_right = page_items
|
||||
.iter()
|
||||
.map(|(_, i)| i.x + i.width)
|
||||
.reduce(f32::max)?;
|
||||
edges.push(max_x_right + 5.0);
|
||||
Some(edges)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let rect_col_edges = {
|
||||
let mut x_vals = Vec::with_capacity(content_rects.len() * 2);
|
||||
for &&(x, _, w, _) in &content_rects {
|
||||
x_vals.push(x);
|
||||
x_vals.push(x + w);
|
||||
}
|
||||
let mut edges = snap_edges(&x_vals, 6.0);
|
||||
edges.sort_by(|a, b| a.total_cmp(b));
|
||||
if (3..=26).contains(&edges.len()) {
|
||||
Some(edges)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// For wired-grid tables whose header text is centered/right-aligned but
|
||||
// whose data is left-aligned, cluster_x_positions can drop the header-only
|
||||
// x-cluster in its singleton-filter pass and merge adjacent data clusters
|
||||
// when the gap is below threshold, losing a column. Rect borders are
|
||||
// ground truth in that case — but only when each rect column actually
|
||||
// holds text. Decorative or background rects (prose laid out in a frame,
|
||||
// cell-fill rects with extra borders) can produce more rect-derived
|
||||
// columns than the text supports; preferring rects there would split a
|
||||
// logical column into spurious sub-columns.
|
||||
let rect_cols_match_text = match (&rect_col_edges, &text_col_edges) {
|
||||
(Some(rect_edges), _) if rect_edges.len() >= 4 => {
|
||||
let num_rect_cols = rect_edges.len() - 1;
|
||||
let mut col_item_counts = vec![0usize; num_rect_cols];
|
||||
for (_, item) in &page_items {
|
||||
let cx = item.x + item.width / 2.0;
|
||||
for c in 0..num_rect_cols {
|
||||
if cx >= rect_edges[c] - 2.0 && cx <= rect_edges[c + 1] + 2.0 {
|
||||
col_item_counts[c] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Require every rect column to hold multiple text items. A rect
|
||||
// column with no (or only one) item is decorative or the rect grid
|
||||
// is detecting a spurious column the data does not need; in those
|
||||
// cases the old text-cluster preference is the safer fallback.
|
||||
col_item_counts.iter().all(|&n| n >= 2)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let (col_edges, columns_from_text) = match (rect_col_edges, text_col_edges) {
|
||||
(Some(rect_edges), text_edges_opt) if rect_cols_match_text => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns (text clusters: {}; rect cols well-distributed)",
|
||||
rect_edges.len() - 1,
|
||||
text_edges_opt
|
||||
.as_ref()
|
||||
.map(|e| (e.len() - 1) as i32)
|
||||
.unwrap_or(-1)
|
||||
);
|
||||
(rect_edges, false)
|
||||
}
|
||||
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
|
||||
debug!(
|
||||
" cell-rect using {} rect-derived columns over {} text clusters",
|
||||
rect_edges.len() - 1,
|
||||
text_edges.len() - 1
|
||||
);
|
||||
(rect_edges, false)
|
||||
}
|
||||
(_, Some(text_edges)) => (text_edges, true),
|
||||
(Some(rect_edges), None) => (rect_edges, false),
|
||||
(None, None) => {
|
||||
debug!(
|
||||
" cell-rect rejected: only {} columns from text clustering",
|
||||
columns.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
if col_edges.len() < 3 {
|
||||
if columns.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Build column edges
|
||||
let mut col_edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
|
||||
let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
|
||||
col_edges.push(min_x - 5.0);
|
||||
for pair in columns.windows(2) {
|
||||
col_edges.push((pair[0] + pair[1]) / 2.0);
|
||||
}
|
||||
let max_x_right = page_items
|
||||
.iter()
|
||||
.map(|(_, i)| i.x + i.width)
|
||||
.reduce(f32::max)?;
|
||||
col_edges.push(max_x_right + 5.0);
|
||||
|
||||
let num_cols = col_edges.len() - 1;
|
||||
let num_rows = row_edges.len() - 1;
|
||||
|
||||
@@ -1720,25 +1617,12 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
page_items.len()
|
||||
);
|
||||
|
||||
let (mut cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
|
||||
let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page);
|
||||
|
||||
if item_indices.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut row_edges = row_edges;
|
||||
let (collapsed_cells, collapsed_row_edges, collapsed_rows) =
|
||||
collapse_multiline_description_rows(cells, row_edges, &col_edges);
|
||||
let has_wrapped_description_rows = collapsed_rows > 0;
|
||||
cells = collapsed_cells;
|
||||
row_edges = collapsed_row_edges;
|
||||
if collapsed_rows > 0 {
|
||||
debug!(
|
||||
" cell-rect collapsed {} wrapped description rows",
|
||||
collapsed_rows
|
||||
);
|
||||
}
|
||||
|
||||
// Validate: >=2 non-empty rows, >=25% density
|
||||
let non_empty_rows = cells
|
||||
.iter()
|
||||
@@ -1752,7 +1636,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
let num_rows = cells.len();
|
||||
let total_cells = (num_cols * num_rows) as f32;
|
||||
let non_empty_cells = cells
|
||||
.iter()
|
||||
@@ -1772,21 +1655,17 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject tables with paragraph-length cells — typically layout
|
||||
// backgrounds (sidebars, banners) where a single big rectangle
|
||||
// contains a wall of prose. Spare multi-row key/value tables where
|
||||
// the value column is a multi-bullet description: those pass every
|
||||
// other gate and shouldn't get killed on cell length alone.
|
||||
// Reject tables with paragraph-length cells (layout backgrounds, not tables)
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" cell-rect rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" cell-rect rejected: max cell length {} > 500",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -1802,36 +1681,13 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
|
||||
// Reject "tables" that are actually prose in a framed region.
|
||||
// Columns here come from text X-position clustering; when prose wraps
|
||||
// inside a bounding-box rect (e.g. chat-transcript figures, two-column
|
||||
// legal-text blocks in forms) the word-boundary gaps cluster into
|
||||
// spurious columns, and the resulting cells hold sentence fragments
|
||||
// riddled with common English function words.
|
||||
//
|
||||
// Apply at any column count >= 2. The 2-col case is the bite — a
|
||||
// paragraph wrapped into 2 justified columns produces the same
|
||||
// surface signal as a real "label / value" table in the
|
||||
// well-distributed-cols check (both cols populated), so we need a
|
||||
// content-based signal to tell them apart.
|
||||
//
|
||||
// Layered checks combine after the 20%-of-cells prose-word
|
||||
// trigger fires:
|
||||
// (a) Long-cell content: prose-in-a-frame averages ~70-100 chars
|
||||
// per non-empty cell (sentence fragments); real data tables
|
||||
// are typically <30 chars, occasionally up to ~55 for
|
||||
// descriptive 4-col tables. The 65-char threshold cleanly
|
||||
// separates them on observed fixtures (accessory_building
|
||||
// prose=74 chars, upstage data=53, greencomp=20). This
|
||||
// overrides the well-distributed relaxation — long cells
|
||||
// are the strongest prose signal even when both cols are
|
||||
// populated.
|
||||
// (b) Two-column text-only scaffold: when both columns were inferred
|
||||
// from text starts rather than rect edges, prose fragments can look
|
||||
// perfectly balanced. Require rect evidence for this relaxed shape.
|
||||
// (c) Well-distributed columns: ≥75% of cols hold ≥2 non-empty
|
||||
// cells. Catches the prose-paragraph-as-many-cols shape
|
||||
// while admitting real "label / value / description /
|
||||
// benefit"-style tables.
|
||||
if num_cols >= 2 {
|
||||
// inside a bounding-box rect (e.g. chat-transcript figures) the
|
||||
// word-boundary gaps cluster into many spurious columns, and the
|
||||
// resulting cells hold sentence fragments riddled with common English
|
||||
// function words. Count cells with any such word and reject when
|
||||
// 20%+ of non-empty cells match — real tabular data (labels, units,
|
||||
// numbers) rarely contains these words.
|
||||
if num_cols >= 4 {
|
||||
const PROSE_WORDS: &[&str] = &[
|
||||
"a", "an", "the", "of", "to", "is", "was", "are", "were", "be", "been", "in", "on",
|
||||
"at", "with", "for", "by", "as", "and", "or", "but", "this", "that", "these", "those",
|
||||
@@ -1841,7 +1697,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
];
|
||||
let mut prose_cells = 0usize;
|
||||
let mut counted = 0usize;
|
||||
let mut total_chars = 0usize;
|
||||
for row in &cells {
|
||||
for cell in row {
|
||||
let t = cell.trim();
|
||||
@@ -1849,7 +1704,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
continue;
|
||||
}
|
||||
counted += 1;
|
||||
total_chars += t.chars().count();
|
||||
let lower = t.to_ascii_lowercase();
|
||||
let has_prose_word = lower
|
||||
.split(|c: char| !c.is_ascii_alphabetic() && c != '\'')
|
||||
@@ -1860,65 +1714,11 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
}
|
||||
}
|
||||
if counted > 0 && prose_cells * 5 >= counted {
|
||||
// (a) Long-cell content: overrides the well-distributed
|
||||
// relaxation. The 2-col prose-in-a-frame case populates
|
||||
// both cols (passes well-distributed) but every cell
|
||||
// holds a sentence fragment, so mean cell length is the
|
||||
// discriminator.
|
||||
const PROSE_MEAN_CHAR_THRESHOLD: usize = 65;
|
||||
let mean_chars = total_chars / counted;
|
||||
if mean_chars > PROSE_MEAN_CHAR_THRESHOLD && !has_wrapped_description_rows {
|
||||
debug!(
|
||||
" cell-rect rejected: prose-in-frame, mean non-empty cell {} chars > {} (prose words {}/{})",
|
||||
mean_chars, PROSE_MEAN_CHAR_THRESHOLD, prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
} else if mean_chars > PROSE_MEAN_CHAR_THRESHOLD {
|
||||
debug!(
|
||||
" cell-rect prose check relaxed: wrapped description rows, mean {} chars (prose words {}/{})",
|
||||
mean_chars, prose_cells, counted
|
||||
);
|
||||
}
|
||||
|
||||
// (b) Two text-derived columns are not enough vector evidence once
|
||||
// the content looks prose-like. Real 2-col rect tables still pass
|
||||
// when the column scaffold comes from drawn cell geometry.
|
||||
if columns_from_text && num_cols == 2 {
|
||||
debug!(
|
||||
" cell-rect rejected: prose-in-frame with text-derived 2-col scaffold (mean {} chars, prose words {}/{})",
|
||||
mean_chars, prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// (c) Well-distributed columns.
|
||||
let filled_cols = (0..num_cols)
|
||||
.filter(|&c| {
|
||||
cells
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
!row.get(c)
|
||||
.map(String::as_str)
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.is_empty()
|
||||
})
|
||||
.count()
|
||||
>= 2
|
||||
})
|
||||
.count();
|
||||
let well_distributed = filled_cols * 4 >= num_cols * 3;
|
||||
if !well_distributed {
|
||||
debug!(
|
||||
" cell-rect rejected: {}/{} cells contain prose function words — likely prose ({}/{} cols filled, mean {} chars)",
|
||||
prose_cells, counted, filled_cols, num_cols, mean_chars
|
||||
);
|
||||
return None;
|
||||
}
|
||||
debug!(
|
||||
" cell-rect prose check relaxed: {}/{} cols filled, mean {} chars — table-with-description-col",
|
||||
filled_cols, num_cols, mean_chars
|
||||
" cell-rect rejected: {}/{} cells contain prose function words — likely prose",
|
||||
prose_cells, counted
|
||||
);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1939,132 +1739,6 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Merge wrapped description-line bands back into their visual data rows.
|
||||
///
|
||||
/// Some Word/PDF exports draw enough rectangle geometry to prove a table exists
|
||||
/// but expose Y bands per wrapped text line instead of per cell row. In the
|
||||
/// common mapping-table shape, a narrow row-label column precedes one wide
|
||||
/// description column, and wrapped continuation bands have content only in that
|
||||
/// wide column. Merge only that high-confidence shape so framed prose still
|
||||
/// falls through the existing prose guards.
|
||||
fn collapse_multiline_description_rows(
|
||||
cells: Vec<Vec<String>>,
|
||||
row_edges: Vec<f32>,
|
||||
col_edges: &[f32],
|
||||
) -> (Vec<Vec<String>>, Vec<f32>, usize) {
|
||||
let num_rows = cells.len();
|
||||
let num_cols = col_edges.len().saturating_sub(1);
|
||||
if num_rows < 3 || num_cols < 3 || row_edges.len() != num_rows + 1 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let table_width = col_edges[num_cols] - col_edges[0];
|
||||
if table_width <= 0.0 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let Some((description_col, description_width)) = (0..num_cols)
|
||||
.map(|c| (c, col_edges[c + 1] - col_edges[c]))
|
||||
.max_by(|a, b| a.1.total_cmp(&b.1))
|
||||
else {
|
||||
return (cells, row_edges, 0);
|
||||
};
|
||||
|
||||
// Require a preceding row-label column. Without it (e.g. a prose frame
|
||||
// split into text-start columns), "one populated wide column" is not enough
|
||||
// evidence to find visual row starts safely.
|
||||
if description_col == 0 || description_width < table_width * 0.35 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let row_has_left_label = |row: &[String]| {
|
||||
row.iter()
|
||||
.take(description_col)
|
||||
.any(|cell| !cell.trim().is_empty())
|
||||
};
|
||||
let labeled_rows = cells.iter().filter(|row| row_has_left_label(row)).count();
|
||||
if labeled_rows < 2 {
|
||||
return (cells, row_edges, 0);
|
||||
}
|
||||
|
||||
let mut merged_rows = 0usize;
|
||||
let mut wrapped_description_rows = 0usize;
|
||||
let mut new_cells: Vec<Vec<String>> = Vec::with_capacity(num_rows);
|
||||
let mut new_edges = Vec::with_capacity(row_edges.len());
|
||||
new_edges.push(row_edges[0]);
|
||||
|
||||
for (row_idx, row) in cells.into_iter().enumerate() {
|
||||
let desc_text = row
|
||||
.get(description_col)
|
||||
.map(String::as_str)
|
||||
.unwrap_or("")
|
||||
.trim();
|
||||
let left_label = row_has_left_label(&row);
|
||||
let non_desc_non_empty = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(col, cell)| *col != description_col && !cell.trim().is_empty())
|
||||
.count();
|
||||
|
||||
// Wrapped continuation bands contain only description-column text.
|
||||
// The preceding label/marker column is empty because the visual row's
|
||||
// label cell spans the whole wrapped block.
|
||||
let is_description_continuation = row_idx > 0
|
||||
&& !desc_text.is_empty()
|
||||
&& !left_label
|
||||
&& non_desc_non_empty == 0
|
||||
&& !new_cells.is_empty();
|
||||
|
||||
// Header cells are often split as "Controls" / "Version" in the first
|
||||
// column while the other header labels sit on the first band.
|
||||
let only_first_col = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.all(|(col, cell)| col == 0 || cell.trim().is_empty());
|
||||
let is_header_continuation = row_idx > 0
|
||||
&& only_first_col
|
||||
&& row
|
||||
.first()
|
||||
.is_some_and(|cell| !cell.trim().is_empty() && cell.chars().count() <= 24)
|
||||
&& !new_cells.is_empty()
|
||||
&& new_cells
|
||||
.last()
|
||||
.is_some_and(|prev| prev.iter().filter(|c| !c.trim().is_empty()).count() >= 2);
|
||||
|
||||
if is_description_continuation || is_header_continuation {
|
||||
if let Some(prev) = new_cells.last_mut() {
|
||||
for (col, cell) in row.iter().enumerate() {
|
||||
let text = cell.trim();
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !prev[col].trim().is_empty() {
|
||||
prev[col].push(' ');
|
||||
}
|
||||
prev[col].push_str(text);
|
||||
}
|
||||
}
|
||||
merged_rows += 1;
|
||||
if is_description_continuation {
|
||||
wrapped_description_rows += 1;
|
||||
}
|
||||
} else {
|
||||
if !new_cells.is_empty() {
|
||||
new_edges.push(row_edges[row_idx]);
|
||||
}
|
||||
new_cells.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
new_edges.push(*row_edges.last().unwrap());
|
||||
|
||||
if merged_rows == 0 || new_cells.len() < 2 || new_edges.len() != new_cells.len() + 1 {
|
||||
return (new_cells, row_edges, 0);
|
||||
}
|
||||
|
||||
(new_cells, new_edges, wrapped_description_rows)
|
||||
}
|
||||
|
||||
/// Detect a table by merging all cluster rects into one group.
|
||||
///
|
||||
/// This handles clip-path PDFs where each column's cell rects form a separate
|
||||
@@ -2200,20 +1874,18 @@ fn detect_merged_cluster_table(
|
||||
return None;
|
||||
}
|
||||
|
||||
// Reject if any cell has excessive text — layout background rects
|
||||
// produce "cells" containing paragraphs, not short data-table values.
|
||||
// Multi-row key/value tables can legitimately have one column of
|
||||
// long descriptive text, so only reject narrow-row layouts here.
|
||||
// Reject if any cell has excessive text — layout background rects produce
|
||||
// "cells" containing paragraphs, not short data-table values.
|
||||
let max_cell_len = cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.map(|c| c.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if max_cell_len > 500 && non_empty_rows < 4 {
|
||||
if max_cell_len > 500 {
|
||||
debug!(
|
||||
" merged-cluster rejected: max cell length {} > 500 ({} rows, layout background)",
|
||||
max_cell_len, non_empty_rows
|
||||
" merged-cluster rejected: max cell length {} > 500 (layout background)",
|
||||
max_cell_len
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -2643,46 +2315,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_row_stripe_accepts_multi_row_key_value_long_cells() {
|
||||
// Multi-row 2-column key/value table where one value cell holds
|
||||
// a paragraph (>500 chars). The old `max_cell_len > 500` check
|
||||
// rejected this shape as a "layout background"; with the
|
||||
// multi-row guard, it should be accepted.
|
||||
let mut rects = Vec::new();
|
||||
let row_h = 25.0_f32;
|
||||
let y_top = 700.0_f32;
|
||||
for i in 0..8 {
|
||||
let y = y_top - (i as f32) * row_h;
|
||||
rects.push((40.0, y, 510.0, row_h));
|
||||
}
|
||||
let mut items = Vec::new();
|
||||
for i in 0..8 {
|
||||
let row_center_y = y_top - (i as f32) * row_h + row_h / 2.0;
|
||||
// Left column: short label
|
||||
items.push(make_item(&format!("Field {}", i), 45.0, row_center_y, 10.0));
|
||||
// Right column: short value, except the last row which is a paragraph
|
||||
let value = if i == 7 {
|
||||
"X".repeat(800)
|
||||
} else {
|
||||
"value".to_string()
|
||||
};
|
||||
items.push(make_item(&value, 300.0, row_center_y, 10.0));
|
||||
}
|
||||
let result = detect_row_stripe_table(&items, &rects, 1);
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"multi-row key/value table with one long cell should be accepted"
|
||||
);
|
||||
let t = result.unwrap();
|
||||
assert!(
|
||||
t.cells.len() >= 4,
|
||||
"expected ≥4 rows, got {}",
|
||||
t.cells.len()
|
||||
);
|
||||
assert_eq!(t.cells[0].len(), 2, "expected 2 columns");
|
||||
}
|
||||
|
||||
// --- propagate_merged_cells ---
|
||||
|
||||
#[test]
|
||||
@@ -3297,232 +2929,6 @@ mod tests {
|
||||
// If tables were detected, that's also acceptable
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_derived_two_col_prose_is_not_cell_rect_table() {
|
||||
let page = 1;
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..8 {
|
||||
rects.push(PdfRect {
|
||||
x: 50.0,
|
||||
y: 100.0 + row as f32 * 20.0,
|
||||
width: 180.0,
|
||||
height: 18.0,
|
||||
page,
|
||||
});
|
||||
}
|
||||
|
||||
let mut items = Vec::new();
|
||||
let left = [
|
||||
"the annual plan was revised",
|
||||
"and the team noted changes",
|
||||
"this section explains limits",
|
||||
"with additional notes below",
|
||||
"the policy was reviewed",
|
||||
"and results are summarized",
|
||||
"this appendix describes scope",
|
||||
"with examples for reference",
|
||||
];
|
||||
let right = [
|
||||
"for each area in the review",
|
||||
"as part of the assessment",
|
||||
"that were applied in context",
|
||||
"to support the conclusion",
|
||||
"for use by the committee",
|
||||
"as shown in the narrative",
|
||||
"that remain under discussion",
|
||||
"to clarify the method",
|
||||
];
|
||||
for row in 0..8 {
|
||||
let y = 104.0 + row as f32 * 20.0;
|
||||
let mut left_item = make_item(left[row], 60.0, y, 9.0);
|
||||
left_item.width = 50.0;
|
||||
items.push(left_item);
|
||||
let mut right_item = make_item(right[row], 150.0, y, 9.0);
|
||||
right_item.width = 50.0;
|
||||
items.push(right_item);
|
||||
}
|
||||
|
||||
let (tables, _hints) = detect_tables_from_rects(&items, &rects, page);
|
||||
assert!(
|
||||
tables.is_empty(),
|
||||
"text-derived two-column prose must not be accepted as a rect table; got {:?}",
|
||||
tables
|
||||
.iter()
|
||||
.map(|t| (t.rows.len(), t.columns.len()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiline_indented_description_rows_collapse_to_visual_rows() {
|
||||
let page = 1;
|
||||
let col_edges = [0.0, 60.0, 420.0, 460.0, 500.0, 540.0];
|
||||
let row_edges = [
|
||||
340.0, 320.0, 300.0, 270.0, 250.0, 230.0, 200.0, 180.0, 160.0,
|
||||
];
|
||||
|
||||
let mut rects = Vec::new();
|
||||
for row in 0..row_edges.len() - 1 {
|
||||
let y_top = row_edges[row];
|
||||
let y_bot = row_edges[row + 1];
|
||||
for col in 0..col_edges.len() - 1 {
|
||||
rects.push((
|
||||
col_edges[col],
|
||||
y_bot,
|
||||
col_edges[col + 1] - col_edges[col],
|
||||
y_top - y_bot,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items = vec![
|
||||
make_item("Controls", 8.0, 330.0, 9.0),
|
||||
make_item("Control", 70.0, 330.0, 9.0),
|
||||
make_item("IG 1", 428.0, 330.0, 9.0),
|
||||
make_item("IG 2", 468.0, 330.0, 9.0),
|
||||
make_item("IG 3", 508.0, 330.0, 9.0),
|
||||
make_item("Version", 8.0, 310.0, 9.0),
|
||||
make_item("v8", 20.0, 285.0, 9.0),
|
||||
make_item(
|
||||
"4.5 Implement and Manage a Firewall on End-User Devices",
|
||||
70.0,
|
||||
285.0,
|
||||
9.0,
|
||||
),
|
||||
make_item("*", 438.0, 285.0, 9.0),
|
||||
make_item("*", 478.0, 285.0, 9.0),
|
||||
make_item("*", 518.0, 285.0, 9.0),
|
||||
make_item("v7", 20.0, 215.0, 9.0),
|
||||
make_item(
|
||||
"9.4 Apply Host-based Firewalls or Port-Filtering",
|
||||
70.0,
|
||||
215.0,
|
||||
9.0,
|
||||
),
|
||||
make_item("*", 478.0, 215.0, 9.0),
|
||||
make_item("*", 518.0, 215.0, 9.0),
|
||||
];
|
||||
items.push(make_item(
|
||||
"Implement and manage a host-based firewall or port-filtering tool",
|
||||
84.0,
|
||||
260.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"on end-user devices with a default-deny rule",
|
||||
84.0,
|
||||
240.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"Apply host-based firewalls or port filtering tools on end systems",
|
||||
84.0,
|
||||
190.0,
|
||||
8.0,
|
||||
));
|
||||
items.push(make_item(
|
||||
"and deny unauthorized network communication",
|
||||
84.0,
|
||||
170.0,
|
||||
8.0,
|
||||
));
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("expected multiline description table");
|
||||
assert_eq!(table.columns.len(), 5);
|
||||
assert_eq!(
|
||||
table.rows.len(),
|
||||
3,
|
||||
"wrapped lines should collapse to header plus two data rows"
|
||||
);
|
||||
assert_eq!(table.cells[0][0], "Controls Version");
|
||||
assert!(table.cells[1][1].contains("host-based firewall"));
|
||||
assert!(table.cells[1][1].contains("default-deny rule"));
|
||||
assert!(table.cells[2][1].contains("deny unauthorized"));
|
||||
}
|
||||
|
||||
/// Wire-bordered 4-column table whose header text is centered/right-aligned
|
||||
/// inside each cell while the data is left-aligned: cluster_x_positions
|
||||
/// merges adjacent columns (data Item→EAN gap is below threshold) and
|
||||
/// drops the header-only x-clusters in the filter pass, leaving only 3
|
||||
/// text-derived columns. Rect borders are 4 columns of ground truth.
|
||||
/// Before the fix the cell-rect path preferred text edges when they were
|
||||
/// the smaller set — losing a column. After the fix, 3+ rect columns
|
||||
/// always win.
|
||||
#[test]
|
||||
fn wired_header_data_misaligned_keeps_all_columns_from_rects() {
|
||||
let page = 1;
|
||||
// 4 cols: Item | EAN | Nombre | Cant
|
||||
let col_xs = [380.0_f32, 410.0, 470.0, 660.0, 700.0];
|
||||
// Header + 9 data rows at 15pt tall each (y descending).
|
||||
let row_ys: Vec<f32> = (0..=10).map(|r| 400.0 - 15.0 * r as f32).collect();
|
||||
|
||||
let mut rects: Vec<(f32, f32, f32, f32)> = Vec::new();
|
||||
for r in 0..10 {
|
||||
let y_top = row_ys[r];
|
||||
let y_bot = row_ys[r + 1];
|
||||
for c in 0..4 {
|
||||
rects.push((col_xs[c], y_bot, col_xs[c + 1] - col_xs[c], y_top - y_bot));
|
||||
}
|
||||
}
|
||||
|
||||
let mut items: Vec<TextItem> = Vec::new();
|
||||
// Header row (y ≈ 392.5): headers sit further to the right than data
|
||||
// because they are centered/right-aligned in the cells.
|
||||
items.push(make_item("Item", 389.0, 392.5, 9.0));
|
||||
items.push(make_item("EAN", 432.0, 392.5, 9.0));
|
||||
items.push(make_item("Nombre", 552.0, 392.5, 9.0));
|
||||
items.push(make_item("Cant", 672.0, 392.5, 9.0));
|
||||
|
||||
let names = [
|
||||
"Arnes Frontal",
|
||||
"Arnes Motor",
|
||||
"Arnes Piso",
|
||||
"Arnes Techo",
|
||||
"Arnes Puerta",
|
||||
"Arnes Tablero",
|
||||
"Arnes Trasero",
|
||||
"Arnes Lateral",
|
||||
"Arnes Sensor",
|
||||
];
|
||||
for r in 0..9 {
|
||||
let y = 377.5 - 15.0 * r as f32;
|
||||
items.push(make_item(&(r + 1).to_string(), 396.0, y, 9.0));
|
||||
items.push(make_item("7701023403016", 410.0, y, 9.0));
|
||||
items.push(make_item(names[r], 480.0, y, 9.0));
|
||||
items.push(make_item("1", 680.0, y, 9.0));
|
||||
}
|
||||
|
||||
let table = detect_row_stripe_table_from_cell_rects(&items, &rects, page)
|
||||
.expect("wired 4-column table with header/data x-misalignment must detect");
|
||||
assert_eq!(
|
||||
table.columns.len(),
|
||||
4,
|
||||
"expected 4 columns from rect borders; cells: {:?}",
|
||||
table.cells
|
||||
);
|
||||
for c in 0..4 {
|
||||
let any_populated = table.cells.iter().any(|row| !row[c].trim().is_empty());
|
||||
assert!(
|
||||
any_populated,
|
||||
"column {} empty across all rows; cells: {:?}",
|
||||
c, table.cells
|
||||
);
|
||||
}
|
||||
// Header row populated in all 4 cells.
|
||||
let header = &table.cells[0];
|
||||
assert_eq!(header[0].trim(), "Item");
|
||||
assert_eq!(header[1].trim(), "EAN");
|
||||
assert_eq!(header[2].trim(), "Nombre");
|
||||
assert_eq!(header[3].trim(), "Cant");
|
||||
// First data row: Item="1", EAN, name, count="1" — no Item↔EAN merge.
|
||||
let data1 = &table.cells[1];
|
||||
assert_eq!(data1[0].trim(), "1");
|
||||
assert_eq!(data1[1].trim(), "7701023403016");
|
||||
assert!(data1[2].trim().contains("Arnes"));
|
||||
assert_eq!(data1[3].trim(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cluster_no_hint_without_items() {
|
||||
// Rects with no text items inside → no failed-cluster hint generated.
|
||||
|
||||
+7
-269
@@ -160,81 +160,6 @@ fn starts_with_uppercase_word(cell: &str) -> bool {
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_uppercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
fn starts_with_lowercase_alpha(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphabetic())
|
||||
.is_some_and(|c| c.is_lowercase())
|
||||
}
|
||||
|
||||
fn starts_with_numbered_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim_start();
|
||||
let digit_count = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
|
||||
|
||||
digit_count > 0
|
||||
&& digit_count <= 3
|
||||
&& trimmed
|
||||
.chars()
|
||||
.nth(digit_count)
|
||||
.is_some_and(|c| matches!(c, '.' | ')' | '-' | ':'))
|
||||
}
|
||||
|
||||
fn alpha_word_count(cell: &str) -> usize {
|
||||
cell.split_whitespace()
|
||||
.filter(|word| word.chars().any(|c| c.is_alphabetic()))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn looks_like_compact_entry_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 3 || trimmed.len() > 80 {
|
||||
return false;
|
||||
}
|
||||
|
||||
if !starts_with_uppercase_alpha(trimmed) && !starts_with_numbered_label(trimmed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let words = alpha_word_count(trimmed);
|
||||
(1..=6).contains(&words)
|
||||
}
|
||||
|
||||
fn looks_like_plain_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 4 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || trimmed.contains(|ch: char| ch.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.len() <= 4 && trimmed.chars().all(|ch| !ch.is_lowercase()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_alphabetic() || ch.is_whitespace() || matches!(ch, '&' | '/' | '-'))
|
||||
&& starts_with_uppercase_alpha(trimmed)
|
||||
&& (1..=4).contains(&alpha_word_count(trimmed))
|
||||
}
|
||||
|
||||
fn ends_like_incomplete_phrase(cell: &str) -> bool {
|
||||
let lower = cell.trim_end().to_ascii_lowercase();
|
||||
lower.ends_with(" and")
|
||||
|| lower.ends_with(" or")
|
||||
|| lower.ends_with(',')
|
||||
|| lower.ends_with('-')
|
||||
|| lower.ends_with('/')
|
||||
}
|
||||
|
||||
/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows
|
||||
fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
let mut cleaned: Vec<Vec<String>> = Vec::new();
|
||||
@@ -260,9 +185,6 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
|
||||
// Check if this is a continuation row (first column is empty but others have content).
|
||||
// A row with only 1 short non-empty cell (besides the first) is more likely a
|
||||
// section sub-header (e.g. "JAN", "FEB") than overflow text — don't merge it.
|
||||
@@ -300,76 +222,31 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
.iter()
|
||||
.filter(|cell| starts_with_uppercase_word(cell))
|
||||
.count();
|
||||
let first_non_empty_col = row.iter().position(|c| !c.trim().is_empty());
|
||||
let first_non_empty_cell = first_non_empty_col
|
||||
.and_then(|idx| row.get(idx))
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let title_like_later_cells = first_non_empty_col
|
||||
.map(|idx| {
|
||||
row.iter()
|
||||
.skip(idx + 1)
|
||||
.map(|c| c.trim())
|
||||
.filter(|c| !c.is_empty() && starts_with_uppercase_alpha(c))
|
||||
.count()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let prev_first_cell_empty = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.is_some_and(|c| c.trim().is_empty());
|
||||
let prev_first_cell = cleaned
|
||||
.last()
|
||||
.and_then(|r| r.first())
|
||||
.map(|c| c.trim())
|
||||
.unwrap_or("");
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
&& uppercase_leading_cells >= non_first_cells.len().saturating_sub(1);
|
||||
// Hierarchical tables often use a row-spanned first column: sub-rows
|
||||
// leave column 0 blank, then start a compact title-like label in
|
||||
// column 1. Wrapped continuations in the existing fixtures start
|
||||
// mid-sentence/lowercase ("continued text here", "with 3.5%...") or
|
||||
// carry lowercase fragments in the later cells, so keep those mergeable.
|
||||
let looks_like_hierarchical_subrow = first_cell.is_empty()
|
||||
&& row.len() >= 3
|
||||
&& first_non_empty_col == Some(1)
|
||||
&& looks_like_compact_entry_label(first_non_empty_cell)
|
||||
&& ((non_first_cells.len() >= 2 && title_like_later_cells > 0)
|
||||
|| (non_first_cells.len() == 1
|
||||
&& prev_first_cell_empty
|
||||
&& alpha_word_count(first_non_empty_cell) >= 2));
|
||||
let looks_like_new_first_column_entry = !first_cell.is_empty()
|
||||
&& (starts_with_numbered_label(first_cell) || starts_with_uppercase_alpha(first_cell))
|
||||
&& filled_cells >= 2
|
||||
&& non_first_cells
|
||||
.iter()
|
||||
.any(|cell| looks_like_compact_entry_label(cell));
|
||||
let looks_like_section_label_row = !first_cell.is_empty()
|
||||
&& filled_cells == 1
|
||||
&& header_filled >= 3
|
||||
&& looks_like_plain_section_label(first_cell);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
&& !is_short_subheader
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& cleaned.len() > 1;
|
||||
|
||||
// Wrapped-cell continuation: row has fewer filled cells than the header
|
||||
// row, suggesting it's overflow text from the previous row's cells.
|
||||
// Only trigger when the previous row has significantly more filled cells.
|
||||
let num_cols = row.len();
|
||||
let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
let prev_filled = cleaned
|
||||
.last()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(0);
|
||||
let header_filled = cleaned
|
||||
.first()
|
||||
.map(|r| r.iter().filter(|c| !c.trim().is_empty()).count())
|
||||
.unwrap_or(num_cols);
|
||||
// Merge when the row has significantly fewer filled cells than header.
|
||||
// For wide tables (5+ cols), require ≤50% of header cells.
|
||||
// For narrow tables (2-4 cols), require fewer than header cells.
|
||||
@@ -380,18 +257,11 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
} else {
|
||||
header_filled.saturating_sub(1)
|
||||
};
|
||||
let continues_wrapped_first_column_label = !first_cell.is_empty()
|
||||
&& starts_with_lowercase_alpha(first_cell)
|
||||
&& ends_like_incomplete_phrase(prev_first_cell);
|
||||
let is_wrapped_continuation = cleaned.len() > 1
|
||||
&& filled_cells <= max_filled_for_merge
|
||||
&& (prev_filled > filled_cells
|
||||
|| (continues_wrapped_first_column_label && prev_filled >= filled_cells))
|
||||
&& prev_filled > filled_cells
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !looks_like_hierarchical_subrow
|
||||
&& !looks_like_new_first_column_entry
|
||||
&& !looks_like_section_label_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -537,44 +407,6 @@ mod tests {
|
||||
assert!(cleaned[1][1].contains("continued text here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_first_column_section_label_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Properties".into(),
|
||||
"Conditions".into(),
|
||||
"Method".into(),
|
||||
"Typical values".into(),
|
||||
"Units".into(),
|
||||
],
|
||||
vec![
|
||||
"Melt Flow Rate".into(),
|
||||
"230 C/2.16 kg".into(),
|
||||
"ASTM D1238".into(),
|
||||
"3.0".into(),
|
||||
"g/10 min".into(),
|
||||
],
|
||||
vec![
|
||||
"Mechanical".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"Tensile Stress at Yield".into(),
|
||||
"50 mm/min".into(),
|
||||
"ASTM D638".into(),
|
||||
"31".into(),
|
||||
"MPa".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 4);
|
||||
assert_eq!(cleaned[2][0], "Mechanical");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_short_subheader_not_merged() {
|
||||
let cells = vec![
|
||||
@@ -627,100 +459,6 @@ mod tests {
|
||||
assert_eq!(cleaned[2][1], "Uncertainty around other copies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_numbered_hierarchy_rows_not_overmerged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"1. Group alpha".into(),
|
||||
"Task setup and".into(),
|
||||
"Begin setup".into(),
|
||||
"Faster start".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"management".into(),
|
||||
"recommended profile".into(),
|
||||
"with saved defaults".into(),
|
||||
],
|
||||
vec![
|
||||
"2. Group beta and".into(),
|
||||
"Storage setup".into(),
|
||||
"Provides upload tools".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"fine-tuning".into(),
|
||||
"".into(),
|
||||
"for filtered inputs".into(),
|
||||
"service".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Label workspace".into(),
|
||||
"Creates review sets".into(),
|
||||
"Lets teams review".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Model training".into(),
|
||||
"".into(),
|
||||
"Supports custom model".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 5);
|
||||
assert_eq!(cleaned[1][0], "1. Group alpha");
|
||||
assert_eq!(cleaned[1][1], "Task setup and management");
|
||||
assert_eq!(cleaned[2][0], "2. Group beta and fine-tuning");
|
||||
assert_eq!(cleaned[2][1], "Storage setup");
|
||||
assert_eq!(cleaned[3][1], "Label workspace");
|
||||
assert_eq!(cleaned[4][1], "Model training");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_partial_hierarchical_subrow_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Group".into(),
|
||||
"Task".into(),
|
||||
"Detail".into(),
|
||||
"Benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"Group A".into(),
|
||||
"Alpha task".into(),
|
||||
"Initial detail".into(),
|
||||
"Initial benefit".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Beta task".into(),
|
||||
"Parallel detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"second line".into(),
|
||||
"additional detail".into(),
|
||||
"".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
|
||||
assert_eq!(cleaned.len(), 3);
|
||||
assert_eq!(cleaned[1][1], "Alpha task");
|
||||
assert_eq!(cleaned[2][0], "");
|
||||
assert_eq!(cleaned[2][1], "Beta task second line");
|
||||
assert_eq!(cleaned[2][2], "Parallel detail additional detail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_full_width_continuation_row_still_merges_when_lowercase() {
|
||||
let cells = vec![
|
||||
|
||||
@@ -460,7 +460,6 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
item_indices.push(item_idx);
|
||||
}
|
||||
}
|
||||
merge_superscript_marker_rows(&mut row_ys, &mut cells);
|
||||
|
||||
// Validate: need reasonable fill rate
|
||||
let total_cells = row_ys.len() * columns.len();
|
||||
@@ -548,536 +547,6 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
Some(Table::new(col_xs, row_ys, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Build a region-scoped two-column key/value table from text baselines.
|
||||
///
|
||||
/// This intentionally lives outside the full-page heuristic detector. Layout
|
||||
/// callers already supplied a table-shaped bbox, and some real table regions
|
||||
/// are plain product/spec forms with only two visual columns. The main column
|
||||
/// fallback starts at four columns to avoid newspaper/prose false positives;
|
||||
/// this path keeps tighter key/value-specific guards instead.
|
||||
pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32) -> Option<Table> {
|
||||
let page_items: Vec<RowItem> = items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, item)| item.page == page && !item.text.trim().is_empty())
|
||||
.map(|(idx, item)| RowItem {
|
||||
index: idx,
|
||||
item: item.clone(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
if page_items.len() < 4 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let median_font_size = median_f32(page_items.iter().map(|ri| ri.item.font_size).collect())
|
||||
.unwrap_or(10.0)
|
||||
.max(1.0);
|
||||
let y_tol = (median_font_size * 0.75).clamp(4.0, 9.0);
|
||||
let rows = group_key_value_visual_rows(page_items, y_tol);
|
||||
if rows.len() < 2 || rows.len() > 80 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let split_x = infer_key_value_split_x(&rows, median_font_size)?;
|
||||
let mut kv_rows: Vec<KeyValueRow> = Vec::new();
|
||||
let mut paired_rows = 0usize;
|
||||
let mut section_rows = 0usize;
|
||||
let mut left_label_like = 0usize;
|
||||
let mut left_starts = Vec::new();
|
||||
let mut right_starts = Vec::new();
|
||||
|
||||
for row in &rows {
|
||||
let mut left_items = Vec::new();
|
||||
let mut right_items = Vec::new();
|
||||
for item in &row.items {
|
||||
if item.item.x < split_x {
|
||||
left_items.push(item);
|
||||
} else {
|
||||
right_items.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
let left = join_row_item_text(&left_items);
|
||||
let right = join_row_item_text(&right_items);
|
||||
if left.is_empty() && right.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut item_indices: Vec<usize> = row.items.iter().map(|ri| ri.index).collect();
|
||||
item_indices.sort_unstable();
|
||||
item_indices.dedup();
|
||||
|
||||
if !left.is_empty() && !right.is_empty() {
|
||||
paired_rows += 1;
|
||||
if looks_like_key_value_label(&left) {
|
||||
left_label_like += 1;
|
||||
}
|
||||
if let Some(x) = left_items.first().map(|ri| ri.item.x) {
|
||||
left_starts.push(x);
|
||||
}
|
||||
if let Some(x) = right_items.first().map(|ri| ri.item.x) {
|
||||
right_starts.push(x);
|
||||
}
|
||||
} else if !left.is_empty() {
|
||||
section_rows += 1;
|
||||
}
|
||||
|
||||
kv_rows.push(KeyValueRow {
|
||||
y: row.y,
|
||||
left,
|
||||
right,
|
||||
item_indices,
|
||||
});
|
||||
}
|
||||
|
||||
if kv_rows.len() < 2 || paired_rows < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let header_inferred = key_value_first_pair_is_header(&kv_rows);
|
||||
let data_pairs = if header_inferred {
|
||||
paired_rows.saturating_sub(1)
|
||||
} else {
|
||||
paired_rows
|
||||
};
|
||||
if data_pairs < 1 {
|
||||
return None;
|
||||
}
|
||||
|
||||
if section_rows > paired_rows * 2 + 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let label_rows_for_score = if header_inferred {
|
||||
paired_rows.saturating_sub(1)
|
||||
} else {
|
||||
paired_rows
|
||||
};
|
||||
let label_like_for_score = if header_inferred && !kv_rows.is_empty() {
|
||||
left_label_like.saturating_sub(1)
|
||||
} else {
|
||||
left_label_like
|
||||
};
|
||||
if label_rows_for_score >= 2 && label_like_for_score * 2 < label_rows_for_score {
|
||||
return None;
|
||||
}
|
||||
|
||||
let left_x = median_f32(left_starts).unwrap_or_else(|| {
|
||||
rows.iter()
|
||||
.flat_map(|row| row.items.iter().map(|ri| ri.item.x))
|
||||
.fold(f32::INFINITY, f32::min)
|
||||
});
|
||||
let right_x = median_f32(right_starts).unwrap_or(split_x);
|
||||
if !left_x.is_finite() || !right_x.is_finite() || right_x - left_x < 40.0 {
|
||||
return None;
|
||||
}
|
||||
let right_cluster_count = significant_side_x_clusters(&rows, split_x, false);
|
||||
let marker_rows = marker_matrix_value_rows(&kv_rows);
|
||||
if (right_cluster_count >= 5 && paired_rows >= 3)
|
||||
|| (right_cluster_count >= 3 && marker_rows >= 3 && marker_rows * 2 >= paired_rows)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
if key_value_rows_look_like_prose(&kv_rows, header_inferred) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut table_rows = Vec::new();
|
||||
let mut cells = Vec::new();
|
||||
let mut item_indices = Vec::new();
|
||||
|
||||
let mut start_idx = 0usize;
|
||||
if header_inferred {
|
||||
let header = &kv_rows[0];
|
||||
table_rows.push(header.y);
|
||||
cells.push(vec![header.left.clone(), header.right.clone()]);
|
||||
item_indices.extend(header.item_indices.iter().copied());
|
||||
start_idx = 1;
|
||||
} else {
|
||||
table_rows.push(kv_rows.first().map(|row| row.y + y_tol).unwrap_or(0.0));
|
||||
cells.push(vec!["Field".to_string(), "Value".to_string()]);
|
||||
}
|
||||
|
||||
for row in kv_rows.iter().skip(start_idx) {
|
||||
if !row.left.is_empty() && !row.right.is_empty() {
|
||||
table_rows.push(row.y);
|
||||
cells.push(vec![row.left.clone(), row.right.clone()]);
|
||||
item_indices.extend(row.item_indices.iter().copied());
|
||||
} else if !row.left.is_empty() {
|
||||
table_rows.push(row.y);
|
||||
cells.push(vec!["Section".to_string(), row.left.clone()]);
|
||||
item_indices.extend(row.item_indices.iter().copied());
|
||||
} else if !row.right.is_empty() {
|
||||
if let Some(last) = cells.last_mut() {
|
||||
if let Some(value) = last.get_mut(1) {
|
||||
if !value.trim().is_empty() {
|
||||
value.push(' ');
|
||||
}
|
||||
value.push_str(&row.right);
|
||||
item_indices.extend(row.item_indices.iter().copied());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cells.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
item_indices.sort_unstable();
|
||||
item_indices.dedup();
|
||||
|
||||
log::debug!(
|
||||
"key-value table: {} rows, pairs={}, sections={}, split_x={:.1}",
|
||||
cells.len(),
|
||||
paired_rows,
|
||||
section_rows,
|
||||
split_x
|
||||
);
|
||||
|
||||
Some(Table::new(
|
||||
vec![left_x, right_x],
|
||||
table_rows,
|
||||
cells,
|
||||
item_indices,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RowItem {
|
||||
index: usize,
|
||||
item: TextItem,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VisualRow {
|
||||
y: f32,
|
||||
items: Vec<RowItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct KeyValueRow {
|
||||
y: f32,
|
||||
left: String,
|
||||
right: String,
|
||||
item_indices: Vec<usize>,
|
||||
}
|
||||
|
||||
fn group_key_value_visual_rows(mut items: Vec<RowItem>, y_tol: f32) -> Vec<VisualRow> {
|
||||
items.sort_by(|a, b| {
|
||||
b.item
|
||||
.y
|
||||
.total_cmp(&a.item.y)
|
||||
.then_with(|| a.item.x.total_cmp(&b.item.x))
|
||||
});
|
||||
|
||||
let mut rows: Vec<VisualRow> = Vec::new();
|
||||
for row_item in items {
|
||||
if let Some(row) = rows
|
||||
.iter_mut()
|
||||
.find(|row| (row.y - row_item.item.y).abs() <= y_tol)
|
||||
{
|
||||
let len = row.items.len() as f32;
|
||||
row.y = (row.y * len + row_item.item.y) / (len + 1.0);
|
||||
row.items.push(row_item);
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push(VisualRow {
|
||||
y: row_item.item.y,
|
||||
items: vec![row_item],
|
||||
});
|
||||
}
|
||||
|
||||
for row in &mut rows {
|
||||
row.items.sort_by(|a, b| a.item.x.total_cmp(&b.item.x));
|
||||
}
|
||||
rows.sort_by(|a, b| b.y.total_cmp(&a.y));
|
||||
rows
|
||||
}
|
||||
|
||||
fn infer_key_value_split_x(rows: &[VisualRow], median_font_size: f32) -> Option<f32> {
|
||||
let min_gap = (median_font_size * 2.0).max(24.0);
|
||||
let mut splits = Vec::new();
|
||||
|
||||
for row in rows {
|
||||
if row.items.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut best_gap = 0.0f32;
|
||||
let mut best_split = None;
|
||||
for pair in row.items.windows(2) {
|
||||
let left = &pair[0].item;
|
||||
let right = &pair[1].item;
|
||||
let left_right = left.x + left.width.max(0.0);
|
||||
let gap = right.x - left_right;
|
||||
if gap > best_gap {
|
||||
best_gap = gap;
|
||||
best_split = Some(left_right + gap / 2.0);
|
||||
}
|
||||
}
|
||||
|
||||
if best_gap >= min_gap {
|
||||
if let Some(split) = best_split {
|
||||
splits.push(split);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if splits.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
median_f32(splits)
|
||||
}
|
||||
|
||||
fn join_row_item_text(items: &[&RowItem]) -> String {
|
||||
let mut parts = Vec::new();
|
||||
for item in items {
|
||||
let trimmed = item.item.text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
parts.push(trimmed);
|
||||
}
|
||||
}
|
||||
normalize_cell_text(&parts.join(" "))
|
||||
}
|
||||
|
||||
fn normalize_cell_text(text: &str) -> String {
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
fn key_value_first_pair_is_header(rows: &[KeyValueRow]) -> bool {
|
||||
let Some(first) = rows.first() else {
|
||||
return false;
|
||||
};
|
||||
if first.left.is_empty() || first.right.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if !looks_like_key_value_header_cell(&first.left)
|
||||
|| !looks_like_key_value_header_cell(&first.right)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
rows.iter()
|
||||
.skip(1)
|
||||
.any(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
}
|
||||
|
||||
fn looks_like_key_value_header_cell(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 2 || trimmed.len() > 40 {
|
||||
return false;
|
||||
}
|
||||
let words = word_count_simple(trimmed);
|
||||
if !(1..=4).contains(&words) {
|
||||
return false;
|
||||
}
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
if matches!(
|
||||
lower.as_str(),
|
||||
"yes" | "no" | "true" | "false" | "none" | "n/a" | "na"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
trimmed.chars().any(|c| c.is_alphabetic())
|
||||
&& !trimmed.chars().any(|c| c.is_ascii_digit())
|
||||
&& !trimmed.ends_with(['.', ',', ';', ':'])
|
||||
}
|
||||
|
||||
fn looks_like_key_value_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.len() < 2 || trimmed.len() > 90 {
|
||||
return false;
|
||||
}
|
||||
let words = word_count_simple(trimmed);
|
||||
if words == 0 || words > 10 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';']) {
|
||||
return false;
|
||||
}
|
||||
trimmed.chars().any(|c| c.is_alphabetic())
|
||||
}
|
||||
|
||||
fn key_value_rows_look_like_prose(rows: &[KeyValueRow], header_inferred: bool) -> bool {
|
||||
let mut long_sentence_cells = 0usize;
|
||||
let mut total_cells = 0usize;
|
||||
let mut total_chars = 0usize;
|
||||
let mut paired_rows = 0usize;
|
||||
let mut solo_prose_rows = 0usize;
|
||||
|
||||
for row in rows.iter().skip(usize::from(header_inferred)) {
|
||||
if !row.left.is_empty() && !row.right.is_empty() {
|
||||
paired_rows += 1;
|
||||
} else {
|
||||
let solo = if row.left.is_empty() {
|
||||
row.right.trim()
|
||||
} else {
|
||||
row.left.trim()
|
||||
};
|
||||
if solo.chars().count() > 70
|
||||
|| word_count_simple(solo) > 9
|
||||
|| (solo.chars().count() > 35 && solo.ends_with(['.', '!', '?']))
|
||||
{
|
||||
solo_prose_rows += 1;
|
||||
}
|
||||
}
|
||||
for cell in [&row.left, &row.right] {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
total_cells += 1;
|
||||
total_chars += trimmed.chars().count();
|
||||
if trimmed.chars().count() > 100
|
||||
|| (trimmed.chars().count() > 55 && trimmed.ends_with(['.', '!', '?']))
|
||||
{
|
||||
long_sentence_cells += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if paired_rows < 1 || total_cells == 0 {
|
||||
return true;
|
||||
}
|
||||
if solo_prose_rows >= 3 {
|
||||
return true;
|
||||
}
|
||||
|
||||
let avg_chars = total_chars as f32 / total_cells as f32;
|
||||
avg_chars > 75.0 || long_sentence_cells * 2 >= total_cells
|
||||
}
|
||||
|
||||
fn marker_matrix_value_rows(rows: &[KeyValueRow]) -> usize {
|
||||
rows.iter()
|
||||
.filter(|row| !row.left.is_empty() && compact_marker_value(&row.right))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn compact_marker_value(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.is_empty() || trimmed.chars().count() > 80 {
|
||||
return false;
|
||||
}
|
||||
if trimmed.chars().any(|ch| ch.is_alphabetic()) {
|
||||
return false;
|
||||
}
|
||||
trimmed
|
||||
.chars()
|
||||
.any(|ch| ch.is_ascii_digit() || matches!(ch, '•' | '●' | '·'))
|
||||
}
|
||||
|
||||
fn significant_side_x_clusters(rows: &[VisualRow], split_x: f32, left_side: bool) -> usize {
|
||||
let mut xs = Vec::new();
|
||||
for row in rows {
|
||||
for item in &row.items {
|
||||
let is_left = item.item.x < split_x;
|
||||
if is_left == left_side {
|
||||
xs.push(item.item.x);
|
||||
}
|
||||
}
|
||||
}
|
||||
xs.sort_by(|a, b| a.total_cmp(b));
|
||||
|
||||
let mut counts = Vec::new();
|
||||
let mut center = None::<f32>;
|
||||
let mut count = 0usize;
|
||||
for x in xs {
|
||||
match center {
|
||||
Some(current) if (x - current).abs() <= 8.0 => {
|
||||
center = Some((current * count as f32 + x) / (count as f32 + 1.0));
|
||||
count += 1;
|
||||
}
|
||||
Some(_) => {
|
||||
counts.push(count);
|
||||
center = Some(x);
|
||||
count = 1;
|
||||
}
|
||||
None => {
|
||||
center = Some(x);
|
||||
count = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if count > 0 {
|
||||
counts.push(count);
|
||||
}
|
||||
|
||||
counts.into_iter().filter(|&count| count >= 2).count()
|
||||
}
|
||||
|
||||
fn word_count_simple(cell: &str) -> usize {
|
||||
cell.split_whitespace()
|
||||
.filter(|word| word.chars().any(|c| c.is_alphanumeric()))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn median_f32(mut values: Vec<f32>) -> Option<f32> {
|
||||
values.retain(|value| value.is_finite());
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
values.sort_by(|a, b| a.total_cmp(b));
|
||||
Some(values[values.len() / 2])
|
||||
}
|
||||
|
||||
fn merge_superscript_marker_rows(row_ys: &mut Vec<f32>, cells: &mut Vec<Vec<String>>) {
|
||||
let mut row_idx = 0;
|
||||
while row_idx < cells.len() {
|
||||
let non_empty: Vec<(usize, String)> = cells[row_idx]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(col_idx, cell)| {
|
||||
let trimmed = cell.trim();
|
||||
(!trimmed.is_empty()).then_some((col_idx, trimmed.to_string()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if non_empty.len() != 1 || !is_superscript_marker_cell(&non_empty[0].1) {
|
||||
row_idx += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let (marker_col, marker) = &non_empty[0];
|
||||
let prev =
|
||||
(row_idx > 0).then(|| (row_idx - 1, (row_ys[row_idx - 1] - row_ys[row_idx]).abs()));
|
||||
let next = (row_idx + 1 < cells.len())
|
||||
.then(|| (row_idx + 1, (row_ys[row_idx] - row_ys[row_idx + 1]).abs()));
|
||||
let target = [prev, next]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|(_, gap)| *gap <= 10.0)
|
||||
.min_by(|(_, gap_a), (_, gap_b)| gap_a.total_cmp(gap_b))
|
||||
.map(|(idx, _)| idx);
|
||||
|
||||
let Some(target_idx) = target else {
|
||||
row_idx += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
let target_cell = &mut cells[target_idx][*marker_col];
|
||||
if target_cell.trim().is_empty() {
|
||||
*target_cell = marker.to_string();
|
||||
} else {
|
||||
target_cell.push_str(marker);
|
||||
}
|
||||
cells.remove(row_idx);
|
||||
row_ys.remove(row_idx);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_superscript_marker_cell(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
!trimmed.is_empty()
|
||||
&& trimmed.chars().count() <= 2
|
||||
&& trimmed
|
||||
.chars()
|
||||
.all(|ch| matches!(ch, '*' | '#' | 'o' | 'O' | '°' | 'º' | '†' | '‡'))
|
||||
}
|
||||
|
||||
/// What kind of structure a detected `Table` represents. Classification is
|
||||
/// computed once at construction so consumers don't have to re-analyze the
|
||||
/// cells (and stay consistent across detection backends).
|
||||
@@ -1220,190 +689,6 @@ mod tests {
|
||||
assert!(md.contains("|Cell 1|"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_superscript_marker_rows() {
|
||||
let mut rows = vec![506.0, 500.0, 480.0];
|
||||
let mut cells = vec![
|
||||
vec!["".into(), "".into(), "*".into()],
|
||||
vec!["Name".into(), "Method".into(), "Typical values".into()],
|
||||
vec!["Flow".into(), "ASTM D1238".into(), "3.0".into()],
|
||||
];
|
||||
|
||||
merge_superscript_marker_rows(&mut rows, &mut cells);
|
||||
|
||||
assert_eq!(rows, vec![500.0, 480.0]);
|
||||
assert_eq!(cells[0][2], "Typical values*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_builder_handles_borderless_specs_table() {
|
||||
let items = vec![
|
||||
make_char("*", 458.1, 544.2, 8.0, 4.4),
|
||||
make_char("Properties", 36.0, 538.6, 12.0, 53.1),
|
||||
make_char("Conditions", 195.8, 538.6, 12.0, 55.0),
|
||||
make_char("Method", 297.2, 538.6, 12.0, 39.4),
|
||||
make_char("Typical values", 384.1, 538.6, 12.0, 74.0),
|
||||
make_char("Units", 510.6, 538.6, 8.0, 17.9),
|
||||
make_char("Rheology", 36.0, 508.3, 10.0, 40.6),
|
||||
make_char("o", 209.8, 492.5, 6.5, 3.5),
|
||||
make_char("Melt Flow Rate", 36.0, 488.0, 10.0, 65.2),
|
||||
make_char("230 ", 190.4, 488.0, 10.0, 19.4),
|
||||
make_char("C/2.16 kg", 213.3, 488.0, 10.0, 42.8),
|
||||
make_char("ASTM D1238", 288.4, 488.0, 10.0, 56.8),
|
||||
make_char("3.0 ", 416.4, 488.0, 10.0, 16.9),
|
||||
make_char("g/10 min", 504.1, 488.0, 10.0, 39.5),
|
||||
make_char("Mechanical", 36.0, 451.5, 10.0, 48.3),
|
||||
make_char("Tensile Stress at Yield", 36.0, 431.3, 10.0, 96.7),
|
||||
make_char("50 mm/min", 197.9, 431.3, 10.0, 50.8),
|
||||
make_char("ASTM D638", 291.2, 431.3, 10.0, 51.3),
|
||||
make_char("31 ", 417.9, 431.3, 10.0, 13.9),
|
||||
make_char("MPa", 514.7, 431.3, 10.0, 18.4),
|
||||
make_char("Elongation at Yield", 36.0, 403.0, 10.0, 82.2),
|
||||
make_char("50 mm/min", 197.9, 403.0, 10.0, 50.8),
|
||||
make_char("ASTM D638", 291.2, 403.0, 10.0, 51.3),
|
||||
make_char("8 ", 420.6, 403.0, 10.0, 8.5),
|
||||
make_char("%", 519.1, 403.0, 10.0, 9.7),
|
||||
make_char("Flexural Modulus", 36.0, 374.6, 10.0, 74.0),
|
||||
make_char("ASTM D790", 291.2, 374.6, 10.0, 51.3),
|
||||
make_char("1400", 412.4, 374.6, 10.0, 21.8),
|
||||
make_char("MPa", 514.7, 374.6, 10.0, 18.4),
|
||||
];
|
||||
|
||||
let table = try_build_table_from_columns(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(
|
||||
md.contains("|Properties|Conditions|Method|Typical values*|Units|"),
|
||||
"{md}"
|
||||
);
|
||||
assert!(md.contains("|Mechanical|||||"), "{md}");
|
||||
assert!(
|
||||
md.contains("|Flexural Modulus||ASTM D790|1400|MPa|"),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_recovers_sectioned_specs_table() {
|
||||
let items = vec![
|
||||
make_char("Ordering Information", 69.0, 700.0, 9.0, 96.0),
|
||||
make_char("Package Contents", 69.0, 680.0, 9.0, 82.0),
|
||||
make_char(
|
||||
"CCH Adapter Panel with 3 m pigtail; installation guide",
|
||||
200.0,
|
||||
680.0,
|
||||
9.0,
|
||||
245.0,
|
||||
),
|
||||
make_char("Units per Delivery", 69.0, 660.0, 9.0, 78.0),
|
||||
make_char("1/1", 200.0, 660.0, 9.0, 18.0),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.contains("|Field|Value|"), "{md}");
|
||||
assert!(md.contains("|Section|Ordering Information|"), "{md}");
|
||||
assert!(
|
||||
md.contains(
|
||||
"|Package Contents|CCH Adapter Panel with 3 m pigtail; installation guide|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
assert!(md.contains("|Units per Delivery|1/1|"), "{md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_preserves_two_column_header() {
|
||||
let items = vec![
|
||||
make_char("Media", 86.0, 700.0, 10.0, 36.0),
|
||||
make_char("Options", 311.0, 700.0, 10.0, 44.0),
|
||||
make_char("BACnet/IP (Annex J)", 86.0, 680.0, 10.0, 115.0),
|
||||
make_char("Register as Foreign Device", 311.0, 680.0, 10.0, 138.0),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.starts_with("|Media|Options|"), "{md}");
|
||||
assert!(
|
||||
md.contains("|BACnet/IP (Annex J)|Register as Foreign Device|"),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_keeps_repeated_spec_sections() {
|
||||
let items = vec![
|
||||
make_char("1.33 DUAL VVT-i", 90.0, 700.0, 9.0, 82.0),
|
||||
make_char("Engine Code", 90.0, 682.0, 9.0, 62.0),
|
||||
make_char("1NR-FE", 406.0, 682.0, 9.0, 42.0),
|
||||
make_char("Type", 90.0, 664.0, 9.0, 24.0),
|
||||
make_char("Four cylinders in-line", 376.0, 664.0, 9.0, 104.0),
|
||||
make_char("1.6 VALVEMATIC", 90.0, 636.0, 9.0, 78.0),
|
||||
make_char("Engine Code", 90.0, 618.0, 9.0, 62.0),
|
||||
make_char("1ZR-FAE", 404.0, 618.0, 9.0, 44.0),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.contains("|Section|1.33 DUAL VVT-i|"), "{md}");
|
||||
assert!(md.contains("|Engine Code|1NR-FE|"), "{md}");
|
||||
assert!(md.contains("|Section|1.6 VALVEMATIC|"), "{md}");
|
||||
assert!(md.contains("|Engine Code|1ZR-FAE|"), "{md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_rejects_split_prose() {
|
||||
let items = vec![
|
||||
make_char(
|
||||
"This paragraph describes an operational process and continues without a field label.",
|
||||
70.0,
|
||||
700.0,
|
||||
10.0,
|
||||
350.0,
|
||||
),
|
||||
make_char(
|
||||
"It was split only because the text wrapped across a wide line.",
|
||||
455.0,
|
||||
700.0,
|
||||
10.0,
|
||||
300.0,
|
||||
),
|
||||
make_char(
|
||||
"Another sentence explains background context rather than a measurable property.",
|
||||
70.0,
|
||||
680.0,
|
||||
10.0,
|
||||
350.0,
|
||||
),
|
||||
make_char(
|
||||
"The neighboring phrase is not a value and should not form a table.",
|
||||
455.0,
|
||||
680.0,
|
||||
10.0,
|
||||
300.0,
|
||||
),
|
||||
make_char(
|
||||
"Finally, this narrative line keeps flowing with normal prose content.",
|
||||
70.0,
|
||||
660.0,
|
||||
10.0,
|
||||
350.0,
|
||||
),
|
||||
make_char(
|
||||
"It has punctuation and complete sentences on both sides of the gap.",
|
||||
455.0,
|
||||
660.0,
|
||||
10.0,
|
||||
300.0,
|
||||
),
|
||||
];
|
||||
|
||||
assert!(try_build_key_value_table_from_rows(&items, 1).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_body_font_table_detected() {
|
||||
let items = vec![
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+10
-1201
File diff suppressed because it is too large
Load Diff
@@ -54,14 +54,9 @@ company other than a life insurance company shall make a return on Form 1120PC.
|
||||
|
||||
annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return.
|
||||
|
||||
(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and
|
||||
(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company.
|
||||
(4) Exception for insurance companies filing their Federal income tax returns
|
||||
electronically. If an insurance company described in paragraph (c)(1), (c)(2), or
|
||||
|
||||
(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e).
|
||||
(5) Definition. For purposes of this section, the term annual statement means
|
||||
the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of
|
||||
||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and|
|
||||
|---|---|
|
||||
||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of|
|
||||
|
||||
Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement.
|
||||
|
||||
@@ -206,4 +201,3 @@ CFR part or section where Current OMB identified or described control No.
|
||||
Deputy Commissioner for Services and Enforcement.
|
||||
|
||||
Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user