Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddaf412a3b | ||
|
|
fbab84fc20 | ||
|
|
bdea4f345a | ||
|
|
8a0f98dee7 | ||
|
|
d8894326e8 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.6.3",
|
||||
"version": "1.7.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -422,6 +422,52 @@ pub fn extract_tables_with_structure_cells(
|
||||
})
|
||||
}
|
||||
|
||||
/// One result from `extractTablesWithStructureAuto` — markdown plus a
|
||||
/// diagnostic flag identifying which path produced it.
|
||||
///
|
||||
/// `fallbackReason` is `null` when the TSR-hybrid path produced the
|
||||
/// markdown directly. When stage 1's quality check fires (the cells
|
||||
/// look like a SLANet detection pathology — phantom rows or multi-row
|
||||
/// content in a single cell), the heuristic table extractor is run on
|
||||
/// the same region instead, and `fallbackReason` carries the diagnostic
|
||||
/// label (`"phantom_empty_row"`, `"multi_row_in_cell"`).
|
||||
#[napi(object)]
|
||||
pub struct TableExtractionResultJs {
|
||||
pub markdown: String,
|
||||
pub fallback_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Auto-fallback variant of [`extractTablesWithStructure`].
|
||||
///
|
||||
/// Runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies, and falls back to the heuristic
|
||||
/// `extractTablesInRegions` for any input where the TSR path looks
|
||||
/// compromised.
|
||||
///
|
||||
/// On clean inputs this returns identical markdown to
|
||||
/// `extractTablesWithStructure`; on flagged inputs the heuristic
|
||||
/// markdown replaces the TSR markdown and `fallbackReason` is set.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure_auto(
|
||||
buffer: Buffer,
|
||||
inputs: Vec<TsrTableInputJs>,
|
||||
) -> Result<Vec<TableExtractionResultJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let parsed = parse_tsr_inputs(&inputs);
|
||||
|
||||
catch_panic("extract_tables_with_structure_auto", move || {
|
||||
let result = pdf_inspector::extract_tables_with_structure_auto_mem(&bytes, &parsed)
|
||||
.map_err(|e| to_napi_err(e, "extract_tables_with_structure_auto"))?;
|
||||
Ok(result
|
||||
.into_iter()
|
||||
.map(|r| TableExtractionResultJs {
|
||||
markdown: r.markdown,
|
||||
fallback_reason: r.fallback_reason,
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_tsr_inputs(inputs: &[TsrTableInputJs]) -> Vec<pdf_inspector::TsrTableInput> {
|
||||
inputs
|
||||
.iter()
|
||||
|
||||
+33
-11
@@ -1,8 +1,12 @@
|
||||
//! CLI tool for detecting PDF type (text-based vs scanned)
|
||||
|
||||
use pdf_inspector::{detect_pdf_type, process_pdf_with_options, PdfOptions, PdfType, ProcessMode};
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, detector::estimate_page_count_from_bytes, process_pdf_with_options,
|
||||
PdfOptions, PdfType, ProcessMode,
|
||||
};
|
||||
use std::env;
|
||||
use std::fmt::Write;
|
||||
use std::fs;
|
||||
use std::process;
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -64,6 +68,32 @@ fn pdf_type_str(pdf_type: &PdfType) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn page_count_hint(pdf_path: &str) -> Option<u32> {
|
||||
fs::read(pdf_path)
|
||||
.ok()
|
||||
.map(|bytes| estimate_page_count_from_bytes(&bytes))
|
||||
.filter(|&count| count > 0)
|
||||
}
|
||||
|
||||
fn print_error(e: &pdf_inspector::PdfError, pdf_path: &str, json_output: bool) {
|
||||
if json_output {
|
||||
if let Some(count) = page_count_hint(pdf_path) {
|
||||
println!(
|
||||
r#"{{"error":"{}","page_count_hint":{}}}"#,
|
||||
json_escape(&e.to_string()),
|
||||
count
|
||||
);
|
||||
} else {
|
||||
println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
|
||||
}
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
if let Some(count) = page_count_hint(pdf_path) {
|
||||
eprintln!("Page count hint: {}", count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
match process_pdf_with_options(pdf_path, PdfOptions::new().mode(ProcessMode::Analyze)) {
|
||||
Ok(result) => {
|
||||
@@ -135,11 +165,7 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if json_output {
|
||||
println!(r#"{{"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
}
|
||||
print_error(&e, pdf_path, json_output);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -236,11 +262,7 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if json_output {
|
||||
println!(r#"{{"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
}
|
||||
print_error(&e, pdf_path, json_output);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
+58
-36
@@ -97,26 +97,9 @@ pub fn detect_pdf_type_with_config<P: AsRef<Path>>(
|
||||
) -> Result<PdfTypeResult, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
|
||||
// First, load metadata only (fast operation)
|
||||
let metadata = match Document::load_metadata(&path) {
|
||||
Ok(m) => m,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_metadata_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, page_count) = crate::load_document_from_path(&path)?;
|
||||
|
||||
// Then load the full document for content inspection
|
||||
// We use filtered loading to skip heavy objects we don't need
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
|
||||
detect_from_document(&doc, metadata.page_count, &config)
|
||||
detect_from_document(&doc, page_count, &config)
|
||||
}
|
||||
|
||||
/// Detect PDF type from memory buffer
|
||||
@@ -131,25 +114,64 @@ pub fn detect_pdf_type_mem_with_config(
|
||||
) -> Result<PdfTypeResult, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
|
||||
// Load metadata first (fast)
|
||||
let metadata = match Document::load_metadata_mem(buffer) {
|
||||
Ok(m) => m,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_metadata_mem_with_password(buffer, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, page_count) = crate::load_document_from_mem(buffer)?;
|
||||
|
||||
// Load document for inspection
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
detect_from_document(&doc, page_count, &config)
|
||||
}
|
||||
|
||||
detect_from_document(&doc, metadata.page_count, &config)
|
||||
/// Heuristic page-count fallback for malformed PDFs that cannot be parsed.
|
||||
///
|
||||
/// This scans raw bytes for page dictionaries (`/Type /Page`) while excluding
|
||||
/// the page tree node (`/Type /Pages`). It is intended as a low-confidence hint
|
||||
/// for diagnostics; parsed page-tree counts remain authoritative.
|
||||
pub fn estimate_page_count_from_bytes(buffer: &[u8]) -> u32 {
|
||||
let mut count = 0u32;
|
||||
let mut pos = 0usize;
|
||||
|
||||
while let Some(rel_idx) = find_bytes(&buffer[pos..], b"/Type") {
|
||||
let mut value_pos = pos + rel_idx + b"/Type".len();
|
||||
value_pos = skip_pdf_whitespace(buffer, value_pos);
|
||||
|
||||
if buffer.get(value_pos) == Some(&b'/') {
|
||||
let name_start = value_pos + 1;
|
||||
let name_end = name_start + b"Page".len();
|
||||
if name_end <= buffer.len()
|
||||
&& &buffer[name_start..name_end] == b"Page"
|
||||
&& buffer
|
||||
.get(name_end)
|
||||
.is_none_or(|b| is_pdf_name_delimiter(*b))
|
||||
{
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pos += rel_idx + b"/Type".len();
|
||||
}
|
||||
|
||||
count
|
||||
}
|
||||
|
||||
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack.windows(needle.len()).position(|w| w == needle)
|
||||
}
|
||||
|
||||
fn skip_pdf_whitespace(buffer: &[u8], mut pos: usize) -> usize {
|
||||
while pos < buffer.len() && is_pdf_whitespace(buffer[pos]) {
|
||||
pos += 1;
|
||||
}
|
||||
pos
|
||||
}
|
||||
|
||||
fn is_pdf_whitespace(byte: u8) -> bool {
|
||||
matches!(byte, b'\0' | b'\t' | b'\n' | 0x0C | b'\r' | b' ')
|
||||
}
|
||||
|
||||
fn is_pdf_name_delimiter(byte: u8) -> bool {
|
||||
is_pdf_whitespace(byte)
|
||||
|| matches!(
|
||||
byte,
|
||||
b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
|
||||
)
|
||||
}
|
||||
|
||||
/// Detection logic on a pre-loaded document.
|
||||
|
||||
+4
-28
@@ -36,26 +36,14 @@ pub(crate) use layout::ColumnRegion;
|
||||
/// Extract text from PDF file as plain string
|
||||
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_path(&path)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
/// Extract text from PDF memory buffer
|
||||
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_mem(buffer)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
@@ -91,13 +79,7 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<PageExtraction, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_path(&path)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let (extraction, _thresholds, _gid_pages) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
@@ -124,13 +106,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<PageExtraction, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_mem(buffer)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let (extraction, _thresholds, _gid_pages) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
|
||||
+430
-22
@@ -931,24 +931,73 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
// Stage 1: strict text fill — each cell gets the items whose centers
|
||||
// fall inside its (normalized) bbox or whose >=60% overlap rule fires.
|
||||
// Track which item indices any cell claimed so the orphan pass below
|
||||
// doesn't double-assign.
|
||||
// Stage 1: exclusive per-item assignment. For each PDF text item,
|
||||
// find the cell(s) whose (band-clamped) bbox satisfies the strict
|
||||
// membership rule (`tsr_region_contains_item`: center inside OR
|
||||
// >=60% overlap on both axes). If multiple cells qualify, assign
|
||||
// the item to the cell whose center is geometrically closest. If
|
||||
// exactly one qualifies, assign to that. If none, the item is an
|
||||
// orphan and stage 2 below tries to recover it.
|
||||
//
|
||||
// The exclusivity (one item → one cell) prevents the cell-overlap
|
||||
// bug where SLANet emits cells whose y-extents overlap between
|
||||
// rows: under the previous "for each cell, gather items" approach,
|
||||
// an item whose center fell in two cells' overlap got duplicated
|
||||
// into both. Closest-center disambiguation routes it to the
|
||||
// correct row.
|
||||
let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||||
for cell in &mut cells {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, coords);
|
||||
let mut matched: Vec<TextItem> = Vec::new();
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if tsr_region_contains_item(item, bounds) {
|
||||
claimed.insert(i);
|
||||
matched.push(item.clone());
|
||||
let mut item_to_cell: std::collections::HashMap<usize, usize> =
|
||||
std::collections::HashMap::new();
|
||||
|
||||
// Pre-compute each cell's bounds + center (in PDF-pt-flipped space)
|
||||
// so we don't redo the work per item.
|
||||
let cell_meta: Vec<Option<(RegionBounds, f32, f32)>> = cells
|
||||
.iter()
|
||||
.map(|cell| {
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
if x1 >= x2 || y1 >= y2 {
|
||||
return None;
|
||||
}
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, coords);
|
||||
let cx = (bounds.x_min + bounds.x_max) * 0.5;
|
||||
let cy = (bounds.y_min + bounds.y_max) * 0.5;
|
||||
Some((bounds, cx, cy))
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (item_idx, item) in items.iter().enumerate() {
|
||||
let item_w = text_utils::effective_width(item);
|
||||
let item_cx = item.x + item_w * 0.5;
|
||||
let item_cy = item.y + item.height * 0.5;
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
for (cell_idx, meta) in cell_meta.iter().enumerate() {
|
||||
let Some((bounds, ccx, ccy)) = meta else {
|
||||
continue;
|
||||
};
|
||||
if !tsr_region_contains_item(item, *bounds) {
|
||||
continue;
|
||||
}
|
||||
let dx = item_cx - ccx;
|
||||
let dy = item_cy - ccy;
|
||||
let dist_sq = dx * dx + dy * dy;
|
||||
if best.is_none_or(|(_, d)| dist_sq < d) {
|
||||
best = Some((cell_idx, dist_sq));
|
||||
}
|
||||
}
|
||||
// Markdown cells must be one line — collapse line breaks produced
|
||||
// by the line-grouping pass.
|
||||
cell.text = collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
if let Some((ci, _)) = best {
|
||||
claimed.insert(item_idx);
|
||||
item_to_cell.insert(item_idx, ci);
|
||||
}
|
||||
}
|
||||
|
||||
// Build per-cell text from the assigned items. Markdown cells must
|
||||
// be one line — collapse line breaks from the line-grouping pass.
|
||||
let mut per_cell_items: Vec<Vec<TextItem>> = vec![Vec::new(); cells.len()];
|
||||
for (&item_idx, &cell_idx) in &item_to_cell {
|
||||
per_cell_items[cell_idx].push(items[item_idx].clone());
|
||||
}
|
||||
for (cell_idx, matched) in per_cell_items.into_iter().enumerate() {
|
||||
cells[cell_idx].text = collect_text_from_matched_items(matched, adaptive_threshold)
|
||||
.replace(['\n', '\r'], " ");
|
||||
}
|
||||
|
||||
@@ -1136,6 +1185,262 @@ pub fn extract_tables_with_structure_mem(
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Markdown for one extracted table plus a diagnostic flag describing
|
||||
/// which path produced it.
|
||||
///
|
||||
/// `fallback_reason` is `None` when the TSR-hybrid path produced the
|
||||
/// markdown directly; `Some(<short identifier>)` when stage 1's quality
|
||||
/// check fired and the heuristic `extract_tables_in_regions_mem` was
|
||||
/// substituted instead. The reason string is stable enough to use as a
|
||||
/// metric label (e.g. `phantom_empty_row`, `multi_row_in_cell`).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TableExtractionResult {
|
||||
pub markdown: String,
|
||||
pub fallback_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Detect quality issues in the TSR-hybrid output for a single input.
|
||||
///
|
||||
/// Returns `Some(reason)` if the cells look like they reflect a known
|
||||
/// SLANet detection pathology that the heuristic table extractor would
|
||||
/// likely handle better. Reasons (also used as metric labels):
|
||||
///
|
||||
/// * `phantom_empty_row` — a row whose every cell is empty, surrounded
|
||||
/// above and below by rows with content. SLANet sometimes emits an
|
||||
/// extra row that doesn't correspond to any visible PDF row.
|
||||
/// * `multi_row_in_cell` — at least one `rowspan==1` cell encloses
|
||||
/// PDF text items that cluster into two distinct visual lines
|
||||
/// separated by a whitespace gap larger than the line height. Cells
|
||||
/// declared as `rowspan>1` are excluded since they are *expected*
|
||||
/// to span multiple lines. SLANet's row under-detection on
|
||||
/// tightly-packed tables produces the rowspan==1-but-multi-line
|
||||
/// pattern (the FNBO failure mode).
|
||||
fn detect_tsr_quality_issue(
|
||||
buffer: &[u8],
|
||||
input: &TsrTableInput,
|
||||
cells: &[tables::StructuredCell],
|
||||
) -> Result<Option<String>, PdfError> {
|
||||
if cells.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Phantom row: cheap, computed from cell metadata alone.
|
||||
let max_row = cells.iter().map(|c| c.row).max().unwrap_or(0);
|
||||
if max_row >= 2 {
|
||||
let mut row_has_content = vec![false; max_row + 1];
|
||||
for cell in cells {
|
||||
if !cell.text.trim().is_empty() {
|
||||
row_has_content[cell.row] = true;
|
||||
}
|
||||
}
|
||||
for r in 1..max_row {
|
||||
if !row_has_content[r] && row_has_content[r - 1] && row_has_content[r + 1] {
|
||||
return Ok(Some("phantom_empty_row".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-row-in-cell: re-extract PDF text items in the page and look
|
||||
// for `rowspan==1` cells that contain items grouped into ≥2 visual
|
||||
// lines separated by a real whitespace gap. This is the FNBO mode:
|
||||
// a tall TSR cell catches text from two adjacent PDF rows that
|
||||
// SLANet failed to separate. Cells declared `rowspan>1` are
|
||||
// expected to be multi-line and are excluded.
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let pages = doc.get_pages();
|
||||
let page_1idx = input.page + 1;
|
||||
let Some(&page_id) = pages.get(&page_1idx) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let page_h = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
let mut needed: HashSet<u32> = HashSet::new();
|
||||
needed.insert(page_1idx);
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((mut items, _rects, _lines), _has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
page_1idx,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
let _ = text_utils::fix_letterspaced_items(&mut items);
|
||||
let coords = if coords_rotated {
|
||||
RegionCoordSpace::Rotated90Ccw
|
||||
} else {
|
||||
RegionCoordSpace::Standard
|
||||
};
|
||||
|
||||
for cell in cells {
|
||||
// rowspan>1 cells are intentionally multi-line — skip them.
|
||||
if cell.rowspan > 1 {
|
||||
continue;
|
||||
}
|
||||
if cell.text.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let [x1, y1, x2, y2] = cell.page_pt_bbox;
|
||||
if x1 >= x2 || y1 >= y2 {
|
||||
continue;
|
||||
}
|
||||
let bounds = region_bounds(x1, y1, x2, y2, page_h, coords);
|
||||
|
||||
// Collect the items inside this cell, with their y-centers and
|
||||
// half-heights so we can cluster them into visual lines.
|
||||
let mut cell_items: Vec<(f32, f32)> = Vec::new();
|
||||
for item in &items {
|
||||
if tsr_region_contains_item(item, bounds) {
|
||||
let cy = item.y + item.height * 0.5;
|
||||
let half_h = (item.height * 0.5).max(2.5);
|
||||
cell_items.push((cy, half_h));
|
||||
}
|
||||
}
|
||||
if cell_items.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
// Sort by y-center descending (top-of-page first in PDF native
|
||||
// coords where y grows upward) — direction doesn't matter, we
|
||||
// just need consecutive items to be neighbors in the sort.
|
||||
cell_items.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||
|
||||
// Walk pairs and see if there's a real whitespace gap between
|
||||
// any two adjacent items — defined as their bounding-box edges
|
||||
// separated by more than half a line height. This rules out
|
||||
// tall glyphs / superscripts / accents on a single visual line.
|
||||
let max_half_h = cell_items
|
||||
.iter()
|
||||
.map(|(_, h)| *h)
|
||||
.fold(0f32, f32::max)
|
||||
.max(2.5);
|
||||
let gap_threshold = max_half_h; // ≈ half a line height
|
||||
let mut found_gap = false;
|
||||
for w in cell_items.windows(2) {
|
||||
let (cy_a, h_a) = w[0];
|
||||
let (cy_b, h_b) = w[1];
|
||||
// Gap = distance between the bottom of the upper item and
|
||||
// the top of the lower item, measured in PDF-native coords
|
||||
// (y grows upward, so the upper item has the larger cy).
|
||||
let upper_bottom = cy_a - h_a;
|
||||
let lower_top = cy_b + h_b;
|
||||
let gap = upper_bottom - lower_top;
|
||||
if gap > gap_threshold {
|
||||
found_gap = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if found_gap {
|
||||
return Ok(Some("multi_row_in_cell".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Auto-fallback variant of [`extract_tables_with_structure_mem`]:
|
||||
/// runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies (phantom rows, multi-row-in-cell text),
|
||||
/// and falls back to the heuristic [`extract_tables_in_regions_mem`]
|
||||
/// for any input where the TSR path looks compromised.
|
||||
///
|
||||
/// On clean inputs this is identical to the markdown variant.
|
||||
/// On flagged inputs the heuristic markdown replaces the TSR markdown
|
||||
/// and the result's `fallback_reason` is set to the diagnostic label.
|
||||
///
|
||||
/// Two failure modes are guarded against per-input:
|
||||
///
|
||||
/// * **Empty heuristic**: if the heuristic returns empty/whitespace
|
||||
/// markdown for a flagged region, the original TSR markdown is
|
||||
/// preserved and `fallback_reason` is suffixed with
|
||||
/// `_heuristic_empty` (e.g. `multi_row_in_cell_heuristic_empty`).
|
||||
/// This avoids replacing a usable wrong-but-non-empty TSR output
|
||||
/// with literally nothing.
|
||||
/// * **Per-input errors**: any failure in detection or heuristic
|
||||
/// extraction for a single input is contained — that input
|
||||
/// returns the raw TSR markdown with `fallback_reason` set to
|
||||
/// an `_error` label so callers can metric on it. Other inputs
|
||||
/// in the same batch are unaffected.
|
||||
///
|
||||
/// Use this from production callers that want self-healing output.
|
||||
/// Use [`extract_tables_with_structure_mem`] when you want raw TSR
|
||||
/// output regardless of quality (e.g. eval harnesses comparing the
|
||||
/// two paths).
|
||||
pub fn extract_tables_with_structure_auto_mem(
|
||||
buffer: &[u8],
|
||||
inputs: &[TsrTableInput],
|
||||
) -> Result<Vec<TableExtractionResult>, PdfError> {
|
||||
let tsr_cells = extract_tables_with_structure_cells_mem(buffer, inputs)?;
|
||||
let mut results = Vec::with_capacity(inputs.len());
|
||||
|
||||
for (i, input) in inputs.iter().enumerate() {
|
||||
let cells = &tsr_cells[i];
|
||||
let tsr_md = if cells.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
tables::cells_to_markdown(cells)
|
||||
};
|
||||
|
||||
let issue = match detect_tsr_quality_issue(buffer, input, cells) {
|
||||
Ok(opt) => opt,
|
||||
Err(_) => {
|
||||
// Detection failed for this input — fall through with
|
||||
// the raw TSR markdown so the rest of the batch is
|
||||
// unaffected. Tag the reason for caller metrics.
|
||||
results.push(TableExtractionResult {
|
||||
markdown: tsr_md,
|
||||
fallback_reason: Some("detection_error".to_string()),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let result = match issue {
|
||||
None => TableExtractionResult {
|
||||
markdown: tsr_md,
|
||||
fallback_reason: None,
|
||||
},
|
||||
Some(reason) => {
|
||||
// Fall back to heuristic on the input's table region.
|
||||
// The crop's PDF-pt bbox IS the table region.
|
||||
let heuristic_md = match extract_tables_in_regions_mem(
|
||||
buffer,
|
||||
&[(input.page, vec![input.crop_pdf_pt_bbox])],
|
||||
) {
|
||||
Ok(pages) => pages
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|p| p.regions.into_iter().next().map(|r| r.text))
|
||||
.unwrap_or_default(),
|
||||
Err(_) => {
|
||||
// Heuristic threw — keep raw TSR markdown.
|
||||
results.push(TableExtractionResult {
|
||||
markdown: tsr_md,
|
||||
fallback_reason: Some(format!("{reason}_heuristic_error")),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if heuristic_md.trim().is_empty() {
|
||||
// Heuristic produced nothing useful — keep TSR
|
||||
// markdown rather than ship empty. The reason
|
||||
// suffix lets callers count this case.
|
||||
TableExtractionResult {
|
||||
markdown: tsr_md,
|
||||
fallback_reason: Some(format!("{reason}_heuristic_empty")),
|
||||
}
|
||||
} else {
|
||||
TableExtractionResult {
|
||||
markdown: heuristic_md,
|
||||
fallback_reason: Some(reason),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Get page height in points from MediaBox.
|
||||
fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option<f32> {
|
||||
let page_dict = doc.get_dictionary(page_id).ok()?;
|
||||
@@ -1381,30 +1686,133 @@ fn tsr_region_contains_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
/// `Document::load_metadata` for page count + `Document::load` for content
|
||||
/// are combined here, but lopdf loads the full doc in `load()` so we extract
|
||||
/// page count from it directly to avoid the metadata-only round-trip.
|
||||
fn load_document_from_path<P: AsRef<Path>>(path: P) -> Result<(Document, u32), PdfError> {
|
||||
pub(crate) fn load_document_from_path<P: AsRef<Path>>(
|
||||
path: P,
|
||||
) -> Result<(Document, u32), PdfError> {
|
||||
let buffer = std::fs::read(&path)?;
|
||||
load_document_from_mem(&buffer)
|
||||
}
|
||||
|
||||
/// Load a PDF from a memory buffer.
|
||||
fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
|
||||
pub(crate) fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
|
||||
// Fix malformed struct element names before parsing. Some PDF generators
|
||||
// write bare names (/S Code) instead of proper PDF names (/S /Code), which
|
||||
// causes lopdf to silently drop the entire object.
|
||||
let fixed = structure_tree::fix_bare_struct_names(buffer);
|
||||
let buf = fixed.as_ref();
|
||||
|
||||
let doc = match Document::load_mem(buf) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))?
|
||||
let doc = match load_document_bytes(buf) {
|
||||
Ok(doc) => doc,
|
||||
Err(first_err) => {
|
||||
for repaired in repair_pdf_container_candidates(buf) {
|
||||
match load_document_bytes(&repaired) {
|
||||
Ok(doc) => {
|
||||
log::debug!("loaded PDF after repairing malformed container bytes");
|
||||
let page_count = doc.get_pages().len() as u32;
|
||||
return Ok((doc, page_count));
|
||||
}
|
||||
Err(e) => {
|
||||
if is_encrypted_lopdf_error(&e) {
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Err(first_err.into());
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let page_count = doc.get_pages().len() as u32;
|
||||
Ok((doc, page_count))
|
||||
}
|
||||
|
||||
fn load_document_bytes(buf: &[u8]) -> Result<Document, lopdf::Error> {
|
||||
match Document::load_mem(buf) {
|
||||
Ok(doc) => Ok(doc),
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
fn repair_pdf_container_candidates(buf: &[u8]) -> Vec<Vec<u8>> {
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
add_repair_candidate(&mut candidates, append_missing_eof_marker(buf), buf);
|
||||
|
||||
let stripped = strip_leading_pdf_container_bytes(buf);
|
||||
if let Some(stripped_buf) = stripped.as_deref() {
|
||||
add_repair_candidate(&mut candidates, Some(stripped_buf.to_vec()), buf);
|
||||
add_repair_candidate(
|
||||
&mut candidates,
|
||||
append_missing_eof_marker(stripped_buf),
|
||||
buf,
|
||||
);
|
||||
}
|
||||
|
||||
candidates
|
||||
}
|
||||
|
||||
fn add_repair_candidate(
|
||||
candidates: &mut Vec<Vec<u8>>,
|
||||
candidate: Option<Vec<u8>>,
|
||||
original: &[u8],
|
||||
) {
|
||||
let Some(candidate) = candidate else {
|
||||
return;
|
||||
};
|
||||
if candidate.as_slice() == original {
|
||||
return;
|
||||
}
|
||||
if candidates.iter().any(|existing| existing == &candidate) {
|
||||
return;
|
||||
}
|
||||
candidates.push(candidate);
|
||||
}
|
||||
|
||||
fn append_missing_eof_marker(buf: &[u8]) -> Option<Vec<u8>> {
|
||||
if contains_recent_eof_marker(buf) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut end = buf.len();
|
||||
while end > 0 && buf[end - 1].is_ascii_whitespace() {
|
||||
end -= 1;
|
||||
}
|
||||
|
||||
if !buf[..end].ends_with(b"%%EO") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut repaired = Vec::with_capacity(end + 2);
|
||||
repaired.extend_from_slice(&buf[..end]);
|
||||
repaired.extend_from_slice(b"F\n");
|
||||
Some(repaired)
|
||||
}
|
||||
|
||||
fn contains_recent_eof_marker(buf: &[u8]) -> bool {
|
||||
let start = buf.len().saturating_sub(1024);
|
||||
buf[start..].windows(b"%%EOF".len()).any(|w| w == b"%%EOF")
|
||||
}
|
||||
|
||||
fn strip_leading_pdf_container_bytes(buf: &[u8]) -> Option<Vec<u8>> {
|
||||
let mut start = if buf.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
3
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
while start < buf.len() && buf[start].is_ascii_whitespace() {
|
||||
start += 1;
|
||||
}
|
||||
|
||||
if start > 0 && buf[start..].starts_with(b"%PDF-") {
|
||||
Some(buf[start..].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Core processing pipeline operating on a pre-loaded document.
|
||||
fn process_document(
|
||||
doc: Document,
|
||||
|
||||
+496
-1
@@ -1,6 +1,6 @@
|
||||
//! Integration tests for pdf-to-markdown library
|
||||
|
||||
use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
@@ -11,6 +11,83 @@ use pdf_inspector::{
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn make_minimal_text_pdf() -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize];
|
||||
|
||||
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(body.as_bytes());
|
||||
pdf.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
1,
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
2,
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
3,
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
|
||||
);
|
||||
|
||||
let content = "BT /F1 12 Tf 100 700 Td (Hello World) Tj 0 -14 Td (Second Line) Tj 0 -14 Td (Third Line) Tj ET";
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
4,
|
||||
&format!(
|
||||
"<< /Length {} >>\nstream\n{}\nendstream",
|
||||
content.len(),
|
||||
content
|
||||
),
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
5,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
);
|
||||
|
||||
let xref_start = pdf.len();
|
||||
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
|
||||
pdf.extend_from_slice(b"0000000000 65535 f \n");
|
||||
for offset in offsets.iter().skip(1) {
|
||||
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
pdf.extend_from_slice(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
|
||||
offsets.len(),
|
||||
xref_start
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
|
||||
pdf
|
||||
}
|
||||
|
||||
fn truncate_eof_marker(mut pdf: Vec<u8>) -> Vec<u8> {
|
||||
assert!(pdf.ends_with(b"%%EOF"));
|
||||
pdf.pop();
|
||||
pdf
|
||||
}
|
||||
|
||||
fn add_leading_tab(mut pdf: Vec<u8>) -> Vec<u8> {
|
||||
pdf.insert(0, b'\t');
|
||||
pdf
|
||||
}
|
||||
|
||||
// Helper to create test TextItems
|
||||
fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
|
||||
use pdf_inspector::types::ItemType;
|
||||
@@ -826,6 +903,73 @@ fn test_bom_prefixed_pdf_header_not_rejected() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_process_pdf_mem_repairs_truncated_eof_marker() {
|
||||
let pdf = truncate_eof_marker(make_minimal_text_pdf());
|
||||
|
||||
let result = process_pdf_mem(&pdf).expect("truncated %%EO marker should be repaired");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert!(
|
||||
result
|
||||
.markdown
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("Hello World"),
|
||||
"repaired PDF should still extract text"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_process_pdf_mem_repairs_leading_tab_and_truncated_eof() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
|
||||
let result = process_pdf_mem(&pdf).expect("leading whitespace + %%EO should be repaired");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert!(
|
||||
result
|
||||
.markdown
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("Hello World"),
|
||||
"repaired PDF should still extract text"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_pdf_type_repairs_container_from_path() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("broken-container.pdf");
|
||||
std::fs::write(&path, pdf).unwrap();
|
||||
|
||||
let result = detect_pdf_type(&path).expect("detector should use shared repair loader");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert_eq!(result.pages_with_text, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_text_mem_uses_container_repair() {
|
||||
let pdf = truncate_eof_marker(make_minimal_text_pdf());
|
||||
|
||||
let text = pdf_inspector::extractor::extract_text_mem(&pdf)
|
||||
.expect("plain text extraction should use shared repair loader");
|
||||
|
||||
assert!(text.contains("Hello World"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_page_count_from_bytes_excludes_pages_tree() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
|
||||
assert_eq!(estimate_page_count_from_bytes(&pdf), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_a_pdf_detect_pdf_type_mem() {
|
||||
// Verify detect_pdf_type_mem is also guarded
|
||||
@@ -1908,6 +2052,357 @@ fn test_extract_tables_with_structure_separator_after_thead() {
|
||||
assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_auto_mem tests (TSR + heuristic fallback)
|
||||
// =========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_auto_passes_through_clean_tsr_output() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cells fit each visible row cleanly. Same shape as the existing
|
||||
// dense-overlap regression test — TSR should produce clean output
|
||||
// and the auto wrapper should pass through with no fallback.
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"expected no fallback, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
assert!(results[0].markdown.contains("Boardwalk"));
|
||||
assert!(!results[0].markdown.contains("Oak Street Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_falls_back_on_multi_row_in_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// TSR returns only 2 rows for what's actually 3 visible PDF rows.
|
||||
// Row 1's cells are tall enough to encompass both Oak Street and
|
||||
// Boardwalk text — the FNBO row-undercount pattern.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row at top-left y=[88, 105] (covers "Branch Name"/"Deposits"
|
||||
// at native y=700, top-left y≈92-103). The "data" row at top-left
|
||||
// y=[105, 145] is intentionally tall — covers BOTH the Oak Street
|
||||
// line (top-left y≈108-119) AND the Boardwalk line (y≈124-135).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(
|
||||
results[0].fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell"),
|
||||
"expected multi_row_in_cell fallback, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
// The heuristic-fallback markdown should preserve all three PDF rows.
|
||||
let md = &results[0].markdown;
|
||||
assert!(md.contains("Oak Street"), "missing Oak Street: {md}");
|
||||
assert!(md.contains("Boardwalk"), "missing Boardwalk: {md}");
|
||||
assert!(md.contains("100"), "missing 100: {md}");
|
||||
assert!(md.contains("200"), "missing 200: {md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_returns_empty_inputs() {
|
||||
use pdf_inspector::extract_tables_with_structure_auto_mem;
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[]).unwrap();
|
||||
assert!(results.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_does_not_fire_on_legit_rowspan_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// 2 columns, 3 rows in the visible PDF. SLANet emits a 2-row table
|
||||
// where the LEFT cell of row 1 is a rowspan=2 cell that legitimately
|
||||
// covers Oak Street + Boardwalk on two visual lines. The right
|
||||
// column has two normal rows. multi_row_in_cell must NOT fire on
|
||||
// the rowspan=2 cell.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
// First data cell explicitly declares rowspan=2.
|
||||
"<td",
|
||||
" rowspan=\"2\"",
|
||||
">",
|
||||
"</td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row, then a tall left cell covering both data lines, plus
|
||||
// two narrow right cells (one per line).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0), // rowspan=2 — covers both lines
|
||||
poly(90.0, 105.0, 180.0, 122.0), // row 1 only
|
||||
poly(90.0, 122.0, 180.0, 145.0), // row 2 only
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"rowspan=2 cell containing 2 visual lines should not trip multi_row_in_cell, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_keeps_tsr_markdown_when_heuristic_returns_empty() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// Same shape as the multi_row_in_cell regression — a tall data cell
|
||||
// that catches Oak Street + Boardwalk. But the crop bbox we pass
|
||||
// points at a strip of the page that has NO text items, so the
|
||||
// heuristic's region will be empty when it tries to extract there.
|
||||
// The auto wrapper must keep the TSR markdown rather than ship "".
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cell bboxes overlap the actual PDF text (so multi_row_in_cell
|
||||
// fires) — but the crop_pdf_pt_bbox we hand to the heuristic is a
|
||||
// wholly-empty region of the page. The heuristic should return "".
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
// Crop is at the BOTTOM of the page where there's no text.
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 50.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
let r = &results[0];
|
||||
assert_eq!(
|
||||
r.fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell_heuristic_empty"),
|
||||
"expected _heuristic_empty suffix, got {:?}",
|
||||
r.fallback_reason,
|
||||
);
|
||||
// TSR markdown should be preserved — non-empty, contains the cell
|
||||
// text we know was assigned by the TSR path.
|
||||
assert!(
|
||||
!r.markdown.trim().is_empty(),
|
||||
"expected TSR markdown to be preserved, got empty",
|
||||
);
|
||||
assert!(
|
||||
r.markdown.contains("Oak Street") || r.markdown.contains("Boardwalk"),
|
||||
"expected TSR markdown to contain at least one row, got: {}",
|
||||
r.markdown,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_isolates_per_input_failures() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let good_tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// A clean input that should pass through with no fallback.
|
||||
let good_input = TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: good_tokens,
|
||||
cell_bboxes: vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
],
|
||||
};
|
||||
// A bad input that targets a non-existent page. The detection
|
||||
// helper short-circuits on missing pages with Ok(None), so this
|
||||
// shouldn't itself crash, but pairing it with a flagged input
|
||||
// exercises the per-input control flow regardless. The point of
|
||||
// this test is that one input's outcome doesn't poison the other.
|
||||
let bad_input = TsrTableInput {
|
||||
page: 9999,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: vec![
|
||||
"<table>".into(),
|
||||
"<tr>".into(),
|
||||
"<td></td>".into(),
|
||||
"</tr>".into(),
|
||||
"</table>".into(),
|
||||
],
|
||||
cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
|
||||
};
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[good_input, bad_input]).unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
// Good input still produces non-empty TSR markdown with no fallback.
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"good input should pass through, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
// Bad input collapses to empty markdown but doesn't take the
|
||||
// batch down with it.
|
||||
assert_eq!(results[1].markdown, "");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_pages_markdown_mem tests
|
||||
// =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user