Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cabc108d5c | ||
|
|
860328bd74 | ||
|
|
0674ba5155 | ||
|
|
d196d435d1 | ||
|
|
fbab84fc20 | ||
|
|
bdea4f345a | ||
|
|
8a0f98dee7 | ||
|
|
d8894326e8 | ||
|
|
9cce4dd161 | ||
|
|
f61d139710 | ||
|
|
3f8fb645c9 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.6.0",
|
||||
"version": "1.7.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+100
@@ -99,6 +99,13 @@ pub struct PageRegionTexts {
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// Vector-grid detection result compatible with `extractTablesWithStructure*`.
|
||||
#[napi(object)]
|
||||
pub struct VectorGridDetectionJs {
|
||||
pub structure_tokens: Vec<String>,
|
||||
pub cell_bboxes: Vec<Vec<f64>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -317,6 +324,53 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect a vector ruled-line / rectangle grid inside one page region.
|
||||
///
|
||||
/// Returns TSR-compatible structure tokens plus crop-pixel cell bboxes, or
|
||||
/// `null` when the region does not contain a valid vector grid.
|
||||
///
|
||||
/// `pageIdx` is 0-indexed. `regionPdfPtBbox` is `[x1,y1,x2,y2]` in PDF
|
||||
/// points with top-left origin. `renderDpi` is the DPI of the crop image that
|
||||
/// will consume the returned cell bboxes.
|
||||
#[napi]
|
||||
pub fn detect_vector_grid_in_region(
|
||||
buffer: Buffer,
|
||||
page_idx: u32,
|
||||
region_pdf_pt_bbox: Vec<f64>,
|
||||
render_dpi: f64,
|
||||
) -> Result<Option<VectorGridDetectionJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let region = if region_pdf_pt_bbox.len() == 4 {
|
||||
[
|
||||
region_pdf_pt_bbox[0] as f32,
|
||||
region_pdf_pt_bbox[1] as f32,
|
||||
region_pdf_pt_bbox[2] as f32,
|
||||
region_pdf_pt_bbox[3] as f32,
|
||||
]
|
||||
} else {
|
||||
[0.0, 0.0, 0.0, 0.0]
|
||||
};
|
||||
|
||||
catch_panic("detect_vector_grid_in_region", move || {
|
||||
let result = pdf_inspector::detect_vector_grid_in_region_mem(
|
||||
&bytes,
|
||||
page_idx,
|
||||
region,
|
||||
render_dpi as f32,
|
||||
)
|
||||
.map_err(|e| to_napi_err(e, "detect_vector_grid_in_region"))?;
|
||||
|
||||
Ok(result.map(|r| VectorGridDetectionJs {
|
||||
structure_tokens: r.structure_tokens,
|
||||
cell_bboxes: r
|
||||
.cell_bboxes
|
||||
.into_iter()
|
||||
.map(|bbox| bbox.into_iter().map(|v| v as f64).collect())
|
||||
.collect(),
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
/// One cropped table region plus its raw structure-recovery output, for
|
||||
/// `extractTablesWithStructure`.
|
||||
///
|
||||
@@ -422,6 +476,52 @@ pub fn extract_tables_with_structure_cells(
|
||||
})
|
||||
}
|
||||
|
||||
/// One result from `extractTablesWithStructureAuto` — markdown plus a
|
||||
/// diagnostic flag identifying which path produced it.
|
||||
///
|
||||
/// `fallbackReason` is `null` when the TSR-hybrid path produced the
|
||||
/// markdown directly. When stage 1's quality check fires (the cells
|
||||
/// look like a SLANet detection pathology — phantom rows or multi-row
|
||||
/// content in a single cell), the heuristic table extractor is run on
|
||||
/// the same region instead, and `fallbackReason` carries the diagnostic
|
||||
/// label (`"phantom_empty_row"`, `"multi_row_in_cell"`).
|
||||
#[napi(object)]
|
||||
pub struct TableExtractionResultJs {
|
||||
pub markdown: String,
|
||||
pub fallback_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Auto-fallback variant of [`extractTablesWithStructure`].
|
||||
///
|
||||
/// Runs the TSR-hybrid path, checks the resulting cells for known
|
||||
/// SLANet detection pathologies, and falls back to the heuristic
|
||||
/// `extractTablesInRegions` for any input where the TSR path looks
|
||||
/// compromised.
|
||||
///
|
||||
/// On clean inputs this returns identical markdown to
|
||||
/// `extractTablesWithStructure`; on flagged inputs the heuristic
|
||||
/// markdown replaces the TSR markdown and `fallbackReason` is set.
|
||||
#[napi]
|
||||
pub fn extract_tables_with_structure_auto(
|
||||
buffer: Buffer,
|
||||
inputs: Vec<TsrTableInputJs>,
|
||||
) -> Result<Vec<TableExtractionResultJs>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let parsed = parse_tsr_inputs(&inputs);
|
||||
|
||||
catch_panic("extract_tables_with_structure_auto", move || {
|
||||
let result = pdf_inspector::extract_tables_with_structure_auto_mem(&bytes, &parsed)
|
||||
.map_err(|e| to_napi_err(e, "extract_tables_with_structure_auto"))?;
|
||||
Ok(result
|
||||
.into_iter()
|
||||
.map(|r| TableExtractionResultJs {
|
||||
markdown: r.markdown,
|
||||
fallback_reason: r.fallback_reason,
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_tsr_inputs(inputs: &[TsrTableInputJs]) -> Vec<pdf_inspector::TsrTableInput> {
|
||||
inputs
|
||||
.iter()
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
detectVectorGridInRegion,
|
||||
extractPagesMarkdown,
|
||||
} from './index.js';
|
||||
|
||||
@@ -90,6 +91,17 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
|
||||
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
|
||||
console.log(' extractTextInRegions: OK');
|
||||
|
||||
// --- detectVectorGridInRegion ---
|
||||
console.log('Testing detectVectorGridInRegion...');
|
||||
const vectorGrid = detectVectorGridInRegion(fixture, 0, [0, 0, 600, 800], 72);
|
||||
assert.ok(vectorGrid === null || typeof vectorGrid === 'object');
|
||||
if (vectorGrid) {
|
||||
assert.ok(Array.isArray(vectorGrid.structureTokens));
|
||||
assert.ok(Array.isArray(vectorGrid.cellBboxes));
|
||||
assert.ok(vectorGrid.cellBboxes.every(bbox => Array.isArray(bbox) && bbox.length === 4));
|
||||
}
|
||||
console.log(' detectVectorGridInRegion: OK');
|
||||
|
||||
// --- extractPagesMarkdown ---
|
||||
console.log('Testing extractPagesMarkdown...');
|
||||
|
||||
|
||||
+33
-11
@@ -1,8 +1,12 @@
|
||||
//! CLI tool for detecting PDF type (text-based vs scanned)
|
||||
|
||||
use pdf_inspector::{detect_pdf_type, process_pdf_with_options, PdfOptions, PdfType, ProcessMode};
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, detector::estimate_page_count_from_bytes, process_pdf_with_options,
|
||||
PdfOptions, PdfType, ProcessMode,
|
||||
};
|
||||
use std::env;
|
||||
use std::fmt::Write;
|
||||
use std::fs;
|
||||
use std::process;
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -64,6 +68,32 @@ fn pdf_type_str(pdf_type: &PdfType) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn page_count_hint(pdf_path: &str) -> Option<u32> {
|
||||
fs::read(pdf_path)
|
||||
.ok()
|
||||
.map(|bytes| estimate_page_count_from_bytes(&bytes))
|
||||
.filter(|&count| count > 0)
|
||||
}
|
||||
|
||||
fn print_error(e: &pdf_inspector::PdfError, pdf_path: &str, json_output: bool) {
|
||||
if json_output {
|
||||
if let Some(count) = page_count_hint(pdf_path) {
|
||||
println!(
|
||||
r#"{{"error":"{}","page_count_hint":{}}}"#,
|
||||
json_escape(&e.to_string()),
|
||||
count
|
||||
);
|
||||
} else {
|
||||
println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
|
||||
}
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
if let Some(count) = page_count_hint(pdf_path) {
|
||||
eprintln!("Page count hint: {}", count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
match process_pdf_with_options(pdf_path, PdfOptions::new().mode(ProcessMode::Analyze)) {
|
||||
Ok(result) => {
|
||||
@@ -135,11 +165,7 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if json_output {
|
||||
println!(r#"{{"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
}
|
||||
print_error(&e, pdf_path, json_output);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -236,11 +262,7 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if json_output {
|
||||
println!(r#"{{"error":"{}"}}"#, e);
|
||||
} else {
|
||||
eprintln!("Error: {}", e);
|
||||
}
|
||||
print_error(&e, pdf_path, json_output);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
+58
-36
@@ -97,26 +97,9 @@ pub fn detect_pdf_type_with_config<P: AsRef<Path>>(
|
||||
) -> Result<PdfTypeResult, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
|
||||
// First, load metadata only (fast operation)
|
||||
let metadata = match Document::load_metadata(&path) {
|
||||
Ok(m) => m,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_metadata_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, page_count) = crate::load_document_from_path(&path)?;
|
||||
|
||||
// Then load the full document for content inspection
|
||||
// We use filtered loading to skip heavy objects we don't need
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
|
||||
detect_from_document(&doc, metadata.page_count, &config)
|
||||
detect_from_document(&doc, page_count, &config)
|
||||
}
|
||||
|
||||
/// Detect PDF type from memory buffer
|
||||
@@ -131,25 +114,64 @@ pub fn detect_pdf_type_mem_with_config(
|
||||
) -> Result<PdfTypeResult, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
|
||||
// Load metadata first (fast)
|
||||
let metadata = match Document::load_metadata_mem(buffer) {
|
||||
Ok(m) => m,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_metadata_mem_with_password(buffer, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, page_count) = crate::load_document_from_mem(buffer)?;
|
||||
|
||||
// Load document for inspection
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
detect_from_document(&doc, page_count, &config)
|
||||
}
|
||||
|
||||
detect_from_document(&doc, metadata.page_count, &config)
|
||||
/// Heuristic page-count fallback for malformed PDFs that cannot be parsed.
|
||||
///
|
||||
/// This scans raw bytes for page dictionaries (`/Type /Page`) while excluding
|
||||
/// the page tree node (`/Type /Pages`). It is intended as a low-confidence hint
|
||||
/// for diagnostics; parsed page-tree counts remain authoritative.
|
||||
pub fn estimate_page_count_from_bytes(buffer: &[u8]) -> u32 {
|
||||
let mut count = 0u32;
|
||||
let mut pos = 0usize;
|
||||
|
||||
while let Some(rel_idx) = find_bytes(&buffer[pos..], b"/Type") {
|
||||
let mut value_pos = pos + rel_idx + b"/Type".len();
|
||||
value_pos = skip_pdf_whitespace(buffer, value_pos);
|
||||
|
||||
if buffer.get(value_pos) == Some(&b'/') {
|
||||
let name_start = value_pos + 1;
|
||||
let name_end = name_start + b"Page".len();
|
||||
if name_end <= buffer.len()
|
||||
&& &buffer[name_start..name_end] == b"Page"
|
||||
&& buffer
|
||||
.get(name_end)
|
||||
.is_none_or(|b| is_pdf_name_delimiter(*b))
|
||||
{
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pos += rel_idx + b"/Type".len();
|
||||
}
|
||||
|
||||
count
|
||||
}
|
||||
|
||||
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack.windows(needle.len()).position(|w| w == needle)
|
||||
}
|
||||
|
||||
fn skip_pdf_whitespace(buffer: &[u8], mut pos: usize) -> usize {
|
||||
while pos < buffer.len() && is_pdf_whitespace(buffer[pos]) {
|
||||
pos += 1;
|
||||
}
|
||||
pos
|
||||
}
|
||||
|
||||
fn is_pdf_whitespace(byte: u8) -> bool {
|
||||
matches!(byte, b'\0' | b'\t' | b'\n' | 0x0C | b'\r' | b' ')
|
||||
}
|
||||
|
||||
fn is_pdf_name_delimiter(byte: u8) -> bool {
|
||||
is_pdf_whitespace(byte)
|
||||
|| matches!(
|
||||
byte,
|
||||
b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
|
||||
)
|
||||
}
|
||||
|
||||
/// Detection logic on a pre-loaded document.
|
||||
|
||||
+4
-28
@@ -36,26 +36,14 @@ pub(crate) use layout::ColumnRegion;
|
||||
/// Extract text from PDF file as plain string
|
||||
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_path(&path)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
/// Extract text from PDF memory buffer
|
||||
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_mem(buffer)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
@@ -91,13 +79,7 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<PageExtraction, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_with_password(&path, "")?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_path(&path)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let (extraction, _thresholds, _gid_pages) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
@@ -124,13 +106,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<PageExtraction, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let (doc, _) = crate::load_document_from_mem(buffer)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let (extraction, _thresholds, _gid_pages) =
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
|
||||
|
||||
+1763
-27
File diff suppressed because it is too large
Load Diff
+235
-1
@@ -12,7 +12,7 @@
|
||||
//! Cell text is supplied separately by the caller (typically by overlap-
|
||||
//! testing PDF text items against each cell's page-PDF-pt bbox).
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
/// A single resolved cell, with both structural metadata and its bbox in
|
||||
/// page PDF-points (top-left origin).
|
||||
@@ -219,6 +219,144 @@ pub(crate) fn cell_px_to_page_pt(
|
||||
]
|
||||
}
|
||||
|
||||
/// Refine TSR cell bboxes into non-overlapping row/column bands.
|
||||
///
|
||||
/// SLANet-style bboxes are often plausible but too tall on dense borderless
|
||||
/// tables. Native PDF text assignment is more reliable when each parsed row
|
||||
/// owns the band between neighboring row centers instead of the full model box.
|
||||
pub(crate) fn normalize_cell_bands(cells: &mut [StructuredCell]) {
|
||||
if cells.len() < 2 {
|
||||
return;
|
||||
}
|
||||
|
||||
let row_bands = derive_axis_bands(cells, Axis::Y);
|
||||
let col_bands = derive_axis_bands(cells, Axis::X);
|
||||
|
||||
for cell in cells {
|
||||
let row_end = cell.row + cell.rowspan.max(1).saturating_sub(1);
|
||||
if let (Some(&(y1, _)), Some(&(_, y2))) =
|
||||
(row_bands.get(&cell.row), row_bands.get(&row_end))
|
||||
{
|
||||
let clamped_y1 = cell.page_pt_bbox[1].max(y1);
|
||||
let clamped_y2 = cell.page_pt_bbox[3].min(y2);
|
||||
if clamped_y1 < clamped_y2 {
|
||||
cell.page_pt_bbox[1] = clamped_y1;
|
||||
cell.page_pt_bbox[3] = clamped_y2;
|
||||
}
|
||||
}
|
||||
|
||||
let col_end = cell.col + cell.colspan.max(1).saturating_sub(1);
|
||||
if let (Some(&(x1, _)), Some(&(_, x2))) =
|
||||
(col_bands.get(&cell.col), col_bands.get(&col_end))
|
||||
{
|
||||
let clamped_x1 = cell.page_pt_bbox[0].max(x1);
|
||||
let clamped_x2 = cell.page_pt_bbox[2].min(x2);
|
||||
if clamped_x1 < clamped_x2 {
|
||||
cell.page_pt_bbox[0] = clamped_x1;
|
||||
cell.page_pt_bbox[2] = clamped_x2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Axis {
|
||||
X,
|
||||
Y,
|
||||
}
|
||||
|
||||
fn derive_axis_bands(cells: &[StructuredCell], axis: Axis) -> HashMap<usize, (f32, f32)> {
|
||||
let mut by_index: HashMap<usize, Vec<(f32, f32)>> = HashMap::new();
|
||||
|
||||
// Prefer non-spanning cells so colspan/rowspan boxes do not skew a single
|
||||
// column/row center. If an axis has no non-spanning examples for an index,
|
||||
// fall back to anchored cells below.
|
||||
for cell in cells {
|
||||
let span = match axis {
|
||||
Axis::X => cell.colspan.max(1),
|
||||
Axis::Y => cell.rowspan.max(1),
|
||||
};
|
||||
if span == 1 {
|
||||
let idx = match axis {
|
||||
Axis::X => cell.col,
|
||||
Axis::Y => cell.row,
|
||||
};
|
||||
by_index
|
||||
.entry(idx)
|
||||
.or_default()
|
||||
.push(axis_bounds(cell.page_pt_bbox, axis));
|
||||
}
|
||||
}
|
||||
|
||||
for cell in cells {
|
||||
let idx = match axis {
|
||||
Axis::X => cell.col,
|
||||
Axis::Y => cell.row,
|
||||
};
|
||||
if !by_index.contains_key(&idx) {
|
||||
by_index
|
||||
.entry(idx)
|
||||
.or_default()
|
||||
.push(axis_bounds(cell.page_pt_bbox, axis));
|
||||
}
|
||||
}
|
||||
|
||||
let mut rows: Vec<(usize, f32, f32, f32)> = by_index
|
||||
.into_iter()
|
||||
.filter_map(|(idx, bounds)| {
|
||||
let mut min_edge = f32::INFINITY;
|
||||
let mut max_edge = f32::NEG_INFINITY;
|
||||
let mut center_sum = 0.0;
|
||||
let mut count = 0usize;
|
||||
for (lo, hi) in bounds {
|
||||
if lo.is_finite() && hi.is_finite() && lo < hi {
|
||||
min_edge = min_edge.min(lo);
|
||||
max_edge = max_edge.max(hi);
|
||||
center_sum += (lo + hi) * 0.5;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
(count > 0).then_some((idx, center_sum / count as f32, min_edge, max_edge))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if rows.len() < 2 {
|
||||
return rows
|
||||
.into_iter()
|
||||
.map(|(idx, _center, lo, hi)| (idx, (lo, hi)))
|
||||
.collect();
|
||||
}
|
||||
|
||||
rows.sort_by_key(|(idx, _, _, _)| *idx);
|
||||
|
||||
let mut bands = HashMap::new();
|
||||
for i in 0..rows.len() {
|
||||
let (idx, _center, min_edge, max_edge) = rows[i];
|
||||
let lo = if i == 0 {
|
||||
min_edge
|
||||
} else {
|
||||
(rows[i - 1].1 + rows[i].1) * 0.5
|
||||
};
|
||||
let hi = if i + 1 == rows.len() {
|
||||
max_edge
|
||||
} else {
|
||||
(rows[i].1 + rows[i + 1].1) * 0.5
|
||||
};
|
||||
if lo.is_finite() && hi.is_finite() && lo < hi {
|
||||
bands.insert(idx, (lo, hi));
|
||||
}
|
||||
}
|
||||
|
||||
bands
|
||||
}
|
||||
|
||||
fn axis_bounds(bbox: [f32; 4], axis: Axis) -> (f32, f32) {
|
||||
match axis {
|
||||
Axis::X => (bbox[0].min(bbox[2]), bbox[0].max(bbox[2])),
|
||||
Axis::Y => (bbox[1].min(bbox[3]), bbox[1].max(bbox[3])),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sanitize cell text for inclusion in a markdown pipe-table cell:
|
||||
/// collapse whitespace runs, drop newlines/tabs (cells must be one line),
|
||||
/// and escape pipes that would otherwise break the table.
|
||||
@@ -448,6 +586,102 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_cell_bands_splits_overlapping_slanet_rows() {
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 100.0, 90.0, 120.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [90.0, 100.0, 170.0, 120.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 116.0, 90.0, 136.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [90.0, 116.0, 170.0, 136.0],
|
||||
},
|
||||
];
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
assert_eq!(cells[0].page_pt_bbox[3], cells[2].page_pt_bbox[1]);
|
||||
assert_eq!(cells[1].page_pt_bbox[3], cells[3].page_pt_bbox[1]);
|
||||
assert!(
|
||||
(cells[0].page_pt_bbox[3] - 118.0).abs() < 0.01,
|
||||
"row separator should be midpoint between row centers: {:?}",
|
||||
cells
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_cell_bands_preserves_colspan_extent() {
|
||||
let mut cells = vec![
|
||||
StructuredCell {
|
||||
row: 0,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 2,
|
||||
is_header: true,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [8.0, 80.0, 172.0, 98.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 0,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [10.0, 96.0, 90.0, 114.0],
|
||||
},
|
||||
StructuredCell {
|
||||
row: 1,
|
||||
col: 1,
|
||||
rowspan: 1,
|
||||
colspan: 1,
|
||||
is_header: false,
|
||||
text: String::new(),
|
||||
page_pt_bbox: [88.0, 96.0, 170.0, 114.0],
|
||||
},
|
||||
];
|
||||
|
||||
normalize_cell_bands(&mut cells);
|
||||
|
||||
assert!(
|
||||
cells[0].page_pt_bbox[0] <= cells[1].page_pt_bbox[0],
|
||||
"spanning cell should retain the first column's left edge"
|
||||
);
|
||||
assert!(
|
||||
cells[0].page_pt_bbox[2] >= cells[2].page_pt_bbox[2],
|
||||
"spanning cell should retain the last column's right edge"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_int_attr_basic() {
|
||||
assert_eq!(parse_int_attr(" colspan=\"4\"", "colspan"), Some(4));
|
||||
|
||||
+834
-5
@@ -1,16 +1,94 @@
|
||||
//! Integration tests for pdf-to-markdown library
|
||||
|
||||
use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
|
||||
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
|
||||
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
|
||||
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
|
||||
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
|
||||
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn make_minimal_text_pdf() -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = vec![0usize];
|
||||
|
||||
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
|
||||
offsets.push(pdf.len());
|
||||
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
|
||||
pdf.extend_from_slice(body.as_bytes());
|
||||
pdf.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
1,
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
2,
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
3,
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
|
||||
);
|
||||
|
||||
let content = "BT /F1 12 Tf 100 700 Td (Hello World) Tj 0 -14 Td (Second Line) Tj 0 -14 Td (Third Line) Tj ET";
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
4,
|
||||
&format!(
|
||||
"<< /Length {} >>\nstream\n{}\nendstream",
|
||||
content.len(),
|
||||
content
|
||||
),
|
||||
);
|
||||
add_object(
|
||||
&mut pdf,
|
||||
&mut offsets,
|
||||
5,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
);
|
||||
|
||||
let xref_start = pdf.len();
|
||||
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
|
||||
pdf.extend_from_slice(b"0000000000 65535 f \n");
|
||||
for offset in offsets.iter().skip(1) {
|
||||
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
pdf.extend_from_slice(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
|
||||
offsets.len(),
|
||||
xref_start
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
|
||||
pdf
|
||||
}
|
||||
|
||||
fn truncate_eof_marker(mut pdf: Vec<u8>) -> Vec<u8> {
|
||||
assert!(pdf.ends_with(b"%%EOF"));
|
||||
pdf.pop();
|
||||
pdf
|
||||
}
|
||||
|
||||
fn add_leading_tab(mut pdf: Vec<u8>) -> Vec<u8> {
|
||||
pdf.insert(0, b'\t');
|
||||
pdf
|
||||
}
|
||||
|
||||
// Helper to create test TextItems
|
||||
fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
|
||||
use pdf_inspector::types::ItemType;
|
||||
@@ -826,6 +904,73 @@ fn test_bom_prefixed_pdf_header_not_rejected() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_process_pdf_mem_repairs_truncated_eof_marker() {
|
||||
let pdf = truncate_eof_marker(make_minimal_text_pdf());
|
||||
|
||||
let result = process_pdf_mem(&pdf).expect("truncated %%EO marker should be repaired");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert!(
|
||||
result
|
||||
.markdown
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("Hello World"),
|
||||
"repaired PDF should still extract text"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_process_pdf_mem_repairs_leading_tab_and_truncated_eof() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
|
||||
let result = process_pdf_mem(&pdf).expect("leading whitespace + %%EO should be repaired");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert!(
|
||||
result
|
||||
.markdown
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("Hello World"),
|
||||
"repaired PDF should still extract text"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_pdf_type_repairs_container_from_path() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("broken-container.pdf");
|
||||
std::fs::write(&path, pdf).unwrap();
|
||||
|
||||
let result = detect_pdf_type(&path).expect("detector should use shared repair loader");
|
||||
|
||||
assert_eq!(result.pdf_type, PdfType::TextBased);
|
||||
assert_eq!(result.page_count, 1);
|
||||
assert_eq!(result.pages_with_text, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_text_mem_uses_container_repair() {
|
||||
let pdf = truncate_eof_marker(make_minimal_text_pdf());
|
||||
|
||||
let text = pdf_inspector::extractor::extract_text_mem(&pdf)
|
||||
.expect("plain text extraction should use shared repair loader");
|
||||
|
||||
assert!(text.contains("Hello World"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_page_count_from_bytes_excludes_pages_tree() {
|
||||
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
|
||||
|
||||
assert_eq!(estimate_page_count_from_bytes(&pdf), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_a_pdf_detect_pdf_type_mem() {
|
||||
// Verify detect_pdf_type_mem is also guarded
|
||||
@@ -1484,6 +1629,281 @@ fn poly(x1: f32, y1: f32, x2: f32, y2: f32) -> Vec<f32> {
|
||||
vec![x1, y1, x2, y1, x2, y2, x1, y2]
|
||||
}
|
||||
|
||||
fn synthetic_dense_table_pdf() -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let operations = vec![
|
||||
Operation::new("BT", vec![]),
|
||||
Operation::new("Tf", vec!["F1".into(), 10.into()]),
|
||||
Operation::new("Td", vec![20.into(), 700.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Branch Name")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Deposits")]),
|
||||
Operation::new("Td", vec![Object::Integer(-100), Object::Real(-16.8)]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Oak Street")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("100")]),
|
||||
Operation::new("Td", vec![Object::Integer(-100), Object::Real(-16.8)]),
|
||||
Operation::new("Tj", vec![Object::string_literal("Boardwalk")]),
|
||||
Operation::new("Td", vec![100.into(), 0.into()]),
|
||||
Operation::new("Tj", vec![Object::string_literal("200")]),
|
||||
Operation::new("ET", vec![]),
|
||||
];
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 200.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
fn synthetic_vector_grid_pdf(two_tables: bool) -> Vec<u8> {
|
||||
use lopdf::content::{Content, Operation};
|
||||
use lopdf::{dictionary, Document, Object, Stream};
|
||||
|
||||
fn push_grid(
|
||||
operations: &mut Vec<Operation>,
|
||||
x_left: i64,
|
||||
x_mid: i64,
|
||||
x_right: i64,
|
||||
y_top: i64,
|
||||
y_mid: i64,
|
||||
y_bottom: i64,
|
||||
) {
|
||||
for y in [y_top, y_mid, y_bottom] {
|
||||
operations.push(Operation::new("m", vec![x_left.into(), y.into()]));
|
||||
operations.push(Operation::new("l", vec![x_right.into(), y.into()]));
|
||||
}
|
||||
for x in [x_left, x_mid, x_right] {
|
||||
operations.push(Operation::new("m", vec![x.into(), y_bottom.into()]));
|
||||
operations.push(Operation::new("l", vec![x.into(), y_top.into()]));
|
||||
}
|
||||
operations.push(Operation::new("S", vec![]));
|
||||
}
|
||||
|
||||
fn push_text(operations: &mut Vec<Operation>, x: i64, y: i64, text: &str) {
|
||||
operations.push(Operation::new(
|
||||
"Tm",
|
||||
vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
|
||||
));
|
||||
operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
|
||||
}
|
||||
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let pages_id = doc.new_object_id();
|
||||
let page_id = doc.new_object_id();
|
||||
let font_id = doc.new_object_id();
|
||||
let content_id = doc.new_object_id();
|
||||
|
||||
doc.objects.insert(
|
||||
font_id,
|
||||
dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
|
||||
let mut operations = Vec::new();
|
||||
push_grid(&mut operations, 50, 130, 210, 740, 710, 670);
|
||||
if two_tables {
|
||||
push_grid(&mut operations, 50, 130, 210, 560, 530, 490);
|
||||
}
|
||||
|
||||
operations.push(Operation::new("BT", vec![]));
|
||||
operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
|
||||
push_text(&mut operations, 70, 724, "A1");
|
||||
push_text(&mut operations, 150, 724, "B1");
|
||||
push_text(&mut operations, 70, 688, "A2");
|
||||
push_text(&mut operations, 150, 688, "B2");
|
||||
if two_tables {
|
||||
push_text(&mut operations, 70, 544, "C1");
|
||||
push_text(&mut operations, 150, 544, "D1");
|
||||
push_text(&mut operations, 70, 508, "C2");
|
||||
push_text(&mut operations, 150, 508, "D2");
|
||||
}
|
||||
operations.push(Operation::new("ET", vec![]));
|
||||
|
||||
let content = Content { operations }.encode().unwrap();
|
||||
doc.objects
|
||||
.insert(content_id, Stream::new(dictionary! {}, content).into());
|
||||
|
||||
doc.objects.insert(
|
||||
page_id,
|
||||
dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
|
||||
"Resources" => dictionary! {
|
||||
"Font" => dictionary! {
|
||||
"F1" => font_id,
|
||||
},
|
||||
},
|
||||
"Contents" => content_id,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page_id.into()],
|
||||
"Count" => 1,
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let catalog_id = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog_id);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
doc.save_to(&mut bytes).unwrap();
|
||||
bytes
|
||||
}
|
||||
|
||||
fn assert_close(actual: f32, expected: f32) {
|
||||
assert!(
|
||||
(actual - expected).abs() < 0.75,
|
||||
"expected {actual} to be close to {expected}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_line_pdf() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_vector_grid_pdf(false);
|
||||
let crop = [50.0_f32, 60.0, 210.0, 130.0];
|
||||
let detected = detect_vector_grid_in_region_mem(&buf, 0, crop, 72.0)
|
||||
.unwrap()
|
||||
.expect("ruled vector table should be detected");
|
||||
|
||||
assert_eq!(detected.cell_bboxes.len(), 4);
|
||||
assert_eq!(
|
||||
detected
|
||||
.structure_tokens
|
||||
.iter()
|
||||
.filter(|tok| tok.as_str() == "<td></td>")
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
assert_eq!(detected.structure_tokens.first().unwrap(), "<table>");
|
||||
assert_eq!(detected.structure_tokens.last().unwrap(), "</table>");
|
||||
|
||||
let first = &detected.cell_bboxes[0];
|
||||
assert_close(first[0], 0.0);
|
||||
assert_close(first[1], 0.0);
|
||||
assert_close(first[2], 80.0);
|
||||
assert_close(first[3], 30.0);
|
||||
|
||||
let markdown = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: crop,
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: detected.structure_tokens,
|
||||
cell_bboxes: detected.cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
|
||||
assert!(markdown.contains("A1"));
|
||||
assert!(markdown.contains("B1"));
|
||||
assert!(markdown.contains("A2"));
|
||||
assert!(markdown.contains("B2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_text_pdf_returns_none() {
|
||||
let buf = make_minimal_text_pdf();
|
||||
let detected =
|
||||
detect_vector_grid_in_region_mem(&buf, 0, [0.0, 0.0, 300.0, 800.0], 72.0).unwrap();
|
||||
assert!(detected.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_vector_grid_in_region_filters_to_requested_table() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_vector_grid_pdf(true);
|
||||
let second_table_crop = [50.0_f32, 240.0, 210.0, 310.0];
|
||||
let detected = detect_vector_grid_in_region_mem(&buf, 0, second_table_crop, 72.0)
|
||||
.unwrap()
|
||||
.expect("second ruled table should be detected");
|
||||
|
||||
assert_eq!(detected.cell_bboxes.len(), 4);
|
||||
let markdown = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: second_table_crop,
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: detected.structure_tokens,
|
||||
cell_bboxes: detected.cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
|
||||
assert!(markdown.contains("C1"));
|
||||
assert!(markdown.contains("D2"));
|
||||
assert!(!markdown.contains("A1"));
|
||||
assert!(!markdown.contains("B2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
@@ -1564,6 +1984,64 @@ fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_dense_overlapping_slanet_boxes() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
|
||||
// Rows are spaced 16.8pt apart, while the SLANet-style boxes are 40pt
|
||||
// tall and overlap adjacent rows. Text must still land in only one row.
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
];
|
||||
|
||||
let mds = extract_tables_with_structure_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let expected = "|Branch Name|Deposits|\n|---|---|\n|Oak Street|100|\n|Boardwalk|200|\n";
|
||||
assert_eq!(mds[0], expected);
|
||||
assert!(!mds[0].contains("Branch Name Oak Street"));
|
||||
assert!(!mds[0].contains("Oak Street Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tables_with_structure_input_order_preserved() {
|
||||
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
|
||||
@@ -1774,6 +2252,357 @@ fn test_extract_tables_with_structure_separator_after_thead() {
|
||||
assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_tables_with_structure_auto_mem tests (TSR + heuristic fallback)
|
||||
// =========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_auto_passes_through_clean_tsr_output() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cells fit each visible row cleanly. Same shape as the existing
|
||||
// dense-overlap regression test — TSR should produce clean output
|
||||
// and the auto wrapper should pass through with no fallback.
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"expected no fallback, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
assert!(results[0].markdown.contains("Boardwalk"));
|
||||
assert!(!results[0].markdown.contains("Oak Street Boardwalk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_falls_back_on_multi_row_in_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// TSR returns only 2 rows for what's actually 3 visible PDF rows.
|
||||
// Row 1's cells are tall enough to encompass both Oak Street and
|
||||
// Boardwalk text — the FNBO row-undercount pattern.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row at top-left y=[88, 105] (covers "Branch Name"/"Deposits"
|
||||
// at native y=700, top-left y≈92-103). The "data" row at top-left
|
||||
// y=[105, 145] is intentionally tall — covers BOTH the Oak Street
|
||||
// line (top-left y≈108-119) AND the Boardwalk line (y≈124-135).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(
|
||||
results[0].fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell"),
|
||||
"expected multi_row_in_cell fallback, got {:?}",
|
||||
results[0].fallback_reason
|
||||
);
|
||||
// The heuristic-fallback markdown should preserve all three PDF rows.
|
||||
let md = &results[0].markdown;
|
||||
assert!(md.contains("Oak Street"), "missing Oak Street: {md}");
|
||||
assert!(md.contains("Boardwalk"), "missing Boardwalk: {md}");
|
||||
assert!(md.contains("100"), "missing 100: {md}");
|
||||
assert!(md.contains("200"), "missing 200: {md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_returns_empty_inputs() {
|
||||
use pdf_inspector::extract_tables_with_structure_auto_mem;
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[]).unwrap();
|
||||
assert!(results.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_does_not_fire_on_legit_rowspan_cell() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// 2 columns, 3 rows in the visible PDF. SLANet emits a 2-row table
|
||||
// where the LEFT cell of row 1 is a rowspan=2 cell that legitimately
|
||||
// covers Oak Street + Boardwalk on two visual lines. The right
|
||||
// column has two normal rows. multi_row_in_cell must NOT fire on
|
||||
// the rowspan=2 cell.
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
// First data cell explicitly declares rowspan=2.
|
||||
"<td",
|
||||
" rowspan=\"2\"",
|
||||
">",
|
||||
"</td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Header row, then a tall left cell covering both data lines, plus
|
||||
// two narrow right cells (one per line).
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0), // rowspan=2 — covers both lines
|
||||
poly(90.0, 105.0, 180.0, 122.0), // row 1 only
|
||||
poly(90.0, 122.0, 180.0, 145.0), // row 2 only
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"rowspan=2 cell containing 2 visual lines should not trip multi_row_in_cell, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_keeps_tsr_markdown_when_heuristic_returns_empty() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
// Same shape as the multi_row_in_cell regression — a tall data cell
|
||||
// that catches Oak Street + Boardwalk. But the crop bbox we pass
|
||||
// points at a strip of the page that has NO text items, so the
|
||||
// heuristic's region will be empty when it tries to extract there.
|
||||
// The auto wrapper must keep the TSR markdown rather than ship "".
|
||||
let tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// Cell bboxes overlap the actual PDF text (so multi_row_in_cell
|
||||
// fires) — but the crop_pdf_pt_bbox we hand to the heuristic is a
|
||||
// wholly-empty region of the page. The heuristic should return "".
|
||||
let cell_bboxes = vec![
|
||||
poly(10.0, 88.0, 100.0, 105.0),
|
||||
poly(90.0, 88.0, 180.0, 105.0),
|
||||
poly(10.0, 105.0, 100.0, 145.0),
|
||||
poly(90.0, 105.0, 180.0, 145.0),
|
||||
];
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(
|
||||
&buf,
|
||||
&[TsrTableInput {
|
||||
page: 0,
|
||||
// Crop is at the BOTTOM of the page where there's no text.
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 50.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: tokens,
|
||||
cell_bboxes,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
let r = &results[0];
|
||||
assert_eq!(
|
||||
r.fallback_reason.as_deref(),
|
||||
Some("multi_row_in_cell_heuristic_empty"),
|
||||
"expected _heuristic_empty suffix, got {:?}",
|
||||
r.fallback_reason,
|
||||
);
|
||||
// TSR markdown should be preserved — non-empty, contains the cell
|
||||
// text we know was assigned by the TSR path.
|
||||
assert!(
|
||||
!r.markdown.trim().is_empty(),
|
||||
"expected TSR markdown to be preserved, got empty",
|
||||
);
|
||||
assert!(
|
||||
r.markdown.contains("Oak Street") || r.markdown.contains("Boardwalk"),
|
||||
"expected TSR markdown to contain at least one row, got: {}",
|
||||
r.markdown,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_auto_isolates_per_input_failures() {
|
||||
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
|
||||
|
||||
let buf = synthetic_dense_table_pdf();
|
||||
let good_tokens: Vec<String> = [
|
||||
"<table>",
|
||||
"<thead>",
|
||||
"<tr>",
|
||||
"<th></th>",
|
||||
"<th></th>",
|
||||
"</tr>",
|
||||
"</thead>",
|
||||
"<tbody>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"<tr>",
|
||||
"<td></td>",
|
||||
"<td></td>",
|
||||
"</tr>",
|
||||
"</tbody>",
|
||||
"</table>",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
// A clean input that should pass through with no fallback.
|
||||
let good_input = TsrTableInput {
|
||||
page: 0,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: good_tokens,
|
||||
cell_bboxes: vec![
|
||||
poly(10.0, 72.0, 100.0, 112.0),
|
||||
poly(90.0, 72.0, 180.0, 112.0),
|
||||
poly(10.0, 88.8, 100.0, 128.8),
|
||||
poly(90.0, 88.8, 180.0, 128.8),
|
||||
poly(10.0, 105.6, 100.0, 145.6),
|
||||
poly(90.0, 105.6, 180.0, 145.6),
|
||||
],
|
||||
};
|
||||
// A bad input that targets a non-existent page. The detection
|
||||
// helper short-circuits on missing pages with Ok(None), so this
|
||||
// shouldn't itself crash, but pairing it with a flagged input
|
||||
// exercises the per-input control flow regardless. The point of
|
||||
// this test is that one input's outcome doesn't poison the other.
|
||||
let bad_input = TsrTableInput {
|
||||
page: 9999,
|
||||
crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
|
||||
render_dpi: 72.0,
|
||||
structure_tokens: vec![
|
||||
"<table>".into(),
|
||||
"<tr>".into(),
|
||||
"<td></td>".into(),
|
||||
"</tr>".into(),
|
||||
"</table>".into(),
|
||||
],
|
||||
cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
|
||||
};
|
||||
|
||||
let results = extract_tables_with_structure_auto_mem(&buf, &[good_input, bad_input]).unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
// Good input still produces non-empty TSR markdown with no fallback.
|
||||
assert!(
|
||||
results[0].fallback_reason.is_none(),
|
||||
"good input should pass through, got reason={:?}",
|
||||
results[0].fallback_reason,
|
||||
);
|
||||
assert!(results[0].markdown.contains("Oak Street"));
|
||||
// Bad input collapses to empty markdown but doesn't take the
|
||||
// batch down with it.
|
||||
assert_eq!(results[1].markdown, "");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_pages_markdown_mem tests
|
||||
// =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user