Compare commits

..
Author SHA1 Message Date
Abimael MartellandClaude Opus 4.7 af60e378af TSR stage 2: reject cross-line orphan stacking, v1.6.3
Closes the residual run-on-cell pattern observed on FNBO branch list
after 1.6.2 deployed:

  |Shawnee Blue Valley Parkway|Kansas|6301 Pflumm, ...|0523.04|
  |Sonoma Plaza|Kansas Kansas|<addr1> <addr2>|0531.05|
  |Mitchell Woonsocket|South Dakota|<addr1>|9628.01|

Cause: when col-N cells across multiple consecutive rows are y-shifted
the same way (a local SLANet drift), the stage 2 orphan pass sees
multiple orphans qualifying for the same nearest empty cell. The cell
gets all of them appended in order, producing "RowA-text RowB-text".

Fix: track the y-coordinate of the first orphan that lands in each
cell. Subsequent orphans only join that cell if their y is within
half-a-row-height of the first orphan's y (same line). Cross-line
orphans skip that cell and look for the next-nearest empty cell on
their own line.

Same-line slack preserves multi-token branch names like
"Blue Valley Parkway" (3 PDF text items at the same y) — all three
stack into the same cell. Cross-row stacking is what gets rejected.

Two new tests:

  - stage2_rejects_cross_line_stacking_into_same_cell
    Two orphans on different rows, both equidistant to the same empty
    cell. First wins; second routes to its own row's cell.

  - stage2_allows_same_line_orphans_to_stack_into_one_cell
    Three same-line orphans (multi-token branch name) all land in the
    same empty cell, joined by spaces.

The existing four stage-2 tests + the cell-bleed regression test from
PR #62 + #63 all pass: 416 lib + 115 integration + 2 doctests.
clippy + fmt clean.

Bump @firecrawl/pdf-inspector to 1.6.3 (patch — refines 1.6.2; no API
changes).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-27 08:03:58 -07:00
10 changed files with 118 additions and 2577 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.8.2",
"version": "1.6.3",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
-102
View File
@@ -99,13 +99,6 @@ pub struct PageRegionTexts {
pub regions: Vec<RegionText>,
}
/// Vector-grid detection result compatible with `extractTablesWithStructure*`.
#[napi(object)]
pub struct VectorGridDetectionJs {
pub structure_tokens: Vec<String>,
pub cell_bboxes: Vec<Vec<f64>>,
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
@@ -324,53 +317,6 @@ pub fn extract_tables_in_regions(
})
}
/// Detect a vector ruled-line / rectangle grid inside one page region.
///
/// Returns TSR-compatible structure tokens plus crop-pixel cell bboxes, or
/// `null` when the region does not contain a valid vector grid.
///
/// `pageIdx` is 0-indexed. `regionPdfPtBbox` is `[x1,y1,x2,y2]` in PDF
/// points with top-left origin. `renderDpi` is the DPI of the crop image that
/// will consume the returned cell bboxes.
#[napi]
pub fn detect_vector_grid_in_region(
buffer: Buffer,
page_idx: u32,
region_pdf_pt_bbox: Vec<f64>,
render_dpi: f64,
) -> Result<Option<VectorGridDetectionJs>> {
let bytes: Vec<u8> = buffer.to_vec();
let region = if region_pdf_pt_bbox.len() == 4 {
[
region_pdf_pt_bbox[0] as f32,
region_pdf_pt_bbox[1] as f32,
region_pdf_pt_bbox[2] as f32,
region_pdf_pt_bbox[3] as f32,
]
} else {
[0.0, 0.0, 0.0, 0.0]
};
catch_panic("detect_vector_grid_in_region", move || {
let result = pdf_inspector::detect_vector_grid_in_region_mem(
&bytes,
page_idx,
region,
render_dpi as f32,
)
.map_err(|e| to_napi_err(e, "detect_vector_grid_in_region"))?;
Ok(result.map(|r| VectorGridDetectionJs {
structure_tokens: r.structure_tokens,
cell_bboxes: r
.cell_bboxes
.into_iter()
.map(|bbox| bbox.into_iter().map(|v| v as f64).collect())
.collect(),
}))
})
}
/// One cropped table region plus its raw structure-recovery output, for
/// `extractTablesWithStructure`.
///
@@ -476,54 +422,6 @@ pub fn extract_tables_with_structure_cells(
})
}
/// One result from `extractTablesWithStructureAuto` — markdown plus a
/// diagnostic flag identifying which path produced it.
///
/// `fallbackReason` is `null` when the TSR-hybrid path produced the
/// markdown directly. When stage 1's quality check fires (the cells
/// look like a SLANet detection pathology — phantom rows or multi-row
/// content in a single cell), the auto path may expand the TSR cells
/// in-place or run the heuristic table extractor on the same region.
/// `fallbackReason` carries the diagnostic label (for example
/// `"multi_row_in_cell_expanded"` or `"phantom_empty_row"`).
#[napi(object)]
pub struct TableExtractionResultJs {
pub markdown: String,
pub fallback_reason: Option<String>,
}
/// Auto-fallback variant of [`extractTablesWithStructure`].
///
/// Runs the TSR-hybrid path, checks the resulting cells for known
/// SLANet detection pathologies, expands multi-row cells in-place when
/// possible, and otherwise falls back to the heuristic
/// `extractTablesInRegions` for inputs where the TSR path looks
/// compromised.
///
/// On clean inputs this returns identical markdown to
/// `extractTablesWithStructure`; on flagged inputs `fallbackReason` is
/// set to the recovery path that produced the result.
#[napi]
pub fn extract_tables_with_structure_auto(
buffer: Buffer,
inputs: Vec<TsrTableInputJs>,
) -> Result<Vec<TableExtractionResultJs>> {
let bytes: Vec<u8> = buffer.to_vec();
let parsed = parse_tsr_inputs(&inputs);
catch_panic("extract_tables_with_structure_auto", move || {
let result = pdf_inspector::extract_tables_with_structure_auto_mem(&bytes, &parsed)
.map_err(|e| to_napi_err(e, "extract_tables_with_structure_auto"))?;
Ok(result
.into_iter()
.map(|r| TableExtractionResultJs {
markdown: r.markdown,
fallback_reason: r.fallback_reason,
})
.collect())
})
}
fn parse_tsr_inputs(inputs: &[TsrTableInputJs]) -> Vec<pdf_inspector::TsrTableInput> {
inputs
.iter()
-12
View File
@@ -7,7 +7,6 @@ import {
extractText,
extractTextWithPositions,
extractTextInRegions,
detectVectorGridInRegion,
extractPagesMarkdown,
} from './index.js';
@@ -91,17 +90,6 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
console.log(' extractTextInRegions: OK');
// --- detectVectorGridInRegion ---
console.log('Testing detectVectorGridInRegion...');
const vectorGrid = detectVectorGridInRegion(fixture, 0, [0, 0, 600, 800], 72);
assert.ok(vectorGrid === null || typeof vectorGrid === 'object');
if (vectorGrid) {
assert.ok(Array.isArray(vectorGrid.structureTokens));
assert.ok(Array.isArray(vectorGrid.cellBboxes));
assert.ok(vectorGrid.cellBboxes.every(bbox => Array.isArray(bbox) && bbox.length === 4));
}
console.log(' detectVectorGridInRegion: OK');
// --- extractPagesMarkdown ---
console.log('Testing extractPagesMarkdown...');
+11 -33
View File
@@ -1,12 +1,8 @@
//! CLI tool for detecting PDF type (text-based vs scanned)
use pdf_inspector::{
detect_pdf_type, detector::estimate_page_count_from_bytes, process_pdf_with_options,
PdfOptions, PdfType, ProcessMode,
};
use pdf_inspector::{detect_pdf_type, process_pdf_with_options, PdfOptions, PdfType, ProcessMode};
use std::env;
use std::fmt::Write;
use std::fs;
use std::process;
use std::time::Instant;
@@ -68,32 +64,6 @@ fn pdf_type_str(pdf_type: &PdfType) -> &'static str {
}
}
fn page_count_hint(pdf_path: &str) -> Option<u32> {
fs::read(pdf_path)
.ok()
.map(|bytes| estimate_page_count_from_bytes(&bytes))
.filter(|&count| count > 0)
}
fn print_error(e: &pdf_inspector::PdfError, pdf_path: &str, json_output: bool) {
if json_output {
if let Some(count) = page_count_hint(pdf_path) {
println!(
r#"{{"error":"{}","page_count_hint":{}}}"#,
json_escape(&e.to_string()),
count
);
} else {
println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
}
} else {
eprintln!("Error: {}", e);
if let Some(count) = page_count_hint(pdf_path) {
eprintln!("Page count hint: {}", count);
}
}
}
fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
match process_pdf_with_options(pdf_path, PdfOptions::new().mode(ProcessMode::Analyze)) {
Ok(result) => {
@@ -165,7 +135,11 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
}
}
Err(e) => {
print_error(&e, pdf_path, json_output);
if json_output {
println!(r#"{{"error":"{}"}}"#, e);
} else {
eprintln!("Error: {}", e);
}
process::exit(1);
}
}
@@ -262,7 +236,11 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
}
}
Err(e) => {
print_error(&e, pdf_path, json_output);
if json_output {
println!(r#"{{"error":"{}"}}"#, e);
} else {
eprintln!("Error: {}", e);
}
process::exit(1);
}
}
+35 -57
View File
@@ -97,9 +97,26 @@ pub fn detect_pdf_type_with_config<P: AsRef<Path>>(
) -> Result<PdfTypeResult, PdfError> {
crate::validate_pdf_file(&path)?;
let (doc, page_count) = crate::load_document_from_path(&path)?;
// First, load metadata only (fast operation)
let metadata = match Document::load_metadata(&path) {
Ok(m) => m,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_metadata_with_password(&path, "")?
}
Err(e) => return Err(e.into()),
};
detect_from_document(&doc, page_count, &config)
// Then load the full document for content inspection
// We use filtered loading to skip heavy objects we don't need
let doc = match Document::load(&path) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_with_password(&path, "")?
}
Err(e) => return Err(e.into()),
};
detect_from_document(&doc, metadata.page_count, &config)
}
/// Detect PDF type from memory buffer
@@ -114,64 +131,25 @@ pub fn detect_pdf_type_mem_with_config(
) -> Result<PdfTypeResult, PdfError> {
crate::validate_pdf_bytes(buffer)?;
let (doc, page_count) = crate::load_document_from_mem(buffer)?;
detect_from_document(&doc, page_count, &config)
}
/// Heuristic page-count fallback for malformed PDFs that cannot be parsed.
///
/// This scans raw bytes for page dictionaries (`/Type /Page`) while excluding
/// the page tree node (`/Type /Pages`). It is intended as a low-confidence hint
/// for diagnostics; parsed page-tree counts remain authoritative.
pub fn estimate_page_count_from_bytes(buffer: &[u8]) -> u32 {
let mut count = 0u32;
let mut pos = 0usize;
while let Some(rel_idx) = find_bytes(&buffer[pos..], b"/Type") {
let mut value_pos = pos + rel_idx + b"/Type".len();
value_pos = skip_pdf_whitespace(buffer, value_pos);
if buffer.get(value_pos) == Some(&b'/') {
let name_start = value_pos + 1;
let name_end = name_start + b"Page".len();
if name_end <= buffer.len()
&& &buffer[name_start..name_end] == b"Page"
&& buffer
.get(name_end)
.is_none_or(|b| is_pdf_name_delimiter(*b))
{
count += 1;
}
// Load metadata first (fast)
let metadata = match Document::load_metadata_mem(buffer) {
Ok(m) => m,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_metadata_mem_with_password(buffer, "")?
}
Err(e) => return Err(e.into()),
};
pos += rel_idx + b"/Type".len();
}
// Load document for inspection
let doc = match Document::load_mem(buffer) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
}
Err(e) => return Err(e.into()),
};
count
}
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).position(|w| w == needle)
}
fn skip_pdf_whitespace(buffer: &[u8], mut pos: usize) -> usize {
while pos < buffer.len() && is_pdf_whitespace(buffer[pos]) {
pos += 1;
}
pos
}
fn is_pdf_whitespace(byte: u8) -> bool {
matches!(byte, b'\0' | b'\t' | b'\n' | 0x0C | b'\r' | b' ')
}
fn is_pdf_name_delimiter(byte: u8) -> bool {
is_pdf_whitespace(byte)
|| matches!(
byte,
b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
)
detect_from_document(&doc, metadata.page_count, &config)
}
/// Detection logic on a pre-loaded document.
+28 -4
View File
@@ -36,14 +36,26 @@ pub(crate) use layout::ColumnRegion;
/// Extract text from PDF file as plain string
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
crate::validate_pdf_file(&path)?;
let (doc, _) = crate::load_document_from_path(&path)?;
let doc = match Document::load(&path) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_with_password(&path, "")?
}
Err(e) => return Err(e.into()),
};
extract_text_from_doc(&doc)
}
/// Extract text from PDF memory buffer
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
crate::validate_pdf_bytes(buffer)?;
let (doc, _) = crate::load_document_from_mem(buffer)?;
let doc = match Document::load_mem(buffer) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
}
Err(e) => return Err(e.into()),
};
extract_text_from_doc(&doc)
}
@@ -79,7 +91,13 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
page_filter: Option<&HashSet<u32>>,
) -> Result<PageExtraction, PdfError> {
crate::validate_pdf_file(&path)?;
let (doc, _) = crate::load_document_from_path(&path)?;
let doc = match Document::load(&path) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_with_password(&path, "")?
}
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
let (extraction, _thresholds, _gid_pages) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
@@ -106,7 +124,13 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
page_filter: Option<&HashSet<u32>>,
) -> Result<PageExtraction, PdfError> {
crate::validate_pdf_bytes(buffer)?;
let (doc, _) = crate::load_document_from_mem(buffer)?;
let doc = match Document::load_mem(buffer) {
Ok(d) => d,
Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
}
Err(e) => return Err(e.into()),
};
let font_cmaps = FontCMaps::from_doc(&doc);
let (extraction, _thresholds, _gid_pages) =
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
+23 -1461
View File
File diff suppressed because it is too large Load Diff
+15 -59
View File
@@ -1587,69 +1587,25 @@ fn detect_row_stripe_table_from_cell_rects(
return None;
}
// Derive columns from text X-position clustering, but prefer rect
// X-edges when they already provide a tighter scaffold. Some PDFs draw
// only the row-index cells in the body plus a full header row; that is
// not dense enough for `try_build_grid`, but the header rects still define
// the real columns. Text starts inside wide cells can otherwise split the
// table into spurious sub-columns.
// Derive columns from text X-position clustering
let columns = cluster_x_positions(&page_items, 15.0);
let text_col_edges = if columns.len() >= 2 {
let mut edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
edges.push(min_x - 5.0);
for pair in columns.windows(2) {
edges.push((pair[0] + pair[1]) / 2.0);
}
let max_x_right = page_items
.iter()
.map(|(_, i)| i.x + i.width)
.reduce(f32::max)?;
edges.push(max_x_right + 5.0);
Some(edges)
} else {
None
};
let rect_col_edges = {
let mut x_vals = Vec::with_capacity(content_rects.len() * 2);
for &&(x, _, w, _) in &content_rects {
x_vals.push(x);
x_vals.push(x + w);
}
let mut edges = snap_edges(&x_vals, 6.0);
edges.sort_by(|a, b| a.total_cmp(b));
if (3..=26).contains(&edges.len()) {
Some(edges)
} else {
None
}
};
let col_edges = match (rect_col_edges, text_col_edges) {
(Some(rect_edges), Some(text_edges)) if rect_edges.len() <= text_edges.len() => {
debug!(
" cell-rect using {} rect-derived columns over {} text clusters",
rect_edges.len() - 1,
text_edges.len() - 1
);
rect_edges
}
(_, Some(text_edges)) => text_edges,
(Some(rect_edges), None) => rect_edges,
(None, None) => {
debug!(
" cell-rect rejected: only {} columns from text clustering",
columns.len()
);
return None;
}
};
if col_edges.len() < 3 {
if columns.len() < 2 {
return None;
}
// Build column edges
let mut col_edges: Vec<f32> = Vec::with_capacity(columns.len() + 1);
let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?;
col_edges.push(min_x - 5.0);
for pair in columns.windows(2) {
col_edges.push((pair[0] + pair[1]) / 2.0);
}
let max_x_right = page_items
.iter()
.map(|(_, i)| i.x + i.width)
.reduce(f32::max)?;
col_edges.push(max_x_right + 5.0);
let num_cols = col_edges.len() - 1;
let num_rows = row_edges.len() - 1;
Binary file not shown.
+5 -848
View File
@@ -1,94 +1,16 @@
//! Integration tests for pdf-to-markdown library
use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
use pdf_inspector::extractor::group_into_lines;
use pdf_inspector::types::TextLine;
use pdf_inspector::{
detect_pdf_type, detect_vector_grid_in_region_mem, extract_pages_markdown,
extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
TextItem,
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
};
use std::collections::HashSet;
fn make_minimal_text_pdf() -> Vec<u8> {
let mut pdf = b"%PDF-1.4\n".to_vec();
let mut offsets = vec![0usize];
fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
offsets.push(pdf.len());
pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
pdf.extend_from_slice(body.as_bytes());
pdf.extend_from_slice(b"\nendobj\n");
}
add_object(
&mut pdf,
&mut offsets,
1,
"<< /Type /Catalog /Pages 2 0 R >>",
);
add_object(
&mut pdf,
&mut offsets,
2,
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
);
add_object(
&mut pdf,
&mut offsets,
3,
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
);
let content = "BT /F1 12 Tf 100 700 Td (Hello World) Tj 0 -14 Td (Second Line) Tj 0 -14 Td (Third Line) Tj ET";
add_object(
&mut pdf,
&mut offsets,
4,
&format!(
"<< /Length {} >>\nstream\n{}\nendstream",
content.len(),
content
),
);
add_object(
&mut pdf,
&mut offsets,
5,
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
);
let xref_start = pdf.len();
pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
pdf.extend_from_slice(b"0000000000 65535 f \n");
for offset in offsets.iter().skip(1) {
pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
pdf.extend_from_slice(
format!(
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
offsets.len(),
xref_start
)
.as_bytes(),
);
pdf
}
fn truncate_eof_marker(mut pdf: Vec<u8>) -> Vec<u8> {
assert!(pdf.ends_with(b"%%EOF"));
pdf.pop();
pdf
}
fn add_leading_tab(mut pdf: Vec<u8>) -> Vec<u8> {
pdf.insert(0, b'\t');
pdf
}
// Helper to create test TextItems
fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
use pdf_inspector::types::ItemType;
@@ -904,73 +826,6 @@ fn test_bom_prefixed_pdf_header_not_rejected() {
}
}
#[test]
fn test_process_pdf_mem_repairs_truncated_eof_marker() {
let pdf = truncate_eof_marker(make_minimal_text_pdf());
let result = process_pdf_mem(&pdf).expect("truncated %%EO marker should be repaired");
assert_eq!(result.pdf_type, PdfType::TextBased);
assert_eq!(result.page_count, 1);
assert!(
result
.markdown
.as_deref()
.unwrap_or_default()
.contains("Hello World"),
"repaired PDF should still extract text"
);
}
#[test]
fn test_process_pdf_mem_repairs_leading_tab_and_truncated_eof() {
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
let result = process_pdf_mem(&pdf).expect("leading whitespace + %%EO should be repaired");
assert_eq!(result.pdf_type, PdfType::TextBased);
assert_eq!(result.page_count, 1);
assert!(
result
.markdown
.as_deref()
.unwrap_or_default()
.contains("Hello World"),
"repaired PDF should still extract text"
);
}
#[test]
fn test_detect_pdf_type_repairs_container_from_path() {
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("broken-container.pdf");
std::fs::write(&path, pdf).unwrap();
let result = detect_pdf_type(&path).expect("detector should use shared repair loader");
assert_eq!(result.pdf_type, PdfType::TextBased);
assert_eq!(result.page_count, 1);
assert_eq!(result.pages_with_text, 1);
}
#[test]
fn test_extract_text_mem_uses_container_repair() {
let pdf = truncate_eof_marker(make_minimal_text_pdf());
let text = pdf_inspector::extractor::extract_text_mem(&pdf)
.expect("plain text extraction should use shared repair loader");
assert!(text.contains("Hello World"));
}
#[test]
fn test_estimate_page_count_from_bytes_excludes_pages_tree() {
let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
assert_eq!(estimate_page_count_from_bytes(&pdf), 1);
}
#[test]
fn test_not_a_pdf_detect_pdf_type_mem() {
// Verify detect_pdf_type_mem is also guarded
@@ -1705,292 +1560,6 @@ fn synthetic_dense_table_pdf() -> Vec<u8> {
bytes
}
fn synthetic_vector_grid_pdf(two_tables: bool) -> Vec<u8> {
use lopdf::content::{Content, Operation};
use lopdf::{dictionary, Document, Object, Stream};
fn push_grid(
operations: &mut Vec<Operation>,
x_left: i64,
x_mid: i64,
x_right: i64,
y_top: i64,
y_mid: i64,
y_bottom: i64,
) {
for y in [y_top, y_mid, y_bottom] {
operations.push(Operation::new("m", vec![x_left.into(), y.into()]));
operations.push(Operation::new("l", vec![x_right.into(), y.into()]));
}
for x in [x_left, x_mid, x_right] {
operations.push(Operation::new("m", vec![x.into(), y_bottom.into()]));
operations.push(Operation::new("l", vec![x.into(), y_top.into()]));
}
operations.push(Operation::new("S", vec![]));
}
fn push_text(operations: &mut Vec<Operation>, x: i64, y: i64, text: &str) {
operations.push(Operation::new(
"Tm",
vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
));
operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
}
let mut doc = Document::with_version("1.5");
let pages_id = doc.new_object_id();
let page_id = doc.new_object_id();
let font_id = doc.new_object_id();
let content_id = doc.new_object_id();
doc.objects.insert(
font_id,
dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
}
.into(),
);
let mut operations = Vec::new();
push_grid(&mut operations, 50, 130, 210, 740, 710, 670);
if two_tables {
push_grid(&mut operations, 50, 130, 210, 560, 530, 490);
}
operations.push(Operation::new("BT", vec![]));
operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
push_text(&mut operations, 70, 724, "A1");
push_text(&mut operations, 150, 724, "B1");
push_text(&mut operations, 70, 688, "A2");
push_text(&mut operations, 150, 688, "B2");
if two_tables {
push_text(&mut operations, 70, 544, "C1");
push_text(&mut operations, 150, 544, "D1");
push_text(&mut operations, 70, 508, "C2");
push_text(&mut operations, 150, 508, "D2");
}
operations.push(Operation::new("ET", vec![]));
let content = Content { operations }.encode().unwrap();
doc.objects
.insert(content_id, Stream::new(dictionary! {}, content).into());
doc.objects.insert(
page_id,
dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
"Resources" => dictionary! {
"Font" => dictionary! {
"F1" => font_id,
},
},
"Contents" => content_id,
}
.into(),
);
doc.objects.insert(
pages_id,
dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
}
.into(),
);
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog_id);
let mut bytes = Vec::new();
doc.save_to(&mut bytes).unwrap();
bytes
}
fn synthetic_vector_grid_three_row_pdf() -> Vec<u8> {
use lopdf::content::{Content, Operation};
use lopdf::{dictionary, Document, Object, Stream};
let mut doc = Document::with_version("1.5");
let pages_id = doc.new_object_id();
let page_id = doc.new_object_id();
let font_id = doc.new_object_id();
let content_id = doc.new_object_id();
doc.objects.insert(
font_id,
dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
}
.into(),
);
let mut operations = Vec::new();
for y in [740, 710, 680, 650] {
operations.push(Operation::new("m", vec![50.into(), y.into()]));
operations.push(Operation::new("l", vec![210.into(), y.into()]));
}
for x in [50, 130, 210] {
operations.push(Operation::new("m", vec![x.into(), 650.into()]));
operations.push(Operation::new("l", vec![x.into(), 740.into()]));
}
operations.push(Operation::new("S", vec![]));
operations.push(Operation::new("BT", vec![]));
operations.push(Operation::new("Tf", vec!["F1".into(), 10.into()]));
for (x, y, text) in [
(70, 724, "Branch"),
(150, 724, "Deposits"),
(70, 694, "Oak"),
(150, 694, "100"),
(70, 664, "Boardwalk"),
(150, 664, "200"),
] {
operations.push(Operation::new(
"Tm",
vec![1.into(), 0.into(), 0.into(), 1.into(), x.into(), y.into()],
));
operations.push(Operation::new("Tj", vec![Object::string_literal(text)]));
}
operations.push(Operation::new("ET", vec![]));
let content = Content { operations }.encode().unwrap();
doc.objects
.insert(content_id, Stream::new(dictionary! {}, content).into());
doc.objects.insert(
page_id,
dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"MediaBox" => vec![0.into(), 0.into(), 300.into(), 800.into()],
"Resources" => dictionary! {
"Font" => dictionary! {
"F1" => font_id,
},
},
"Contents" => content_id,
}
.into(),
);
doc.objects.insert(
pages_id,
dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
}
.into(),
);
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog_id);
let mut bytes = Vec::new();
doc.save_to(&mut bytes).unwrap();
bytes
}
fn assert_close(actual: f32, expected: f32) {
assert!(
(actual - expected).abs() < 0.75,
"expected {actual} to be close to {expected}"
);
}
#[test]
fn test_detect_vector_grid_in_region_line_pdf() {
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
let buf = synthetic_vector_grid_pdf(false);
let crop = [50.0_f32, 60.0, 210.0, 130.0];
let detected = detect_vector_grid_in_region_mem(&buf, 0, crop, 72.0)
.unwrap()
.expect("ruled vector table should be detected");
assert_eq!(detected.cell_bboxes.len(), 4);
assert_eq!(
detected
.structure_tokens
.iter()
.filter(|tok| tok.as_str() == "<td></td>")
.count(),
4
);
assert_eq!(detected.structure_tokens.first().unwrap(), "<table>");
assert_eq!(detected.structure_tokens.last().unwrap(), "</table>");
let first = &detected.cell_bboxes[0];
assert_close(first[0], 0.0);
assert_close(first[1], 0.0);
assert_close(first[2], 80.0);
assert_close(first[3], 30.0);
let markdown = extract_tables_with_structure_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: crop,
render_dpi: 72.0,
structure_tokens: detected.structure_tokens,
cell_bboxes: detected.cell_bboxes,
}],
)
.unwrap()
.remove(0);
assert!(markdown.contains("A1"));
assert!(markdown.contains("B1"));
assert!(markdown.contains("A2"));
assert!(markdown.contains("B2"));
}
#[test]
fn test_detect_vector_grid_in_region_text_pdf_returns_none() {
let buf = make_minimal_text_pdf();
let detected =
detect_vector_grid_in_region_mem(&buf, 0, [0.0, 0.0, 300.0, 800.0], 72.0).unwrap();
assert!(detected.is_none());
}
#[test]
fn test_detect_vector_grid_in_region_filters_to_requested_table() {
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
let buf = synthetic_vector_grid_pdf(true);
let second_table_crop = [50.0_f32, 240.0, 210.0, 310.0];
let detected = detect_vector_grid_in_region_mem(&buf, 0, second_table_crop, 72.0)
.unwrap()
.expect("second ruled table should be detected");
assert_eq!(detected.cell_bboxes.len(), 4);
let markdown = extract_tables_with_structure_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: second_table_crop,
render_dpi: 72.0,
structure_tokens: detected.structure_tokens,
cell_bboxes: detected.cell_bboxes,
}],
)
.unwrap()
.remove(0);
assert!(markdown.contains("C1"));
assert!(markdown.contains("D2"));
assert!(!markdown.contains("A1"));
assert!(!markdown.contains("B2"));
}
#[test]
fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
@@ -2339,418 +1908,6 @@ fn test_extract_tables_with_structure_separator_after_thead() {
assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
}
// =========================================================================
// extract_tables_with_structure_auto_mem tests (TSR + heuristic fallback)
// =========================================================================
#[test]
fn test_auto_passes_through_clean_tsr_output() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_dense_table_pdf();
let tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
// Cells fit each visible row cleanly. Same shape as the existing
// dense-overlap regression test — TSR should produce clean output
// and the auto wrapper should pass through with no fallback.
let cell_bboxes = vec![
poly(10.0, 72.0, 100.0, 112.0),
poly(90.0, 72.0, 180.0, 112.0),
poly(10.0, 88.8, 100.0, 128.8),
poly(90.0, 88.8, 180.0, 128.8),
poly(10.0, 105.6, 100.0, 145.6),
poly(90.0, 105.6, 180.0, 145.6),
];
let results = extract_tables_with_structure_auto_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
render_dpi: 72.0,
structure_tokens: tokens,
cell_bboxes,
}],
)
.unwrap();
assert_eq!(results.len(), 1);
assert!(
results[0].fallback_reason.is_none(),
"expected no fallback, got {:?}",
results[0].fallback_reason
);
assert!(results[0].markdown.contains("Oak Street"));
assert!(results[0].markdown.contains("Boardwalk"));
assert!(!results[0].markdown.contains("Oak Street Boardwalk"));
}
#[test]
fn test_auto_expands_multi_row_in_cell() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_dense_table_pdf();
// TSR returns only 2 rows for what's actually 3 visible PDF rows.
// Row 1's cells are tall enough to encompass both Oak Street and
// Boardwalk text — the FNBO row-undercount pattern.
let tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
// Header row at top-left y=[88, 105] (covers "Branch Name"/"Deposits"
// at native y=700, top-left y≈92-103). The "data" row at top-left
// y=[105, 145] is intentionally tall — covers BOTH the Oak Street
// line (top-left y≈108-119) AND the Boardwalk line (y≈124-135).
let cell_bboxes = vec![
poly(10.0, 88.0, 100.0, 105.0),
poly(90.0, 88.0, 180.0, 105.0),
poly(10.0, 105.0, 100.0, 145.0),
poly(90.0, 105.0, 180.0, 145.0),
];
let results = extract_tables_with_structure_auto_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
render_dpi: 72.0,
structure_tokens: tokens,
cell_bboxes,
}],
)
.unwrap();
assert_eq!(results.len(), 1);
assert_eq!(
results[0].fallback_reason.as_deref(),
Some("multi_row_in_cell_expanded"),
"expected multi_row_in_cell_expanded, got {:?}",
results[0].fallback_reason
);
// The in-place expansion should preserve all three PDF rows.
let md = &results[0].markdown;
assert!(md.contains("Oak Street"), "missing Oak Street: {md}");
assert!(md.contains("Boardwalk"), "missing Boardwalk: {md}");
assert!(md.contains("100"), "missing 100: {md}");
assert!(md.contains("200"), "missing 200: {md}");
assert!(
!md.contains("Oak Street Boardwalk"),
"rows should not remain compressed: {md}"
);
}
#[test]
fn test_auto_expands_under_counted_vector_grid_rows() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_vector_grid_three_row_pdf();
let tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
let crop = [50.0, 60.0, 210.0, 150.0];
let cell_bboxes = vec![
poly(0.0, 0.0, 80.0, 30.0),
poly(80.0, 0.0, 160.0, 30.0),
poly(0.0, 30.0, 80.0, 90.0),
poly(80.0, 30.0, 160.0, 90.0),
];
let results = extract_tables_with_structure_auto_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: crop,
render_dpi: 72.0,
structure_tokens: tokens,
cell_bboxes,
}],
)
.unwrap();
assert_eq!(results.len(), 1);
assert_eq!(
results[0].fallback_reason.as_deref(),
Some("multi_row_in_cell_expanded")
);
let md = &results[0].markdown;
assert!(md.contains("|Branch|Deposits|"), "missing header: {md}");
assert!(md.contains("|Oak|100|"), "missing row 1: {md}");
assert!(md.contains("|Boardwalk|200|"), "missing row 2: {md}");
assert!(
!md.contains("Oak Boardwalk"),
"rows stayed compressed: {md}"
);
}
#[test]
fn test_auto_returns_empty_inputs() {
use pdf_inspector::extract_tables_with_structure_auto_mem;
let buf = synthetic_dense_table_pdf();
let results = extract_tables_with_structure_auto_mem(&buf, &[]).unwrap();
assert!(results.is_empty());
}
#[test]
fn test_auto_does_not_fire_on_legit_rowspan_cell() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_dense_table_pdf();
// 2 columns, 3 rows in the visible PDF. SLANet emits a 2-row table
// where the LEFT cell of row 1 is a rowspan=2 cell that legitimately
// covers Oak Street + Boardwalk on two visual lines. The right
// column has two normal rows. multi_row_in_cell must NOT fire on
// the rowspan=2 cell.
let tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
// First data cell explicitly declares rowspan=2.
"<td",
" rowspan=\"2\"",
">",
"</td>",
"<td></td>",
"</tr>",
"<tr>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
// Header row, then a tall left cell covering both data lines, plus
// two narrow right cells (one per line).
let cell_bboxes = vec![
poly(10.0, 88.0, 100.0, 105.0),
poly(90.0, 88.0, 180.0, 105.0),
poly(10.0, 105.0, 100.0, 145.0), // rowspan=2 — covers both lines
poly(90.0, 105.0, 180.0, 122.0), // row 1 only
poly(90.0, 122.0, 180.0, 145.0), // row 2 only
];
let results = extract_tables_with_structure_auto_mem(
&buf,
&[TsrTableInput {
page: 0,
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
render_dpi: 72.0,
structure_tokens: tokens,
cell_bboxes,
}],
)
.unwrap();
assert_eq!(results.len(), 1);
assert!(
results[0].fallback_reason.is_none(),
"rowspan=2 cell containing 2 visual lines should not trip multi_row_in_cell, got reason={:?}",
results[0].fallback_reason,
);
}
#[test]
fn test_auto_expands_when_heuristic_region_is_empty() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_dense_table_pdf();
// Same shape as the multi_row_in_cell regression — a tall data cell
// that catches Oak Street + Boardwalk. The crop bbox we pass points
// at a strip of the page that has NO text items, so the old heuristic
// fallback would be empty. Expansion uses the cell bboxes directly.
let tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
// Cell bboxes overlap the actual PDF text (so multi_row_in_cell
// fires) — but the crop_pdf_pt_bbox we hand to the heuristic is a
// wholly-empty region of the page. The heuristic should return "".
let cell_bboxes = vec![
poly(10.0, 88.0, 100.0, 105.0),
poly(90.0, 88.0, 180.0, 105.0),
poly(10.0, 105.0, 100.0, 145.0),
poly(90.0, 105.0, 180.0, 145.0),
];
let results = extract_tables_with_structure_auto_mem(
&buf,
&[TsrTableInput {
page: 0,
// Crop is at the BOTTOM of the page where there's no text.
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 50.0],
render_dpi: 72.0,
structure_tokens: tokens,
cell_bboxes,
}],
)
.unwrap();
assert_eq!(results.len(), 1);
let r = &results[0];
assert_eq!(
r.fallback_reason.as_deref(),
Some("multi_row_in_cell_expanded"),
"expected expansion despite empty heuristic region, got {:?}",
r.fallback_reason,
);
assert!(
r.markdown.contains("|Oak Street|100|"),
"missing row 1: {}",
r.markdown
);
assert!(
r.markdown.contains("|Boardwalk|200|"),
"missing row 2: {}",
r.markdown
);
}
#[test]
fn test_auto_isolates_per_input_failures() {
use pdf_inspector::{extract_tables_with_structure_auto_mem, TsrTableInput};
let buf = synthetic_dense_table_pdf();
let good_tokens: Vec<String> = [
"<table>",
"<thead>",
"<tr>",
"<th></th>",
"<th></th>",
"</tr>",
"</thead>",
"<tbody>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"<tr>",
"<td></td>",
"<td></td>",
"</tr>",
"</tbody>",
"</table>",
]
.into_iter()
.map(String::from)
.collect();
// A clean input that should pass through with no fallback.
let good_input = TsrTableInput {
page: 0,
crop_pdf_pt_bbox: [0.0, 0.0, 200.0, 800.0],
render_dpi: 72.0,
structure_tokens: good_tokens,
cell_bboxes: vec![
poly(10.0, 72.0, 100.0, 112.0),
poly(90.0, 72.0, 180.0, 112.0),
poly(10.0, 88.8, 100.0, 128.8),
poly(90.0, 88.8, 180.0, 128.8),
poly(10.0, 105.6, 100.0, 145.6),
poly(90.0, 105.6, 180.0, 145.6),
],
};
// A bad input that targets a non-existent page. The detection
// helper short-circuits on missing pages with Ok(None), so this
// shouldn't itself crash, but pairing it with a flagged input
// exercises the per-input control flow regardless. The point of
// this test is that one input's outcome doesn't poison the other.
let bad_input = TsrTableInput {
page: 9999,
crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
render_dpi: 72.0,
structure_tokens: vec![
"<table>".into(),
"<tr>".into(),
"<td></td>".into(),
"</tr>".into(),
"</table>".into(),
],
cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
};
let results = extract_tables_with_structure_auto_mem(&buf, &[good_input, bad_input]).unwrap();
assert_eq!(results.len(), 2);
// Good input still produces non-empty TSR markdown with no fallback.
assert!(
results[0].fallback_reason.is_none(),
"good input should pass through, got reason={:?}",
results[0].fallback_reason,
);
assert!(results[0].markdown.contains("Oak Street"));
// Bad input collapses to empty markdown but doesn't take the
// batch down with it.
assert_eq!(results[1].markdown, "");
}
// =========================================================================
// extract_pages_markdown_mem tests
// =========================================================================