Compare commits

...
Author SHA1 Message Date
Abimael Martell ed3ab81f8e bump version to 0.7.1 2026-04-14 17:23:10 -07:00
Abimael MartellandClaude Opus 4.6 e165206fef fix: improve heuristic table detection for numeric columns and multi-line headers
Two fixes for tables that have clean extractable text but fail heuristic
structure detection:

1. Numeric column merge pass (grid.rs): After initial X-position
   clustering, adjacent clusters are merged when one is sparse (header
   text) and the other is dense with >50% numeric items (data column).
   Multi-line wrapped headers often land slightly offset from their
   data column — the merge closes gaps within 1.5× the clustering
   threshold. New is_numeric_text() helper matches decimals, percentages,
   negative numbers, and comma-separated thousands.

2. Duplicate-header skip (detect_heuristic.rs): Spanning super-headers
   like "First Degree | First Degree | Higher Degree" contain duplicate
   cells that trigger looks_like_partial_table_ex rejection. Now skips
   rows with duplicate cells when a better header candidate exists
   within the next 3 rows (higher fill ratio or numeric cells).

Tested on BITS Pilani university report (430 pages, 314 table pages).
Page 4 (multi-line header + numeric data) previously returned
needs_ocr=true; now correctly detects the table structure.

Eval: 197 PDFs, zero regressions, all 104+ tests pass, zero clippy.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-14 17:20:26 -07:00
Abimael MartellandClaude Opus 4.6 20f24d1f8d extractPagesMarkdown: return classification metadata (0.7.0) (#32)
Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled
Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled
Publish npm package / Publish to npm (push) Has been cancelled
Combine per-page markdown extraction with layout classification into a
single parse. extractPagesMarkdown now returns PagesExtractionResult with
pages_with_tables, pages_with_columns, pages_needing_ocr, and is_complex
alongside the per-page markdown — eliminating redundant PDF parses for
callers that need both.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-14 13:08:07 -07:00
Abimael MartellandClaude Opus 4.6 abb0b925fb Add extractPagesMarkdown for per-page markdown extraction (#31)
Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled
Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled
Publish npm package / Publish to npm (push) Has been cancelled
* add extract_pages_markdown_mem for per-page markdown extraction

Enables hybrid OCR pipelines to skip GPU render+layout for simple text
pages by providing per-page markdown with needs_ocr flags. Font stats
are computed document-wide for consistent header detection.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* bump napi package version to 0.6.0

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-14 12:37:32 -07:00
Abimael MartellandClaude Opus 4.6 00c5c18e2a napi: use string enums for PdfType and ItemType (0.5.0) (#29)
Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled
Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled
Publish npm package / Publish to npm (push) Has been cancelled
Replace stringly-typed pdf_type and item_type fields with
#[napi(string_enum)] enums for proper TypeScript type checking.
Add link_url field to TextItem instead of encoding URL in the
item_type string.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-14 11:25:26 -07:00
7 changed files with 614 additions and 43 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "firecrawl-pdf-inspector",
"version": "0.4.3",
"version": "0.7.1",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
+114 -27
View File
@@ -5,6 +5,28 @@ use napi_derive::napi;
use std::collections::HashSet;
use std::panic;
// ---------------------------------------------------------------------------
// Enums
// ---------------------------------------------------------------------------
/// PDF document type classification.
#[napi(string_enum)]
pub enum PdfType {
TextBased,
Scanned,
ImageBased,
Mixed,
}
/// Type of a positioned text item.
#[napi(string_enum)]
pub enum ItemType {
Text,
Image,
Link,
FormField,
}
// ---------------------------------------------------------------------------
// Result types
// ---------------------------------------------------------------------------
@@ -12,7 +34,7 @@ use std::panic;
/// Full PDF processing result with markdown and metadata.
#[napi(object)]
pub struct PdfResult {
pub pdf_type: String,
pub pdf_type: PdfType,
pub markdown: Option<String>,
pub page_count: u32,
pub processing_time_ms: u32,
@@ -29,7 +51,7 @@ pub struct PdfResult {
/// Lightweight PDF classification result.
#[napi(object)]
pub struct PdfClassification {
pub pdf_type: String,
pub pdf_type: PdfType,
pub page_count: u32,
/// 0-indexed page numbers that need OCR.
pub pages_needing_ocr: Vec<u32>,
@@ -49,7 +71,9 @@ pub struct TextItem {
pub page: u32,
pub is_bold: bool,
pub is_italic: bool,
pub item_type: String,
pub item_type: ItemType,
/// URL for link items, `None` for other types.
pub link_url: Option<String>,
}
/// A page's regions for text extraction: (page_index_0based, bboxes).
@@ -79,18 +103,18 @@ pub struct PageRegionTexts {
// Helpers
// ---------------------------------------------------------------------------
fn pdf_type_string(t: pdf_inspector::PdfType) -> String {
fn convert_pdf_type(t: pdf_inspector::PdfType) -> PdfType {
match t {
pdf_inspector::PdfType::TextBased => "TextBased".to_string(),
pdf_inspector::PdfType::Scanned => "Scanned".to_string(),
pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(),
pdf_inspector::PdfType::Mixed => "Mixed".to_string(),
pdf_inspector::PdfType::TextBased => PdfType::TextBased,
pdf_inspector::PdfType::Scanned => PdfType::Scanned,
pdf_inspector::PdfType::ImageBased => PdfType::ImageBased,
pdf_inspector::PdfType::Mixed => PdfType::Mixed,
}
}
fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
PdfResult {
pdf_type: pdf_type_string(r.pdf_type),
pdf_type: convert_pdf_type(r.pdf_type),
markdown: r.markdown,
page_count: r.page_count,
processing_time_ms: r.processing_time_ms as u32,
@@ -104,12 +128,12 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
}
}
fn item_type_string(t: &pdf_inspector::types::ItemType) -> String {
fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
match t {
pdf_inspector::types::ItemType::Text => "text".into(),
pdf_inspector::types::ItemType::Image => "image".into(),
pdf_inspector::types::ItemType::Link(url) => format!("link:{url}"),
pdf_inspector::types::ItemType::FormField => "form_field".into(),
pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
pdf_inspector::types::ItemType::Image => (ItemType::Image, None),
pdf_inspector::types::ItemType::Link(url) => (ItemType::Link, Some(url.clone())),
pdf_inspector::types::ItemType::FormField => (ItemType::FormField, None),
}
}
@@ -181,7 +205,7 @@ pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
let result =
pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
Ok(PdfClassification {
pdf_type: pdf_type_string(result.pdf_type),
pdf_type: convert_pdf_type(result.pdf_type),
page_count: result.page_count,
pages_needing_ocr: result.pages_needing_ocr,
confidence: result.confidence as f64,
@@ -222,18 +246,22 @@ pub fn extract_text_with_positions(
Ok(items
.into_iter()
.map(|item| TextItem {
text: item.text,
x: item.x as f64,
y: item.y as f64,
width: item.width as f64,
height: item.height as f64,
font: item.font,
font_size: item.font_size as f64,
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
item_type: item_type_string(&item.item_type),
.map(|item| {
let (item_type, link_url) = convert_item_type(&item.item_type);
TextItem {
text: item.text,
x: item.x as f64,
y: item.y as f64,
width: item.width as f64,
height: item.height as f64,
font: item.font,
font_size: item.font_size as f64,
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
item_type,
link_url,
}
})
.collect())
})
@@ -289,6 +317,65 @@ pub fn extract_tables_in_regions(
})
}
/// Per-page markdown extraction result.
#[napi(object)]
pub struct PageMarkdownResult {
/// 0-indexed page number.
pub page: u32,
/// Formatted markdown for this page.
pub markdown: String,
/// `true` when text on this page is unreliable.
pub needs_ocr: bool,
}
/// Combined per-page markdown extraction and layout classification result.
#[napi(object)]
pub struct PagesExtractionResult {
/// Per-page markdown results.
pub pages: Vec<PageMarkdownResult>,
/// 1-indexed pages where tables were detected.
pub pages_with_tables: Vec<u32>,
/// 1-indexed pages where multi-column layout was detected.
pub pages_with_columns: Vec<u32>,
/// 1-indexed pages that need OCR (scanned/image-based).
pub pages_needing_ocr: Vec<u32>,
/// True if any page has tables or columns.
pub is_complex: bool,
}
/// Extract formatted markdown for specific pages of a PDF, with layout
/// classification metadata.
///
/// Returns per-page markdown and classification data (tables, columns,
/// OCR needs) from a single parse. Font statistics are computed from the
/// full document so header detection is consistent across pages.
#[napi]
pub fn extract_pages_markdown(
buffer: Buffer,
pages: Vec<u32>,
) -> Result<PagesExtractionResult> {
let bytes: Vec<u8> = buffer.to_vec();
catch_panic("extract_pages_markdown", move || {
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
Ok(PagesExtractionResult {
pages: result
.pages
.into_iter()
.map(|r| PageMarkdownResult {
page: r.page,
markdown: r.markdown,
needs_ocr: r.needs_ocr,
})
.collect(),
pages_with_tables: result.pages_with_tables,
pages_with_columns: result.pages_with_columns,
pages_needing_ocr: result.pages_needing_ocr,
is_complex: result.is_complex,
})
})
}
fn parse_page_regions(page_regions: &[PageRegions]) -> Vec<(u32, Vec<[f32; 4]>)> {
page_regions
.iter()
+143
View File
@@ -299,6 +299,149 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result<PdfClassification, PdfError> {
})
}
// =========================================================================
// Per-page markdown extraction
// =========================================================================
/// Per-page markdown extraction result.
#[derive(Debug)]
pub struct PageMarkdown {
/// 0-indexed page number.
pub page: u32,
/// Formatted markdown for this page.
pub markdown: String,
/// `true` when text on this page is unreliable (GID-encoded fonts,
/// encoding issues, garbage text, or empty extraction).
pub needs_ocr: bool,
}
/// Combined per-page markdown extraction and layout classification result.
#[derive(Debug)]
pub struct PagesExtractionResult {
/// Per-page markdown results.
pub pages: Vec<PageMarkdown>,
/// 1-indexed pages where tables were detected.
pub pages_with_tables: Vec<u32>,
/// 1-indexed pages where multi-column layout was detected.
pub pages_with_columns: Vec<u32>,
/// 1-indexed pages that need OCR (scanned/image-based).
pub pages_needing_ocr: Vec<u32>,
/// True if any page has tables or columns.
pub is_complex: bool,
}
/// Extract formatted markdown for specific pages of a PDF, with layout
/// classification metadata.
///
/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
/// this returns per-page markdown so callers can mix direct extraction
/// (for simple text pages) with GPU OCR (for complex/scanned pages).
///
/// Font statistics are computed from the full document so header
/// detection thresholds are consistent regardless of which pages are
/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
/// fonts, encoding issues, or garbage text.
///
/// Layout complexity (tables, columns) is computed from the full document
/// at near-zero cost since the items/rects/lines are already in memory.
pub fn extract_pages_markdown_mem(
buffer: &[u8],
pages: &[u32],
) -> Result<PagesExtractionResult, PdfError> {
validate_pdf_bytes(buffer)?;
let (doc, page_count) = load_document_from_mem(buffer)?;
let font_cmaps = FontCMaps::from_doc(&doc);
// Extract ALL pages to get accurate, document-wide font stats.
let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
// Compute layout complexity from full document (near-zero cost).
let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
// Compute font stats from full document (cross-page consistency).
let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
let mut results = Vec::with_capacity(pages.len());
let mut pages_needing_ocr = Vec::new();
for &page_0idx in pages {
// Out-of-range pages → empty + needs_ocr
if page_0idx >= page_count {
pages_needing_ocr.push(page_0idx + 1);
results.push(PageMarkdown {
page: page_0idx,
markdown: String::new(),
needs_ocr: true,
});
continue;
}
let page_1idx = page_0idx + 1;
// Filter items/rects for this page only
let page_items: Vec<TextItem> = all_items
.iter()
.filter(|i| i.page == page_1idx)
.cloned()
.collect();
let page_rects: Vec<PdfRect> = all_rects
.iter()
.filter(|r| r.page == page_1idx)
.cloned()
.collect();
let has_gid = gid_pages.contains(&page_1idx);
// Build markdown with document-wide font stats
let options = MarkdownOptions {
base_font_size: Some(font_stats.most_common_size),
include_page_numbers: false,
strip_headers_footers: false,
..MarkdownOptions::default()
};
let md = markdown::to_markdown_from_items_with_rects_and_lines(
page_items,
options,
&page_rects,
&[],
&page_thresholds,
None,
&[],
);
let needs_ocr = md.trim().is_empty()
|| has_gid
|| is_garbage_text(&md)
|| is_cid_garbage(&md)
|| detect_encoding_issues(&md);
if needs_ocr {
pages_needing_ocr.push(page_1idx);
}
results.push(PageMarkdown {
page: page_0idx,
markdown: if needs_ocr { String::new() } else { md },
needs_ocr,
});
}
Ok(PagesExtractionResult {
pages: results,
pages_with_tables: complexity.pages_with_tables,
pages_with_columns: complexity.pages_with_columns,
pages_needing_ocr,
is_complex: complexity.is_complex,
})
}
// =========================================================================
// Region-based text extraction (for hybrid OCR pipelines)
// =========================================================================
/// Result for a single region's text extraction.
#[derive(Debug)]
pub struct RegionText {
+27
View File
@@ -1144,6 +1144,33 @@ pub(crate) fn find_first_table_row(
continue;
}
// Skip rows that have duplicate non-empty cells. These are spanning
// super-headers (e.g., "First Degree | First Degree | Higher Degree")
// that sit above the real column header row. Using them as the markdown
// header produces duplicate column names that downstream validation
// rejects. Only skip if a subsequent row looks like a better header
// (denser fill or has data).
if filled_count >= 2 && !has_data {
let mut text_counts: std::collections::HashMap<&str, usize> =
std::collections::HashMap::new();
for cell in &filled_cells {
*text_counts.entry(cell.trim()).or_insert(0) += 1;
}
let has_duplicates = text_counts.values().any(|&count| count >= 2);
if has_duplicates {
// Check if a later row is a better header candidate
let has_better_below = cells.iter().skip(row_idx + 1).take(3).any(|r| {
let next_filled = r.iter().filter(|c| !c.trim().is_empty()).count();
let next_fill = next_filled as f32 / total_cols as f32;
let next_numeric = r.iter().filter(|c| looks_like_number(c.trim())).count();
next_fill >= 0.4 || next_numeric >= 2
});
if has_better_below {
continue;
}
}
}
// Data rows are definitely table content
if has_data {
first_table_row = row_idx;
+132 -12
View File
@@ -82,33 +82,42 @@ pub(crate) fn find_column_boundaries(
}
}
let mut columns = Vec::new();
let mut cluster_items: Vec<f32> = vec![x_positions[0]];
// Track cluster membership: for each cluster, store the list of x positions
let mut cluster_xs: Vec<Vec<f32>> = vec![vec![x_positions[0]]];
for &x in &x_positions[1..] {
let last_cluster = cluster_xs.last().unwrap();
// For dense columns (gap-histogram triggered), use edge-based clustering:
// compare with the last item to avoid center-drift that merges adjacent
// narrow columns. For normal tables, use center-based (original behavior).
let reference = if use_edge_clustering {
*cluster_items.last().unwrap()
*last_cluster.last().unwrap()
} else {
cluster_items.iter().sum::<f32>() / cluster_items.len() as f32
last_cluster.iter().sum::<f32>() / last_cluster.len() as f32
};
if x - reference > cluster_threshold {
let cluster_center = cluster_items.iter().sum::<f32>() / cluster_items.len() as f32;
columns.push(cluster_center);
cluster_items = vec![x];
cluster_xs.push(vec![x]);
} else {
cluster_items.push(x);
cluster_xs.last_mut().unwrap().push(x);
}
}
// Don't forget last cluster
if !cluster_items.is_empty() {
columns.push(cluster_items.iter().sum::<f32>() / cluster_items.len() as f32);
// Numeric column merge pass: when a sparse cluster (few items, typically
// header text) is adjacent to a dense numeric cluster and within 1.5×
// threshold, merge them. This fixes tables where multi-line wrapped
// headers have slightly different X positions than the data columns,
// causing the header and data to split into separate clusters.
let columns_before_merge = cluster_xs.len();
if columns_before_merge >= 3 {
cluster_xs = merge_numeric_adjacent_clusters(cluster_xs, items, cluster_threshold);
}
let columns: Vec<f32> = cluster_xs
.iter()
.map(|xs| xs.iter().sum::<f32>() / xs.len() as f32)
.collect();
// Filter columns - each should have multiple items
let min_items_per_col = (items.len() / columns.len().max(1) / 4).max(2);
let columns: Vec<f32> = columns
@@ -123,8 +132,9 @@ pub(crate) fn find_column_boundaries(
.collect();
log::debug!(
" find_column_boundaries: {} columns before filter, threshold={:.1}, {} items",
" find_column_boundaries: {} columns (merged from {}), threshold={:.1}, {} items",
columns.len(),
columns_before_merge,
cluster_threshold,
items.len()
);
@@ -148,6 +158,116 @@ pub(crate) fn find_column_boundaries(
columns
}
/// Check if a text string looks like a number (digits, decimals, sign, comma).
fn is_numeric_text(s: &str) -> bool {
let s = s.trim();
if s.is_empty() {
return false;
}
// Match patterns like: 8.23, -1.05, 9.99, 7.12, 100, 3,456.78, +5%, ---
// But NOT: BIO, Department, Core Courses
s.chars()
.all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+' || c == '%')
&& s.chars().any(|c| c.is_ascii_digit())
}
/// Merge adjacent X-position clusters when one is a sparse header cluster
/// and the other is a dense numeric data cluster. This prevents multi-line
/// wrapped headers from splitting a logical column into two clusters.
fn merge_numeric_adjacent_clusters(
mut clusters: Vec<Vec<f32>>,
items: &[(usize, &TextItem)],
threshold: f32,
) -> Vec<Vec<f32>> {
// For each cluster, compute: center, item count, numeric fraction
struct ClusterInfo {
center: f32,
count: usize,
numeric_frac: f32,
}
let compute_info = |xs: &[f32]| -> ClusterInfo {
let center = xs.iter().sum::<f32>() / xs.len() as f32;
// Count items and numeric fraction for items near this cluster center
let mut total = 0;
let mut numeric = 0;
for (_, item) in items {
if (item.x - center).abs() < threshold {
total += 1;
if is_numeric_text(&item.text) {
numeric += 1;
}
}
}
ClusterInfo {
center,
count: total,
numeric_frac: if total > 0 {
numeric as f32 / total as f32
} else {
0.0
},
}
};
// Merge distance: allow merging clusters that are slightly beyond the
// original threshold. Use 1.5× threshold to catch header-vs-data splits.
let merge_dist = threshold * 1.5;
// Iterate and merge adjacent pairs. Use a simple left-to-right scan.
let mut merged = true;
while merged {
merged = false;
let mut i = 0;
while i + 1 < clusters.len() {
let info_a = compute_info(&clusters[i]);
let info_b = compute_info(&clusters[i + 1]);
let dist = (info_b.center - info_a.center).abs();
if dist > merge_dist {
i += 1;
continue;
}
// Determine if one cluster is sparse (header) and the other
// is dense and numeric (data). A cluster is "sparse" if it has
// significantly fewer items than the other.
let (sparse, dense) = if info_a.count < info_b.count {
(&info_a, &info_b)
} else {
(&info_b, &info_a)
};
// Merge if the dense cluster is predominantly numeric (>50%)
// and the sparse cluster has at most 1/3 the items of the dense one.
let should_merge =
dense.numeric_frac > 0.50 && sparse.count <= dense.count / 2 && sparse.count <= 5;
if should_merge {
log::debug!(
" merging column clusters: center {:.1} ({} items, {:.0}% numeric) + {:.1} ({} items, {:.0}% numeric), dist={:.1}",
info_a.center,
info_a.count,
info_a.numeric_frac * 100.0,
info_b.center,
info_b.count,
info_b.numeric_frac * 100.0,
dist,
);
// Merge cluster i+1 into cluster i
let next = clusters.remove(i + 1);
clusters[i].extend(next);
merged = true;
// Don't increment i — check if the merged cluster can merge further
} else {
i += 1;
}
}
}
clusters
}
/// Find row boundaries by clustering Y positions
pub(crate) fn find_row_boundaries(items: &[(usize, &TextItem)]) -> Vec<f32> {
let mut y_positions: Vec<f32> = items.iter().map(|(_, i)| i.y).collect();
Binary file not shown.
+197 -3
View File
@@ -4,9 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
use pdf_inspector::extractor::group_into_lines;
use pdf_inspector::types::TextLine;
use pdf_inspector::{
detect_pdf_type, extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
TextItem,
};
use std::collections::HashSet;
@@ -1436,3 +1437,196 @@ fn test_extract_tables_in_regions_nonexistent_page() {
assert!(region.needs_ocr);
assert!(region.text.is_empty());
}
#[test]
fn test_bits_pilani_page4_table_detection() {
// Page 4 (0-indexed 3) has a table with multi-line wrapped headers and
// numeric data columns. The heuristic detector previously failed because:
// 1. Header items at different X positions than data created extra column
// clusters (6 cols instead of 4)
// 2. Spanning super-header row ("First Degree | First Degree") produced
// duplicate header cells that looks_like_partial_table_ex rejected
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
let results =
extract_tables_in_regions_mem(&buf, &[(3, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
assert_eq!(results.len(), 1);
let region = &results[0].regions[0];
assert!(
!region.needs_ocr,
"Page 4 table should be detected, got needs_ocr=true"
);
assert!(
region.text.contains("BIO"),
"Should contain department name BIO"
);
assert!(region.text.contains("8.23"), "Should contain numeric data");
}
#[test]
fn test_bits_pilani_page8_table_detection() {
// Page 8 (0-indexed 7) has a numbered-row table that already worked.
// Verify it still works after changes.
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
let results =
extract_tables_in_regions_mem(&buf, &[(7, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
assert_eq!(results.len(), 1);
let region = &results[0].regions[0];
assert!(!region.needs_ocr, "Page 8 table should still be detected");
}
// =========================================================================
// extract_pages_markdown_mem tests
// =========================================================================
#[test]
fn test_extract_pages_markdown_basic() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
assert_eq!(result.pages.len(), 2);
assert_eq!(result.pages[0].page, 0);
assert_eq!(result.pages[1].page, 1);
// Text-based PDF should produce non-empty markdown
assert!(!result.pages[0].markdown.is_empty());
assert!(!result.pages[0].needs_ocr);
}
#[test]
fn test_extract_pages_markdown_page_ordering() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
// Request pages in non-sequential order
let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
assert_eq!(result.pages.len(), 2);
// Results should match input order, not document order
assert_eq!(result.pages[0].page, 1);
assert_eq!(result.pages[1].page, 0);
}
#[test]
fn test_extract_pages_markdown_out_of_range() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
assert_eq!(result.pages.len(), 1);
assert_eq!(result.pages[0].page, 9999);
assert!(result.pages[0].markdown.is_empty());
assert!(result.pages[0].needs_ocr);
assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed
}
#[test]
fn test_extract_pages_markdown_empty_pages_list() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
assert!(result.pages.is_empty());
}
#[test]
fn test_extract_pages_markdown_single_page() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
assert_eq!(result.pages.len(), 1);
assert_eq!(result.pages[0].page, 0);
assert!(!result.pages[0].markdown.is_empty());
assert!(!result.pages[0].needs_ocr);
}
#[test]
fn test_extract_pages_markdown_invalid_buffer() {
let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
assert!(result.is_err());
}
#[test]
fn test_extract_pages_markdown_gid_pages_need_ocr() {
// shinagawa_identity_h.pdf has GID-encoded fonts
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
assert_eq!(result.pages.len(), 1);
assert!(result.pages[0].needs_ocr);
assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
}
#[test]
fn test_extract_pages_markdown_classification_with_tables() {
// nexo-price-en.pdf is known to have tables
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let page_count = process_pdf_mem(&buf).unwrap().page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
assert!(
!result.pages_with_tables.is_empty(),
"nexo-price-en.pdf should have pages with tables"
);
assert!(result.is_complex);
}
#[test]
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
// bare_name_struct.pdf is a simple document with a heading and code block
let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
assert!(result.pages_with_tables.is_empty());
assert!(result.pages_with_columns.is_empty());
assert!(!result.is_complex);
}
#[test]
fn test_extract_pages_markdown_classification_matches_process_pdf() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let full = process_pdf_mem(&buf).unwrap();
let page_count = full.page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
assert_eq!(
result.pages_with_tables, full.layout.pages_with_tables,
"pages_with_tables should match process_pdf"
);
assert_eq!(
result.pages_with_columns, full.layout.pages_with_columns,
"pages_with_columns should match process_pdf"
);
}
#[test]
fn test_extract_pages_markdown_consistency_with_process_pdf() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
// Get full process_pdf output
let full = process_pdf_mem(&buf).unwrap();
let full_md = full.markdown.unwrap_or_default();
// Get per-page output for all pages
let page_count = full.page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
// Concatenated per-page markdown should contain substantial overlap with
// the full output (exact match not expected due to header/footer stripping
// and cross-page paragraph merging differences)
let concat: String = result
.pages
.iter()
.map(|p| p.markdown.as_str())
.collect::<Vec<_>>()
.join("\n");
// Both should be non-empty for a text-based PDF
assert!(!full_md.is_empty());
assert!(!concat.is_empty());
// The per-page version should contain at least 50% of the full content's
// length (accounting for header/footer stripping differences)
assert!(
concat.len() * 2 >= full_md.len(),
"per-page concat ({} chars) is too short vs full ({} chars)",
concat.len(),
full_md.len()
);
}