Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ed3ab81f8e | ||
|
|
e165206fef | ||
|
|
20f24d1f8d | ||
|
|
abb0b925fb | ||
|
|
00c5c18e2a |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "firecrawl-pdf-inspector",
|
||||
"version": "0.4.3",
|
||||
"version": "0.7.1",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+114
-27
@@ -5,6 +5,28 @@ use napi_derive::napi;
|
||||
use std::collections::HashSet;
|
||||
use std::panic;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Enums
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// PDF document type classification.
|
||||
#[napi(string_enum)]
|
||||
pub enum PdfType {
|
||||
TextBased,
|
||||
Scanned,
|
||||
ImageBased,
|
||||
Mixed,
|
||||
}
|
||||
|
||||
/// Type of a positioned text item.
|
||||
#[napi(string_enum)]
|
||||
pub enum ItemType {
|
||||
Text,
|
||||
Image,
|
||||
Link,
|
||||
FormField,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -12,7 +34,7 @@ use std::panic;
|
||||
/// Full PDF processing result with markdown and metadata.
|
||||
#[napi(object)]
|
||||
pub struct PdfResult {
|
||||
pub pdf_type: String,
|
||||
pub pdf_type: PdfType,
|
||||
pub markdown: Option<String>,
|
||||
pub page_count: u32,
|
||||
pub processing_time_ms: u32,
|
||||
@@ -29,7 +51,7 @@ pub struct PdfResult {
|
||||
/// Lightweight PDF classification result.
|
||||
#[napi(object)]
|
||||
pub struct PdfClassification {
|
||||
pub pdf_type: String,
|
||||
pub pdf_type: PdfType,
|
||||
pub page_count: u32,
|
||||
/// 0-indexed page numbers that need OCR.
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
@@ -49,7 +71,9 @@ pub struct TextItem {
|
||||
pub page: u32,
|
||||
pub is_bold: bool,
|
||||
pub is_italic: bool,
|
||||
pub item_type: String,
|
||||
pub item_type: ItemType,
|
||||
/// URL for link items, `None` for other types.
|
||||
pub link_url: Option<String>,
|
||||
}
|
||||
|
||||
/// A page's regions for text extraction: (page_index_0based, bboxes).
|
||||
@@ -79,18 +103,18 @@ pub struct PageRegionTexts {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn pdf_type_string(t: pdf_inspector::PdfType) -> String {
|
||||
fn convert_pdf_type(t: pdf_inspector::PdfType) -> PdfType {
|
||||
match t {
|
||||
pdf_inspector::PdfType::TextBased => "TextBased".to_string(),
|
||||
pdf_inspector::PdfType::Scanned => "Scanned".to_string(),
|
||||
pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(),
|
||||
pdf_inspector::PdfType::Mixed => "Mixed".to_string(),
|
||||
pdf_inspector::PdfType::TextBased => PdfType::TextBased,
|
||||
pdf_inspector::PdfType::Scanned => PdfType::Scanned,
|
||||
pdf_inspector::PdfType::ImageBased => PdfType::ImageBased,
|
||||
pdf_inspector::PdfType::Mixed => PdfType::Mixed,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
PdfResult {
|
||||
pdf_type: pdf_type_string(r.pdf_type),
|
||||
pdf_type: convert_pdf_type(r.pdf_type),
|
||||
markdown: r.markdown,
|
||||
page_count: r.page_count,
|
||||
processing_time_ms: r.processing_time_ms as u32,
|
||||
@@ -104,12 +128,12 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn item_type_string(t: &pdf_inspector::types::ItemType) -> String {
|
||||
fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
|
||||
match t {
|
||||
pdf_inspector::types::ItemType::Text => "text".into(),
|
||||
pdf_inspector::types::ItemType::Image => "image".into(),
|
||||
pdf_inspector::types::ItemType::Link(url) => format!("link:{url}"),
|
||||
pdf_inspector::types::ItemType::FormField => "form_field".into(),
|
||||
pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
|
||||
pdf_inspector::types::ItemType::Image => (ItemType::Image, None),
|
||||
pdf_inspector::types::ItemType::Link(url) => (ItemType::Link, Some(url.clone())),
|
||||
pdf_inspector::types::ItemType::FormField => (ItemType::FormField, None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +205,7 @@ pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
|
||||
let result =
|
||||
pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: pdf_type_string(result.pdf_type),
|
||||
pdf_type: convert_pdf_type(result.pdf_type),
|
||||
page_count: result.page_count,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
confidence: result.confidence as f64,
|
||||
@@ -222,18 +246,22 @@ pub fn extract_text_with_positions(
|
||||
|
||||
Ok(items
|
||||
.into_iter()
|
||||
.map(|item| TextItem {
|
||||
text: item.text,
|
||||
x: item.x as f64,
|
||||
y: item.y as f64,
|
||||
width: item.width as f64,
|
||||
height: item.height as f64,
|
||||
font: item.font,
|
||||
font_size: item.font_size as f64,
|
||||
page: item.page,
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
item_type: item_type_string(&item.item_type),
|
||||
.map(|item| {
|
||||
let (item_type, link_url) = convert_item_type(&item.item_type);
|
||||
TextItem {
|
||||
text: item.text,
|
||||
x: item.x as f64,
|
||||
y: item.y as f64,
|
||||
width: item.width as f64,
|
||||
height: item.height as f64,
|
||||
font: item.font,
|
||||
font_size: item.font_size as f64,
|
||||
page: item.page,
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
item_type,
|
||||
link_url,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
@@ -289,6 +317,65 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[napi(object)]
|
||||
pub struct PageMarkdownResult {
|
||||
/// 0-indexed page number.
|
||||
pub page: u32,
|
||||
/// Formatted markdown for this page.
|
||||
pub markdown: String,
|
||||
/// `true` when text on this page is unreliable.
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
/// Combined per-page markdown extraction and layout classification result.
|
||||
#[napi(object)]
|
||||
pub struct PagesExtractionResult {
|
||||
/// Per-page markdown results.
|
||||
pub pages: Vec<PageMarkdownResult>,
|
||||
/// 1-indexed pages where tables were detected.
|
||||
pub pages_with_tables: Vec<u32>,
|
||||
/// 1-indexed pages where multi-column layout was detected.
|
||||
pub pages_with_columns: Vec<u32>,
|
||||
/// 1-indexed pages that need OCR (scanned/image-based).
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// True if any page has tables or columns.
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Returns per-page markdown and classification data (tables, columns,
|
||||
/// OCR needs) from a single parse. Font statistics are computed from the
|
||||
/// full document so header detection is consistent across pages.
|
||||
#[napi]
|
||||
pub fn extract_pages_markdown(
|
||||
buffer: Buffer,
|
||||
pages: Vec<u32>,
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|r| PageMarkdownResult {
|
||||
page: r.page,
|
||||
markdown: r.markdown,
|
||||
needs_ocr: r.needs_ocr,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: result.pages_with_tables,
|
||||
pages_with_columns: result.pages_with_columns,
|
||||
pages_needing_ocr: result.pages_needing_ocr,
|
||||
is_complex: result.is_complex,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_page_regions(page_regions: &[PageRegions]) -> Vec<(u32, Vec<[f32; 4]>)> {
|
||||
page_regions
|
||||
.iter()
|
||||
|
||||
+143
@@ -299,6 +299,149 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result<PdfClassification, PdfError> {
|
||||
})
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Per-page markdown extraction
|
||||
// =========================================================================
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[derive(Debug)]
|
||||
pub struct PageMarkdown {
|
||||
/// 0-indexed page number.
|
||||
pub page: u32,
|
||||
/// Formatted markdown for this page.
|
||||
pub markdown: String,
|
||||
/// `true` when text on this page is unreliable (GID-encoded fonts,
|
||||
/// encoding issues, garbage text, or empty extraction).
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
/// Combined per-page markdown extraction and layout classification result.
|
||||
#[derive(Debug)]
|
||||
pub struct PagesExtractionResult {
|
||||
/// Per-page markdown results.
|
||||
pub pages: Vec<PageMarkdown>,
|
||||
/// 1-indexed pages where tables were detected.
|
||||
pub pages_with_tables: Vec<u32>,
|
||||
/// 1-indexed pages where multi-column layout was detected.
|
||||
pub pages_with_columns: Vec<u32>,
|
||||
/// 1-indexed pages that need OCR (scanned/image-based).
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// True if any page has tables or columns.
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
|
||||
/// this returns per-page markdown so callers can mix direct extraction
|
||||
/// (for simple text pages) with GPU OCR (for complex/scanned pages).
|
||||
///
|
||||
/// Font statistics are computed from the full document so header
|
||||
/// detection thresholds are consistent regardless of which pages are
|
||||
/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
|
||||
/// fonts, encoding issues, or garbage text.
|
||||
///
|
||||
/// Layout complexity (tables, columns) is computed from the full document
|
||||
/// at near-zero cost since the items/rects/lines are already in memory.
|
||||
pub fn extract_pages_markdown_mem(
|
||||
buffer: &[u8],
|
||||
pages: &[u32],
|
||||
) -> Result<PagesExtractionResult, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, page_count) = load_document_from_mem(buffer)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
|
||||
// Extract ALL pages to get accurate, document-wide font stats.
|
||||
let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
|
||||
extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
|
||||
|
||||
// Compute layout complexity from full document (near-zero cost).
|
||||
let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
|
||||
|
||||
// Compute font stats from full document (cross-page consistency).
|
||||
let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
|
||||
|
||||
let mut results = Vec::with_capacity(pages.len());
|
||||
let mut pages_needing_ocr = Vec::new();
|
||||
|
||||
for &page_0idx in pages {
|
||||
// Out-of-range pages → empty + needs_ocr
|
||||
if page_0idx >= page_count {
|
||||
pages_needing_ocr.push(page_0idx + 1);
|
||||
results.push(PageMarkdown {
|
||||
page: page_0idx,
|
||||
markdown: String::new(),
|
||||
needs_ocr: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let page_1idx = page_0idx + 1;
|
||||
|
||||
// Filter items/rects for this page only
|
||||
let page_items: Vec<TextItem> = all_items
|
||||
.iter()
|
||||
.filter(|i| i.page == page_1idx)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let page_rects: Vec<PdfRect> = all_rects
|
||||
.iter()
|
||||
.filter(|r| r.page == page_1idx)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let has_gid = gid_pages.contains(&page_1idx);
|
||||
|
||||
// Build markdown with document-wide font stats
|
||||
let options = MarkdownOptions {
|
||||
base_font_size: Some(font_stats.most_common_size),
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: false,
|
||||
..MarkdownOptions::default()
|
||||
};
|
||||
|
||||
let md = markdown::to_markdown_from_items_with_rects_and_lines(
|
||||
page_items,
|
||||
options,
|
||||
&page_rects,
|
||||
&[],
|
||||
&page_thresholds,
|
||||
None,
|
||||
&[],
|
||||
);
|
||||
|
||||
let needs_ocr = md.trim().is_empty()
|
||||
|| has_gid
|
||||
|| is_garbage_text(&md)
|
||||
|| is_cid_garbage(&md)
|
||||
|| detect_encoding_issues(&md);
|
||||
|
||||
if needs_ocr {
|
||||
pages_needing_ocr.push(page_1idx);
|
||||
}
|
||||
|
||||
results.push(PageMarkdown {
|
||||
page: page_0idx,
|
||||
markdown: if needs_ocr { String::new() } else { md },
|
||||
needs_ocr,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(PagesExtractionResult {
|
||||
pages: results,
|
||||
pages_with_tables: complexity.pages_with_tables,
|
||||
pages_with_columns: complexity.pages_with_columns,
|
||||
pages_needing_ocr,
|
||||
is_complex: complexity.is_complex,
|
||||
})
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Region-based text extraction (for hybrid OCR pipelines)
|
||||
// =========================================================================
|
||||
|
||||
/// Result for a single region's text extraction.
|
||||
#[derive(Debug)]
|
||||
pub struct RegionText {
|
||||
|
||||
@@ -1144,6 +1144,33 @@ pub(crate) fn find_first_table_row(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip rows that have duplicate non-empty cells. These are spanning
|
||||
// super-headers (e.g., "First Degree | First Degree | Higher Degree")
|
||||
// that sit above the real column header row. Using them as the markdown
|
||||
// header produces duplicate column names that downstream validation
|
||||
// rejects. Only skip if a subsequent row looks like a better header
|
||||
// (denser fill or has data).
|
||||
if filled_count >= 2 && !has_data {
|
||||
let mut text_counts: std::collections::HashMap<&str, usize> =
|
||||
std::collections::HashMap::new();
|
||||
for cell in &filled_cells {
|
||||
*text_counts.entry(cell.trim()).or_insert(0) += 1;
|
||||
}
|
||||
let has_duplicates = text_counts.values().any(|&count| count >= 2);
|
||||
if has_duplicates {
|
||||
// Check if a later row is a better header candidate
|
||||
let has_better_below = cells.iter().skip(row_idx + 1).take(3).any(|r| {
|
||||
let next_filled = r.iter().filter(|c| !c.trim().is_empty()).count();
|
||||
let next_fill = next_filled as f32 / total_cols as f32;
|
||||
let next_numeric = r.iter().filter(|c| looks_like_number(c.trim())).count();
|
||||
next_fill >= 0.4 || next_numeric >= 2
|
||||
});
|
||||
if has_better_below {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Data rows are definitely table content
|
||||
if has_data {
|
||||
first_table_row = row_idx;
|
||||
|
||||
+132
-12
@@ -82,33 +82,42 @@ pub(crate) fn find_column_boundaries(
|
||||
}
|
||||
}
|
||||
|
||||
let mut columns = Vec::new();
|
||||
let mut cluster_items: Vec<f32> = vec![x_positions[0]];
|
||||
// Track cluster membership: for each cluster, store the list of x positions
|
||||
let mut cluster_xs: Vec<Vec<f32>> = vec![vec![x_positions[0]]];
|
||||
|
||||
for &x in &x_positions[1..] {
|
||||
let last_cluster = cluster_xs.last().unwrap();
|
||||
// For dense columns (gap-histogram triggered), use edge-based clustering:
|
||||
// compare with the last item to avoid center-drift that merges adjacent
|
||||
// narrow columns. For normal tables, use center-based (original behavior).
|
||||
let reference = if use_edge_clustering {
|
||||
*cluster_items.last().unwrap()
|
||||
*last_cluster.last().unwrap()
|
||||
} else {
|
||||
cluster_items.iter().sum::<f32>() / cluster_items.len() as f32
|
||||
last_cluster.iter().sum::<f32>() / last_cluster.len() as f32
|
||||
};
|
||||
|
||||
if x - reference > cluster_threshold {
|
||||
let cluster_center = cluster_items.iter().sum::<f32>() / cluster_items.len() as f32;
|
||||
columns.push(cluster_center);
|
||||
cluster_items = vec![x];
|
||||
cluster_xs.push(vec![x]);
|
||||
} else {
|
||||
cluster_items.push(x);
|
||||
cluster_xs.last_mut().unwrap().push(x);
|
||||
}
|
||||
}
|
||||
|
||||
// Don't forget last cluster
|
||||
if !cluster_items.is_empty() {
|
||||
columns.push(cluster_items.iter().sum::<f32>() / cluster_items.len() as f32);
|
||||
// Numeric column merge pass: when a sparse cluster (few items, typically
|
||||
// header text) is adjacent to a dense numeric cluster and within 1.5×
|
||||
// threshold, merge them. This fixes tables where multi-line wrapped
|
||||
// headers have slightly different X positions than the data columns,
|
||||
// causing the header and data to split into separate clusters.
|
||||
let columns_before_merge = cluster_xs.len();
|
||||
if columns_before_merge >= 3 {
|
||||
cluster_xs = merge_numeric_adjacent_clusters(cluster_xs, items, cluster_threshold);
|
||||
}
|
||||
|
||||
let columns: Vec<f32> = cluster_xs
|
||||
.iter()
|
||||
.map(|xs| xs.iter().sum::<f32>() / xs.len() as f32)
|
||||
.collect();
|
||||
|
||||
// Filter columns - each should have multiple items
|
||||
let min_items_per_col = (items.len() / columns.len().max(1) / 4).max(2);
|
||||
let columns: Vec<f32> = columns
|
||||
@@ -123,8 +132,9 @@ pub(crate) fn find_column_boundaries(
|
||||
.collect();
|
||||
|
||||
log::debug!(
|
||||
" find_column_boundaries: {} columns before filter, threshold={:.1}, {} items",
|
||||
" find_column_boundaries: {} columns (merged from {}), threshold={:.1}, {} items",
|
||||
columns.len(),
|
||||
columns_before_merge,
|
||||
cluster_threshold,
|
||||
items.len()
|
||||
);
|
||||
@@ -148,6 +158,116 @@ pub(crate) fn find_column_boundaries(
|
||||
columns
|
||||
}
|
||||
|
||||
/// Check if a text string looks like a number (digits, decimals, sign, comma).
|
||||
fn is_numeric_text(s: &str) -> bool {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// Match patterns like: 8.23, -1.05, 9.99, 7.12, 100, 3,456.78, +5%, ---
|
||||
// But NOT: BIO, Department, Core Courses
|
||||
s.chars()
|
||||
.all(|c| c.is_ascii_digit() || c == '.' || c == ',' || c == '-' || c == '+' || c == '%')
|
||||
&& s.chars().any(|c| c.is_ascii_digit())
|
||||
}
|
||||
|
||||
/// Merge adjacent X-position clusters when one is a sparse header cluster
|
||||
/// and the other is a dense numeric data cluster. This prevents multi-line
|
||||
/// wrapped headers from splitting a logical column into two clusters.
|
||||
fn merge_numeric_adjacent_clusters(
|
||||
mut clusters: Vec<Vec<f32>>,
|
||||
items: &[(usize, &TextItem)],
|
||||
threshold: f32,
|
||||
) -> Vec<Vec<f32>> {
|
||||
// For each cluster, compute: center, item count, numeric fraction
|
||||
struct ClusterInfo {
|
||||
center: f32,
|
||||
count: usize,
|
||||
numeric_frac: f32,
|
||||
}
|
||||
|
||||
let compute_info = |xs: &[f32]| -> ClusterInfo {
|
||||
let center = xs.iter().sum::<f32>() / xs.len() as f32;
|
||||
// Count items and numeric fraction for items near this cluster center
|
||||
let mut total = 0;
|
||||
let mut numeric = 0;
|
||||
for (_, item) in items {
|
||||
if (item.x - center).abs() < threshold {
|
||||
total += 1;
|
||||
if is_numeric_text(&item.text) {
|
||||
numeric += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
ClusterInfo {
|
||||
center,
|
||||
count: total,
|
||||
numeric_frac: if total > 0 {
|
||||
numeric as f32 / total as f32
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
// Merge distance: allow merging clusters that are slightly beyond the
|
||||
// original threshold. Use 1.5× threshold to catch header-vs-data splits.
|
||||
let merge_dist = threshold * 1.5;
|
||||
|
||||
// Iterate and merge adjacent pairs. Use a simple left-to-right scan.
|
||||
let mut merged = true;
|
||||
while merged {
|
||||
merged = false;
|
||||
let mut i = 0;
|
||||
while i + 1 < clusters.len() {
|
||||
let info_a = compute_info(&clusters[i]);
|
||||
let info_b = compute_info(&clusters[i + 1]);
|
||||
let dist = (info_b.center - info_a.center).abs();
|
||||
|
||||
if dist > merge_dist {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Determine if one cluster is sparse (header) and the other
|
||||
// is dense and numeric (data). A cluster is "sparse" if it has
|
||||
// significantly fewer items than the other.
|
||||
let (sparse, dense) = if info_a.count < info_b.count {
|
||||
(&info_a, &info_b)
|
||||
} else {
|
||||
(&info_b, &info_a)
|
||||
};
|
||||
|
||||
// Merge if the dense cluster is predominantly numeric (>50%)
|
||||
// and the sparse cluster has at most 1/3 the items of the dense one.
|
||||
let should_merge =
|
||||
dense.numeric_frac > 0.50 && sparse.count <= dense.count / 2 && sparse.count <= 5;
|
||||
|
||||
if should_merge {
|
||||
log::debug!(
|
||||
" merging column clusters: center {:.1} ({} items, {:.0}% numeric) + {:.1} ({} items, {:.0}% numeric), dist={:.1}",
|
||||
info_a.center,
|
||||
info_a.count,
|
||||
info_a.numeric_frac * 100.0,
|
||||
info_b.center,
|
||||
info_b.count,
|
||||
info_b.numeric_frac * 100.0,
|
||||
dist,
|
||||
);
|
||||
// Merge cluster i+1 into cluster i
|
||||
let next = clusters.remove(i + 1);
|
||||
clusters[i].extend(next);
|
||||
merged = true;
|
||||
// Don't increment i — check if the merged cluster can merge further
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusters
|
||||
}
|
||||
|
||||
/// Find row boundaries by clustering Y positions
|
||||
pub(crate) fn find_row_boundaries(items: &[(usize, &TextItem)]) -> Vec<f32> {
|
||||
let mut y_positions: Vec<f32> = items.iter().map(|(_, i)| i.y).collect();
|
||||
|
||||
BIN
Binary file not shown.
+197
-3
@@ -4,9 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
|
||||
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
|
||||
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
|
||||
detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -1436,3 +1437,196 @@ fn test_extract_tables_in_regions_nonexistent_page() {
|
||||
assert!(region.needs_ocr);
|
||||
assert!(region.text.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bits_pilani_page4_table_detection() {
|
||||
// Page 4 (0-indexed 3) has a table with multi-line wrapped headers and
|
||||
// numeric data columns. The heuristic detector previously failed because:
|
||||
// 1. Header items at different X positions than data created extra column
|
||||
// clusters (6 cols instead of 4)
|
||||
// 2. Spanning super-header row ("First Degree | First Degree") produced
|
||||
// duplicate header cells that looks_like_partial_table_ex rejected
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
let results =
|
||||
extract_tables_in_regions_mem(&buf, &[(3, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
let region = &results[0].regions[0];
|
||||
assert!(
|
||||
!region.needs_ocr,
|
||||
"Page 4 table should be detected, got needs_ocr=true"
|
||||
);
|
||||
assert!(
|
||||
region.text.contains("BIO"),
|
||||
"Should contain department name BIO"
|
||||
);
|
||||
assert!(region.text.contains("8.23"), "Should contain numeric data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bits_pilani_page8_table_detection() {
|
||||
// Page 8 (0-indexed 7) has a numbered-row table that already worked.
|
||||
// Verify it still works after changes.
|
||||
let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
|
||||
let results =
|
||||
extract_tables_in_regions_mem(&buf, &[(7, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
|
||||
assert_eq!(results.len(), 1);
|
||||
let region = &results[0].regions[0];
|
||||
assert!(!region.needs_ocr, "Page 8 table should still be detected");
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// extract_pages_markdown_mem tests
|
||||
// =========================================================================
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_basic() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
assert_eq!(result.pages[1].page, 1);
|
||||
// Text-based PDF should produce non-empty markdown
|
||||
assert!(!result.pages[0].markdown.is_empty());
|
||||
assert!(!result.pages[0].needs_ocr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_page_ordering() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
// Request pages in non-sequential order
|
||||
let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
// Results should match input order, not document order
|
||||
assert_eq!(result.pages[0].page, 1);
|
||||
assert_eq!(result.pages[1].page, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_out_of_range() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 9999);
|
||||
assert!(result.pages[0].markdown.is_empty());
|
||||
assert!(result.pages[0].needs_ocr);
|
||||
assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_empty_pages_list() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
|
||||
assert!(result.pages.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_single_page() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
assert!(!result.pages[0].markdown.is_empty());
|
||||
assert!(!result.pages[0].needs_ocr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_invalid_buffer() {
|
||||
let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_gid_pages_need_ocr() {
|
||||
// shinagawa_identity_h.pdf has GID-encoded fonts
|
||||
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(result.pages[0].needs_ocr);
|
||||
assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_classification_with_tables() {
|
||||
// nexo-price-en.pdf is known to have tables
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let page_count = process_pdf_mem(&buf).unwrap().page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
assert!(
|
||||
!result.pages_with_tables.is_empty(),
|
||||
"nexo-price-en.pdf should have pages with tables"
|
||||
);
|
||||
assert!(result.is_complex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
|
||||
// bare_name_struct.pdf is a simple document with a heading and code block
|
||||
let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert!(result.pages_with_tables.is_empty());
|
||||
assert!(result.pages_with_columns.is_empty());
|
||||
assert!(!result.is_complex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_classification_matches_process_pdf() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let full = process_pdf_mem(&buf).unwrap();
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result.pages_with_tables, full.layout.pages_with_tables,
|
||||
"pages_with_tables should match process_pdf"
|
||||
);
|
||||
assert_eq!(
|
||||
result.pages_with_columns, full.layout.pages_with_columns,
|
||||
"pages_with_columns should match process_pdf"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_consistency_with_process_pdf() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
|
||||
// Get full process_pdf output
|
||||
let full = process_pdf_mem(&buf).unwrap();
|
||||
let full_md = full.markdown.unwrap_or_default();
|
||||
|
||||
// Get per-page output for all pages
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
// Concatenated per-page markdown should contain substantial overlap with
|
||||
// the full output (exact match not expected due to header/footer stripping
|
||||
// and cross-page paragraph merging differences)
|
||||
let concat: String = result
|
||||
.pages
|
||||
.iter()
|
||||
.map(|p| p.markdown.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
// Both should be non-empty for a text-based PDF
|
||||
assert!(!full_md.is_empty());
|
||||
assert!(!concat.is_empty());
|
||||
|
||||
// The per-page version should contain at least 50% of the full content's
|
||||
// length (accounting for header/footer stripping differences)
|
||||
assert!(
|
||||
concat.len() * 2 >= full_md.len(),
|
||||
"per-page concat ({} chars) is too short vs full ({} chars)",
|
||||
concat.len(),
|
||||
full_md.len()
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user