From 20f24d1f8d4ade4f76224ef7c930c83f4423390a Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 14 Apr 2026 13:08:07 -0700 Subject: [PATCH] extractPagesMarkdown: return classification metadata (0.7.0) (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combine per-page markdown extraction with layout classification into a single parse. extractPagesMarkdown now returns PagesExtractionResult with pages_with_tables, pages_with_columns, pages_needing_ocr, and is_complex alongside the per-page markdown — eliminating redundant PDF parses for callers that need both. Co-authored-by: Claude Opus 4.6 (1M context) --- napi/package.json | 2 +- napi/src/lib.rs | 56 ++++++++++++++------ src/lib.rs | 42 +++++++++++++-- tests/integration_tests.rs | 101 +++++++++++++++++++++++++++---------- 4 files changed, 153 insertions(+), 48 deletions(-) diff --git a/napi/package.json b/napi/package.json index 79279c9..1eaafcf 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-pdf-inspector", - "version": "0.6.0", + "version": "0.7.0", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/napi/src/lib.rs b/napi/src/lib.rs index c07933a..5689b5d 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -328,27 +328,51 @@ pub struct PageMarkdownResult { pub needs_ocr: bool, } -/// Extract formatted markdown for specific pages of a PDF. +/// Combined per-page markdown extraction and layout classification result. +#[napi(object)] +pub struct PagesExtractionResult { + /// Per-page markdown results. + pub pages: Vec, + /// 1-indexed pages where tables were detected. + pub pages_with_tables: Vec, + /// 1-indexed pages where multi-column layout was detected. + pub pages_with_columns: Vec, + /// 1-indexed pages that need OCR (scanned/image-based). + pub pages_needing_ocr: Vec, + /// True if any page has tables or columns. + pub is_complex: bool, +} + +/// Extract formatted markdown for specific pages of a PDF, with layout +/// classification metadata. /// -/// Returns per-page markdown so callers can mix direct extraction -/// (for simple text pages) with GPU OCR (for complex/scanned pages). -/// -/// Font statistics are computed from the full document so header -/// detection is consistent across pages. +/// Returns per-page markdown and classification data (tables, columns, +/// OCR needs) from a single parse. Font statistics are computed from the +/// full document so header detection is consistent across pages. #[napi] -pub fn extract_pages_markdown(buffer: Buffer, pages: Vec) -> Result> { +pub fn extract_pages_markdown( + buffer: Buffer, + pages: Vec, +) -> Result { let bytes: Vec = buffer.to_vec(); catch_panic("extract_pages_markdown", move || { - let results = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages) + let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages) .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?; - Ok(results - .into_iter() - .map(|r| PageMarkdownResult { - page: r.page, - markdown: r.markdown, - needs_ocr: r.needs_ocr, - }) - .collect()) + Ok(PagesExtractionResult { + pages: result + .pages + .into_iter() + .map(|r| PageMarkdownResult { + page: r.page, + markdown: r.markdown, + needs_ocr: r.needs_ocr, + }) + .collect(), + pages_with_tables: result.pages_with_tables, + pages_with_columns: result.pages_with_columns, + pages_needing_ocr: result.pages_needing_ocr, + is_complex: result.is_complex, + }) }) } diff --git a/src/lib.rs b/src/lib.rs index df98fbd..afc7161 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -315,7 +315,23 @@ pub struct PageMarkdown { pub needs_ocr: bool, } -/// Extract formatted markdown for specific pages of a PDF. +/// Combined per-page markdown extraction and layout classification result. +#[derive(Debug)] +pub struct PagesExtractionResult { + /// Per-page markdown results. + pub pages: Vec, + /// 1-indexed pages where tables were detected. + pub pages_with_tables: Vec, + /// 1-indexed pages where multi-column layout was detected. + pub pages_with_columns: Vec, + /// 1-indexed pages that need OCR (scanned/image-based). + pub pages_needing_ocr: Vec, + /// True if any page has tables or columns. + pub is_complex: bool, +} + +/// Extract formatted markdown for specific pages of a PDF, with layout +/// classification metadata. /// /// Unlike [`process_pdf_mem`] which returns one concatenated markdown string, /// this returns per-page markdown so callers can mix direct extraction @@ -325,26 +341,34 @@ pub struct PageMarkdown { /// detection thresholds are consistent regardless of which pages are /// requested. Per-page `needs_ocr` is set when the page has GID-encoded /// fonts, encoding issues, or garbage text. +/// +/// Layout complexity (tables, columns) is computed from the full document +/// at near-zero cost since the items/rects/lines are already in memory. pub fn extract_pages_markdown_mem( buffer: &[u8], pages: &[u32], -) -> Result, PdfError> { +) -> Result { validate_pdf_bytes(buffer)?; let (doc, page_count) = load_document_from_mem(buffer)?; let font_cmaps = FontCMaps::from_doc(&doc); // Extract ALL pages to get accurate, document-wide font stats. - let ((all_items, all_rects, _all_lines), page_thresholds, gid_pages) = + let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) = extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?; + // Compute layout complexity from full document (near-zero cost). + let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines); + // Compute font stats from full document (cross-page consistency). let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items); let mut results = Vec::with_capacity(pages.len()); + let mut pages_needing_ocr = Vec::new(); for &page_0idx in pages { // Out-of-range pages → empty + needs_ocr if page_0idx >= page_count { + pages_needing_ocr.push(page_0idx + 1); results.push(PageMarkdown { page: page_0idx, markdown: String::new(), @@ -394,6 +418,10 @@ pub fn extract_pages_markdown_mem( || is_cid_garbage(&md) || detect_encoding_issues(&md); + if needs_ocr { + pages_needing_ocr.push(page_1idx); + } + results.push(PageMarkdown { page: page_0idx, markdown: if needs_ocr { String::new() } else { md }, @@ -401,7 +429,13 @@ pub fn extract_pages_markdown_mem( }); } - Ok(results) + Ok(PagesExtractionResult { + pages: results, + pages_with_tables: complexity.pages_with_tables, + pages_with_columns: complexity.pages_with_columns, + pages_needing_ocr, + is_complex: complexity.is_complex, + }) } // ========================================================================= diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index b112900..9823ec2 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1445,55 +1445,56 @@ fn test_extract_tables_in_regions_nonexistent_page() { #[test] fn test_extract_pages_markdown_basic() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let results = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap(); + let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap(); - assert_eq!(results.len(), 2); - assert_eq!(results[0].page, 0); - assert_eq!(results[1].page, 1); + assert_eq!(result.pages.len(), 2); + assert_eq!(result.pages[0].page, 0); + assert_eq!(result.pages[1].page, 1); // Text-based PDF should produce non-empty markdown - assert!(!results[0].markdown.is_empty()); - assert!(!results[0].needs_ocr); + assert!(!result.pages[0].markdown.is_empty()); + assert!(!result.pages[0].needs_ocr); } #[test] fn test_extract_pages_markdown_page_ordering() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); // Request pages in non-sequential order - let results = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap(); - assert_eq!(results.len(), 2); + assert_eq!(result.pages.len(), 2); // Results should match input order, not document order - assert_eq!(results[0].page, 1); - assert_eq!(results[1].page, 0); + assert_eq!(result.pages[0].page, 1); + assert_eq!(result.pages[1].page, 0); } #[test] fn test_extract_pages_markdown_out_of_range() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let results = extract_pages_markdown_mem(&buf, &[9999]).unwrap(); + let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].page, 9999); - assert!(results[0].markdown.is_empty()); - assert!(results[0].needs_ocr); + assert_eq!(result.pages.len(), 1); + assert_eq!(result.pages[0].page, 9999); + assert!(result.pages[0].markdown.is_empty()); + assert!(result.pages[0].needs_ocr); + assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed } #[test] fn test_extract_pages_markdown_empty_pages_list() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let results = extract_pages_markdown_mem(&buf, &[]).unwrap(); - assert!(results.is_empty()); + let result = extract_pages_markdown_mem(&buf, &[]).unwrap(); + assert!(result.pages.is_empty()); } #[test] fn test_extract_pages_markdown_single_page() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let results = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!(results[0].page, 0); - assert!(!results[0].markdown.is_empty()); - assert!(!results[0].needs_ocr); + assert_eq!(result.pages.len(), 1); + assert_eq!(result.pages[0].page, 0); + assert!(!result.pages[0].markdown.is_empty()); + assert!(!result.pages[0].needs_ocr); } #[test] @@ -1506,10 +1507,55 @@ fn test_extract_pages_markdown_invalid_buffer() { fn test_extract_pages_markdown_gid_pages_need_ocr() { // shinagawa_identity_h.pdf has GID-encoded fonts let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap(); - let results = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); - assert_eq!(results.len(), 1); - assert!(results[0].needs_ocr); + assert_eq!(result.pages.len(), 1); + assert!(result.pages[0].needs_ocr); + assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed +} + +#[test] +fn test_extract_pages_markdown_classification_with_tables() { + // nexo-price-en.pdf is known to have tables + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let page_count = process_pdf_mem(&buf).unwrap().page_count; + let page_indices: Vec = (0..page_count).collect(); + let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + + assert!( + !result.pages_with_tables.is_empty(), + "nexo-price-en.pdf should have pages with tables" + ); + assert!(result.is_complex); +} + +#[test] +fn test_extract_pages_markdown_simple_pdf_no_complexity() { + // bare_name_struct.pdf is a simple document with a heading and code block + let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap(); + let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + + assert!(result.pages_with_tables.is_empty()); + assert!(result.pages_with_columns.is_empty()); + assert!(!result.is_complex); +} + +#[test] +fn test_extract_pages_markdown_classification_matches_process_pdf() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let full = process_pdf_mem(&buf).unwrap(); + let page_count = full.page_count; + let page_indices: Vec = (0..page_count).collect(); + let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + + assert_eq!( + result.pages_with_tables, full.layout.pages_with_tables, + "pages_with_tables should match process_pdf" + ); + assert_eq!( + result.pages_with_columns, full.layout.pages_with_columns, + "pages_with_columns should match process_pdf" + ); } #[test] @@ -1523,12 +1569,13 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() { // Get per-page output for all pages let page_count = full.page_count; let page_indices: Vec = (0..page_count).collect(); - let per_page = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); // Concatenated per-page markdown should contain substantial overlap with // the full output (exact match not expected due to header/footer stripping // and cross-page paragraph merging differences) - let concat: String = per_page + let concat: String = result + .pages .iter() .map(|p| p.markdown.as_str()) .collect::>()