From abb0b925fbc2f3621cfc7905283b92261717c775 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 14 Apr 2026 12:37:32 -0700 Subject: [PATCH] Add extractPagesMarkdown for per-page markdown extraction (#31) * add extract_pages_markdown_mem for per-page markdown extraction Enables hybrid OCR pipelines to skip GPU render+layout for simple text pages by providing per-page markdown with needs_ocr flags. Font stats are computed document-wide for consistent header detection. Co-Authored-By: Claude Opus 4.6 (1M context) * bump napi package version to 0.6.0 Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- napi/package.json | 2 +- napi/src/lib.rs | 35 +++++++++++ src/lib.rs | 109 ++++++++++++++++++++++++++++++++++ tests/integration_tests.rs | 117 ++++++++++++++++++++++++++++++++++++- 4 files changed, 259 insertions(+), 4 deletions(-) diff --git a/napi/package.json b/napi/package.json index c5b7fb8..79279c9 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-pdf-inspector", - "version": "0.5.0", + "version": "0.6.0", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 5f824c6..c07933a 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -317,6 +317,41 @@ pub fn extract_tables_in_regions( }) } +/// Per-page markdown extraction result. +#[napi(object)] +pub struct PageMarkdownResult { + /// 0-indexed page number. + pub page: u32, + /// Formatted markdown for this page. + pub markdown: String, + /// `true` when text on this page is unreliable. + pub needs_ocr: bool, +} + +/// Extract formatted markdown for specific pages of a PDF. +/// +/// Returns per-page markdown so callers can mix direct extraction +/// (for simple text pages) with GPU OCR (for complex/scanned pages). +/// +/// Font statistics are computed from the full document so header +/// detection is consistent across pages. +#[napi] +pub fn extract_pages_markdown(buffer: Buffer, pages: Vec) -> Result> { + let bytes: Vec = buffer.to_vec(); + catch_panic("extract_pages_markdown", move || { + let results = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages) + .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?; + Ok(results + .into_iter() + .map(|r| PageMarkdownResult { + page: r.page, + markdown: r.markdown, + needs_ocr: r.needs_ocr, + }) + .collect()) + }) +} + fn parse_page_regions(page_regions: &[PageRegions]) -> Vec<(u32, Vec<[f32; 4]>)> { page_regions .iter() diff --git a/src/lib.rs b/src/lib.rs index 7356971..df98fbd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -299,6 +299,115 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result { }) } +// ========================================================================= +// Per-page markdown extraction +// ========================================================================= + +/// Per-page markdown extraction result. +#[derive(Debug)] +pub struct PageMarkdown { + /// 0-indexed page number. + pub page: u32, + /// Formatted markdown for this page. + pub markdown: String, + /// `true` when text on this page is unreliable (GID-encoded fonts, + /// encoding issues, garbage text, or empty extraction). + pub needs_ocr: bool, +} + +/// Extract formatted markdown for specific pages of a PDF. +/// +/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string, +/// this returns per-page markdown so callers can mix direct extraction +/// (for simple text pages) with GPU OCR (for complex/scanned pages). +/// +/// Font statistics are computed from the full document so header +/// detection thresholds are consistent regardless of which pages are +/// requested. Per-page `needs_ocr` is set when the page has GID-encoded +/// fonts, encoding issues, or garbage text. +pub fn extract_pages_markdown_mem( + buffer: &[u8], + pages: &[u32], +) -> Result, PdfError> { + validate_pdf_bytes(buffer)?; + let (doc, page_count) = load_document_from_mem(buffer)?; + let font_cmaps = FontCMaps::from_doc(&doc); + + // Extract ALL pages to get accurate, document-wide font stats. + let ((all_items, all_rects, _all_lines), page_thresholds, gid_pages) = + extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?; + + // Compute font stats from full document (cross-page consistency). + let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items); + + let mut results = Vec::with_capacity(pages.len()); + + for &page_0idx in pages { + // Out-of-range pages → empty + needs_ocr + if page_0idx >= page_count { + results.push(PageMarkdown { + page: page_0idx, + markdown: String::new(), + needs_ocr: true, + }); + continue; + } + + let page_1idx = page_0idx + 1; + + // Filter items/rects for this page only + let page_items: Vec = all_items + .iter() + .filter(|i| i.page == page_1idx) + .cloned() + .collect(); + + let page_rects: Vec = all_rects + .iter() + .filter(|r| r.page == page_1idx) + .cloned() + .collect(); + + let has_gid = gid_pages.contains(&page_1idx); + + // Build markdown with document-wide font stats + let options = MarkdownOptions { + base_font_size: Some(font_stats.most_common_size), + include_page_numbers: false, + strip_headers_footers: false, + ..MarkdownOptions::default() + }; + + let md = markdown::to_markdown_from_items_with_rects_and_lines( + page_items, + options, + &page_rects, + &[], + &page_thresholds, + None, + &[], + ); + + let needs_ocr = md.trim().is_empty() + || has_gid + || is_garbage_text(&md) + || is_cid_garbage(&md) + || detect_encoding_issues(&md); + + results.push(PageMarkdown { + page: page_0idx, + markdown: if needs_ocr { String::new() } else { md }, + needs_ocr, + }); + } + + Ok(results) +} + +// ========================================================================= +// Region-based text extraction (for hybrid OCR pipelines) +// ========================================================================= + /// Result for a single region's text extraction. #[derive(Debug)] pub struct RegionText { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 5ba7d81..b112900 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -4,9 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy}; use pdf_inspector::extractor::group_into_lines; use pdf_inspector::types::TextLine; use pdf_inspector::{ - detect_pdf_type, extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem, - extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown, - MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem, + detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text, + extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem, + process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType, + TextItem, }; use std::collections::HashSet; @@ -1436,3 +1437,113 @@ fn test_extract_tables_in_regions_nonexistent_page() { assert!(region.needs_ocr); assert!(region.text.is_empty()); } + +// ========================================================================= +// extract_pages_markdown_mem tests +// ========================================================================= + +#[test] +fn test_extract_pages_markdown_basic() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let results = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap(); + + assert_eq!(results.len(), 2); + assert_eq!(results[0].page, 0); + assert_eq!(results[1].page, 1); + // Text-based PDF should produce non-empty markdown + assert!(!results[0].markdown.is_empty()); + assert!(!results[0].needs_ocr); +} + +#[test] +fn test_extract_pages_markdown_page_ordering() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + // Request pages in non-sequential order + let results = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap(); + + assert_eq!(results.len(), 2); + // Results should match input order, not document order + assert_eq!(results[0].page, 1); + assert_eq!(results[1].page, 0); +} + +#[test] +fn test_extract_pages_markdown_out_of_range() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let results = extract_pages_markdown_mem(&buf, &[9999]).unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].page, 9999); + assert!(results[0].markdown.is_empty()); + assert!(results[0].needs_ocr); +} + +#[test] +fn test_extract_pages_markdown_empty_pages_list() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let results = extract_pages_markdown_mem(&buf, &[]).unwrap(); + assert!(results.is_empty()); +} + +#[test] +fn test_extract_pages_markdown_single_page() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let results = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].page, 0); + assert!(!results[0].markdown.is_empty()); + assert!(!results[0].needs_ocr); +} + +#[test] +fn test_extract_pages_markdown_invalid_buffer() { + let result = extract_pages_markdown_mem(b"not a pdf", &[0]); + assert!(result.is_err()); +} + +#[test] +fn test_extract_pages_markdown_gid_pages_need_ocr() { + // shinagawa_identity_h.pdf has GID-encoded fonts + let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap(); + let results = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + + assert_eq!(results.len(), 1); + assert!(results[0].needs_ocr); +} + +#[test] +fn test_extract_pages_markdown_consistency_with_process_pdf() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + + // Get full process_pdf output + let full = process_pdf_mem(&buf).unwrap(); + let full_md = full.markdown.unwrap_or_default(); + + // Get per-page output for all pages + let page_count = full.page_count; + let page_indices: Vec = (0..page_count).collect(); + let per_page = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + + // Concatenated per-page markdown should contain substantial overlap with + // the full output (exact match not expected due to header/footer stripping + // and cross-page paragraph merging differences) + let concat: String = per_page + .iter() + .map(|p| p.markdown.as_str()) + .collect::>() + .join("\n"); + + // Both should be non-empty for a text-based PDF + assert!(!full_md.is_empty()); + assert!(!concat.is_empty()); + + // The per-page version should contain at least 50% of the full content's + // length (accounting for header/footer stripping differences) + assert!( + concat.len() * 2 >= full_md.len(), + "per-page concat ({} chars) is too short vs full ({} chars)", + concat.len(), + full_md.len() + ); +}