extractPagesMarkdown: return classification metadata (0.7.0) (#32)

Combine per-page markdown extraction with layout classification into a
single parse. extractPagesMarkdown now returns PagesExtractionResult with
pages_with_tables, pages_with_columns, pages_needing_ocr, and is_complex
alongside the per-page markdown — eliminating redundant PDF parses for
callers that need both.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-pdf-inspector",
-  "version": "0.4.3",
+  "version": "0.7.0",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -5,6 +5,28 @@ use napi_derive::napi;
 use std::collections::HashSet;
 use std::panic;
 
+// ---------------------------------------------------------------------------
+// Enums
+// ---------------------------------------------------------------------------
+
+/// PDF document type classification.
+#[napi(string_enum)]
+pub enum PdfType {
+    TextBased,
+    Scanned,
+    ImageBased,
+    Mixed,
+}
+
+/// Type of a positioned text item.
+#[napi(string_enum)]
+pub enum ItemType {
+    Text,
+    Image,
+    Link,
+    FormField,
+}
+
 // ---------------------------------------------------------------------------
 // Result types
 // ---------------------------------------------------------------------------
@@ -12,7 +34,7 @@ use std::panic;
 /// Full PDF processing result with markdown and metadata.
 #[napi(object)]
 pub struct PdfResult {
-    pub pdf_type: String,
+    pub pdf_type: PdfType,
     pub markdown: Option<String>,
     pub page_count: u32,
     pub processing_time_ms: u32,
@@ -29,7 +51,7 @@ pub struct PdfResult {
 /// Lightweight PDF classification result.
 #[napi(object)]
 pub struct PdfClassification {
-    pub pdf_type: String,
+    pub pdf_type: PdfType,
     pub page_count: u32,
     /// 0-indexed page numbers that need OCR.
     pub pages_needing_ocr: Vec<u32>,
@@ -49,7 +71,9 @@ pub struct TextItem {
     pub page: u32,
     pub is_bold: bool,
     pub is_italic: bool,
-    pub item_type: String,
+    pub item_type: ItemType,
+    /// URL for link items, `None` for other types.
+    pub link_url: Option<String>,
 }
 
 /// A page's regions for text extraction: (page_index_0based, bboxes).
@@ -79,18 +103,18 @@ pub struct PageRegionTexts {
 // Helpers
 // ---------------------------------------------------------------------------
 
-fn pdf_type_string(t: pdf_inspector::PdfType) -> String {
+fn convert_pdf_type(t: pdf_inspector::PdfType) -> PdfType {
     match t {
-        pdf_inspector::PdfType::TextBased => "TextBased".to_string(),
+        pdf_inspector::PdfType::TextBased => PdfType::TextBased,
-        pdf_inspector::PdfType::Scanned => "Scanned".to_string(),
+        pdf_inspector::PdfType::Scanned => PdfType::Scanned,
-        pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(),
+        pdf_inspector::PdfType::ImageBased => PdfType::ImageBased,
-        pdf_inspector::PdfType::Mixed => "Mixed".to_string(),
+        pdf_inspector::PdfType::Mixed => PdfType::Mixed,
     }
 }
 
 fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
     PdfResult {
-        pdf_type: pdf_type_string(r.pdf_type),
+        pdf_type: convert_pdf_type(r.pdf_type),
         markdown: r.markdown,
         page_count: r.page_count,
         processing_time_ms: r.processing_time_ms as u32,
@@ -104,12 +128,12 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
     }
 }
 
-fn item_type_string(t: &pdf_inspector::types::ItemType) -> String {
+fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
     match t {
-        pdf_inspector::types::ItemType::Text => "text".into(),
+        pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
-        pdf_inspector::types::ItemType::Image => "image".into(),
+        pdf_inspector::types::ItemType::Image => (ItemType::Image, None),
-        pdf_inspector::types::ItemType::Link(url) => format!("link:{url}"),
+        pdf_inspector::types::ItemType::Link(url) => (ItemType::Link, Some(url.clone())),
-        pdf_inspector::types::ItemType::FormField => "form_field".into(),
+        pdf_inspector::types::ItemType::FormField => (ItemType::FormField, None),
     }
 }
 
@@ -181,7 +205,7 @@ pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
         let result =
             pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
         Ok(PdfClassification {
-            pdf_type: pdf_type_string(result.pdf_type),
+            pdf_type: convert_pdf_type(result.pdf_type),
             page_count: result.page_count,
             pages_needing_ocr: result.pages_needing_ocr,
             confidence: result.confidence as f64,
@@ -222,7 +246,9 @@ pub fn extract_text_with_positions(
 
         Ok(items
             .into_iter()
-            .map(|item| TextItem {
+            .map(|item| {
+                let (item_type, link_url) = convert_item_type(&item.item_type);
+                TextItem {
                     text: item.text,
                     x: item.x as f64,
                     y: item.y as f64,
@@ -233,7 +259,9 @@ pub fn extract_text_with_positions(
                     page: item.page,
                     is_bold: item.is_bold,
                     is_italic: item.is_italic,
-                item_type: item_type_string(&item.item_type),
+                    item_type,
+                    link_url,
+                }
             })
             .collect())
     })
@@ -289,6 +317,65 @@ pub fn extract_tables_in_regions(
     })
 }
 
+/// Per-page markdown extraction result.
+#[napi(object)]
+pub struct PageMarkdownResult {
+    /// 0-indexed page number.
+    pub page: u32,
+    /// Formatted markdown for this page.
+    pub markdown: String,
+    /// `true` when text on this page is unreliable.
+    pub needs_ocr: bool,
+}
+
+/// Combined per-page markdown extraction and layout classification result.
+#[napi(object)]
+pub struct PagesExtractionResult {
+    /// Per-page markdown results.
+    pub pages: Vec<PageMarkdownResult>,
+    /// 1-indexed pages where tables were detected.
+    pub pages_with_tables: Vec<u32>,
+    /// 1-indexed pages where multi-column layout was detected.
+    pub pages_with_columns: Vec<u32>,
+    /// 1-indexed pages that need OCR (scanned/image-based).
+    pub pages_needing_ocr: Vec<u32>,
+    /// True if any page has tables or columns.
+    pub is_complex: bool,
+}
+
+/// Extract formatted markdown for specific pages of a PDF, with layout
+/// classification metadata.
+///
+/// Returns per-page markdown and classification data (tables, columns,
+/// OCR needs) from a single parse. Font statistics are computed from the
+/// full document so header detection is consistent across pages.
+#[napi]
+pub fn extract_pages_markdown(
+    buffer: Buffer,
+    pages: Vec<u32>,
+) -> Result<PagesExtractionResult> {
+    let bytes: Vec<u8> = buffer.to_vec();
+    catch_panic("extract_pages_markdown", move || {
+        let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
+            .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
+        Ok(PagesExtractionResult {
+            pages: result
+                .pages
+                .into_iter()
+                .map(|r| PageMarkdownResult {
+                    page: r.page,
+                    markdown: r.markdown,
+                    needs_ocr: r.needs_ocr,
+                })
+                .collect(),
+            pages_with_tables: result.pages_with_tables,
+            pages_with_columns: result.pages_with_columns,
+            pages_needing_ocr: result.pages_needing_ocr,
+            is_complex: result.is_complex,
+        })
+    })
+}
+
 fn parse_page_regions(page_regions: &[PageRegions]) -> Vec<(u32, Vec<[f32; 4]>)> {
     page_regions
         .iter()

src/lib.rs

@@ -299,6 +299,149 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result<PdfClassification, PdfError> {
     })
 }
 
+// =========================================================================
+// Per-page markdown extraction
+// =========================================================================
+
+/// Per-page markdown extraction result.
+#[derive(Debug)]
+pub struct PageMarkdown {
+    /// 0-indexed page number.
+    pub page: u32,
+    /// Formatted markdown for this page.
+    pub markdown: String,
+    /// `true` when text on this page is unreliable (GID-encoded fonts,
+    /// encoding issues, garbage text, or empty extraction).
+    pub needs_ocr: bool,
+}
+
+/// Combined per-page markdown extraction and layout classification result.
+#[derive(Debug)]
+pub struct PagesExtractionResult {
+    /// Per-page markdown results.
+    pub pages: Vec<PageMarkdown>,
+    /// 1-indexed pages where tables were detected.
+    pub pages_with_tables: Vec<u32>,
+    /// 1-indexed pages where multi-column layout was detected.
+    pub pages_with_columns: Vec<u32>,
+    /// 1-indexed pages that need OCR (scanned/image-based).
+    pub pages_needing_ocr: Vec<u32>,
+    /// True if any page has tables or columns.
+    pub is_complex: bool,
+}
+
+/// Extract formatted markdown for specific pages of a PDF, with layout
+/// classification metadata.
+///
+/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
+/// this returns per-page markdown so callers can mix direct extraction
+/// (for simple text pages) with GPU OCR (for complex/scanned pages).
+///
+/// Font statistics are computed from the full document so header
+/// detection thresholds are consistent regardless of which pages are
+/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
+/// fonts, encoding issues, or garbage text.
+///
+/// Layout complexity (tables, columns) is computed from the full document
+/// at near-zero cost since the items/rects/lines are already in memory.
+pub fn extract_pages_markdown_mem(
+    buffer: &[u8],
+    pages: &[u32],
+) -> Result<PagesExtractionResult, PdfError> {
+    validate_pdf_bytes(buffer)?;
+    let (doc, page_count) = load_document_from_mem(buffer)?;
+    let font_cmaps = FontCMaps::from_doc(&doc);
+
+    // Extract ALL pages to get accurate, document-wide font stats.
+    let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
+        extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
+
+    // Compute layout complexity from full document (near-zero cost).
+    let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
+
+    // Compute font stats from full document (cross-page consistency).
+    let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
+
+    let mut results = Vec::with_capacity(pages.len());
+    let mut pages_needing_ocr = Vec::new();
+
+    for &page_0idx in pages {
+        // Out-of-range pages → empty + needs_ocr
+        if page_0idx >= page_count {
+            pages_needing_ocr.push(page_0idx + 1);
+            results.push(PageMarkdown {
+                page: page_0idx,
+                markdown: String::new(),
+                needs_ocr: true,
+            });
+            continue;
+        }
+
+        let page_1idx = page_0idx + 1;
+
+        // Filter items/rects for this page only
+        let page_items: Vec<TextItem> = all_items
+            .iter()
+            .filter(|i| i.page == page_1idx)
+            .cloned()
+            .collect();
+
+        let page_rects: Vec<PdfRect> = all_rects
+            .iter()
+            .filter(|r| r.page == page_1idx)
+            .cloned()
+            .collect();
+
+        let has_gid = gid_pages.contains(&page_1idx);
+
+        // Build markdown with document-wide font stats
+        let options = MarkdownOptions {
+            base_font_size: Some(font_stats.most_common_size),
+            include_page_numbers: false,
+            strip_headers_footers: false,
+            ..MarkdownOptions::default()
+        };
+
+        let md = markdown::to_markdown_from_items_with_rects_and_lines(
+            page_items,
+            options,
+            &page_rects,
+            &[],
+            &page_thresholds,
+            None,
+            &[],
+        );
+
+        let needs_ocr = md.trim().is_empty()
+            || has_gid
+            || is_garbage_text(&md)
+            || is_cid_garbage(&md)
+            || detect_encoding_issues(&md);
+
+        if needs_ocr {
+            pages_needing_ocr.push(page_1idx);
+        }
+
+        results.push(PageMarkdown {
+            page: page_0idx,
+            markdown: if needs_ocr { String::new() } else { md },
+            needs_ocr,
+        });
+    }
+
+    Ok(PagesExtractionResult {
+        pages: results,
+        pages_with_tables: complexity.pages_with_tables,
+        pages_with_columns: complexity.pages_with_columns,
+        pages_needing_ocr,
+        is_complex: complexity.is_complex,
+    })
+}
+
+// =========================================================================
+// Region-based text extraction (for hybrid OCR pipelines)
+// =========================================================================
+
 /// Result for a single region's text extraction.
 #[derive(Debug)]
 pub struct RegionText {

tests/integration_tests.rs

@@ -4,9 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
 use pdf_inspector::extractor::group_into_lines;
 use pdf_inspector::types::TextLine;
 use pdf_inspector::{
-    detect_pdf_type, extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
+    detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
-    extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
+    extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
-    MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
+    process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
+    TextItem,
 };
 use std::collections::HashSet;
 
@@ -1436,3 +1437,160 @@ fn test_extract_tables_in_regions_nonexistent_page() {
     assert!(region.needs_ocr);
     assert!(region.text.is_empty());
 }
+
+// =========================================================================
+// extract_pages_markdown_mem tests
+// =========================================================================
+
+#[test]
+fn test_extract_pages_markdown_basic() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
+
+    assert_eq!(result.pages.len(), 2);
+    assert_eq!(result.pages[0].page, 0);
+    assert_eq!(result.pages[1].page, 1);
+    // Text-based PDF should produce non-empty markdown
+    assert!(!result.pages[0].markdown.is_empty());
+    assert!(!result.pages[0].needs_ocr);
+}
+
+#[test]
+fn test_extract_pages_markdown_page_ordering() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    // Request pages in non-sequential order
+    let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
+
+    assert_eq!(result.pages.len(), 2);
+    // Results should match input order, not document order
+    assert_eq!(result.pages[0].page, 1);
+    assert_eq!(result.pages[1].page, 0);
+}
+
+#[test]
+fn test_extract_pages_markdown_out_of_range() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert_eq!(result.pages[0].page, 9999);
+    assert!(result.pages[0].markdown.is_empty());
+    assert!(result.pages[0].needs_ocr);
+    assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed
+}
+
+#[test]
+fn test_extract_pages_markdown_empty_pages_list() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
+    assert!(result.pages.is_empty());
+}
+
+#[test]
+fn test_extract_pages_markdown_single_page() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert_eq!(result.pages[0].page, 0);
+    assert!(!result.pages[0].markdown.is_empty());
+    assert!(!result.pages[0].needs_ocr);
+}
+
+#[test]
+fn test_extract_pages_markdown_invalid_buffer() {
+    let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
+    assert!(result.is_err());
+}
+
+#[test]
+fn test_extract_pages_markdown_gid_pages_need_ocr() {
+    // shinagawa_identity_h.pdf has GID-encoded fonts
+    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert!(result.pages[0].needs_ocr);
+    assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
+}
+
+#[test]
+fn test_extract_pages_markdown_classification_with_tables() {
+    // nexo-price-en.pdf is known to have tables
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let page_count = process_pdf_mem(&buf).unwrap().page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    assert!(
+        !result.pages_with_tables.is_empty(),
+        "nexo-price-en.pdf should have pages with tables"
+    );
+    assert!(result.is_complex);
+}
+
+#[test]
+fn test_extract_pages_markdown_simple_pdf_no_complexity() {
+    // bare_name_struct.pdf is a simple document with a heading and code block
+    let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert!(result.pages_with_tables.is_empty());
+    assert!(result.pages_with_columns.is_empty());
+    assert!(!result.is_complex);
+}
+
+#[test]
+fn test_extract_pages_markdown_classification_matches_process_pdf() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let full = process_pdf_mem(&buf).unwrap();
+    let page_count = full.page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    assert_eq!(
+        result.pages_with_tables, full.layout.pages_with_tables,
+        "pages_with_tables should match process_pdf"
+    );
+    assert_eq!(
+        result.pages_with_columns, full.layout.pages_with_columns,
+        "pages_with_columns should match process_pdf"
+    );
+}
+
+#[test]
+fn test_extract_pages_markdown_consistency_with_process_pdf() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+
+    // Get full process_pdf output
+    let full = process_pdf_mem(&buf).unwrap();
+    let full_md = full.markdown.unwrap_or_default();
+
+    // Get per-page output for all pages
+    let page_count = full.page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    // Concatenated per-page markdown should contain substantial overlap with
+    // the full output (exact match not expected due to header/footer stripping
+    // and cross-page paragraph merging differences)
+    let concat: String = result
+        .pages
+        .iter()
+        .map(|p| p.markdown.as_str())
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    // Both should be non-empty for a text-based PDF
+    assert!(!full_md.is_empty());
+    assert!(!concat.is_empty());
+
+    // The per-page version should contain at least 50% of the full content's
+    // length (accounting for header/footer stripping differences)
+    assert!(
+        concat.len() * 2 >= full_md.len(),
+        "per-page concat ({} chars) is too short vs full ({} chars)",
+        concat.len(),
+        full_md.len()
+    );
+}