From 4b5ae91f54147ba2aca04c3571a129a337d7664d Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 20 Apr 2026 17:19:03 -0700 Subject: [PATCH] feat: expose per-page markdown extraction to Python and Node (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: expose per-page markdown extraction to Python and Node (#49) Implements the feature requested in issue #49: a list-of-pages markdown output from the Python API. Matching the existing project pattern, the feature lives in the Rust core and is surfaced through every binding. - Rust core: `extract_pages_markdown` (path) and `extract_pages_markdown_mem` (bytes) now take `Option<&[u32]>` — `None` returns every page in document order; a slice restricts and preserves caller order. - Python: new `extract_pages_markdown(path, pages=None)` and `extract_pages_markdown_bytes(data, pages=None)` functions plus `PageMarkdown` / `PagesExtractionResult` classes; stub file updated. - Node: `extractPagesMarkdown(buffer, pages?)` — `pages` is now optional. - Tests: 2 new Rust integration tests, 9 new Python tests, 2 new Node assertions. All 372 unit + 107 integration + 53 Python tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: bump version from 1.3.0 to 1.4.0 Minor bump for the new per-page markdown extraction API exposed through the Python and Node bindings. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .gitignore | 5 ++ docs/python.md | 14 +++++ docs/rust-api.md | 25 ++++++++ napi/package.json | 2 +- napi/src/lib.rs | 15 +++-- napi/test.mjs | 23 +++++++ pdf_inspector.pyi | 50 +++++++++++++++ src/lib.rs | 36 +++++++++-- src/python.rs | 121 +++++++++++++++++++++++++++++++++++++ tests/integration_tests.rs | 68 ++++++++++++++++----- tests/test_python.py | 72 ++++++++++++++++++++++ 11 files changed, 406 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 0c09176..bb2e7af 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,8 @@ scripts/ # Test output test_output/ +# Python +__pycache__/ +*.pyc +.pytest_cache/ + diff --git a/docs/python.md b/docs/python.md index 8416460..3affc6b 100644 --- a/docs/python.md +++ b/docs/python.md @@ -42,6 +42,14 @@ text = pdf_inspector.extract_text("document.pdf") items = pdf_inspector.extract_text_with_positions("document.pdf") for item in items[:5]: print(f"'{item.text}' at ({item.x:.0f}, {item.y:.0f}) size={item.font_size}") + +# Per-page markdown (one Markdown string per page, plus layout metadata) +result = pdf_inspector.extract_pages_markdown("document.pdf") +for page in result.pages: + print(f"Page {page.page}: {len(page.markdown)} chars, needs_ocr={page.needs_ocr}") + +# Restrict to specific 0-indexed pages (preserves caller order) +result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2]) ``` ## API reference @@ -60,6 +68,8 @@ for item in items[:5]: | `extract_text_with_positions_bytes(data, pages=None)` | Text with positions from bytes | | `extract_text_in_regions(path, page_regions)` | Extract text in bounding-box regions | | `extract_text_in_regions_bytes(data, page_regions)` | Region extraction from bytes | +| `extract_pages_markdown(path, pages=None)` | Per-page Markdown + layout metadata (all pages by default) | +| `extract_pages_markdown_bytes(data, pages=None)` | Per-page Markdown from bytes | ## Types @@ -72,3 +82,7 @@ for item in items[:5]: **`RegionText` fields:** `text`, `needs_ocr` **`PageRegionTexts` fields:** `page` (0-indexed), `regions` (list of RegionText) + +**`PageMarkdown` fields:** `page` (0-indexed), `markdown`, `needs_ocr` + +**`PagesExtractionResult` fields:** `pages` (list of PageMarkdown), `pages_with_tables` (1-indexed), `pages_with_columns` (1-indexed), `pages_needing_ocr` (1-indexed), `is_complex` diff --git a/docs/rust-api.md b/docs/rust-api.md index 02eec00..68404af 100644 --- a/docs/rust-api.md +++ b/docs/rust-api.md @@ -79,6 +79,27 @@ let bytes = std::fs::read("document.pdf")?; let result = process_pdf_mem(&bytes)?; ``` +Extract per-page Markdown (one string per page, plus document-wide layout +metadata): + +```rust +use pdf_inspector::extract_pages_markdown; + +// Pass `None` for every page in document order, or a slice of 0-indexed +// pages to restrict the output (caller-supplied order is preserved). +let result = extract_pages_markdown("document.pdf", None)?; + +for page in &result.pages { + if page.needs_ocr { + // Route this page to OCR + } else { + println!("Page {}: {}", page.page, page.markdown); + } +} + +println!("Complex layout? {}", result.is_complex); +``` + ## Processing modes | Mode | What it does | Returns | @@ -102,6 +123,8 @@ let result = process_pdf_mem(&bytes)?; | `to_markdown(text, options)` | Convert plain text to Markdown | | `to_markdown_from_items(items, options)` | Markdown from pre-extracted `TextItem`s | | `to_markdown_from_items_with_rects(items, options, rects)` | Markdown with rectangle-based table detection | +| `extract_pages_markdown(path, pages)` | Per-page Markdown + layout metadata (file) | +| `extract_pages_markdown_mem(bytes, pages)` | Per-page Markdown from bytes | Low-level detection functions are also available via the `detector` module (`detect_pdf_type`, `detect_pdf_type_with_config`, etc.) for callers who need `PdfTypeResult` instead of `PdfProcessResult`. @@ -119,4 +142,6 @@ Low-level detection functions are also available via the `detector` module (`det | `LayoutComplexity` | Layout analysis: is_complex, pages_with_tables, pages_with_columns | | `TextItem` | Text with position, font info, and page number | | `MarkdownOptions` | Configuration for Markdown formatting (page numbers, etc.) | +| `PageMarkdown` | Per-page result: page (0-indexed), markdown, needs_ocr | +| `PagesExtractionResult` | Per-page output + 1-indexed pages_with_tables / pages_with_columns / pages_needing_ocr, is_complex | | `PdfError` | `Io`, `Parse`, `Encrypted`, `InvalidStructure`, `NotAPdf` | diff --git a/napi/package.json b/napi/package.json index eee532b..38ef201 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.3.0", + "version": "1.4.0", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 5689b5d..0a9a2e6 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -343,21 +343,26 @@ pub struct PagesExtractionResult { pub is_complex: bool, } -/// Extract formatted markdown for specific pages of a PDF, with layout -/// classification metadata. +/// Extract formatted markdown for pages of a PDF, with layout classification +/// metadata. /// /// Returns per-page markdown and classification data (tables, columns, /// OCR needs) from a single parse. Font statistics are computed from the /// full document so header detection is consistent across pages. +/// +/// Omit `pages` (or pass `undefined`) to return every page in document +/// order. Pass an array of 0-indexed page numbers to restrict output to +/// those pages, in caller-supplied order. #[napi] pub fn extract_pages_markdown( buffer: Buffer, - pages: Vec, + pages: Option>, ) -> Result { let bytes: Vec = buffer.to_vec(); catch_panic("extract_pages_markdown", move || { - let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages) - .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?; + let result = + pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref()) + .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?; Ok(PagesExtractionResult { pages: result .pages diff --git a/napi/test.mjs b/napi/test.mjs index 64f1490..f6946b2 100644 --- a/napi/test.mjs +++ b/napi/test.mjs @@ -7,6 +7,7 @@ import { extractText, extractTextWithPositions, extractTextInRegions, + extractPagesMarkdown, } from './index.js'; const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf'); @@ -89,6 +90,28 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string'); assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean'); console.log(' extractTextInRegions: OK'); +// --- extractPagesMarkdown --- +console.log('Testing extractPagesMarkdown...'); + +// omit pages → every page in document order +const allPages = extractPagesMarkdown(fixture); +assert.equal(allPages.pages.length, 3); +assert.deepEqual(allPages.pages.map(p => p.page), [0, 1, 2]); +assert.ok(typeof allPages.pages[0].markdown === 'string'); +assert.equal(typeof allPages.pages[0].needsOcr, 'boolean'); +assert.ok(Array.isArray(allPages.pagesWithTables)); +assert.ok(Array.isArray(allPages.pagesWithColumns)); +assert.ok(Array.isArray(allPages.pagesNeedingOcr)); +assert.equal(typeof allPages.isComplex, 'boolean'); +console.log(' extractPagesMarkdown (no pages arg): OK'); + +// selected pages preserve caller order +const picked = extractPagesMarkdown(fixture, [2, 0]); +assert.equal(picked.pages.length, 2); +assert.equal(picked.pages[0].page, 2); +assert.equal(picked.pages[1].page, 0); +console.log(' extractPagesMarkdown with pages: OK'); + // --- Error handling --- console.log('Testing error handling...'); assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/); diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index 041e321..fa89917 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -52,6 +52,28 @@ class PageRegionTexts: """0-indexed page number.""" regions: list[RegionText] +class PageMarkdown: + """Per-page markdown extraction result.""" + page: int + """0-indexed page number.""" + markdown: str + """Formatted markdown for this page (empty string when needs_ocr is True).""" + needs_ocr: bool + """True when text on this page is unreliable and OCR should be used instead.""" + +class PagesExtractionResult: + """Per-page markdown output with document-wide layout classification.""" + pages: list[PageMarkdown] + """Per-page markdown results, in the order requested.""" + pages_with_tables: list[int] + """1-indexed pages where tables were detected.""" + pages_with_columns: list[int] + """1-indexed pages where multi-column layout was detected.""" + pages_needing_ocr: list[int] + """1-indexed pages that need OCR.""" + is_complex: bool + """True if any page has tables or multi-column layout.""" + def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult: """Process a PDF: detect type, extract text, convert to Markdown.""" ... @@ -115,3 +137,31 @@ def extract_text_in_regions_bytes( page_regions: List of (page_0indexed, [[x1, y1, x2, y2], ...]) tuples. """ ... + +def extract_pages_markdown( + path: str, + pages: Optional[list[int]] = None, +) -> PagesExtractionResult: + """Extract formatted markdown for pages of a PDF, with layout classification. + + Args: + path: Path to the PDF file. + pages: Optional list of 0-indexed pages. When ``None`` (default), every + page is returned in document order. Otherwise, output matches the + caller-supplied order. + + Returns: + PagesExtractionResult with per-page markdown and document-wide layout + classification (tables, columns, OCR needs). + """ + ... + +def extract_pages_markdown_bytes( + data: bytes, + pages: Optional[list[int]] = None, +) -> PagesExtractionResult: + """Extract formatted markdown for pages of a PDF from bytes. + + See :func:`extract_pages_markdown` for details. + """ + ... diff --git a/src/lib.rs b/src/lib.rs index 2006f6f..5fcafeb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -336,13 +336,17 @@ pub struct PagesExtractionResult { pub is_complex: bool, } -/// Extract formatted markdown for specific pages of a PDF, with layout +/// Extract formatted markdown for pages of a PDF, with layout /// classification metadata. /// /// Unlike [`process_pdf_mem`] which returns one concatenated markdown string, /// this returns per-page markdown so callers can mix direct extraction /// (for simple text pages) with GPU OCR (for complex/scanned pages). /// +/// When `pages` is `None`, every page (0-indexed, in document order) is +/// returned. When `Some(&[...])`, only the listed 0-indexed pages are +/// returned, in the caller's order. +/// /// Font statistics are computed from the full document so header /// detection thresholds are consistent regardless of which pages are /// requested. Per-page `needs_ocr` is set when the page has GID-encoded @@ -352,7 +356,7 @@ pub struct PagesExtractionResult { /// at near-zero cost since the items/rects/lines are already in memory. pub fn extract_pages_markdown_mem( buffer: &[u8], - pages: &[u32], + pages: Option<&[u32]>, ) -> Result { validate_pdf_bytes(buffer)?; let (doc, page_count) = load_document_from_mem(buffer)?; @@ -368,10 +372,20 @@ pub fn extract_pages_markdown_mem( // Compute font stats from full document (cross-page consistency). let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items); - let mut results = Vec::with_capacity(pages.len()); + // When caller doesn't specify pages, return every page in document order. + let all_pages: Vec; + let pages_slice: &[u32] = match pages { + Some(p) => p, + None => { + all_pages = (0..page_count).collect(); + &all_pages + } + }; + + let mut results = Vec::with_capacity(pages_slice.len()); let mut pages_needing_ocr = Vec::new(); - for &page_0idx in pages { + for &page_0idx in pages_slice { // Out-of-range pages → empty + needs_ocr if page_0idx >= page_count { pages_needing_ocr.push(page_0idx + 1); @@ -444,6 +458,20 @@ pub fn extract_pages_markdown_mem( }) } +/// Path-based wrapper for [`extract_pages_markdown_mem`]. +/// +/// Reads the PDF from disk and extracts per-page markdown. Pass `None` for +/// `pages` to return every page in document order, or `Some(&[...])` to +/// restrict to specific 0-indexed pages (in caller-supplied order). +pub fn extract_pages_markdown>( + path: P, + pages: Option<&[u32]>, +) -> Result { + validate_pdf_file(&path)?; + let buffer = std::fs::read(path.as_ref())?; + extract_pages_markdown_mem(&buffer, pages) +} + // ========================================================================= // Region-based text extraction (for hybrid OCR pipelines) // ========================================================================= diff --git a/src/python.rs b/src/python.rs index 6110d1c..b7b1fe4 100644 --- a/src/python.rs +++ b/src/python.rs @@ -146,6 +146,67 @@ impl PyPageRegionTexts { // Text item wrapper // --------------------------------------------------------------------------- +/// Per-page markdown extraction result. +#[pyclass(name = "PageMarkdown")] +#[derive(Clone)] +pub struct PyPageMarkdown { + /// 0-indexed page number. + #[pyo3(get)] + pub page: u32, + /// Formatted markdown for this page. + #[pyo3(get)] + pub markdown: String, + /// True when text on this page is unreliable (GID-encoded fonts, + /// encoding issues, garbage text, or empty extraction). + #[pyo3(get)] + pub needs_ocr: bool, +} + +#[pymethods] +impl PyPageMarkdown { + fn __repr__(&self) -> String { + format!( + "PageMarkdown(page={}, markdown='{}', needs_ocr={})", + self.page, + self.markdown.chars().take(40).collect::(), + self.needs_ocr + ) + } +} + +/// Combined per-page markdown extraction and layout classification result. +#[pyclass(name = "PagesExtractionResult")] +#[derive(Clone)] +pub struct PyPagesExtractionResult { + /// Per-page markdown results, in the order requested. + #[pyo3(get)] + pub pages: Vec, + /// 1-indexed pages where tables were detected. + #[pyo3(get)] + pub pages_with_tables: Vec, + /// 1-indexed pages where multi-column layout was detected. + #[pyo3(get)] + pub pages_with_columns: Vec, + /// 1-indexed pages that need OCR (scanned/image-based or unreliable text). + #[pyo3(get)] + pub pages_needing_ocr: Vec, + /// True if any page has tables or columns. + #[pyo3(get)] + pub is_complex: bool, +} + +#[pymethods] +impl PyPagesExtractionResult { + fn __repr__(&self) -> String { + format!( + "PagesExtractionResult(pages={}, pages_with_tables={:?}, is_complex={})", + self.pages.len(), + self.pages_with_tables, + self.is_complex + ) + } +} + /// A positioned text item extracted from a PDF. #[pyclass(name = "TextItem")] #[derive(Clone)] @@ -280,6 +341,24 @@ fn parse_page_regions( .collect() } +fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResult { + PyPagesExtractionResult { + pages: r + .pages + .into_iter() + .map(|p| PyPageMarkdown { + page: p.page, + markdown: p.markdown, + needs_ocr: p.needs_ocr, + }) + .collect(), + pages_with_tables: r.pages_with_tables, + pages_with_columns: r.pages_with_columns, + pages_needing_ocr: r.pages_needing_ocr, + is_complex: r.is_complex, + } +} + fn convert_region_results(results: Vec) -> Vec { results .into_iter() @@ -442,6 +521,44 @@ fn extract_text_in_regions_bytes( Ok(convert_region_results(results)) } +/// Extract formatted markdown for pages of a PDF file, with layout +/// classification metadata. +/// +/// Returns per-page markdown and classification data (tables, columns, +/// OCR needs) from a single parse. Font statistics are computed from the +/// full document so header detection is consistent across pages. +/// +/// Args: +/// path: Path to the PDF file. +/// pages: Optional list of 0-indexed pages. When None (default), every +/// page is returned in document order. When provided, output +/// matches the caller-supplied order. +/// +/// Returns: +/// PagesExtractionResult with per-page markdown and classification data. +#[pyfunction] +#[pyo3(signature = (path, pages=None))] +fn extract_pages_markdown( + path: &str, + pages: Option>, +) -> PyResult { + let result = crate::extract_pages_markdown(path, pages.as_deref()).map_err(to_py_err)?; + Ok(to_py_pages_result(result)) +} + +/// Extract formatted markdown for pages of a PDF from bytes. +/// +/// See [`extract_pages_markdown`] for details. +#[pyfunction] +#[pyo3(signature = (data, pages=None))] +fn extract_pages_markdown_bytes( + data: &[u8], + pages: Option>, +) -> PyResult { + let result = crate::extract_pages_markdown_mem(data, pages.as_deref()).map_err(to_py_err)?; + Ok(to_py_pages_result(result)) +} + /// Python module definition. #[pymodule] fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -450,6 +567,8 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(process_pdf, m)?)?; m.add_function(wrap_pyfunction!(process_pdf_bytes, m)?)?; m.add_function(wrap_pyfunction!(detect_pdf, m)?)?; @@ -462,5 +581,7 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(extract_text_with_positions_bytes, m)?)?; m.add_function(wrap_pyfunction!(extract_text_in_regions, m)?)?; m.add_function(wrap_pyfunction!(extract_text_in_regions_bytes, m)?)?; + m.add_function(wrap_pyfunction!(extract_pages_markdown, m)?)?; + m.add_function(wrap_pyfunction!(extract_pages_markdown_bytes, m)?)?; Ok(()) } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index f7bff48..60389ef 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -4,10 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy}; use pdf_inspector::extractor::group_into_lines; use pdf_inspector::types::TextLine; use pdf_inspector::{ - detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text, - extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem, - process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType, - TextItem, + detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem, + extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem, + extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown, + MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem, }; use std::collections::HashSet; @@ -1481,7 +1481,7 @@ fn test_bits_pilani_page8_table_detection() { #[test] fn test_extract_pages_markdown_basic() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[0, 1])).unwrap(); assert_eq!(result.pages.len(), 2); assert_eq!(result.pages[0].page, 0); @@ -1495,7 +1495,7 @@ fn test_extract_pages_markdown_basic() { fn test_extract_pages_markdown_page_ordering() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); // Request pages in non-sequential order - let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[1, 0])).unwrap(); assert_eq!(result.pages.len(), 2); // Results should match input order, not document order @@ -1506,7 +1506,7 @@ fn test_extract_pages_markdown_page_ordering() { #[test] fn test_extract_pages_markdown_out_of_range() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[9999])).unwrap(); assert_eq!(result.pages.len(), 1); assert_eq!(result.pages[0].page, 9999); @@ -1518,14 +1518,14 @@ fn test_extract_pages_markdown_out_of_range() { #[test] fn test_extract_pages_markdown_empty_pages_list() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[])).unwrap(); assert!(result.pages.is_empty()); } #[test] fn test_extract_pages_markdown_single_page() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap(); assert_eq!(result.pages.len(), 1); assert_eq!(result.pages[0].page, 0); @@ -1535,7 +1535,7 @@ fn test_extract_pages_markdown_single_page() { #[test] fn test_extract_pages_markdown_invalid_buffer() { - let result = extract_pages_markdown_mem(b"not a pdf", &[0]); + let result = extract_pages_markdown_mem(b"not a pdf", Some(&[0])); assert!(result.is_err()); } @@ -1543,7 +1543,7 @@ fn test_extract_pages_markdown_invalid_buffer() { fn test_extract_pages_markdown_gid_pages_need_ocr() { // shinagawa_identity_h.pdf has GID-encoded fonts let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap(); assert_eq!(result.pages.len(), 1); assert!(result.pages[0].needs_ocr); @@ -1556,7 +1556,7 @@ fn test_extract_pages_markdown_classification_with_tables() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); let page_count = process_pdf_mem(&buf).unwrap().page_count; let page_indices: Vec = (0..page_count).collect(); - let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap(); assert!( !result.pages_with_tables.is_empty(), @@ -1569,7 +1569,7 @@ fn test_extract_pages_markdown_classification_with_tables() { fn test_extract_pages_markdown_simple_pdf_no_complexity() { // bare_name_struct.pdf is a simple document with a heading and code block let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap(); - let result = extract_pages_markdown_mem(&buf, &[0]).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap(); assert!(result.pages_with_tables.is_empty()); assert!(result.pages_with_columns.is_empty()); @@ -1582,7 +1582,7 @@ fn test_extract_pages_markdown_classification_matches_process_pdf() { let full = process_pdf_mem(&buf).unwrap(); let page_count = full.page_count; let page_indices: Vec = (0..page_count).collect(); - let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap(); assert_eq!( result.pages_with_tables, full.layout.pages_with_tables, @@ -1605,7 +1605,7 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() { // Get per-page output for all pages let page_count = full.page_count; let page_indices: Vec = (0..page_count).collect(); - let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap(); + let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap(); // Concatenated per-page markdown should contain substantial overlap with // the full output (exact match not expected due to header/footer stripping @@ -1630,3 +1630,41 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() { full_md.len() ); } + +#[test] +fn test_extract_pages_markdown_none_returns_all_pages() { + let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); + let page_count = process_pdf_mem(&buf).unwrap().page_count; + + let result = extract_pages_markdown_mem(&buf, None).unwrap(); + + assert_eq!(result.pages.len() as u32, page_count); + for (i, page) in result.pages.iter().enumerate() { + assert_eq!(page.page, i as u32, "pages should be in document order"); + } +} + +#[test] +fn test_extract_pages_markdown_path_api() { + let path = "tests/fixtures/nexo-price-en.pdf"; + let buf = std::fs::read(path).unwrap(); + + let via_path = extract_pages_markdown(path, Some(&[0])).unwrap(); + let via_mem = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap(); + + assert_eq!(via_path.pages.len(), via_mem.pages.len()); + assert_eq!(via_path.pages[0].markdown, via_mem.pages[0].markdown); + assert_eq!(via_path.pages[0].needs_ocr, via_mem.pages[0].needs_ocr); + assert_eq!(via_path.is_complex, via_mem.is_complex); +} + +#[test] +fn test_extract_pages_markdown_path_none_returns_all_pages() { + let path = "tests/fixtures/nexo-price-en.pdf"; + let page_count = process_pdf_mem(&std::fs::read(path).unwrap()) + .unwrap() + .page_count; + + let result = extract_pages_markdown(path, None).unwrap(); + assert_eq!(result.pages.len() as u32, page_count); +} diff --git a/tests/test_python.py b/tests/test_python.py index 6e46dcd..a17e1d7 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -270,6 +270,78 @@ class TestExtractTextInRegions: ) +# --------------------------------------------------------------------------- +# extract_pages_markdown / extract_pages_markdown_bytes +# --------------------------------------------------------------------------- + + +class TestExtractPagesMarkdown: + def test_default_returns_all_pages(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf") + ) + assert len(result.pages) == 3 + assert [p.page for p in result.pages] == [0, 1, 2] + assert all(isinstance(p.markdown, str) for p in result.pages) + + def test_bytes_default_returns_all_pages(self): + data = fixture_bytes("thermo-freon12.pdf") + result = pdf_inspector.extract_pages_markdown_bytes(data) + assert len(result.pages) == 3 + + def test_selected_pages_preserve_order(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf"), pages=[2, 0] + ) + assert [p.page for p in result.pages] == [2, 0] + + def test_bytes_selected_pages_preserve_order(self): + data = fixture_bytes("thermo-freon12.pdf") + result = pdf_inspector.extract_pages_markdown_bytes(data, pages=[1]) + assert len(result.pages) == 1 + assert result.pages[0].page == 1 + + def test_page_fields(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf"), pages=[0] + ) + page = result.pages[0] + assert isinstance(page.page, int) + assert isinstance(page.markdown, str) + assert isinstance(page.needs_ocr, bool) + assert not page.needs_ocr # text-based fixture + assert len(page.markdown) > 0 + + def test_result_fields(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf") + ) + assert isinstance(result.pages, list) + assert isinstance(result.pages_with_tables, list) + assert isinstance(result.pages_with_columns, list) + assert isinstance(result.pages_needing_ocr, list) + assert isinstance(result.is_complex, bool) + + def test_out_of_range_page_marks_needs_ocr(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf"), pages=[9999] + ) + assert len(result.pages) == 1 + assert result.pages[0].needs_ocr + assert result.pages[0].markdown == "" + + def test_repr(self): + result = pdf_inspector.extract_pages_markdown( + fixture_path("thermo-freon12.pdf"), pages=[0] + ) + assert "PagesExtractionResult" in repr(result) + assert "PageMarkdown" in repr(result.pages[0]) + + def test_not_a_pdf(self): + with pytest.raises(ValueError): + pdf_inspector.extract_pages_markdown_bytes(b"not a pdf") + + # --------------------------------------------------------------------------- # Error handling # ---------------------------------------------------------------------------