Compare commits

...
Author SHA1 Message Date
Abimael MartellandClaude Opus 4.7 9059c3a2ad chore: bump version from 1.3.0 to 1.4.0
Minor bump for the new per-page markdown extraction API exposed through
the Python and Node bindings.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-20 17:17:24 -07:00
Abimael MartellandClaude Opus 4.7 fb7f289b3d feat: expose per-page markdown extraction to Python and Node (#49)
Implements the feature requested in issue #49: a list-of-pages markdown output
from the Python API. Matching the existing project pattern, the feature lives
in the Rust core and is surfaced through every binding.

- Rust core: `extract_pages_markdown` (path) and `extract_pages_markdown_mem`
  (bytes) now take `Option<&[u32]>` — `None` returns every page in document
  order; a slice restricts and preserves caller order.
- Python: new `extract_pages_markdown(path, pages=None)` and
  `extract_pages_markdown_bytes(data, pages=None)` functions plus
  `PageMarkdown` / `PagesExtractionResult` classes; stub file updated.
- Node: `extractPagesMarkdown(buffer, pages?)` — `pages` is now optional.
- Tests: 2 new Rust integration tests, 9 new Python tests, 2 new Node
  assertions. All 372 unit + 107 integration + 53 Python tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-20 17:00:02 -07:00
11 changed files with 406 additions and 25 deletions
+5
View File
@@ -35,3 +35,8 @@ scripts/
# Test output
test_output/
# Python
__pycache__/
*.pyc
.pytest_cache/
+14
View File
@@ -42,6 +42,14 @@ text = pdf_inspector.extract_text("document.pdf")
items = pdf_inspector.extract_text_with_positions("document.pdf")
for item in items[:5]:
print(f"'{item.text}' at ({item.x:.0f}, {item.y:.0f}) size={item.font_size}")
# Per-page markdown (one Markdown string per page, plus layout metadata)
result = pdf_inspector.extract_pages_markdown("document.pdf")
for page in result.pages:
print(f"Page {page.page}: {len(page.markdown)} chars, needs_ocr={page.needs_ocr}")
# Restrict to specific 0-indexed pages (preserves caller order)
result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
```
## API reference
@@ -60,6 +68,8 @@ for item in items[:5]:
| `extract_text_with_positions_bytes(data, pages=None)` | Text with positions from bytes |
| `extract_text_in_regions(path, page_regions)` | Extract text in bounding-box regions |
| `extract_text_in_regions_bytes(data, page_regions)` | Region extraction from bytes |
| `extract_pages_markdown(path, pages=None)` | Per-page Markdown + layout metadata (all pages by default) |
| `extract_pages_markdown_bytes(data, pages=None)` | Per-page Markdown from bytes |
## Types
@@ -72,3 +82,7 @@ for item in items[:5]:
**`RegionText` fields:** `text`, `needs_ocr`
**`PageRegionTexts` fields:** `page` (0-indexed), `regions` (list of RegionText)
**`PageMarkdown` fields:** `page` (0-indexed), `markdown`, `needs_ocr`
**`PagesExtractionResult` fields:** `pages` (list of PageMarkdown), `pages_with_tables` (1-indexed), `pages_with_columns` (1-indexed), `pages_needing_ocr` (1-indexed), `is_complex`
+25
View File
@@ -79,6 +79,27 @@ let bytes = std::fs::read("document.pdf")?;
let result = process_pdf_mem(&bytes)?;
```
Extract per-page Markdown (one string per page, plus document-wide layout
metadata):
```rust
use pdf_inspector::extract_pages_markdown;
// Pass `None` for every page in document order, or a slice of 0-indexed
// pages to restrict the output (caller-supplied order is preserved).
let result = extract_pages_markdown("document.pdf", None)?;
for page in &result.pages {
if page.needs_ocr {
// Route this page to OCR
} else {
println!("Page {}: {}", page.page, page.markdown);
}
}
println!("Complex layout? {}", result.is_complex);
```
## Processing modes
| Mode | What it does | Returns |
@@ -102,6 +123,8 @@ let result = process_pdf_mem(&bytes)?;
| `to_markdown(text, options)` | Convert plain text to Markdown |
| `to_markdown_from_items(items, options)` | Markdown from pre-extracted `TextItem`s |
| `to_markdown_from_items_with_rects(items, options, rects)` | Markdown with rectangle-based table detection |
| `extract_pages_markdown(path, pages)` | Per-page Markdown + layout metadata (file) |
| `extract_pages_markdown_mem(bytes, pages)` | Per-page Markdown from bytes |
Low-level detection functions are also available via the `detector` module (`detect_pdf_type`, `detect_pdf_type_with_config`, etc.) for callers who need `PdfTypeResult` instead of `PdfProcessResult`.
@@ -119,4 +142,6 @@ Low-level detection functions are also available via the `detector` module (`det
| `LayoutComplexity` | Layout analysis: is_complex, pages_with_tables, pages_with_columns |
| `TextItem` | Text with position, font info, and page number |
| `MarkdownOptions` | Configuration for Markdown formatting (page numbers, etc.) |
| `PageMarkdown` | Per-page result: page (0-indexed), markdown, needs_ocr |
| `PagesExtractionResult` | Per-page output + 1-indexed pages_with_tables / pages_with_columns / pages_needing_ocr, is_complex |
| `PdfError` | `Io`, `Parse`, `Encrypted`, `InvalidStructure`, `NotAPdf` |
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.3.0",
"version": "1.4.0",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
+10 -5
View File
@@ -343,21 +343,26 @@ pub struct PagesExtractionResult {
pub is_complex: bool,
}
/// Extract formatted markdown for specific pages of a PDF, with layout
/// classification metadata.
/// Extract formatted markdown for pages of a PDF, with layout classification
/// metadata.
///
/// Returns per-page markdown and classification data (tables, columns,
/// OCR needs) from a single parse. Font statistics are computed from the
/// full document so header detection is consistent across pages.
///
/// Omit `pages` (or pass `undefined`) to return every page in document
/// order. Pass an array of 0-indexed page numbers to restrict output to
/// those pages, in caller-supplied order.
#[napi]
pub fn extract_pages_markdown(
buffer: Buffer,
pages: Vec<u32>,
pages: Option<Vec<u32>>,
) -> Result<PagesExtractionResult> {
let bytes: Vec<u8> = buffer.to_vec();
catch_panic("extract_pages_markdown", move || {
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
let result =
pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
Ok(PagesExtractionResult {
pages: result
.pages
+23
View File
@@ -7,6 +7,7 @@ import {
extractText,
extractTextWithPositions,
extractTextInRegions,
extractPagesMarkdown,
} from './index.js';
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
@@ -89,6 +90,28 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
console.log(' extractTextInRegions: OK');
// --- extractPagesMarkdown ---
console.log('Testing extractPagesMarkdown...');
// omit pages → every page in document order
const allPages = extractPagesMarkdown(fixture);
assert.equal(allPages.pages.length, 3);
assert.deepEqual(allPages.pages.map(p => p.page), [0, 1, 2]);
assert.ok(typeof allPages.pages[0].markdown === 'string');
assert.equal(typeof allPages.pages[0].needsOcr, 'boolean');
assert.ok(Array.isArray(allPages.pagesWithTables));
assert.ok(Array.isArray(allPages.pagesWithColumns));
assert.ok(Array.isArray(allPages.pagesNeedingOcr));
assert.equal(typeof allPages.isComplex, 'boolean');
console.log(' extractPagesMarkdown (no pages arg): OK');
// selected pages preserve caller order
const picked = extractPagesMarkdown(fixture, [2, 0]);
assert.equal(picked.pages.length, 2);
assert.equal(picked.pages[0].page, 2);
assert.equal(picked.pages[1].page, 0);
console.log(' extractPagesMarkdown with pages: OK');
// --- Error handling ---
console.log('Testing error handling...');
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
+50
View File
@@ -52,6 +52,28 @@ class PageRegionTexts:
"""0-indexed page number."""
regions: list[RegionText]
class PageMarkdown:
"""Per-page markdown extraction result."""
page: int
"""0-indexed page number."""
markdown: str
"""Formatted markdown for this page (empty string when needs_ocr is True)."""
needs_ocr: bool
"""True when text on this page is unreliable and OCR should be used instead."""
class PagesExtractionResult:
"""Per-page markdown output with document-wide layout classification."""
pages: list[PageMarkdown]
"""Per-page markdown results, in the order requested."""
pages_with_tables: list[int]
"""1-indexed pages where tables were detected."""
pages_with_columns: list[int]
"""1-indexed pages where multi-column layout was detected."""
pages_needing_ocr: list[int]
"""1-indexed pages that need OCR."""
is_complex: bool
"""True if any page has tables or multi-column layout."""
def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult:
"""Process a PDF: detect type, extract text, convert to Markdown."""
...
@@ -115,3 +137,31 @@ def extract_text_in_regions_bytes(
page_regions: List of (page_0indexed, [[x1, y1, x2, y2], ...]) tuples.
"""
...
def extract_pages_markdown(
path: str,
pages: Optional[list[int]] = None,
) -> PagesExtractionResult:
"""Extract formatted markdown for pages of a PDF, with layout classification.
Args:
path: Path to the PDF file.
pages: Optional list of 0-indexed pages. When ``None`` (default), every
page is returned in document order. Otherwise, output matches the
caller-supplied order.
Returns:
PagesExtractionResult with per-page markdown and document-wide layout
classification (tables, columns, OCR needs).
"""
...
def extract_pages_markdown_bytes(
data: bytes,
pages: Optional[list[int]] = None,
) -> PagesExtractionResult:
"""Extract formatted markdown for pages of a PDF from bytes.
See :func:`extract_pages_markdown` for details.
"""
...
+32 -4
View File
@@ -336,13 +336,17 @@ pub struct PagesExtractionResult {
pub is_complex: bool,
}
/// Extract formatted markdown for specific pages of a PDF, with layout
/// Extract formatted markdown for pages of a PDF, with layout
/// classification metadata.
///
/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
/// this returns per-page markdown so callers can mix direct extraction
/// (for simple text pages) with GPU OCR (for complex/scanned pages).
///
/// When `pages` is `None`, every page (0-indexed, in document order) is
/// returned. When `Some(&[...])`, only the listed 0-indexed pages are
/// returned, in the caller's order.
///
/// Font statistics are computed from the full document so header
/// detection thresholds are consistent regardless of which pages are
/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
@@ -352,7 +356,7 @@ pub struct PagesExtractionResult {
/// at near-zero cost since the items/rects/lines are already in memory.
pub fn extract_pages_markdown_mem(
buffer: &[u8],
pages: &[u32],
pages: Option<&[u32]>,
) -> Result<PagesExtractionResult, PdfError> {
validate_pdf_bytes(buffer)?;
let (doc, page_count) = load_document_from_mem(buffer)?;
@@ -368,10 +372,20 @@ pub fn extract_pages_markdown_mem(
// Compute font stats from full document (cross-page consistency).
let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
let mut results = Vec::with_capacity(pages.len());
// When caller doesn't specify pages, return every page in document order.
let all_pages: Vec<u32>;
let pages_slice: &[u32] = match pages {
Some(p) => p,
None => {
all_pages = (0..page_count).collect();
&all_pages
}
};
let mut results = Vec::with_capacity(pages_slice.len());
let mut pages_needing_ocr = Vec::new();
for &page_0idx in pages {
for &page_0idx in pages_slice {
// Out-of-range pages → empty + needs_ocr
if page_0idx >= page_count {
pages_needing_ocr.push(page_0idx + 1);
@@ -444,6 +458,20 @@ pub fn extract_pages_markdown_mem(
})
}
/// Path-based wrapper for [`extract_pages_markdown_mem`].
///
/// Reads the PDF from disk and extracts per-page markdown. Pass `None` for
/// `pages` to return every page in document order, or `Some(&[...])` to
/// restrict to specific 0-indexed pages (in caller-supplied order).
pub fn extract_pages_markdown<P: AsRef<Path>>(
path: P,
pages: Option<&[u32]>,
) -> Result<PagesExtractionResult, PdfError> {
validate_pdf_file(&path)?;
let buffer = std::fs::read(path.as_ref())?;
extract_pages_markdown_mem(&buffer, pages)
}
// =========================================================================
// Region-based text extraction (for hybrid OCR pipelines)
// =========================================================================
+121
View File
@@ -146,6 +146,67 @@ impl PyPageRegionTexts {
// Text item wrapper
// ---------------------------------------------------------------------------
/// Per-page markdown extraction result.
#[pyclass(name = "PageMarkdown")]
#[derive(Clone)]
pub struct PyPageMarkdown {
/// 0-indexed page number.
#[pyo3(get)]
pub page: u32,
/// Formatted markdown for this page.
#[pyo3(get)]
pub markdown: String,
/// True when text on this page is unreliable (GID-encoded fonts,
/// encoding issues, garbage text, or empty extraction).
#[pyo3(get)]
pub needs_ocr: bool,
}
#[pymethods]
impl PyPageMarkdown {
fn __repr__(&self) -> String {
format!(
"PageMarkdown(page={}, markdown='{}', needs_ocr={})",
self.page,
self.markdown.chars().take(40).collect::<String>(),
self.needs_ocr
)
}
}
/// Combined per-page markdown extraction and layout classification result.
#[pyclass(name = "PagesExtractionResult")]
#[derive(Clone)]
pub struct PyPagesExtractionResult {
/// Per-page markdown results, in the order requested.
#[pyo3(get)]
pub pages: Vec<PyPageMarkdown>,
/// 1-indexed pages where tables were detected.
#[pyo3(get)]
pub pages_with_tables: Vec<u32>,
/// 1-indexed pages where multi-column layout was detected.
#[pyo3(get)]
pub pages_with_columns: Vec<u32>,
/// 1-indexed pages that need OCR (scanned/image-based or unreliable text).
#[pyo3(get)]
pub pages_needing_ocr: Vec<u32>,
/// True if any page has tables or columns.
#[pyo3(get)]
pub is_complex: bool,
}
#[pymethods]
impl PyPagesExtractionResult {
fn __repr__(&self) -> String {
format!(
"PagesExtractionResult(pages={}, pages_with_tables={:?}, is_complex={})",
self.pages.len(),
self.pages_with_tables,
self.is_complex
)
}
}
/// A positioned text item extracted from a PDF.
#[pyclass(name = "TextItem")]
#[derive(Clone)]
@@ -280,6 +341,24 @@ fn parse_page_regions(
.collect()
}
fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResult {
PyPagesExtractionResult {
pages: r
.pages
.into_iter()
.map(|p| PyPageMarkdown {
page: p.page,
markdown: p.markdown,
needs_ocr: p.needs_ocr,
})
.collect(),
pages_with_tables: r.pages_with_tables,
pages_with_columns: r.pages_with_columns,
pages_needing_ocr: r.pages_needing_ocr,
is_complex: r.is_complex,
}
}
fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRegionTexts> {
results
.into_iter()
@@ -442,6 +521,44 @@ fn extract_text_in_regions_bytes(
Ok(convert_region_results(results))
}
/// Extract formatted markdown for pages of a PDF file, with layout
/// classification metadata.
///
/// Returns per-page markdown and classification data (tables, columns,
/// OCR needs) from a single parse. Font statistics are computed from the
/// full document so header detection is consistent across pages.
///
/// Args:
/// path: Path to the PDF file.
/// pages: Optional list of 0-indexed pages. When None (default), every
/// page is returned in document order. When provided, output
/// matches the caller-supplied order.
///
/// Returns:
/// PagesExtractionResult with per-page markdown and classification data.
#[pyfunction]
#[pyo3(signature = (path, pages=None))]
fn extract_pages_markdown(
path: &str,
pages: Option<Vec<u32>>,
) -> PyResult<PyPagesExtractionResult> {
let result = crate::extract_pages_markdown(path, pages.as_deref()).map_err(to_py_err)?;
Ok(to_py_pages_result(result))
}
/// Extract formatted markdown for pages of a PDF from bytes.
///
/// See [`extract_pages_markdown`] for details.
#[pyfunction]
#[pyo3(signature = (data, pages=None))]
fn extract_pages_markdown_bytes(
data: &[u8],
pages: Option<Vec<u32>>,
) -> PyResult<PyPagesExtractionResult> {
let result = crate::extract_pages_markdown_mem(data, pages.as_deref()).map_err(to_py_err)?;
Ok(to_py_pages_result(result))
}
/// Python module definition.
#[pymodule]
fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
@@ -450,6 +567,8 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyTextItem>()?;
m.add_class::<PyRegionText>()?;
m.add_class::<PyPageRegionTexts>()?;
m.add_class::<PyPageMarkdown>()?;
m.add_class::<PyPagesExtractionResult>()?;
m.add_function(wrap_pyfunction!(process_pdf, m)?)?;
m.add_function(wrap_pyfunction!(process_pdf_bytes, m)?)?;
m.add_function(wrap_pyfunction!(detect_pdf, m)?)?;
@@ -462,5 +581,7 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(extract_text_with_positions_bytes, m)?)?;
m.add_function(wrap_pyfunction!(extract_text_in_regions, m)?)?;
m.add_function(wrap_pyfunction!(extract_text_in_regions_bytes, m)?)?;
m.add_function(wrap_pyfunction!(extract_pages_markdown, m)?)?;
m.add_function(wrap_pyfunction!(extract_pages_markdown_bytes, m)?)?;
Ok(())
}
+53 -15
View File
@@ -4,10 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
use pdf_inspector::extractor::group_into_lines;
use pdf_inspector::types::TextLine;
use pdf_inspector::{
detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
TextItem,
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
};
use std::collections::HashSet;
@@ -1481,7 +1481,7 @@ fn test_bits_pilani_page8_table_detection() {
#[test]
fn test_extract_pages_markdown_basic() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[0, 1])).unwrap();
assert_eq!(result.pages.len(), 2);
assert_eq!(result.pages[0].page, 0);
@@ -1495,7 +1495,7 @@ fn test_extract_pages_markdown_basic() {
fn test_extract_pages_markdown_page_ordering() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
// Request pages in non-sequential order
let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[1, 0])).unwrap();
assert_eq!(result.pages.len(), 2);
// Results should match input order, not document order
@@ -1506,7 +1506,7 @@ fn test_extract_pages_markdown_page_ordering() {
#[test]
fn test_extract_pages_markdown_out_of_range() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[9999])).unwrap();
assert_eq!(result.pages.len(), 1);
assert_eq!(result.pages[0].page, 9999);
@@ -1518,14 +1518,14 @@ fn test_extract_pages_markdown_out_of_range() {
#[test]
fn test_extract_pages_markdown_empty_pages_list() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[])).unwrap();
assert!(result.pages.is_empty());
}
#[test]
fn test_extract_pages_markdown_single_page() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
assert_eq!(result.pages.len(), 1);
assert_eq!(result.pages[0].page, 0);
@@ -1535,7 +1535,7 @@ fn test_extract_pages_markdown_single_page() {
#[test]
fn test_extract_pages_markdown_invalid_buffer() {
let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
let result = extract_pages_markdown_mem(b"not a pdf", Some(&[0]));
assert!(result.is_err());
}
@@ -1543,7 +1543,7 @@ fn test_extract_pages_markdown_invalid_buffer() {
fn test_extract_pages_markdown_gid_pages_need_ocr() {
// shinagawa_identity_h.pdf has GID-encoded fonts
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
assert_eq!(result.pages.len(), 1);
assert!(result.pages[0].needs_ocr);
@@ -1556,7 +1556,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let page_count = process_pdf_mem(&buf).unwrap().page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
assert!(
!result.pages_with_tables.is_empty(),
@@ -1569,7 +1569,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
// bare_name_struct.pdf is a simple document with a heading and code block
let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
assert!(result.pages_with_tables.is_empty());
assert!(result.pages_with_columns.is_empty());
@@ -1582,7 +1582,7 @@ fn test_extract_pages_markdown_classification_matches_process_pdf() {
let full = process_pdf_mem(&buf).unwrap();
let page_count = full.page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
assert_eq!(
result.pages_with_tables, full.layout.pages_with_tables,
@@ -1605,7 +1605,7 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
// Get per-page output for all pages
let page_count = full.page_count;
let page_indices: Vec<u32> = (0..page_count).collect();
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
// Concatenated per-page markdown should contain substantial overlap with
// the full output (exact match not expected due to header/footer stripping
@@ -1630,3 +1630,41 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
full_md.len()
);
}
#[test]
fn test_extract_pages_markdown_none_returns_all_pages() {
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
let page_count = process_pdf_mem(&buf).unwrap().page_count;
let result = extract_pages_markdown_mem(&buf, None).unwrap();
assert_eq!(result.pages.len() as u32, page_count);
for (i, page) in result.pages.iter().enumerate() {
assert_eq!(page.page, i as u32, "pages should be in document order");
}
}
#[test]
fn test_extract_pages_markdown_path_api() {
let path = "tests/fixtures/nexo-price-en.pdf";
let buf = std::fs::read(path).unwrap();
let via_path = extract_pages_markdown(path, Some(&[0])).unwrap();
let via_mem = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
assert_eq!(via_path.pages.len(), via_mem.pages.len());
assert_eq!(via_path.pages[0].markdown, via_mem.pages[0].markdown);
assert_eq!(via_path.pages[0].needs_ocr, via_mem.pages[0].needs_ocr);
assert_eq!(via_path.is_complex, via_mem.is_complex);
}
#[test]
fn test_extract_pages_markdown_path_none_returns_all_pages() {
let path = "tests/fixtures/nexo-price-en.pdf";
let page_count = process_pdf_mem(&std::fs::read(path).unwrap())
.unwrap()
.page_count;
let result = extract_pages_markdown(path, None).unwrap();
assert_eq!(result.pages.len() as u32, page_count);
}
+72
View File
@@ -270,6 +270,78 @@ class TestExtractTextInRegions:
)
# ---------------------------------------------------------------------------
# extract_pages_markdown / extract_pages_markdown_bytes
# ---------------------------------------------------------------------------
class TestExtractPagesMarkdown:
def test_default_returns_all_pages(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf")
)
assert len(result.pages) == 3
assert [p.page for p in result.pages] == [0, 1, 2]
assert all(isinstance(p.markdown, str) for p in result.pages)
def test_bytes_default_returns_all_pages(self):
data = fixture_bytes("thermo-freon12.pdf")
result = pdf_inspector.extract_pages_markdown_bytes(data)
assert len(result.pages) == 3
def test_selected_pages_preserve_order(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf"), pages=[2, 0]
)
assert [p.page for p in result.pages] == [2, 0]
def test_bytes_selected_pages_preserve_order(self):
data = fixture_bytes("thermo-freon12.pdf")
result = pdf_inspector.extract_pages_markdown_bytes(data, pages=[1])
assert len(result.pages) == 1
assert result.pages[0].page == 1
def test_page_fields(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf"), pages=[0]
)
page = result.pages[0]
assert isinstance(page.page, int)
assert isinstance(page.markdown, str)
assert isinstance(page.needs_ocr, bool)
assert not page.needs_ocr # text-based fixture
assert len(page.markdown) > 0
def test_result_fields(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf")
)
assert isinstance(result.pages, list)
assert isinstance(result.pages_with_tables, list)
assert isinstance(result.pages_with_columns, list)
assert isinstance(result.pages_needing_ocr, list)
assert isinstance(result.is_complex, bool)
def test_out_of_range_page_marks_needs_ocr(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf"), pages=[9999]
)
assert len(result.pages) == 1
assert result.pages[0].needs_ocr
assert result.pages[0].markdown == ""
def test_repr(self):
result = pdf_inspector.extract_pages_markdown(
fixture_path("thermo-freon12.pdf"), pages=[0]
)
assert "PagesExtractionResult" in repr(result)
assert "PageMarkdown" in repr(result.pages[0])
def test_not_a_pdf(self):
with pytest.raises(ValueError):
pdf_inspector.extract_pages_markdown_bytes(b"not a pdf")
# ---------------------------------------------------------------------------
# Error handling
# ---------------------------------------------------------------------------