Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9059c3a2ad | ||
|
|
fb7f289b3d |
@@ -35,3 +35,8 @@ scripts/
|
||||
# Test output
|
||||
test_output/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
|
||||
|
||||
@@ -42,6 +42,14 @@ text = pdf_inspector.extract_text("document.pdf")
|
||||
items = pdf_inspector.extract_text_with_positions("document.pdf")
|
||||
for item in items[:5]:
|
||||
print(f"'{item.text}' at ({item.x:.0f}, {item.y:.0f}) size={item.font_size}")
|
||||
|
||||
# Per-page markdown (one Markdown string per page, plus layout metadata)
|
||||
result = pdf_inspector.extract_pages_markdown("document.pdf")
|
||||
for page in result.pages:
|
||||
print(f"Page {page.page}: {len(page.markdown)} chars, needs_ocr={page.needs_ocr}")
|
||||
|
||||
# Restrict to specific 0-indexed pages (preserves caller order)
|
||||
result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
|
||||
```
|
||||
|
||||
## API reference
|
||||
@@ -60,6 +68,8 @@ for item in items[:5]:
|
||||
| `extract_text_with_positions_bytes(data, pages=None)` | Text with positions from bytes |
|
||||
| `extract_text_in_regions(path, page_regions)` | Extract text in bounding-box regions |
|
||||
| `extract_text_in_regions_bytes(data, page_regions)` | Region extraction from bytes |
|
||||
| `extract_pages_markdown(path, pages=None)` | Per-page Markdown + layout metadata (all pages by default) |
|
||||
| `extract_pages_markdown_bytes(data, pages=None)` | Per-page Markdown from bytes |
|
||||
|
||||
## Types
|
||||
|
||||
@@ -72,3 +82,7 @@ for item in items[:5]:
|
||||
**`RegionText` fields:** `text`, `needs_ocr`
|
||||
|
||||
**`PageRegionTexts` fields:** `page` (0-indexed), `regions` (list of RegionText)
|
||||
|
||||
**`PageMarkdown` fields:** `page` (0-indexed), `markdown`, `needs_ocr`
|
||||
|
||||
**`PagesExtractionResult` fields:** `pages` (list of PageMarkdown), `pages_with_tables` (1-indexed), `pages_with_columns` (1-indexed), `pages_needing_ocr` (1-indexed), `is_complex`
|
||||
|
||||
@@ -79,6 +79,27 @@ let bytes = std::fs::read("document.pdf")?;
|
||||
let result = process_pdf_mem(&bytes)?;
|
||||
```
|
||||
|
||||
Extract per-page Markdown (one string per page, plus document-wide layout
|
||||
metadata):
|
||||
|
||||
```rust
|
||||
use pdf_inspector::extract_pages_markdown;
|
||||
|
||||
// Pass `None` for every page in document order, or a slice of 0-indexed
|
||||
// pages to restrict the output (caller-supplied order is preserved).
|
||||
let result = extract_pages_markdown("document.pdf", None)?;
|
||||
|
||||
for page in &result.pages {
|
||||
if page.needs_ocr {
|
||||
// Route this page to OCR
|
||||
} else {
|
||||
println!("Page {}: {}", page.page, page.markdown);
|
||||
}
|
||||
}
|
||||
|
||||
println!("Complex layout? {}", result.is_complex);
|
||||
```
|
||||
|
||||
## Processing modes
|
||||
|
||||
| Mode | What it does | Returns |
|
||||
@@ -102,6 +123,8 @@ let result = process_pdf_mem(&bytes)?;
|
||||
| `to_markdown(text, options)` | Convert plain text to Markdown |
|
||||
| `to_markdown_from_items(items, options)` | Markdown from pre-extracted `TextItem`s |
|
||||
| `to_markdown_from_items_with_rects(items, options, rects)` | Markdown with rectangle-based table detection |
|
||||
| `extract_pages_markdown(path, pages)` | Per-page Markdown + layout metadata (file) |
|
||||
| `extract_pages_markdown_mem(bytes, pages)` | Per-page Markdown from bytes |
|
||||
|
||||
Low-level detection functions are also available via the `detector` module (`detect_pdf_type`, `detect_pdf_type_with_config`, etc.) for callers who need `PdfTypeResult` instead of `PdfProcessResult`.
|
||||
|
||||
@@ -119,4 +142,6 @@ Low-level detection functions are also available via the `detector` module (`det
|
||||
| `LayoutComplexity` | Layout analysis: is_complex, pages_with_tables, pages_with_columns |
|
||||
| `TextItem` | Text with position, font info, and page number |
|
||||
| `MarkdownOptions` | Configuration for Markdown formatting (page numbers, etc.) |
|
||||
| `PageMarkdown` | Per-page result: page (0-indexed), markdown, needs_ocr |
|
||||
| `PagesExtractionResult` | Per-page output + 1-indexed pages_with_tables / pages_with_columns / pages_needing_ocr, is_complex |
|
||||
| `PdfError` | `Io`, `Parse`, `Encrypted`, `InvalidStructure`, `NotAPdf` |
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.3.0",
|
||||
"version": "1.4.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+10
-5
@@ -343,21 +343,26 @@ pub struct PagesExtractionResult {
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
/// Extract formatted markdown for pages of a PDF, with layout classification
|
||||
/// metadata.
|
||||
///
|
||||
/// Returns per-page markdown and classification data (tables, columns,
|
||||
/// OCR needs) from a single parse. Font statistics are computed from the
|
||||
/// full document so header detection is consistent across pages.
|
||||
///
|
||||
/// Omit `pages` (or pass `undefined`) to return every page in document
|
||||
/// order. Pass an array of 0-indexed page numbers to restrict output to
|
||||
/// those pages, in caller-supplied order.
|
||||
#[napi]
|
||||
pub fn extract_pages_markdown(
|
||||
buffer: Buffer,
|
||||
pages: Vec<u32>,
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
let result =
|
||||
pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
extractPagesMarkdown,
|
||||
} from './index.js';
|
||||
|
||||
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
|
||||
@@ -89,6 +90,28 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
|
||||
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
|
||||
console.log(' extractTextInRegions: OK');
|
||||
|
||||
// --- extractPagesMarkdown ---
|
||||
console.log('Testing extractPagesMarkdown...');
|
||||
|
||||
// omit pages → every page in document order
|
||||
const allPages = extractPagesMarkdown(fixture);
|
||||
assert.equal(allPages.pages.length, 3);
|
||||
assert.deepEqual(allPages.pages.map(p => p.page), [0, 1, 2]);
|
||||
assert.ok(typeof allPages.pages[0].markdown === 'string');
|
||||
assert.equal(typeof allPages.pages[0].needsOcr, 'boolean');
|
||||
assert.ok(Array.isArray(allPages.pagesWithTables));
|
||||
assert.ok(Array.isArray(allPages.pagesWithColumns));
|
||||
assert.ok(Array.isArray(allPages.pagesNeedingOcr));
|
||||
assert.equal(typeof allPages.isComplex, 'boolean');
|
||||
console.log(' extractPagesMarkdown (no pages arg): OK');
|
||||
|
||||
// selected pages preserve caller order
|
||||
const picked = extractPagesMarkdown(fixture, [2, 0]);
|
||||
assert.equal(picked.pages.length, 2);
|
||||
assert.equal(picked.pages[0].page, 2);
|
||||
assert.equal(picked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdown with pages: OK');
|
||||
|
||||
// --- Error handling ---
|
||||
console.log('Testing error handling...');
|
||||
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
|
||||
|
||||
@@ -52,6 +52,28 @@ class PageRegionTexts:
|
||||
"""0-indexed page number."""
|
||||
regions: list[RegionText]
|
||||
|
||||
class PageMarkdown:
|
||||
"""Per-page markdown extraction result."""
|
||||
page: int
|
||||
"""0-indexed page number."""
|
||||
markdown: str
|
||||
"""Formatted markdown for this page (empty string when needs_ocr is True)."""
|
||||
needs_ocr: bool
|
||||
"""True when text on this page is unreliable and OCR should be used instead."""
|
||||
|
||||
class PagesExtractionResult:
|
||||
"""Per-page markdown output with document-wide layout classification."""
|
||||
pages: list[PageMarkdown]
|
||||
"""Per-page markdown results, in the order requested."""
|
||||
pages_with_tables: list[int]
|
||||
"""1-indexed pages where tables were detected."""
|
||||
pages_with_columns: list[int]
|
||||
"""1-indexed pages where multi-column layout was detected."""
|
||||
pages_needing_ocr: list[int]
|
||||
"""1-indexed pages that need OCR."""
|
||||
is_complex: bool
|
||||
"""True if any page has tables or multi-column layout."""
|
||||
|
||||
def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult:
|
||||
"""Process a PDF: detect type, extract text, convert to Markdown."""
|
||||
...
|
||||
@@ -115,3 +137,31 @@ def extract_text_in_regions_bytes(
|
||||
page_regions: List of (page_0indexed, [[x1, y1, x2, y2], ...]) tuples.
|
||||
"""
|
||||
...
|
||||
|
||||
def extract_pages_markdown(
|
||||
path: str,
|
||||
pages: Optional[list[int]] = None,
|
||||
) -> PagesExtractionResult:
|
||||
"""Extract formatted markdown for pages of a PDF, with layout classification.
|
||||
|
||||
Args:
|
||||
path: Path to the PDF file.
|
||||
pages: Optional list of 0-indexed pages. When ``None`` (default), every
|
||||
page is returned in document order. Otherwise, output matches the
|
||||
caller-supplied order.
|
||||
|
||||
Returns:
|
||||
PagesExtractionResult with per-page markdown and document-wide layout
|
||||
classification (tables, columns, OCR needs).
|
||||
"""
|
||||
...
|
||||
|
||||
def extract_pages_markdown_bytes(
|
||||
data: bytes,
|
||||
pages: Optional[list[int]] = None,
|
||||
) -> PagesExtractionResult:
|
||||
"""Extract formatted markdown for pages of a PDF from bytes.
|
||||
|
||||
See :func:`extract_pages_markdown` for details.
|
||||
"""
|
||||
...
|
||||
|
||||
+32
-4
@@ -336,13 +336,17 @@ pub struct PagesExtractionResult {
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// Extract formatted markdown for pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
|
||||
/// this returns per-page markdown so callers can mix direct extraction
|
||||
/// (for simple text pages) with GPU OCR (for complex/scanned pages).
|
||||
///
|
||||
/// When `pages` is `None`, every page (0-indexed, in document order) is
|
||||
/// returned. When `Some(&[...])`, only the listed 0-indexed pages are
|
||||
/// returned, in the caller's order.
|
||||
///
|
||||
/// Font statistics are computed from the full document so header
|
||||
/// detection thresholds are consistent regardless of which pages are
|
||||
/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
|
||||
@@ -352,7 +356,7 @@ pub struct PagesExtractionResult {
|
||||
/// at near-zero cost since the items/rects/lines are already in memory.
|
||||
pub fn extract_pages_markdown_mem(
|
||||
buffer: &[u8],
|
||||
pages: &[u32],
|
||||
pages: Option<&[u32]>,
|
||||
) -> Result<PagesExtractionResult, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, page_count) = load_document_from_mem(buffer)?;
|
||||
@@ -368,10 +372,20 @@ pub fn extract_pages_markdown_mem(
|
||||
// Compute font stats from full document (cross-page consistency).
|
||||
let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
|
||||
|
||||
let mut results = Vec::with_capacity(pages.len());
|
||||
// When caller doesn't specify pages, return every page in document order.
|
||||
let all_pages: Vec<u32>;
|
||||
let pages_slice: &[u32] = match pages {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
all_pages = (0..page_count).collect();
|
||||
&all_pages
|
||||
}
|
||||
};
|
||||
|
||||
let mut results = Vec::with_capacity(pages_slice.len());
|
||||
let mut pages_needing_ocr = Vec::new();
|
||||
|
||||
for &page_0idx in pages {
|
||||
for &page_0idx in pages_slice {
|
||||
// Out-of-range pages → empty + needs_ocr
|
||||
if page_0idx >= page_count {
|
||||
pages_needing_ocr.push(page_0idx + 1);
|
||||
@@ -444,6 +458,20 @@ pub fn extract_pages_markdown_mem(
|
||||
})
|
||||
}
|
||||
|
||||
/// Path-based wrapper for [`extract_pages_markdown_mem`].
|
||||
///
|
||||
/// Reads the PDF from disk and extracts per-page markdown. Pass `None` for
|
||||
/// `pages` to return every page in document order, or `Some(&[...])` to
|
||||
/// restrict to specific 0-indexed pages (in caller-supplied order).
|
||||
pub fn extract_pages_markdown<P: AsRef<Path>>(
|
||||
path: P,
|
||||
pages: Option<&[u32]>,
|
||||
) -> Result<PagesExtractionResult, PdfError> {
|
||||
validate_pdf_file(&path)?;
|
||||
let buffer = std::fs::read(path.as_ref())?;
|
||||
extract_pages_markdown_mem(&buffer, pages)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Region-based text extraction (for hybrid OCR pipelines)
|
||||
// =========================================================================
|
||||
|
||||
+121
@@ -146,6 +146,67 @@ impl PyPageRegionTexts {
|
||||
// Text item wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[pyclass(name = "PageMarkdown")]
|
||||
#[derive(Clone)]
|
||||
pub struct PyPageMarkdown {
|
||||
/// 0-indexed page number.
|
||||
#[pyo3(get)]
|
||||
pub page: u32,
|
||||
/// Formatted markdown for this page.
|
||||
#[pyo3(get)]
|
||||
pub markdown: String,
|
||||
/// True when text on this page is unreliable (GID-encoded fonts,
|
||||
/// encoding issues, garbage text, or empty extraction).
|
||||
#[pyo3(get)]
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyPageMarkdown {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"PageMarkdown(page={}, markdown='{}', needs_ocr={})",
|
||||
self.page,
|
||||
self.markdown.chars().take(40).collect::<String>(),
|
||||
self.needs_ocr
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Combined per-page markdown extraction and layout classification result.
|
||||
#[pyclass(name = "PagesExtractionResult")]
|
||||
#[derive(Clone)]
|
||||
pub struct PyPagesExtractionResult {
|
||||
/// Per-page markdown results, in the order requested.
|
||||
#[pyo3(get)]
|
||||
pub pages: Vec<PyPageMarkdown>,
|
||||
/// 1-indexed pages where tables were detected.
|
||||
#[pyo3(get)]
|
||||
pub pages_with_tables: Vec<u32>,
|
||||
/// 1-indexed pages where multi-column layout was detected.
|
||||
#[pyo3(get)]
|
||||
pub pages_with_columns: Vec<u32>,
|
||||
/// 1-indexed pages that need OCR (scanned/image-based or unreliable text).
|
||||
#[pyo3(get)]
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// True if any page has tables or columns.
|
||||
#[pyo3(get)]
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyPagesExtractionResult {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"PagesExtractionResult(pages={}, pages_with_tables={:?}, is_complex={})",
|
||||
self.pages.len(),
|
||||
self.pages_with_tables,
|
||||
self.is_complex
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A positioned text item extracted from a PDF.
|
||||
#[pyclass(name = "TextItem")]
|
||||
#[derive(Clone)]
|
||||
@@ -280,6 +341,24 @@ fn parse_page_regions(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResult {
|
||||
PyPagesExtractionResult {
|
||||
pages: r
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|p| PyPageMarkdown {
|
||||
page: p.page,
|
||||
markdown: p.markdown,
|
||||
needs_ocr: p.needs_ocr,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: r.pages_with_tables,
|
||||
pages_with_columns: r.pages_with_columns,
|
||||
pages_needing_ocr: r.pages_needing_ocr,
|
||||
is_complex: r.is_complex,
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRegionTexts> {
|
||||
results
|
||||
.into_iter()
|
||||
@@ -442,6 +521,44 @@ fn extract_text_in_regions_bytes(
|
||||
Ok(convert_region_results(results))
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF file, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Returns per-page markdown and classification data (tables, columns,
|
||||
/// OCR needs) from a single parse. Font statistics are computed from the
|
||||
/// full document so header detection is consistent across pages.
|
||||
///
|
||||
/// Args:
|
||||
/// path: Path to the PDF file.
|
||||
/// pages: Optional list of 0-indexed pages. When None (default), every
|
||||
/// page is returned in document order. When provided, output
|
||||
/// matches the caller-supplied order.
|
||||
///
|
||||
/// Returns:
|
||||
/// PagesExtractionResult with per-page markdown and classification data.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (path, pages=None))]
|
||||
fn extract_pages_markdown(
|
||||
path: &str,
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> PyResult<PyPagesExtractionResult> {
|
||||
let result = crate::extract_pages_markdown(path, pages.as_deref()).map_err(to_py_err)?;
|
||||
Ok(to_py_pages_result(result))
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF from bytes.
|
||||
///
|
||||
/// See [`extract_pages_markdown`] for details.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (data, pages=None))]
|
||||
fn extract_pages_markdown_bytes(
|
||||
data: &[u8],
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> PyResult<PyPagesExtractionResult> {
|
||||
let result = crate::extract_pages_markdown_mem(data, pages.as_deref()).map_err(to_py_err)?;
|
||||
Ok(to_py_pages_result(result))
|
||||
}
|
||||
|
||||
/// Python module definition.
|
||||
#[pymodule]
|
||||
fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
@@ -450,6 +567,8 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<PyTextItem>()?;
|
||||
m.add_class::<PyRegionText>()?;
|
||||
m.add_class::<PyPageRegionTexts>()?;
|
||||
m.add_class::<PyPageMarkdown>()?;
|
||||
m.add_class::<PyPagesExtractionResult>()?;
|
||||
m.add_function(wrap_pyfunction!(process_pdf, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(process_pdf_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(detect_pdf, m)?)?;
|
||||
@@ -462,5 +581,7 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(extract_text_with_positions_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_text_in_regions, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_text_in_regions_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_pages_markdown, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_pages_markdown_bytes, m)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+53
-15
@@ -4,10 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
|
||||
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
|
||||
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
|
||||
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -1481,7 +1481,7 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_basic() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0, 1])).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
@@ -1495,7 +1495,7 @@ fn test_extract_pages_markdown_basic() {
|
||||
fn test_extract_pages_markdown_page_ordering() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
// Request pages in non-sequential order
|
||||
let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[1, 0])).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
// Results should match input order, not document order
|
||||
@@ -1506,7 +1506,7 @@ fn test_extract_pages_markdown_page_ordering() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_out_of_range() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[9999])).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 9999);
|
||||
@@ -1518,14 +1518,14 @@ fn test_extract_pages_markdown_out_of_range() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_empty_pages_list() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[])).unwrap();
|
||||
assert!(result.pages.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_single_page() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
@@ -1535,7 +1535,7 @@ fn test_extract_pages_markdown_single_page() {
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_invalid_buffer() {
|
||||
let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
|
||||
let result = extract_pages_markdown_mem(b"not a pdf", Some(&[0]));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -1543,7 +1543,7 @@ fn test_extract_pages_markdown_invalid_buffer() {
|
||||
fn test_extract_pages_markdown_gid_pages_need_ocr() {
|
||||
// shinagawa_identity_h.pdf has GID-encoded fonts
|
||||
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(result.pages[0].needs_ocr);
|
||||
@@ -1556,7 +1556,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let page_count = process_pdf_mem(&buf).unwrap().page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
|
||||
assert!(
|
||||
!result.pages_with_tables.is_empty(),
|
||||
@@ -1569,7 +1569,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
|
||||
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
|
||||
// bare_name_struct.pdf is a simple document with a heading and code block
|
||||
let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
|
||||
assert!(result.pages_with_tables.is_empty());
|
||||
assert!(result.pages_with_columns.is_empty());
|
||||
@@ -1582,7 +1582,7 @@ fn test_extract_pages_markdown_classification_matches_process_pdf() {
|
||||
let full = process_pdf_mem(&buf).unwrap();
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result.pages_with_tables, full.layout.pages_with_tables,
|
||||
@@ -1605,7 +1605,7 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
|
||||
// Get per-page output for all pages
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
|
||||
// Concatenated per-page markdown should contain substantial overlap with
|
||||
// the full output (exact match not expected due to header/footer stripping
|
||||
@@ -1630,3 +1630,41 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
|
||||
full_md.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_none_returns_all_pages() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let page_count = process_pdf_mem(&buf).unwrap().page_count;
|
||||
|
||||
let result = extract_pages_markdown_mem(&buf, None).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len() as u32, page_count);
|
||||
for (i, page) in result.pages.iter().enumerate() {
|
||||
assert_eq!(page.page, i as u32, "pages should be in document order");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_path_api() {
|
||||
let path = "tests/fixtures/nexo-price-en.pdf";
|
||||
let buf = std::fs::read(path).unwrap();
|
||||
|
||||
let via_path = extract_pages_markdown(path, Some(&[0])).unwrap();
|
||||
let via_mem = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
|
||||
assert_eq!(via_path.pages.len(), via_mem.pages.len());
|
||||
assert_eq!(via_path.pages[0].markdown, via_mem.pages[0].markdown);
|
||||
assert_eq!(via_path.pages[0].needs_ocr, via_mem.pages[0].needs_ocr);
|
||||
assert_eq!(via_path.is_complex, via_mem.is_complex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_path_none_returns_all_pages() {
|
||||
let path = "tests/fixtures/nexo-price-en.pdf";
|
||||
let page_count = process_pdf_mem(&std::fs::read(path).unwrap())
|
||||
.unwrap()
|
||||
.page_count;
|
||||
|
||||
let result = extract_pages_markdown(path, None).unwrap();
|
||||
assert_eq!(result.pages.len() as u32, page_count);
|
||||
}
|
||||
|
||||
@@ -270,6 +270,78 @@ class TestExtractTextInRegions:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_pages_markdown / extract_pages_markdown_bytes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractPagesMarkdown:
|
||||
def test_default_returns_all_pages(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf")
|
||||
)
|
||||
assert len(result.pages) == 3
|
||||
assert [p.page for p in result.pages] == [0, 1, 2]
|
||||
assert all(isinstance(p.markdown, str) for p in result.pages)
|
||||
|
||||
def test_bytes_default_returns_all_pages(self):
|
||||
data = fixture_bytes("thermo-freon12.pdf")
|
||||
result = pdf_inspector.extract_pages_markdown_bytes(data)
|
||||
assert len(result.pages) == 3
|
||||
|
||||
def test_selected_pages_preserve_order(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[2, 0]
|
||||
)
|
||||
assert [p.page for p in result.pages] == [2, 0]
|
||||
|
||||
def test_bytes_selected_pages_preserve_order(self):
|
||||
data = fixture_bytes("thermo-freon12.pdf")
|
||||
result = pdf_inspector.extract_pages_markdown_bytes(data, pages=[1])
|
||||
assert len(result.pages) == 1
|
||||
assert result.pages[0].page == 1
|
||||
|
||||
def test_page_fields(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[0]
|
||||
)
|
||||
page = result.pages[0]
|
||||
assert isinstance(page.page, int)
|
||||
assert isinstance(page.markdown, str)
|
||||
assert isinstance(page.needs_ocr, bool)
|
||||
assert not page.needs_ocr # text-based fixture
|
||||
assert len(page.markdown) > 0
|
||||
|
||||
def test_result_fields(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf")
|
||||
)
|
||||
assert isinstance(result.pages, list)
|
||||
assert isinstance(result.pages_with_tables, list)
|
||||
assert isinstance(result.pages_with_columns, list)
|
||||
assert isinstance(result.pages_needing_ocr, list)
|
||||
assert isinstance(result.is_complex, bool)
|
||||
|
||||
def test_out_of_range_page_marks_needs_ocr(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[9999]
|
||||
)
|
||||
assert len(result.pages) == 1
|
||||
assert result.pages[0].needs_ocr
|
||||
assert result.pages[0].markdown == ""
|
||||
|
||||
def test_repr(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[0]
|
||||
)
|
||||
assert "PagesExtractionResult" in repr(result)
|
||||
assert "PageMarkdown" in repr(result.pages[0])
|
||||
|
||||
def test_not_a_pdf(self):
|
||||
with pytest.raises(ValueError):
|
||||
pdf_inspector.extract_pages_markdown_bytes(b"not a pdf")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user