Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b88562b973 |
@@ -35,8 +35,3 @@ scripts/
|
||||
# Test output
|
||||
test_output/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
|
||||
|
||||
@@ -61,8 +61,7 @@ src/
|
||||
|
||||
- **Unit tests**: inline `#[cfg(test)] mod tests` in each module with synthetic data.
|
||||
- **Integration tests**: `tests/integration_tests.rs` with fixture PDFs in `tests/fixtures/`.
|
||||
- **Regression suite**: sibling repo `pdf-evals` with 187+ snapshot PDFs. Run `cargo build --release` then `bench.py test` in that repo before committing.
|
||||
- **Semantic quality**: run `bench.py score` in `pdf-evals` for the semantic verdict (TEDS + MHS + reading order + char/word + list preservation, composited). Character-level diff alone misclassifies structural improvements (e.g., column-detection rewrites) as regressions — `score` is the tie-breaker. See `pdf-evals/CLAUDE.md` "Semantic scoring".
|
||||
- **Regression suite**: sibling repo `pdf-evals` with 179+ snapshot PDFs. Run `cargo build --release` then `bench.py test` in that repo before committing.
|
||||
|
||||
## Debugging
|
||||
|
||||
|
||||
@@ -42,14 +42,6 @@ text = pdf_inspector.extract_text("document.pdf")
|
||||
items = pdf_inspector.extract_text_with_positions("document.pdf")
|
||||
for item in items[:5]:
|
||||
print(f"'{item.text}' at ({item.x:.0f}, {item.y:.0f}) size={item.font_size}")
|
||||
|
||||
# Per-page markdown (one Markdown string per page, plus layout metadata)
|
||||
result = pdf_inspector.extract_pages_markdown("document.pdf")
|
||||
for page in result.pages:
|
||||
print(f"Page {page.page}: {len(page.markdown)} chars, needs_ocr={page.needs_ocr}")
|
||||
|
||||
# Restrict to specific 0-indexed pages (preserves caller order)
|
||||
result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
|
||||
```
|
||||
|
||||
## API reference
|
||||
@@ -68,8 +60,6 @@ result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
|
||||
| `extract_text_with_positions_bytes(data, pages=None)` | Text with positions from bytes |
|
||||
| `extract_text_in_regions(path, page_regions)` | Extract text in bounding-box regions |
|
||||
| `extract_text_in_regions_bytes(data, page_regions)` | Region extraction from bytes |
|
||||
| `extract_pages_markdown(path, pages=None)` | Per-page Markdown + layout metadata (all pages by default) |
|
||||
| `extract_pages_markdown_bytes(data, pages=None)` | Per-page Markdown from bytes |
|
||||
|
||||
## Types
|
||||
|
||||
@@ -82,7 +72,3 @@ result = pdf_inspector.extract_pages_markdown("document.pdf", pages=[0, 2])
|
||||
**`RegionText` fields:** `text`, `needs_ocr`
|
||||
|
||||
**`PageRegionTexts` fields:** `page` (0-indexed), `regions` (list of RegionText)
|
||||
|
||||
**`PageMarkdown` fields:** `page` (0-indexed), `markdown`, `needs_ocr`
|
||||
|
||||
**`PagesExtractionResult` fields:** `pages` (list of PageMarkdown), `pages_with_tables` (1-indexed), `pages_with_columns` (1-indexed), `pages_needing_ocr` (1-indexed), `is_complex`
|
||||
|
||||
@@ -79,27 +79,6 @@ let bytes = std::fs::read("document.pdf")?;
|
||||
let result = process_pdf_mem(&bytes)?;
|
||||
```
|
||||
|
||||
Extract per-page Markdown (one string per page, plus document-wide layout
|
||||
metadata):
|
||||
|
||||
```rust
|
||||
use pdf_inspector::extract_pages_markdown;
|
||||
|
||||
// Pass `None` for every page in document order, or a slice of 0-indexed
|
||||
// pages to restrict the output (caller-supplied order is preserved).
|
||||
let result = extract_pages_markdown("document.pdf", None)?;
|
||||
|
||||
for page in &result.pages {
|
||||
if page.needs_ocr {
|
||||
// Route this page to OCR
|
||||
} else {
|
||||
println!("Page {}: {}", page.page, page.markdown);
|
||||
}
|
||||
}
|
||||
|
||||
println!("Complex layout? {}", result.is_complex);
|
||||
```
|
||||
|
||||
## Processing modes
|
||||
|
||||
| Mode | What it does | Returns |
|
||||
@@ -123,8 +102,6 @@ println!("Complex layout? {}", result.is_complex);
|
||||
| `to_markdown(text, options)` | Convert plain text to Markdown |
|
||||
| `to_markdown_from_items(items, options)` | Markdown from pre-extracted `TextItem`s |
|
||||
| `to_markdown_from_items_with_rects(items, options, rects)` | Markdown with rectangle-based table detection |
|
||||
| `extract_pages_markdown(path, pages)` | Per-page Markdown + layout metadata (file) |
|
||||
| `extract_pages_markdown_mem(bytes, pages)` | Per-page Markdown from bytes |
|
||||
|
||||
Low-level detection functions are also available via the `detector` module (`detect_pdf_type`, `detect_pdf_type_with_config`, etc.) for callers who need `PdfTypeResult` instead of `PdfProcessResult`.
|
||||
|
||||
@@ -142,6 +119,4 @@ Low-level detection functions are also available via the `detector` module (`det
|
||||
| `LayoutComplexity` | Layout analysis: is_complex, pages_with_tables, pages_with_columns |
|
||||
| `TextItem` | Text with position, font info, and page number |
|
||||
| `MarkdownOptions` | Configuration for Markdown formatting (page numbers, etc.) |
|
||||
| `PageMarkdown` | Per-page result: page (0-indexed), markdown, needs_ocr |
|
||||
| `PagesExtractionResult` | Per-page output + 1-indexed pages_with_tables / pages_with_columns / pages_needing_ocr, is_complex |
|
||||
| `PdfError` | `Io`, `Parse`, `Encrypted`, `InvalidStructure`, `NotAPdf` |
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.4.0",
|
||||
"version": "1.2.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+5
-10
@@ -343,26 +343,21 @@ pub struct PagesExtractionResult {
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF, with layout classification
|
||||
/// metadata.
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Returns per-page markdown and classification data (tables, columns,
|
||||
/// OCR needs) from a single parse. Font statistics are computed from the
|
||||
/// full document so header detection is consistent across pages.
|
||||
///
|
||||
/// Omit `pages` (or pass `undefined`) to return every page in document
|
||||
/// order. Pass an array of 0-indexed page numbers to restrict output to
|
||||
/// those pages, in caller-supplied order.
|
||||
#[napi]
|
||||
pub fn extract_pages_markdown(
|
||||
buffer: Buffer,
|
||||
pages: Option<Vec<u32>>,
|
||||
pages: Vec<u32>,
|
||||
) -> Result<PagesExtractionResult> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
catch_panic("extract_pages_markdown", move || {
|
||||
let result =
|
||||
pdf_inspector::extract_pages_markdown_mem(&bytes, pages.as_deref())
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
|
||||
.map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
|
||||
Ok(PagesExtractionResult {
|
||||
pages: result
|
||||
.pages
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
extractText,
|
||||
extractTextWithPositions,
|
||||
extractTextInRegions,
|
||||
extractPagesMarkdown,
|
||||
} from './index.js';
|
||||
|
||||
const fixture = readFileSync('../tests/fixtures/thermo-freon12.pdf');
|
||||
@@ -90,28 +89,6 @@ assert.equal(typeof regionResults[0].regions[0].text, 'string');
|
||||
assert.equal(typeof regionResults[0].regions[0].needsOcr, 'boolean');
|
||||
console.log(' extractTextInRegions: OK');
|
||||
|
||||
// --- extractPagesMarkdown ---
|
||||
console.log('Testing extractPagesMarkdown...');
|
||||
|
||||
// omit pages → every page in document order
|
||||
const allPages = extractPagesMarkdown(fixture);
|
||||
assert.equal(allPages.pages.length, 3);
|
||||
assert.deepEqual(allPages.pages.map(p => p.page), [0, 1, 2]);
|
||||
assert.ok(typeof allPages.pages[0].markdown === 'string');
|
||||
assert.equal(typeof allPages.pages[0].needsOcr, 'boolean');
|
||||
assert.ok(Array.isArray(allPages.pagesWithTables));
|
||||
assert.ok(Array.isArray(allPages.pagesWithColumns));
|
||||
assert.ok(Array.isArray(allPages.pagesNeedingOcr));
|
||||
assert.equal(typeof allPages.isComplex, 'boolean');
|
||||
console.log(' extractPagesMarkdown (no pages arg): OK');
|
||||
|
||||
// selected pages preserve caller order
|
||||
const picked = extractPagesMarkdown(fixture, [2, 0]);
|
||||
assert.equal(picked.pages.length, 2);
|
||||
assert.equal(picked.pages[0].page, 2);
|
||||
assert.equal(picked.pages[1].page, 0);
|
||||
console.log(' extractPagesMarkdown with pages: OK');
|
||||
|
||||
// --- Error handling ---
|
||||
console.log('Testing error handling...');
|
||||
assert.throws(() => processPdf(Buffer.from('not a pdf')), /process_pdf/);
|
||||
|
||||
@@ -52,28 +52,6 @@ class PageRegionTexts:
|
||||
"""0-indexed page number."""
|
||||
regions: list[RegionText]
|
||||
|
||||
class PageMarkdown:
|
||||
"""Per-page markdown extraction result."""
|
||||
page: int
|
||||
"""0-indexed page number."""
|
||||
markdown: str
|
||||
"""Formatted markdown for this page (empty string when needs_ocr is True)."""
|
||||
needs_ocr: bool
|
||||
"""True when text on this page is unreliable and OCR should be used instead."""
|
||||
|
||||
class PagesExtractionResult:
|
||||
"""Per-page markdown output with document-wide layout classification."""
|
||||
pages: list[PageMarkdown]
|
||||
"""Per-page markdown results, in the order requested."""
|
||||
pages_with_tables: list[int]
|
||||
"""1-indexed pages where tables were detected."""
|
||||
pages_with_columns: list[int]
|
||||
"""1-indexed pages where multi-column layout was detected."""
|
||||
pages_needing_ocr: list[int]
|
||||
"""1-indexed pages that need OCR."""
|
||||
is_complex: bool
|
||||
"""True if any page has tables or multi-column layout."""
|
||||
|
||||
def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult:
|
||||
"""Process a PDF: detect type, extract text, convert to Markdown."""
|
||||
...
|
||||
@@ -137,31 +115,3 @@ def extract_text_in_regions_bytes(
|
||||
page_regions: List of (page_0indexed, [[x1, y1, x2, y2], ...]) tuples.
|
||||
"""
|
||||
...
|
||||
|
||||
def extract_pages_markdown(
|
||||
path: str,
|
||||
pages: Optional[list[int]] = None,
|
||||
) -> PagesExtractionResult:
|
||||
"""Extract formatted markdown for pages of a PDF, with layout classification.
|
||||
|
||||
Args:
|
||||
path: Path to the PDF file.
|
||||
pages: Optional list of 0-indexed pages. When ``None`` (default), every
|
||||
page is returned in document order. Otherwise, output matches the
|
||||
caller-supplied order.
|
||||
|
||||
Returns:
|
||||
PagesExtractionResult with per-page markdown and document-wide layout
|
||||
classification (tables, columns, OCR needs).
|
||||
"""
|
||||
...
|
||||
|
||||
def extract_pages_markdown_bytes(
|
||||
data: bytes,
|
||||
pages: Optional[list[int]] = None,
|
||||
) -> PagesExtractionResult:
|
||||
"""Extract formatted markdown for pages of a PDF from bytes.
|
||||
|
||||
See :func:`extract_pages_markdown` for details.
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -626,33 +626,6 @@ fn find_relative_valleys(
|
||||
valleys
|
||||
}
|
||||
|
||||
/// Detect whether a side of a gutter consists predominantly of list-marker
|
||||
/// glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇). A column of bullets on the left margin
|
||||
/// creates a spurious histogram valley between the bullet and the content.
|
||||
/// Treating it as a real column splits each list item's text across two
|
||||
/// "columns," so we reject these candidates.
|
||||
fn is_list_marker_column(items: &[&&TextItem]) -> bool {
|
||||
const LIST_MARKERS: &[char] = &['•', '●', '○', '◦', '▪', '▫', '◆', '◇', '■', '□'];
|
||||
if items.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let marker_count = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
let t = i.text.trim();
|
||||
let mut chars = t.chars();
|
||||
match (chars.next(), chars.next()) {
|
||||
(Some(c), None) => LIST_MARKERS.contains(&c),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
.count();
|
||||
// Require ≥80% of items on this side to be standalone markers. A handful
|
||||
// of non-marker items (stray page numbers, footnote refs) shouldn't
|
||||
// defeat the check.
|
||||
marker_count as f32 / items.len() as f32 >= 0.8
|
||||
}
|
||||
|
||||
/// Validate valley candidates with vertical consistency checks and build column regions.
|
||||
///
|
||||
/// When `center_assign` is true, items are assigned to columns based on their
|
||||
@@ -721,19 +694,6 @@ fn validate_and_build_columns(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reject valleys where the smaller side is just a column of list
|
||||
// markers (bullets aligned at the left margin). This is a common
|
||||
// pattern in PDFs where ● starts each list item: histogram detection
|
||||
// sees the gap between bullet and content as a gutter.
|
||||
let smaller_items: &[&&TextItem] = if left_items.len() <= right_items.len() {
|
||||
&left_items
|
||||
} else {
|
||||
&right_items
|
||||
};
|
||||
if is_list_marker_column(smaller_items) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check vertical overlap
|
||||
if y_range > 0.0 {
|
||||
let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
|
||||
@@ -1820,63 +1780,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bullet_marker_column_not_detected_as_column() {
|
||||
// Pattern: every line is `● <content>`, with ● at x=90 and content
|
||||
// starting at x=104. Histogram detection sees a gutter between them
|
||||
// and would split the page into a "bullet column" and "content column",
|
||||
// scrambling every list item.
|
||||
let mut items = Vec::new();
|
||||
for i in 0..15 {
|
||||
let y = 750.0 - i as f32 * 30.0;
|
||||
items.push(make_item(1, 90.0, y, "●"));
|
||||
items.push(make_item(
|
||||
1,
|
||||
104.0,
|
||||
y,
|
||||
"FullContentLineTextHere________________",
|
||||
));
|
||||
}
|
||||
// Pad with content to satisfy min item count for column detection.
|
||||
for i in 0..15 {
|
||||
let y = 300.0 - i as f32 * 14.0;
|
||||
items.push(make_item(1, 72.0, y, "FootnoteText_____________________"));
|
||||
}
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(
|
||||
cols.len(),
|
||||
1,
|
||||
"Bullet markers aligned at left margin should not be treated as their own column"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_detects_bullets() {
|
||||
let items = vec![
|
||||
make_item(1, 90.0, 100.0, "●"),
|
||||
make_item(1, 90.0, 114.0, "●"),
|
||||
make_item(1, 90.0, 128.0, "●"),
|
||||
make_item(1, 90.0, 142.0, "●"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_rejects_prose() {
|
||||
let items = vec![
|
||||
make_item(1, 30.0, 100.0, "Regular prose line"),
|
||||
make_item(1, 30.0, 114.0, "Another sentence"),
|
||||
make_item(1, 30.0, 128.0, "Third line"),
|
||||
make_item(1, 30.0, 142.0, "Fourth line"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(!is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn premask_narrow_line_not_masked() {
|
||||
// Items that form a line spanning only ~40% of column width → not masked
|
||||
|
||||
+7
-42
@@ -336,17 +336,13 @@ pub struct PagesExtractionResult {
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF, with layout
|
||||
/// Extract formatted markdown for specific pages of a PDF, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
|
||||
/// this returns per-page markdown so callers can mix direct extraction
|
||||
/// (for simple text pages) with GPU OCR (for complex/scanned pages).
|
||||
///
|
||||
/// When `pages` is `None`, every page (0-indexed, in document order) is
|
||||
/// returned. When `Some(&[...])`, only the listed 0-indexed pages are
|
||||
/// returned, in the caller's order.
|
||||
///
|
||||
/// Font statistics are computed from the full document so header
|
||||
/// detection thresholds are consistent regardless of which pages are
|
||||
/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
|
||||
@@ -356,7 +352,7 @@ pub struct PagesExtractionResult {
|
||||
/// at near-zero cost since the items/rects/lines are already in memory.
|
||||
pub fn extract_pages_markdown_mem(
|
||||
buffer: &[u8],
|
||||
pages: Option<&[u32]>,
|
||||
pages: &[u32],
|
||||
) -> Result<PagesExtractionResult, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, page_count) = load_document_from_mem(buffer)?;
|
||||
@@ -372,20 +368,10 @@ pub fn extract_pages_markdown_mem(
|
||||
// Compute font stats from full document (cross-page consistency).
|
||||
let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
|
||||
|
||||
// When caller doesn't specify pages, return every page in document order.
|
||||
let all_pages: Vec<u32>;
|
||||
let pages_slice: &[u32] = match pages {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
all_pages = (0..page_count).collect();
|
||||
&all_pages
|
||||
}
|
||||
};
|
||||
|
||||
let mut results = Vec::with_capacity(pages_slice.len());
|
||||
let mut results = Vec::with_capacity(pages.len());
|
||||
let mut pages_needing_ocr = Vec::new();
|
||||
|
||||
for &page_0idx in pages_slice {
|
||||
for &page_0idx in pages {
|
||||
// Out-of-range pages → empty + needs_ocr
|
||||
if page_0idx >= page_count {
|
||||
pages_needing_ocr.push(page_0idx + 1);
|
||||
@@ -458,20 +444,6 @@ pub fn extract_pages_markdown_mem(
|
||||
})
|
||||
}
|
||||
|
||||
/// Path-based wrapper for [`extract_pages_markdown_mem`].
|
||||
///
|
||||
/// Reads the PDF from disk and extracts per-page markdown. Pass `None` for
|
||||
/// `pages` to return every page in document order, or `Some(&[...])` to
|
||||
/// restrict to specific 0-indexed pages (in caller-supplied order).
|
||||
pub fn extract_pages_markdown<P: AsRef<Path>>(
|
||||
path: P,
|
||||
pages: Option<&[u32]>,
|
||||
) -> Result<PagesExtractionResult, PdfError> {
|
||||
validate_pdf_file(&path)?;
|
||||
let buffer = std::fs::read(path.as_ref())?;
|
||||
extract_pages_markdown_mem(&buffer, pages)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Region-based text extraction (for hybrid OCR pipelines)
|
||||
// =========================================================================
|
||||
@@ -1810,26 +1782,19 @@ fn compute_layout_complexity(
|
||||
markdown::filter_lines_to_band(lines, page, x_lo, x_hi)
|
||||
};
|
||||
|
||||
// TOC pages route through the table detector but render as flat
|
||||
// lists. They aren't tables in any user-facing sense, so don't
|
||||
// count them toward LayoutComplexity (would also trip the
|
||||
// table-page guard in column detection below).
|
||||
let has_data_table =
|
||||
|tables: &[tables::Table]| tables.iter().any(|t| t.kind == tables::TableKind::Data);
|
||||
|
||||
let (rect_tables, _) = tables::detect_tables_from_rects(&band_items, &band_rects, page);
|
||||
if has_data_table(&rect_tables) {
|
||||
if !rect_tables.is_empty() {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
let line_tables = tables::detect_tables_from_lines(&band_items, &band_lines, page);
|
||||
if has_data_table(&line_tables) {
|
||||
if !line_tables.is_empty() {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
// Heuristic fallback for borderless tables
|
||||
let heuristic_tables = tables::detect_tables(&band_items, base_size, false);
|
||||
if has_data_table(&heuristic_tables) {
|
||||
if !heuristic_tables.is_empty() {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -64,22 +64,6 @@ pub(crate) fn is_caption_line(text: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if text starts with an unambiguous bullet marker (●, •, ○, ◦).
|
||||
///
|
||||
/// Narrower than [`is_list_item`]: it excludes numbered/lettered patterns
|
||||
/// like `1.` or `a)`, which legitimately appear as section headings in many
|
||||
/// documents. Used by the heading classifier to reject bullet lines without
|
||||
/// also demoting numbered headings.
|
||||
pub(crate) fn starts_with_bullet_marker(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
trimmed.starts_with("• ")
|
||||
|| trimmed.starts_with("● ")
|
||||
|| trimmed.starts_with("○ ")
|
||||
|| trimmed.starts_with("◦ ")
|
||||
|| trimmed.starts_with("- ")
|
||||
|| trimmed.starts_with("* ")
|
||||
}
|
||||
|
||||
/// Check if text looks like a list item
|
||||
pub(crate) fn is_list_item(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
@@ -131,16 +115,6 @@ pub(crate) fn format_list_item(text: &str) -> String {
|
||||
if let Some(rest) = trimmed.strip_prefix(*bullet) {
|
||||
return format!("- {}", rest.trim_start());
|
||||
}
|
||||
// Bullet inside a leading bold/italic run (e.g. "**● Label:** rest").
|
||||
// The run wraps both the marker and the following label because both
|
||||
// use a bold font in the PDF.
|
||||
for wrapper in ["**", "*"] {
|
||||
if let Some(after_open) = trimmed.strip_prefix(wrapper) {
|
||||
if let Some(rest) = after_open.strip_prefix(*bullet) {
|
||||
return format!("- {}{}", wrapper, rest.trim_start());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
|
||||
@@ -224,42 +198,3 @@ pub(crate) fn is_monospace_font(font_name: &str) -> bool {
|
||||
|
||||
patterns.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn format_list_item_plain_bullet() {
|
||||
assert_eq!(format_list_item("● Item"), "- Item");
|
||||
assert_eq!(format_list_item("• Item"), "- Item");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_list_item_bullet_inside_bold() {
|
||||
// PDF that uses bold font for both the marker and the label produces
|
||||
// a single bold run like "**● Label:** rest"; the bullet must still
|
||||
// be stripped and the bold wrapper preserved on the label.
|
||||
assert_eq!(
|
||||
format_list_item("**● Fraud: Willing cooperation;**"),
|
||||
"- **Fraud: Willing cooperation;**"
|
||||
);
|
||||
assert_eq!(
|
||||
format_list_item("**● Label:** rest of line"),
|
||||
"- **Label:** rest of line"
|
||||
);
|
||||
assert_eq!(format_list_item("*● Italic:* rest"), "- *Italic:* rest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_list_item_already_dash() {
|
||||
assert_eq!(format_list_item("- existing"), "- existing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_item_with_bullet_space() {
|
||||
assert!(is_list_item("● Item"));
|
||||
assert!(is_list_item("• Item"));
|
||||
assert!(is_list_item("- Item"));
|
||||
}
|
||||
}
|
||||
|
||||
+2
-149
@@ -9,9 +9,7 @@ use super::analysis::{
|
||||
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders,
|
||||
};
|
||||
use super::classify::{
|
||||
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
||||
};
|
||||
use super::classify::{format_list_item, is_caption_line, is_list_item, is_monospace_font};
|
||||
use super::postprocess::clean_markdown;
|
||||
use super::preprocess::{merge_drop_caps, merge_heading_lines};
|
||||
use super::MarkdownOptions;
|
||||
@@ -586,30 +584,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
.as_ref()
|
||||
.and_then(struct_role_heading_level)
|
||||
.filter(|level| !overused_heading_levels.contains(level));
|
||||
|
||||
// Protect wrapped list items: when inside a list, a visually-continuing
|
||||
// line (same indent, line-wrap spacing) must not be reclassified as a
|
||||
// heading by the font heuristic — PDFs often bold the lead phrase of a
|
||||
// list item across multiple wrap lines, and an all-bold middle line
|
||||
// would otherwise split one item into a heading + stray body text.
|
||||
// We gate on the document's paragraph threshold so genuine section
|
||||
// headings that follow a numbered paragraph (y_gap > para_threshold)
|
||||
// remain detectable.
|
||||
let looks_like_list_continuation = in_list
|
||||
&& match (last_list_x, line.items.first().map(|i| i.x)) {
|
||||
(Some(list_x), Some(curr_x)) => {
|
||||
let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0;
|
||||
let y_ok = y_gap >= 0.0 && y_gap <= para_threshold;
|
||||
x_ok && y_ok && !is_list_item(plain_trimmed)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let heuristic_heading = if options.detect_headers
|
||||
&& !looks_like_list_continuation
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !starts_with_bullet_marker(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
||||
@@ -664,16 +641,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Structure-tree list item (LI only — LBody is a continuation, not a new item).
|
||||
// Some tagged PDFs use a "flat" style where every wrapped line in a list item
|
||||
// gets its own MCID tagged directly under LI. When we're already inside a list
|
||||
// and the line has no visible bullet marker, treat it as a continuation (falls
|
||||
// through to the continuation logic below) rather than a new list item.
|
||||
// Structure-tree list item (LI only — LBody is a continuation, not a new item)
|
||||
if struct_role
|
||||
.as_ref()
|
||||
.is_some_and(|r| matches!(r, StructRole::LI))
|
||||
&& !is_list_item(plain_trimmed)
|
||||
&& !in_list
|
||||
{
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
@@ -1118,59 +1090,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_li_flat_continuation_lines_merge() {
|
||||
// Regression: some tagged PDFs put each wrapped visual line of a list
|
||||
// item under its own MCID, all tagged directly as LI. Continuation
|
||||
// lines (no bullet marker) must merge into the bulleted parent item,
|
||||
// not each become their own list item.
|
||||
let make = |text: &str, mcid: i64, x: f32, y: f32| {
|
||||
let mut item = make_item(text, 1, Some(mcid));
|
||||
item.x = x;
|
||||
item.y = y;
|
||||
item
|
||||
};
|
||||
let lines = vec![
|
||||
make_line(vec![make("● First item that wraps onto", 0, 90.0, 322.0)]),
|
||||
make_line(vec![make("a continuation line.", 1, 108.0, 306.0)]),
|
||||
make_line(vec![make("● Second bullet also wraps", 2, 90.0, 290.0)]),
|
||||
make_line(vec![make("to a second line here.", 3, 108.0, 274.0)]),
|
||||
];
|
||||
|
||||
let mut page_roles = HashMap::new();
|
||||
for mcid in 0..4 {
|
||||
page_roles.insert(mcid, StructRole::LI);
|
||||
}
|
||||
let mut roles = HashMap::new();
|
||||
roles.insert(1u32, page_roles);
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
Some(&roles),
|
||||
);
|
||||
|
||||
assert!(
|
||||
md.contains("- First item that wraps onto a continuation line."),
|
||||
"continuation should merge into first bullet: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("- Second bullet also wraps to a second line here."),
|
||||
"continuation should merge into second bullet: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("- a continuation line."),
|
||||
"continuation line should not get its own bullet: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("- to a second line here."),
|
||||
"continuation line should not get its own bullet: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_blockquote() {
|
||||
let lines = vec![make_line(vec![make_item("Quoted text", 1, Some(0))])];
|
||||
@@ -1473,70 +1392,4 @@ mod tests {
|
||||
overused
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrapped_bold_lead_in_list_item_not_heading() {
|
||||
// Regression: numbered-list items whose bold "lead" phrase wraps onto
|
||||
// a second line (e.g. definitions in system cards) must not have the
|
||||
// wrapped line reclassified as a heading. The middle line is
|
||||
// all_bold + standalone (in_paragraph=false while in_list), which
|
||||
// previously tripped the rarity heuristic and emitted #### in the
|
||||
// middle of the item, splitting the body into stray bullets.
|
||||
let make = |text: &str, x: f32, y: f32, bold: bool| {
|
||||
let mut item = make_item(text, 1, None);
|
||||
item.x = x;
|
||||
item.y = y;
|
||||
item.is_bold = bold;
|
||||
item
|
||||
};
|
||||
|
||||
let lines = vec![
|
||||
// "1. **bold lead phrase start**"
|
||||
make_line(vec![
|
||||
make("1. ", 72.0, 700.0, false),
|
||||
make(
|
||||
"Chemical and biological weapons threat model 1 (CB-1): Non-novel",
|
||||
90.0,
|
||||
700.0,
|
||||
true,
|
||||
),
|
||||
]),
|
||||
// wrapped continuation of the bold lead — all_bold, same indent
|
||||
make_line(vec![make(
|
||||
"chemical/biological weapons production capabilities: A model has CB-1",
|
||||
90.0,
|
||||
686.0,
|
||||
true,
|
||||
)]),
|
||||
// body text of the same list item
|
||||
make_line(vec![make(
|
||||
"capabilities if it has the ability to significantly help individuals.",
|
||||
90.0,
|
||||
672.0,
|
||||
false,
|
||||
)]),
|
||||
];
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(
|
||||
!md.contains("#### "),
|
||||
"wrapped bold lead must not become a heading: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.lines().filter(|l| l.starts_with("- ")).count() == 0,
|
||||
"continuation body must not become a stray bullet: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("1. ") && md.contains("A model has CB-1"),
|
||||
"numbered list item should remain intact: {md}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
-121
@@ -146,67 +146,6 @@ impl PyPageRegionTexts {
|
||||
// Text item wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[pyclass(name = "PageMarkdown")]
|
||||
#[derive(Clone)]
|
||||
pub struct PyPageMarkdown {
|
||||
/// 0-indexed page number.
|
||||
#[pyo3(get)]
|
||||
pub page: u32,
|
||||
/// Formatted markdown for this page.
|
||||
#[pyo3(get)]
|
||||
pub markdown: String,
|
||||
/// True when text on this page is unreliable (GID-encoded fonts,
|
||||
/// encoding issues, garbage text, or empty extraction).
|
||||
#[pyo3(get)]
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyPageMarkdown {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"PageMarkdown(page={}, markdown='{}', needs_ocr={})",
|
||||
self.page,
|
||||
self.markdown.chars().take(40).collect::<String>(),
|
||||
self.needs_ocr
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Combined per-page markdown extraction and layout classification result.
|
||||
#[pyclass(name = "PagesExtractionResult")]
|
||||
#[derive(Clone)]
|
||||
pub struct PyPagesExtractionResult {
|
||||
/// Per-page markdown results, in the order requested.
|
||||
#[pyo3(get)]
|
||||
pub pages: Vec<PyPageMarkdown>,
|
||||
/// 1-indexed pages where tables were detected.
|
||||
#[pyo3(get)]
|
||||
pub pages_with_tables: Vec<u32>,
|
||||
/// 1-indexed pages where multi-column layout was detected.
|
||||
#[pyo3(get)]
|
||||
pub pages_with_columns: Vec<u32>,
|
||||
/// 1-indexed pages that need OCR (scanned/image-based or unreliable text).
|
||||
#[pyo3(get)]
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// True if any page has tables or columns.
|
||||
#[pyo3(get)]
|
||||
pub is_complex: bool,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PyPagesExtractionResult {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"PagesExtractionResult(pages={}, pages_with_tables={:?}, is_complex={})",
|
||||
self.pages.len(),
|
||||
self.pages_with_tables,
|
||||
self.is_complex
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A positioned text item extracted from a PDF.
|
||||
#[pyclass(name = "TextItem")]
|
||||
#[derive(Clone)]
|
||||
@@ -341,24 +280,6 @@ fn parse_page_regions(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResult {
|
||||
PyPagesExtractionResult {
|
||||
pages: r
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|p| PyPageMarkdown {
|
||||
page: p.page,
|
||||
markdown: p.markdown,
|
||||
needs_ocr: p.needs_ocr,
|
||||
})
|
||||
.collect(),
|
||||
pages_with_tables: r.pages_with_tables,
|
||||
pages_with_columns: r.pages_with_columns,
|
||||
pages_needing_ocr: r.pages_needing_ocr,
|
||||
is_complex: r.is_complex,
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRegionTexts> {
|
||||
results
|
||||
.into_iter()
|
||||
@@ -521,44 +442,6 @@ fn extract_text_in_regions_bytes(
|
||||
Ok(convert_region_results(results))
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF file, with layout
|
||||
/// classification metadata.
|
||||
///
|
||||
/// Returns per-page markdown and classification data (tables, columns,
|
||||
/// OCR needs) from a single parse. Font statistics are computed from the
|
||||
/// full document so header detection is consistent across pages.
|
||||
///
|
||||
/// Args:
|
||||
/// path: Path to the PDF file.
|
||||
/// pages: Optional list of 0-indexed pages. When None (default), every
|
||||
/// page is returned in document order. When provided, output
|
||||
/// matches the caller-supplied order.
|
||||
///
|
||||
/// Returns:
|
||||
/// PagesExtractionResult with per-page markdown and classification data.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (path, pages=None))]
|
||||
fn extract_pages_markdown(
|
||||
path: &str,
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> PyResult<PyPagesExtractionResult> {
|
||||
let result = crate::extract_pages_markdown(path, pages.as_deref()).map_err(to_py_err)?;
|
||||
Ok(to_py_pages_result(result))
|
||||
}
|
||||
|
||||
/// Extract formatted markdown for pages of a PDF from bytes.
|
||||
///
|
||||
/// See [`extract_pages_markdown`] for details.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (data, pages=None))]
|
||||
fn extract_pages_markdown_bytes(
|
||||
data: &[u8],
|
||||
pages: Option<Vec<u32>>,
|
||||
) -> PyResult<PyPagesExtractionResult> {
|
||||
let result = crate::extract_pages_markdown_mem(data, pages.as_deref()).map_err(to_py_err)?;
|
||||
Ok(to_py_pages_result(result))
|
||||
}
|
||||
|
||||
/// Python module definition.
|
||||
#[pymodule]
|
||||
fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
@@ -567,8 +450,6 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<PyTextItem>()?;
|
||||
m.add_class::<PyRegionText>()?;
|
||||
m.add_class::<PyPageRegionTexts>()?;
|
||||
m.add_class::<PyPageMarkdown>()?;
|
||||
m.add_class::<PyPagesExtractionResult>()?;
|
||||
m.add_function(wrap_pyfunction!(process_pdf, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(process_pdf_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(detect_pdf, m)?)?;
|
||||
@@ -581,7 +462,5 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(extract_text_with_positions_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_text_in_regions, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_text_in_regions_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_pages_markdown, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(extract_pages_markdown_bytes, m)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -581,14 +581,8 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
// Validation 1: some rows should have content in first column.
|
||||
// Use a lower threshold (25%) for tables with wrapped cells where
|
||||
// continuation lines leave the first column empty.
|
||||
// Skip when cells form a narrow TOC pattern: hierarchical entries indented
|
||||
// across multiple X levels leave the leftmost column sparse (only top-level
|
||||
// chapters land there) but the structure is still a valid TOC. Narrow only
|
||||
// (<=5 cols) — wide multi-column TOCs (e.g. 2-up indices) would render
|
||||
// poorly through format_toc_as_list, which assumes one entry per row.
|
||||
let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
|
||||
let is_narrow_toc = columns.len() <= 5 && is_table_of_contents(&cells);
|
||||
if rows_with_first_col < rows.len() / 4 && !is_narrow_toc {
|
||||
if rows_with_first_col < rows.len() / 4 {
|
||||
log::debug!(
|
||||
" validation 1 fail: {}/{} rows have first col",
|
||||
rows_with_first_col,
|
||||
@@ -659,12 +653,8 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
return None;
|
||||
}
|
||||
|
||||
// Validation 8: Reject paragraph-like content falsely detected as tables.
|
||||
// TOC pages with deep indentation (top-level chapters in col 0, subsections
|
||||
// in cols 1-3, page numbers in last col) leave most cells empty and trip
|
||||
// the paragraph heuristic; TOC shape is a safer signal here. Narrow only
|
||||
// — see narrow-TOC rationale at validation 1.
|
||||
if is_paragraph_content(&cells) && !is_narrow_toc {
|
||||
// Validation 8: Reject paragraph-like content falsely detected as tables
|
||||
if is_paragraph_content(&cells) {
|
||||
log::debug!(" validation 9 fail: paragraph content");
|
||||
return None;
|
||||
}
|
||||
@@ -687,7 +677,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
item_indices.len()
|
||||
);
|
||||
|
||||
Some(Table::new(columns, rows, cells, item_indices))
|
||||
Some(Table {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if this looks like a key-value pair layout rather than a table
|
||||
@@ -911,7 +906,7 @@ fn looks_like_number(s: &str) -> bool {
|
||||
/// Check if this looks like a Table of Contents (either style).
|
||||
///
|
||||
/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
|
||||
pub fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||
pub(super) fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||
is_dot_leader_toc(cells) || is_tabular_toc(cells)
|
||||
}
|
||||
|
||||
@@ -1606,62 +1601,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_table_of_contents_accepts_hierarchical_indented_toc() {
|
||||
// Mythos system card pages 4-5: top-level chapters indent at col 0,
|
||||
// subsections at cols 1-2, leaving col 0 mostly empty (only ~10% of
|
||||
// rows). Validation 1 was rejecting these even though the structure
|
||||
// is unambiguously a TOC.
|
||||
let cells = vec![
|
||||
vec!["Abstract".to_string(), String::new(), "3".to_string()],
|
||||
vec![
|
||||
"1 Introduction".to_string(),
|
||||
String::new(),
|
||||
"10".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1 Model training".to_string(),
|
||||
"11".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1.1 Training data".to_string(),
|
||||
"11".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1.2 Crowd workers".to_string(),
|
||||
"12".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.2 Release decision".to_string(),
|
||||
"13".to_string(),
|
||||
],
|
||||
vec![
|
||||
"2 RSP evaluations".to_string(),
|
||||
String::new(),
|
||||
"16".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"2.1 RSP risk assessment".to_string(),
|
||||
"16".to_string(),
|
||||
],
|
||||
vec![String::new(), "2.1.1 Context".to_string(), "16".to_string()],
|
||||
vec![
|
||||
String::new(),
|
||||
"2.2 CB evaluations".to_string(),
|
||||
"20".to_string(),
|
||||
],
|
||||
];
|
||||
assert!(
|
||||
is_table_of_contents(&cells),
|
||||
"hierarchical TOC with sparse col 0 should still be detected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_table_of_contents_rejects_dotless_toc() {
|
||||
// Tabular TOC without leader dots: first column starts with dotted
|
||||
|
||||
@@ -265,12 +265,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
|
||||
);
|
||||
|
||||
vec![Table::new(
|
||||
col_edges,
|
||||
row_edges_desc[..num_rows].to_vec(),
|
||||
vec![Table {
|
||||
columns: col_edges,
|
||||
rows: row_edges_desc[..num_rows].to_vec(),
|
||||
cells,
|
||||
item_indices,
|
||||
)]
|
||||
}]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1037,7 +1037,12 @@ fn try_build_grid(
|
||||
(columns, cells)
|
||||
};
|
||||
|
||||
GridResult::Ok(Table::new(columns, rows, cells, item_indices))
|
||||
GridResult::Ok(Table {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges.
|
||||
@@ -1437,7 +1442,12 @@ fn detect_row_stripe_table(
|
||||
content_ratio * 100.0
|
||||
);
|
||||
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect a table from cell-background rects that failed grid detection.
|
||||
@@ -1681,7 +1691,12 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
non_empty_cells as f32 / total_cells * 100.0
|
||||
);
|
||||
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect a table by merging all cluster rects into one group.
|
||||
@@ -1860,7 +1875,12 @@ fn detect_merged_cluster_table(
|
||||
content_ratio * 100.0
|
||||
);
|
||||
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Cluster text item X positions into column centers with a given minimum threshold.
|
||||
|
||||
@@ -188,12 +188,12 @@ pub fn detect_tables_from_struct_tree(
|
||||
all_item_indices.sort_unstable();
|
||||
all_item_indices.dedup();
|
||||
|
||||
tables.push(Table::new(
|
||||
col_positions,
|
||||
row_positions,
|
||||
tables.push(Table {
|
||||
columns: col_positions,
|
||||
rows: row_positions,
|
||||
cells,
|
||||
all_item_indices,
|
||||
));
|
||||
item_indices: all_item_indices,
|
||||
});
|
||||
}
|
||||
|
||||
tables
|
||||
|
||||
+21
-21
@@ -1,19 +1,26 @@
|
||||
//! Table-to-markdown formatting and cell cleanup.
|
||||
|
||||
use super::{Table, TableKind};
|
||||
use super::detect_heuristic::is_table_of_contents;
|
||||
use super::Table;
|
||||
|
||||
pub fn table_to_markdown(table: &Table) -> String {
|
||||
if table.cells.is_empty() || table.cells[0].is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// TOCs render poorly as markdown tables — emit a flat per-row text list
|
||||
// instead so the page numbers stay aligned with their section titles
|
||||
// rather than drifting to a separate column. Format from raw cells
|
||||
// because continuation-row merging in clean_table_cells collapses
|
||||
// separate TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench")
|
||||
// into one line where sub-entries leave column 0 empty.
|
||||
if table.kind == TableKind::Toc {
|
||||
// Detect TOC on the raw cells: clean_table_cells merges rows in ways
|
||||
// that can make genuine data tables superficially resemble a TOC
|
||||
// (short numeric cells, few columns) — but the raw detection here
|
||||
// preserves the original multi-column structure and only matches the
|
||||
// true TOC pattern.
|
||||
//
|
||||
// Tables of contents render poorly as markdown tables — emit a flat
|
||||
// per-row text list instead so the page numbers stay aligned with
|
||||
// their section titles rather than drifting to a separate column.
|
||||
// Format from raw cells: continuation-row merging collapses separate
|
||||
// TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench") into a
|
||||
// single line because sub-entries leave column 0 empty.
|
||||
if is_table_of_contents(&table.cells) {
|
||||
return format_toc_as_list(&table.cells, &[]);
|
||||
}
|
||||
|
||||
@@ -463,7 +470,6 @@ mod tests {
|
||||
vec!["Bob".into(), "25".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("|Name|"));
|
||||
@@ -479,7 +485,6 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["Only".into(), "Row".into()]],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("|Only|"));
|
||||
@@ -493,7 +498,6 @@ mod tests {
|
||||
rows: vec![],
|
||||
cells: vec![],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
assert_eq!(table_to_markdown(&table), "");
|
||||
}
|
||||
@@ -509,7 +513,6 @@ mod tests {
|
||||
vec!["(1)".into(), "Footnote text".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("(1) Footnote text"));
|
||||
@@ -525,7 +528,6 @@ mod tests {
|
||||
vec!["太郎".into(), "25".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("名前"));
|
||||
@@ -539,7 +541,6 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec![]],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
assert_eq!(table_to_markdown(&table), "");
|
||||
}
|
||||
@@ -549,10 +550,10 @@ mod tests {
|
||||
// A TOC-shaped table with section numbers in col 0 and page numbers
|
||||
// in the last column should render as a flat list, not a markdown
|
||||
// table, so the page numbers stay on the same line as their titles.
|
||||
let table = Table::new(
|
||||
vec![50.0, 80.0, 300.0],
|
||||
vec![500.0; 5],
|
||||
vec![
|
||||
let table = Table {
|
||||
columns: vec![50.0, 80.0, 300.0],
|
||||
rows: vec![500.0; 5],
|
||||
cells: vec![
|
||||
vec![
|
||||
"4.3".into(),
|
||||
"Case studies and targeted evaluations".into(),
|
||||
@@ -571,9 +572,8 @@ mod tests {
|
||||
vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
|
||||
vec!["4.5".into(), "White-box analyses".into(), "113".into()],
|
||||
],
|
||||
vec![],
|
||||
);
|
||||
assert_eq!(table.kind, TableKind::Toc);
|
||||
item_indices: vec![],
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(
|
||||
!md.contains("|---|"),
|
||||
|
||||
@@ -499,7 +499,6 @@ pub(crate) fn recover_header_row(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tables::TableKind;
|
||||
use crate::types::ItemType;
|
||||
|
||||
fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem {
|
||||
@@ -757,7 +756,6 @@ mod tests {
|
||||
rows: vec![500.0, 480.0],
|
||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||
item_indices: vec![2, 3],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
recover_header_row(&mut table, &all_items, 9.0);
|
||||
@@ -776,7 +774,6 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["A".into(), "B".into()]],
|
||||
item_indices: vec![0, 1],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -797,7 +794,6 @@ mod tests {
|
||||
rows: vec![500.0, 480.0],
|
||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||
item_indices: vec![2, 3],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -818,7 +814,6 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["A".into(), "B".into()]],
|
||||
item_indices: vec![1, 2],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -834,7 +829,6 @@ mod tests {
|
||||
rows: vec![],
|
||||
cells: vec![],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
recover_header_row(&mut table, &all_items, 9.0);
|
||||
|
||||
+11
-49
@@ -11,7 +11,6 @@ mod format;
|
||||
mod grid;
|
||||
|
||||
pub use detect_heuristic::detect_tables;
|
||||
pub(crate) use detect_heuristic::is_table_of_contents;
|
||||
pub use detect_lines::detect_tables_from_lines;
|
||||
pub(crate) use detect_rects::cluster_rects;
|
||||
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
||||
@@ -167,12 +166,12 @@ pub(crate) fn try_build_rect_guided_table(
|
||||
used_indices.sort_unstable();
|
||||
used_indices.dedup();
|
||||
|
||||
Some(Table::new(
|
||||
col_boundaries,
|
||||
row_boundaries,
|
||||
Some(Table {
|
||||
columns: col_boundaries,
|
||||
rows: row_boundaries,
|
||||
cells,
|
||||
used_indices,
|
||||
))
|
||||
item_indices: used_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// Split a TextItem whose text contains multiple whitespace-separated tokens
|
||||
@@ -542,22 +541,12 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
multi_col_rows
|
||||
);
|
||||
|
||||
Some(Table::new(col_xs, row_ys, cells, item_indices))
|
||||
}
|
||||
|
||||
/// What kind of structure a detected `Table` represents. Classification is
|
||||
/// computed once at construction so consumers don't have to re-analyze the
|
||||
/// cells (and stay consistent across detection backends).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum TableKind {
|
||||
/// A real data table — renders as markdown table syntax.
|
||||
#[default]
|
||||
Data,
|
||||
/// A table of contents — renders as a flat list with tab-aligned page
|
||||
/// numbers via `format_toc_as_list`. Detected through the table pipeline
|
||||
/// because TOCs share row/column structure with tables, but they are not
|
||||
/// data tables and shouldn't appear in `pages_with_tables` etc.
|
||||
Toc,
|
||||
Some(Table {
|
||||
columns: col_xs,
|
||||
rows: row_ys,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
}
|
||||
|
||||
/// A detected table.
|
||||
@@ -571,31 +560,6 @@ pub struct Table {
|
||||
pub cells: Vec<Vec<String>>,
|
||||
/// Items that belong to this table
|
||||
pub item_indices: Vec<usize>,
|
||||
/// Data table vs TOC. Set by `Table::new` from `cells`.
|
||||
pub kind: TableKind,
|
||||
}
|
||||
|
||||
impl Table {
|
||||
/// Build a table and classify it (data vs TOC) from its cells.
|
||||
pub fn new(
|
||||
columns: Vec<f32>,
|
||||
rows: Vec<f32>,
|
||||
cells: Vec<Vec<String>>,
|
||||
item_indices: Vec<usize>,
|
||||
) -> Self {
|
||||
let kind = if is_table_of_contents(&cells) {
|
||||
TableKind::Toc
|
||||
} else {
|
||||
TableKind::Data
|
||||
};
|
||||
Self {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -678,7 +642,6 @@ mod tests {
|
||||
vec!["Cell 1".into(), "Cell 2".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let md = table_to_markdown(&table);
|
||||
@@ -1039,7 +1002,6 @@ mod tests {
|
||||
vec!["3".into(), "5/2".into(), "Item C".into(), "300".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
@@ -520,18 +520,6 @@ impl ToUnicodeCMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the maximum source CID across all mappings (char_map + ranges).
|
||||
fn max_source_cid(&self) -> Option<u16> {
|
||||
let char_max = self.char_map.keys().copied().max();
|
||||
let range_max = self.ranges.iter().map(|&(_, end, _)| end).max();
|
||||
match (char_max, range_max) {
|
||||
(Some(a), Some(b)) => Some(a.max(b)),
|
||||
(a @ Some(_), None) => a,
|
||||
(None, b @ Some(_)) => b,
|
||||
(None, None) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Remap a CMap that references pre-subsetting GIDs to sequential post-subsetting GIDs.
|
||||
/// Collects all source CIDs, sorts them, and reassigns to 1, 2, 3, ...
|
||||
pub fn remap_to_sequential(&self) -> ToUnicodeCMap {
|
||||
@@ -669,81 +657,6 @@ fn get_w_array_start_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> O
|
||||
}
|
||||
}
|
||||
|
||||
/// Return true if the CIDFont's W (widths) array explicitly covers the given CID.
|
||||
///
|
||||
/// The W array uses two formats (PDF 32000-1:2008, §9.7.4.3):
|
||||
/// 1. `c [w1 w2 ... wn]` — widths for CIDs c, c+1, ..., c+n-1
|
||||
/// 2. `c_first c_last w` — CIDs c_first..c_last all have width w
|
||||
fn w_array_covers_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document, target: u16) -> bool {
|
||||
let Ok(w_obj) = cid_font_dict.get(b"W") else {
|
||||
return false;
|
||||
};
|
||||
let arr = match w_obj {
|
||||
Object::Array(arr) => arr,
|
||||
Object::Reference(r) => match doc.get_object(*r) {
|
||||
Ok(Object::Array(arr)) => arr,
|
||||
_ => return false,
|
||||
},
|
||||
_ => return false,
|
||||
};
|
||||
|
||||
let resolve_int = |o: &Object| -> Option<i64> {
|
||||
match o {
|
||||
Object::Integer(n) => Some(*n),
|
||||
Object::Reference(r) => match doc.get_object(*r) {
|
||||
Ok(Object::Integer(n)) => Some(*n),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
let resolve_arr = |o: &Object| -> Option<Vec<Object>> {
|
||||
match o {
|
||||
Object::Array(a) => Some(a.clone()),
|
||||
Object::Reference(r) => match doc.get_object(*r) {
|
||||
Ok(Object::Array(a)) => Some(a.clone()),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
let target = target as i64;
|
||||
let mut i = 0usize;
|
||||
while i < arr.len() {
|
||||
let Some(first) = resolve_int(&arr[i]) else {
|
||||
break;
|
||||
};
|
||||
i += 1;
|
||||
if i >= arr.len() {
|
||||
break;
|
||||
}
|
||||
// Peek at arr[i] to decide format.
|
||||
if let Some(widths) = resolve_arr(&arr[i]) {
|
||||
// Format 1: c [w1 ... wn]
|
||||
let last = first + widths.len() as i64 - 1;
|
||||
if target >= first && target <= last {
|
||||
return true;
|
||||
}
|
||||
i += 1;
|
||||
} else if let Some(last) = resolve_int(&arr[i]) {
|
||||
// Format 2: c_first c_last w
|
||||
i += 1;
|
||||
if i < arr.len() {
|
||||
i += 1; // skip the width value
|
||||
}
|
||||
if target >= first && target <= last {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
// Unknown token — abort parsing safely
|
||||
break;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract CIDToGIDMap as a vector of GIDs (u16) indexed by CID.
|
||||
fn get_cid_to_gid_map(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> Option<Vec<u16>> {
|
||||
let obj = cid_font_dict.get(b"CIDToGIDMap").ok()?;
|
||||
@@ -839,20 +752,6 @@ fn try_remap_subset_cmap(
|
||||
_ => return (cmap, None),
|
||||
};
|
||||
|
||||
// If the W array actually covers the CMap's max source CID, the CMap is
|
||||
// aligned with the font — no sequential renumbering happened. A sparse W
|
||||
// array starting at CID 0 (for .notdef) with additional high-CID entries
|
||||
// matching the CMap is the normal subset layout, not a mismatch.
|
||||
if let Some(max_cid) = cmap.max_source_cid() {
|
||||
if w_array_covers_cid(cid_font_dict, doc, max_cid) {
|
||||
debug!(
|
||||
"Subset remap skipped for obj={}: W array covers CMap max CID {}",
|
||||
obj_num, max_cid
|
||||
);
|
||||
return (cmap, None);
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Subset GID mismatch detected for obj={}: W starts at CID {}, CMap min CID {}. Remapping to sequential.",
|
||||
obj_num, w_start, min_cid
|
||||
@@ -2818,187 +2717,4 @@ endbfchar
|
||||
assert_eq!(remapped.unwrap().char_map.len(), 50);
|
||||
assert_eq!(fallback.unwrap().char_map.len(), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_source_cid() {
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<0000><FFFF>
|
||||
endcodespacerange
|
||||
2 beginbfchar
|
||||
<0003> <0020>
|
||||
<0031> <004E>
|
||||
endbfchar
|
||||
1 beginbfrange
|
||||
<0208> <0227> <0430>
|
||||
endbfrange
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
assert_eq!(cmap.min_source_cid(), Some(0x0003));
|
||||
assert_eq!(cmap.max_source_cid(), Some(0x0227));
|
||||
}
|
||||
|
||||
/// Helper: build a minimal CIDFont dict with a W array and check coverage.
|
||||
fn cid_font_dict_with_w(w_items: Vec<lopdf::Object>) -> lopdf::Dictionary {
|
||||
let mut d = lopdf::Dictionary::new();
|
||||
d.set("W", lopdf::Object::Array(w_items));
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_w_array_covers_cid_format1() {
|
||||
// Format 1: `c [w1 w2 ... wn]` — widths for CIDs c..c+n-1.
|
||||
// Mimics the 16.pdf Tahoma W array: 0[1000] 3[313] 5[401] 11[383 383] 16[363 303 382]
|
||||
let doc = Document::new();
|
||||
let d = cid_font_dict_with_w(vec![
|
||||
lopdf::Object::Integer(0),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(1000)]),
|
||||
lopdf::Object::Integer(3),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(313)]),
|
||||
lopdf::Object::Integer(5),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(401)]),
|
||||
lopdf::Object::Integer(11),
|
||||
lopdf::Object::Array(vec![
|
||||
lopdf::Object::Integer(383),
|
||||
lopdf::Object::Integer(383),
|
||||
]),
|
||||
lopdf::Object::Integer(16),
|
||||
lopdf::Object::Array(vec![
|
||||
lopdf::Object::Integer(363),
|
||||
lopdf::Object::Integer(303),
|
||||
lopdf::Object::Integer(382),
|
||||
]),
|
||||
lopdf::Object::Integer(570),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(667); 26]),
|
||||
]);
|
||||
|
||||
assert!(w_array_covers_cid(&d, &doc, 0));
|
||||
assert!(w_array_covers_cid(&d, &doc, 3));
|
||||
assert!(w_array_covers_cid(&d, &doc, 5));
|
||||
assert!(w_array_covers_cid(&d, &doc, 11));
|
||||
assert!(w_array_covers_cid(&d, &doc, 12));
|
||||
assert!(w_array_covers_cid(&d, &doc, 16));
|
||||
assert!(w_array_covers_cid(&d, &doc, 18));
|
||||
assert!(w_array_covers_cid(&d, &doc, 570));
|
||||
assert!(w_array_covers_cid(&d, &doc, 595));
|
||||
// Gaps are NOT covered
|
||||
assert!(!w_array_covers_cid(&d, &doc, 1));
|
||||
assert!(!w_array_covers_cid(&d, &doc, 4));
|
||||
assert!(!w_array_covers_cid(&d, &doc, 19));
|
||||
assert!(!w_array_covers_cid(&d, &doc, 596));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_w_array_covers_cid_format2() {
|
||||
// Format 2: `c_first c_last w` — CIDs c_first..c_last all have width w.
|
||||
let doc = Document::new();
|
||||
let d = cid_font_dict_with_w(vec![
|
||||
lopdf::Object::Integer(100),
|
||||
lopdf::Object::Integer(120),
|
||||
lopdf::Object::Integer(500),
|
||||
]);
|
||||
|
||||
assert!(w_array_covers_cid(&d, &doc, 100));
|
||||
assert!(w_array_covers_cid(&d, &doc, 110));
|
||||
assert!(w_array_covers_cid(&d, &doc, 120));
|
||||
assert!(!w_array_covers_cid(&d, &doc, 99));
|
||||
assert!(!w_array_covers_cid(&d, &doc, 121));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_w_array_covers_cid_missing_w() {
|
||||
let doc = Document::new();
|
||||
let d = lopdf::Dictionary::new();
|
||||
assert!(!w_array_covers_cid(&d, &doc, 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_try_remap_skipped_when_w_covers_cmap() {
|
||||
// Simulates 16.pdf: CMap's max source CID (0x0279 = 633) is explicitly
|
||||
// in the W array, so no subset-renumbering happened — remap must NOT fire.
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<0000><FFFF>
|
||||
endcodespacerange
|
||||
2 beginbfchar
|
||||
<0003> <0020>
|
||||
<0031> <004E>
|
||||
endbfchar
|
||||
2 beginbfrange
|
||||
<023A> <0253> <0410>
|
||||
<0255> <0279> <042B>
|
||||
endbfrange
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
let mut doc = Document::new();
|
||||
// Build a CIDFont dict with Identity CIDToGIDMap and a W array that
|
||||
// covers CID 633 via `597 [widths...]`.
|
||||
let mut cid_font = lopdf::Dictionary::new();
|
||||
cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
|
||||
cid_font.set(
|
||||
"W",
|
||||
lopdf::Object::Array(vec![
|
||||
lopdf::Object::Integer(0),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(750)]),
|
||||
lopdf::Object::Integer(597),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(500); 37]), // 597..633
|
||||
]),
|
||||
);
|
||||
let cid_font_id = doc.add_object(cid_font);
|
||||
|
||||
// Build the Type0 font dict with Identity-H + DescendantFonts ref.
|
||||
let mut font_dict = lopdf::Dictionary::new();
|
||||
font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
|
||||
font_dict.set(
|
||||
"DescendantFonts",
|
||||
lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
|
||||
);
|
||||
|
||||
let (primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 123);
|
||||
assert!(
|
||||
remapped.is_none(),
|
||||
"Remap must be skipped when W covers CMap max CID (this is 16.pdf)"
|
||||
);
|
||||
assert_eq!(primary.lookup(0x0003), Some(" ".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_try_remap_fires_for_true_subset_mismatch() {
|
||||
// True mismatch: CMap has high CIDs (512-544) but W only lists low sequential CIDs.
|
||||
let cmap_content = r#"
|
||||
1 begincodespacerange
|
||||
<0000><FFFF>
|
||||
endcodespacerange
|
||||
1 beginbfrange
|
||||
<0200> <0220> <0410>
|
||||
endbfrange
|
||||
"#;
|
||||
let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
|
||||
|
||||
let mut doc = Document::new();
|
||||
let mut cid_font = lopdf::Dictionary::new();
|
||||
cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
|
||||
cid_font.set(
|
||||
"W",
|
||||
lopdf::Object::Array(vec![
|
||||
lopdf::Object::Integer(0),
|
||||
lopdf::Object::Array(vec![lopdf::Object::Integer(500); 34]), // 0..33
|
||||
]),
|
||||
);
|
||||
let cid_font_id = doc.add_object(cid_font);
|
||||
|
||||
let mut font_dict = lopdf::Dictionary::new();
|
||||
font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
|
||||
font_dict.set(
|
||||
"DescendantFonts",
|
||||
lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
|
||||
);
|
||||
|
||||
let (_primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 456);
|
||||
assert!(
|
||||
remapped.is_some(),
|
||||
"Remap must fire when CMap's CIDs are outside W array coverage"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+15
-53
@@ -4,10 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
|
||||
use pdf_inspector::extractor::group_into_lines;
|
||||
use pdf_inspector::types::TextLine;
|
||||
use pdf_inspector::{
|
||||
detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
|
||||
extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
|
||||
extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
|
||||
MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
|
||||
detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
|
||||
extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
|
||||
process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
|
||||
TextItem,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -1481,7 +1481,7 @@ fn test_bits_pilani_page8_table_detection() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_basic() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0, 1])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
@@ -1495,7 +1495,7 @@ fn test_extract_pages_markdown_basic() {
|
||||
fn test_extract_pages_markdown_page_ordering() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
// Request pages in non-sequential order
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[1, 0])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 2);
|
||||
// Results should match input order, not document order
|
||||
@@ -1506,7 +1506,7 @@ fn test_extract_pages_markdown_page_ordering() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_out_of_range() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[9999])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 9999);
|
||||
@@ -1518,14 +1518,14 @@ fn test_extract_pages_markdown_out_of_range() {
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_empty_pages_list() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
|
||||
assert!(result.pages.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_single_page() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert_eq!(result.pages[0].page, 0);
|
||||
@@ -1535,7 +1535,7 @@ fn test_extract_pages_markdown_single_page() {
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_invalid_buffer() {
|
||||
let result = extract_pages_markdown_mem(b"not a pdf", Some(&[0]));
|
||||
let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -1543,7 +1543,7 @@ fn test_extract_pages_markdown_invalid_buffer() {
|
||||
fn test_extract_pages_markdown_gid_pages_need_ocr() {
|
||||
// shinagawa_identity_h.pdf has GID-encoded fonts
|
||||
let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len(), 1);
|
||||
assert!(result.pages[0].needs_ocr);
|
||||
@@ -1556,7 +1556,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let page_count = process_pdf_mem(&buf).unwrap().page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
assert!(
|
||||
!result.pages_with_tables.is_empty(),
|
||||
@@ -1569,7 +1569,7 @@ fn test_extract_pages_markdown_classification_with_tables() {
|
||||
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
|
||||
// bare_name_struct.pdf is a simple document with a heading and code block
|
||||
let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
|
||||
|
||||
assert!(result.pages_with_tables.is_empty());
|
||||
assert!(result.pages_with_columns.is_empty());
|
||||
@@ -1582,7 +1582,7 @@ fn test_extract_pages_markdown_classification_matches_process_pdf() {
|
||||
let full = process_pdf_mem(&buf).unwrap();
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result.pages_with_tables, full.layout.pages_with_tables,
|
||||
@@ -1605,7 +1605,7 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
|
||||
// Get per-page output for all pages
|
||||
let page_count = full.page_count;
|
||||
let page_indices: Vec<u32> = (0..page_count).collect();
|
||||
let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();
|
||||
let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
|
||||
|
||||
// Concatenated per-page markdown should contain substantial overlap with
|
||||
// the full output (exact match not expected due to header/footer stripping
|
||||
@@ -1630,41 +1630,3 @@ fn test_extract_pages_markdown_consistency_with_process_pdf() {
|
||||
full_md.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_none_returns_all_pages() {
|
||||
let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
|
||||
let page_count = process_pdf_mem(&buf).unwrap().page_count;
|
||||
|
||||
let result = extract_pages_markdown_mem(&buf, None).unwrap();
|
||||
|
||||
assert_eq!(result.pages.len() as u32, page_count);
|
||||
for (i, page) in result.pages.iter().enumerate() {
|
||||
assert_eq!(page.page, i as u32, "pages should be in document order");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_path_api() {
|
||||
let path = "tests/fixtures/nexo-price-en.pdf";
|
||||
let buf = std::fs::read(path).unwrap();
|
||||
|
||||
let via_path = extract_pages_markdown(path, Some(&[0])).unwrap();
|
||||
let via_mem = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();
|
||||
|
||||
assert_eq!(via_path.pages.len(), via_mem.pages.len());
|
||||
assert_eq!(via_path.pages[0].markdown, via_mem.pages[0].markdown);
|
||||
assert_eq!(via_path.pages[0].needs_ocr, via_mem.pages[0].needs_ocr);
|
||||
assert_eq!(via_path.is_complex, via_mem.is_complex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_pages_markdown_path_none_returns_all_pages() {
|
||||
let path = "tests/fixtures/nexo-price-en.pdf";
|
||||
let page_count = process_pdf_mem(&std::fs::read(path).unwrap())
|
||||
.unwrap()
|
||||
.page_count;
|
||||
|
||||
let result = extract_pages_markdown(path, None).unwrap();
|
||||
assert_eq!(result.pages.len() as u32, page_count);
|
||||
}
|
||||
|
||||
@@ -270,78 +270,6 @@ class TestExtractTextInRegions:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_pages_markdown / extract_pages_markdown_bytes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractPagesMarkdown:
|
||||
def test_default_returns_all_pages(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf")
|
||||
)
|
||||
assert len(result.pages) == 3
|
||||
assert [p.page for p in result.pages] == [0, 1, 2]
|
||||
assert all(isinstance(p.markdown, str) for p in result.pages)
|
||||
|
||||
def test_bytes_default_returns_all_pages(self):
|
||||
data = fixture_bytes("thermo-freon12.pdf")
|
||||
result = pdf_inspector.extract_pages_markdown_bytes(data)
|
||||
assert len(result.pages) == 3
|
||||
|
||||
def test_selected_pages_preserve_order(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[2, 0]
|
||||
)
|
||||
assert [p.page for p in result.pages] == [2, 0]
|
||||
|
||||
def test_bytes_selected_pages_preserve_order(self):
|
||||
data = fixture_bytes("thermo-freon12.pdf")
|
||||
result = pdf_inspector.extract_pages_markdown_bytes(data, pages=[1])
|
||||
assert len(result.pages) == 1
|
||||
assert result.pages[0].page == 1
|
||||
|
||||
def test_page_fields(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[0]
|
||||
)
|
||||
page = result.pages[0]
|
||||
assert isinstance(page.page, int)
|
||||
assert isinstance(page.markdown, str)
|
||||
assert isinstance(page.needs_ocr, bool)
|
||||
assert not page.needs_ocr # text-based fixture
|
||||
assert len(page.markdown) > 0
|
||||
|
||||
def test_result_fields(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf")
|
||||
)
|
||||
assert isinstance(result.pages, list)
|
||||
assert isinstance(result.pages_with_tables, list)
|
||||
assert isinstance(result.pages_with_columns, list)
|
||||
assert isinstance(result.pages_needing_ocr, list)
|
||||
assert isinstance(result.is_complex, bool)
|
||||
|
||||
def test_out_of_range_page_marks_needs_ocr(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[9999]
|
||||
)
|
||||
assert len(result.pages) == 1
|
||||
assert result.pages[0].needs_ocr
|
||||
assert result.pages[0].markdown == ""
|
||||
|
||||
def test_repr(self):
|
||||
result = pdf_inspector.extract_pages_markdown(
|
||||
fixture_path("thermo-freon12.pdf"), pages=[0]
|
||||
)
|
||||
assert "PagesExtractionResult" in repr(result)
|
||||
assert "PageMarkdown" in repr(result.pages[0])
|
||||
|
||||
def test_not_a_pdf(self):
|
||||
with pytest.raises(ValueError):
|
||||
pdf_inspector.extract_pages_markdown_bytes(b"not a pdf")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user