feat: detect tables from PDF line path operators (m/l/S)

Many IRS forms and government PDFs draw table gridlines using path
operators (m/l/S) instead of rectangle (re) operators. This adds
line-based table detection to capture these tables.

- Add PdfLine type for line segments from path operators
- Capture m/l/h/S/s/B/b/f/n path operators in content_stream.rs
- Thread Vec<PdfLine> through extraction pipeline
- New detect_lines.rs: classify lines, snap to grid, validate and
  assign items with extensive false-positive filters
- Integrate in markdown pipeline: rects first, then lines as fallback

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-03 12:58:59 -08:00
co-authored by Claude Opus 4.6
parent 8adfacd3e3
commit 2b5fcd7bbb
7 changed files with 571 additions and 23 deletions
+12 -5
View File
@@ -42,7 +42,7 @@ pub use markdown::{
to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions,
};
pub use process_mode::ProcessMode;
pub use types::{LayoutComplexity, PdfRect, TextItem};
pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
use lopdf::Document;
use std::collections::HashSet;
@@ -351,16 +351,17 @@ fn process_document(
};
let (markdown, layout, has_encoding_issues) = match extracted {
Some((items, rects)) => {
let layout = compute_layout_complexity(&items, &rects);
Some((items, rects, lines)) => {
let layout = compute_layout_complexity(&items, &rects, &lines);
let md = if options.mode == ProcessMode::Analyze {
None
} else {
Some(to_markdown_from_items_with_rects(
Some(markdown::to_markdown_from_items_with_rects_and_lines(
items,
options.markdown,
&rects,
&lines,
))
};
@@ -427,18 +428,24 @@ fn detect_encoding_issues(markdown: &str) -> bool {
fn compute_layout_complexity(
items: &[types::TextItem],
rects: &[types::PdfRect],
lines: &[types::PdfLine],
) -> LayoutComplexity {
// --- Collect unique pages ---
let mut seen_pages: Vec<u32> = items.iter().map(|i| i.page).collect();
seen_pages.sort();
seen_pages.dedup();
// --- Tables: use the real rect-based table detector per page ---
// --- Tables: use rect-based then line-based detectors per page ---
let mut pages_with_tables: Vec<u32> = Vec::new();
for &page in &seen_pages {
let (tables, _) = tables::detect_tables_from_rects(items, rects, page);
if !tables.is_empty() {
pages_with_tables.push(page);
continue;
}
let line_tables = tables::detect_tables_from_lines(items, lines, page);
if !line_tables.is_empty() {
pages_with_tables.push(page);
}
}