feat: detect tables from PDF line path operators (m/l/S)

Many IRS forms and government PDFs draw table gridlines using path
operators (m/l/S) instead of rectangle (re) operators. This adds
line-based table detection to capture these tables.

- Add PdfLine type for line segments from path operators
- Capture m/l/h/S/s/B/b/f/n path operators in content_stream.rs
- Thread Vec<PdfLine> through extraction pipeline
- New detect_lines.rs: classify lines, snap to grid, validate and
  assign items with extensive false-positive filters
- Integrate in markdown pipeline: rects first, then lines as fallback

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-03 12:58:59 -08:00
co-authored by Claude Opus 4.6
parent 8adfacd3e3
commit 2b5fcd7bbb
7 changed files with 571 additions and 23 deletions
+104 -3
View File
@@ -7,7 +7,7 @@ use crate::text_utils::{
decode_text_string, effective_font_size, expand_ligatures, is_bold_font, is_italic_font, decode_text_string, effective_font_size, expand_ligatures, is_bold_font, is_italic_font,
}; };
use crate::tounicode::FontCMaps; use crate::tounicode::FontCMaps;
use crate::types::{ItemType, PdfRect, TextItem}; use crate::types::{ItemType, PdfLine, PdfRect, TextItem};
use crate::PdfError; use crate::PdfError;
use log::trace; use log::trace;
use lopdf::{Document, Encoding, Object, ObjectId}; use lopdf::{Document, Encoding, Object, ObjectId};
@@ -25,11 +25,17 @@ pub(crate) fn extract_page_text_items(
page_id: ObjectId, page_id: ObjectId,
page_num: u32, page_num: u32,
font_cmaps: &FontCMaps, font_cmaps: &FontCMaps,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> { ) -> Result<(Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>), PdfError> {
use lopdf::content::Content; use lopdf::content::Content;
let mut items = Vec::new(); let mut items = Vec::new();
let mut rects: Vec<PdfRect> = Vec::new(); let mut rects: Vec<PdfRect> = Vec::new();
let mut lines: Vec<PdfLine> = Vec::new();
// Path construction state for m/l/h → S/s line extraction
let mut path_subpath_start: Option<(f32, f32)> = None;
let mut path_current: Option<(f32, f32)> = None;
let mut pending_lines: Vec<(f32, f32, f32, f32)> = Vec::new();
// Get fonts for encoding // Get fonts for encoding
let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
@@ -632,10 +638,105 @@ pub(crate) fn extract_page_text_items(
}); });
} }
} }
// ── Path construction operators ──────────────────────
"m" => {
// moveto: start a new subpath
if op.operands.len() >= 2 {
let px = get_number(&op.operands[0]).unwrap_or(0.0);
let py = get_number(&op.operands[1]).unwrap_or(0.0);
path_subpath_start = Some((px, py));
path_current = Some((px, py));
}
}
"l" => {
// lineto: add segment from current point
if op.operands.len() >= 2 {
if let Some((cx, cy)) = path_current {
let px = get_number(&op.operands[0]).unwrap_or(0.0);
let py = get_number(&op.operands[1]).unwrap_or(0.0);
pending_lines.push((cx, cy, px, py));
path_current = Some((px, py));
}
}
}
"h" => {
// closepath: segment back to subpath start
if let (Some((cx, cy)), Some((sx, sy))) = (path_current, path_subpath_start) {
if (cx - sx).abs() > 0.01 || (cy - sy).abs() > 0.01 {
pending_lines.push((cx, cy, sx, sy));
}
path_current = path_subpath_start;
}
}
// ── Path painting operators ──────────────────────────
"S" | "s" => {
// stroke / close-and-stroke: emit pending lines
if op.operator == "s" {
// close first
if let (Some((cx, cy)), Some((sx, sy))) = (path_current, path_subpath_start) {
if (cx - sx).abs() > 0.01 || (cy - sy).abs() > 0.01 {
pending_lines.push((cx, cy, sx, sy));
}
}
}
for (x1, y1, x2, y2) in pending_lines.drain(..) {
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
lines.push(PdfLine {
x1: x1d,
y1: y1d,
x2: x2d,
y2: y2d,
page: page_num,
});
}
path_subpath_start = None;
path_current = None;
}
"B" | "B*" | "b" | "b*" => {
// fill+stroke: emit lines AND clear state
if op.operator == "b" || op.operator == "b*" {
// close first
if let (Some((cx, cy)), Some((sx, sy))) = (path_current, path_subpath_start) {
if (cx - sx).abs() > 0.01 || (cy - sy).abs() > 0.01 {
pending_lines.push((cx, cy, sx, sy));
}
}
}
for (x1, y1, x2, y2) in pending_lines.drain(..) {
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
lines.push(PdfLine {
x1: x1d,
y1: y1d,
x2: x2d,
y2: y2d,
page: page_num,
});
}
path_subpath_start = None;
path_current = None;
}
"f" | "F" | "f*" => {
// fill-only: discard path without emitting lines
pending_lines.clear();
path_subpath_start = None;
path_current = None;
}
"n" => {
// end path (no-op): discard
pending_lines.clear();
path_subpath_start = None;
path_current = None;
}
_ => {} _ => {}
} }
} }
let items = super::merge_text_items(items); let items = super::merge_text_items(items);
Ok((items, rects)) Ok((items, rects, lines))
} }
+14 -11
View File
@@ -10,7 +10,7 @@ mod xobjects;
use crate::text_utils::is_rtl_text; use crate::text_utils::is_rtl_text;
use crate::tounicode::FontCMaps; use crate::tounicode::FontCMaps;
use crate::types::{PdfRect, TextItem}; use crate::types::{PdfLine, PdfRect, TextItem};
use crate::PdfError; use crate::PdfError;
use log::debug; use log::debug;
use lopdf::{Document, Object, ObjectId}; use lopdf::{Document, Object, ObjectId};
@@ -78,7 +78,7 @@ pub fn extract_text_with_positions_pages<P: AsRef<Path>>(
path: P, path: P,
page_filter: Option<&HashSet<u32>>, page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> { ) -> Result<Vec<TextItem>, PdfError> {
let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?; let (items, _rects, _lines) = extract_text_with_positions_and_rects(path, page_filter)?;
Ok(items) Ok(items)
} }
@@ -86,7 +86,7 @@ pub fn extract_text_with_positions_pages<P: AsRef<Path>>(
pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>( pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
path: P, path: P,
page_filter: Option<&HashSet<u32>>, page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> { ) -> Result<(Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>), PdfError> {
crate::validate_pdf_file(&path)?; crate::validate_pdf_file(&path)?;
let doc = match Document::load(&path) { let doc = match Document::load(&path) {
Ok(d) => d, Ok(d) => d,
@@ -109,7 +109,7 @@ pub fn extract_text_with_positions_mem_pages(
buffer: &[u8], buffer: &[u8],
page_filter: Option<&HashSet<u32>>, page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> { ) -> Result<Vec<TextItem>, PdfError> {
let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?; let (items, _rects, _lines) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?;
Ok(items) Ok(items)
} }
@@ -117,7 +117,7 @@ pub fn extract_text_with_positions_mem_pages(
pub(crate) fn extract_text_with_positions_mem_and_rects( pub(crate) fn extract_text_with_positions_mem_and_rects(
buffer: &[u8], buffer: &[u8],
page_filter: Option<&HashSet<u32>>, page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> { ) -> Result<(Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>), PdfError> {
crate::validate_pdf_bytes(buffer)?; crate::validate_pdf_bytes(buffer)?;
let doc = match Document::load_mem(buffer) { let doc = match Document::load_mem(buffer) {
Ok(d) => d, Ok(d) => d,
@@ -134,15 +134,16 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
// Orchestration // Orchestration
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/// Extract positioned text and rectangles from a pre-loaded document. /// Extract positioned text, rectangles, and line segments from a pre-loaded document.
pub(crate) fn extract_positioned_text_from_doc( pub(crate) fn extract_positioned_text_from_doc(
doc: &Document, doc: &Document,
font_cmaps: &FontCMaps, font_cmaps: &FontCMaps,
page_filter: Option<&HashSet<u32>>, page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> { ) -> Result<(Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>), PdfError> {
let pages = doc.get_pages(); let pages = doc.get_pages();
let mut all_items = Vec::new(); let mut all_items = Vec::new();
let mut all_rects = Vec::new(); let mut all_rects = Vec::new();
let mut all_lines = Vec::new();
// Build page ObjectId → page number map for form field extraction // Build page ObjectId → page number map for form field extraction
let page_id_to_num: HashMap<ObjectId, u32> = let page_id_to_num: HashMap<ObjectId, u32> =
@@ -154,12 +155,13 @@ pub(crate) fn extract_positioned_text_from_doc(
continue; continue;
} }
} }
let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
debug!( debug!(
"page {}: {} text items, {} rects", "page {}: {} text items, {} rects, {} lines",
page_num, page_num,
items.len(), items.len(),
rects.len() rects.len(),
lines.len()
); );
if log::log_enabled!(log::Level::Trace) { if log::log_enabled!(log::Level::Trace) {
for item in &items { for item in &items {
@@ -181,6 +183,7 @@ pub(crate) fn extract_positioned_text_from_doc(
} }
all_items.extend(items); all_items.extend(items);
all_rects.extend(rects); all_rects.extend(rects);
all_lines.extend(lines);
// Extract hyperlinks from page annotations // Extract hyperlinks from page annotations
let links = extract_page_links(doc, page_id, *page_num); let links = extract_page_links(doc, page_id, *page_num);
@@ -191,7 +194,7 @@ pub(crate) fn extract_positioned_text_from_doc(
let form_items = extract_form_fields(doc, &page_id_to_num); let form_items = extract_form_fields(doc, &page_id_to_num);
all_items.extend(form_items); all_items.extend(form_items);
Ok((all_items, all_rects)) Ok((all_items, all_rects, all_lines))
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
+12 -5
View File
@@ -42,7 +42,7 @@ pub use markdown::{
to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions, to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions,
}; };
pub use process_mode::ProcessMode; pub use process_mode::ProcessMode;
pub use types::{LayoutComplexity, PdfRect, TextItem}; pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
use lopdf::Document; use lopdf::Document;
use std::collections::HashSet; use std::collections::HashSet;
@@ -351,16 +351,17 @@ fn process_document(
}; };
let (markdown, layout, has_encoding_issues) = match extracted { let (markdown, layout, has_encoding_issues) = match extracted {
Some((items, rects)) => { Some((items, rects, lines)) => {
let layout = compute_layout_complexity(&items, &rects); let layout = compute_layout_complexity(&items, &rects, &lines);
let md = if options.mode == ProcessMode::Analyze { let md = if options.mode == ProcessMode::Analyze {
None None
} else { } else {
Some(to_markdown_from_items_with_rects( Some(markdown::to_markdown_from_items_with_rects_and_lines(
items, items,
options.markdown, options.markdown,
&rects, &rects,
&lines,
)) ))
}; };
@@ -427,18 +428,24 @@ fn detect_encoding_issues(markdown: &str) -> bool {
fn compute_layout_complexity( fn compute_layout_complexity(
items: &[types::TextItem], items: &[types::TextItem],
rects: &[types::PdfRect], rects: &[types::PdfRect],
lines: &[types::PdfLine],
) -> LayoutComplexity { ) -> LayoutComplexity {
// --- Collect unique pages --- // --- Collect unique pages ---
let mut seen_pages: Vec<u32> = items.iter().map(|i| i.page).collect(); let mut seen_pages: Vec<u32> = items.iter().map(|i| i.page).collect();
seen_pages.sort(); seen_pages.sort();
seen_pages.dedup(); seen_pages.dedup();
// --- Tables: use the real rect-based table detector per page --- // --- Tables: use rect-based then line-based detectors per page ---
let mut pages_with_tables: Vec<u32> = Vec::new(); let mut pages_with_tables: Vec<u32> = Vec::new();
for &page in &seen_pages { for &page in &seen_pages {
let (tables, _) = tables::detect_tables_from_rects(items, rects, page); let (tables, _) = tables::detect_tables_from_rects(items, rects, page);
if !tables.is_empty() { if !tables.is_empty() {
pages_with_tables.push(page); pages_with_tables.push(page);
continue;
}
let line_tables = tables::detect_tables_from_lines(items, lines, page);
if !line_tables.is_empty() {
pages_with_tables.push(page);
} }
} }
+38 -4
View File
@@ -141,7 +141,22 @@ pub fn to_markdown_from_items_with_rects(
options: MarkdownOptions, options: MarkdownOptions,
rects: &[crate::types::PdfRect], rects: &[crate::types::PdfRect],
) -> String { ) -> String {
use crate::tables::{detect_tables, detect_tables_from_rects, table_to_markdown}; to_markdown_from_items_with_rects_and_lines(items, options, rects, &[])
}
/// Convert positioned text items to markdown, using rectangles and line segments for table detection.
///
/// Line-based detection runs first (strongest structural evidence), then rect-based,
/// then heuristic fallback on unclaimed items.
pub(crate) fn to_markdown_from_items_with_rects_and_lines(
items: Vec<TextItem>,
options: MarkdownOptions,
rects: &[crate::types::PdfRect],
pdf_lines: &[crate::types::PdfLine],
) -> String {
use crate::tables::{
detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown,
};
use crate::types::ItemType; use crate::types::ItemType;
if items.is_empty() { if items.is_empty() {
@@ -215,10 +230,10 @@ pub fn to_markdown_from_items_with_rects(
let group = page_groups.get(&page).unwrap(); let group = page_groups.get(&page).unwrap();
let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect(); let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();
// Track which local indices are claimed by rect-based tables // Track which local indices are claimed by structural table detection
let mut rect_claimed: HashSet<usize> = HashSet::new(); let mut rect_claimed: HashSet<usize> = HashSet::new();
// Try rectangle-based table detection first // 1. Rect-based detection first (well-tested, high precision)
let (rect_tables, hint_regions) = detect_tables_from_rects(&page_items, rects, page); let (rect_tables, hint_regions) = detect_tables_from_rects(&page_items, rects, page);
for table in &rect_tables { for table in &rect_tables {
for &idx in &table.item_indices { for &idx in &table.item_indices {
@@ -235,7 +250,26 @@ pub fn to_markdown_from_items_with_rects(
.push((table_y, table_md)); .push((table_y, table_md));
} }
// Helper: run heuristic on a subset of items, remapping indices back to page-space // 2. Line-based detection on unclaimed items (when rects didn't find tables)
if rect_claimed.is_empty() {
let line_tables = detect_tables_from_lines(&page_items, pdf_lines, page);
for table in &line_tables {
for &idx in &table.item_indices {
rect_claimed.insert(idx);
if let Some(&(global_idx, _)) = group.get(idx) {
table_items.insert(global_idx);
}
}
let table_y = table.rows.first().copied().unwrap_or(0.0);
let table_md = table_to_markdown(table);
page_tables
.entry(page)
.or_default()
.push((table_y, table_md));
}
}
// 3. Heuristic fallback on unclaimed items
let mut run_heuristic = let mut run_heuristic =
|subset_items: &[TextItem], index_map: &[usize], min_items: usize| { |subset_items: &[TextItem], index_map: &[usize], min_items: usize| {
if subset_items.len() < min_items { if subset_items.len() < min_items {
+391
View File
@@ -0,0 +1,391 @@
//! Line-based table detection.
//!
//! Detects tables from PDF path operators (`m`/`l`/`S`) that draw ruled
//! gridlines. Many IRS forms and government PDFs use these instead of
//! `re` (rectangle) operators.
use crate::tables::Table;
use crate::types::{PdfLine, TextItem};
use super::detect_rects::{assign_items_to_grid, snap_edges};
/// Detect tables from line segments on a given page.
///
/// Lines are classified as horizontal or vertical, snapped into grid edges,
/// and validated before assigning text items to the resulting grid.
pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32) -> Vec<Table> {
// Filter lines for this page
let page_lines: Vec<&PdfLine> = lines.iter().filter(|l| l.page == page).collect();
if page_lines.is_empty() {
return Vec::new();
}
// Classify lines as horizontal or vertical (within 2° of axis)
let mut horizontals: Vec<(f32, f32, f32)> = Vec::new(); // (y, x_min, x_max)
let mut verticals: Vec<(f32, f32, f32)> = Vec::new(); // (x, y_min, y_max)
let angle_tolerance = 2.0_f32.to_radians().tan(); // ~0.035
for line in &page_lines {
let dx = (line.x2 - line.x1).abs();
let dy = (line.y2 - line.y1).abs();
let length = (dx * dx + dy * dy).sqrt();
// Skip very short lines (decorations, tick marks)
if length < 20.0 {
continue;
}
if dx > 0.01 && dy / dx <= angle_tolerance {
// Horizontal line
let y = (line.y1 + line.y2) / 2.0;
let x_min = line.x1.min(line.x2);
let x_max = line.x1.max(line.x2);
horizontals.push((y, x_min, x_max));
} else if dy > 0.01 && dx / dy <= angle_tolerance {
// Vertical line
let x = (line.x1 + line.x2) / 2.0;
let y_min = line.y1.min(line.y2);
let y_max = line.y1.max(line.y2);
verticals.push((x, y_min, y_max));
}
// Diagonal lines are ignored
}
if horizontals.len() < 3 || verticals.len() < 2 {
return Vec::new();
}
log::debug!(
"detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
page,
horizontals.len(),
verticals.len(),
page_lines.len()
);
// Snap Y-values of horizontal lines → row edges
let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
let row_edges = snap_edges(&h_ys, 3.0);
// Snap X-values of vertical lines → column edges
let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
let col_edges = snap_edges(&v_xs, 3.0);
// Require at least 2 columns (3 col edges) and 2 rows (3 row edges).
// A single column of horizontal lines is just separator rules, not a table.
if row_edges.len() < 3 || col_edges.len() < 3 {
return Vec::new();
}
// Cap grid size: >20 columns is almost certainly a diagram, not a table
if col_edges.len() > 21 || row_edges.len() > 80 {
return Vec::new();
}
let table_x_min = col_edges.first().copied().unwrap_or(0.0);
let table_x_max = col_edges.last().copied().unwrap_or(0.0);
let table_width = table_x_max - table_x_min;
if table_width < 50.0 {
return Vec::new();
}
let table_y_min = row_edges.first().copied().unwrap_or(0.0);
let table_y_max = row_edges.last().copied().unwrap_or(0.0);
let table_height = (table_y_max - table_y_min).abs();
if table_height < 20.0 {
return Vec::new();
}
// Reject page-spanning frames: if the grid covers >90% of a standard page
// dimension in both axes, it's a border frame, not a table.
// Standard pages are ~595×842 (A4) or ~612×792 (Letter).
if table_width > 500.0 && table_height > 700.0 {
return Vec::new();
}
// Validate horizontal lines: at least 3 should span >50% of table width.
// This filters out short segments that accidentally align.
let spanning_h = horizontals
.iter()
.filter(|(_, x_min, x_max)| (x_max - x_min) > table_width * 0.5)
.count();
if spanning_h < 3 {
return Vec::new();
}
// Validate vertical lines: at least 2 should span >30% of table height.
// Real table columns extend most of the table height.
let spanning_v = verticals
.iter()
.filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
.count();
if spanning_v < 2 {
return Vec::new();
}
// Row edges need to be in descending order (top of page = higher Y first)
let mut row_edges_desc = row_edges;
row_edges_desc.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
log::debug!(
"detect_lines p{}: {} row_edges, {} col_edges, table=({:.0},{:.0})-({:.0},{:.0}), spanning_h={}, spanning_v={}",
page, row_edges_desc.len(), col_edges.len(),
table_x_min, table_y_min, table_x_max, table_y_max,
spanning_h, spanning_v
);
// Assign items to grid
let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges_desc, page);
// Require at least 2 non-empty rows
let non_empty_rows = cells
.iter()
.filter(|row| row.iter().any(|cell| !cell.is_empty()))
.count();
if non_empty_rows < 2 {
return Vec::new();
}
// Content density: at least 15% of cells should have content
let num_cols_grid = cells.first().map_or(0, |r| r.len());
let total_cells = cells.len() * num_cols_grid;
if total_cells > 0 {
let filled_cells = cells
.iter()
.flat_map(|row| row.iter())
.filter(|cell| !cell.is_empty())
.count();
let density = filled_cells as f32 / total_cells as f32;
if density < 0.15 {
return Vec::new();
}
}
// Require that at least 2 distinct columns have content.
// Charts/diagrams have text concentrated on axes (1 column);
// real tables spread data across multiple columns.
let cols_with_content = (0..num_cols_grid)
.filter(|&c| {
cells
.iter()
.any(|row| row.get(c).is_some_and(|cell| !cell.is_empty()))
})
.count();
if cols_with_content < 2 {
return Vec::new();
}
// The grid must capture a meaningful portion of the page's text items.
// Chart/graph grids on textbook pages capture scattered labels but miss
// the bulk of the page content (explanatory text, problem statements).
let page_item_count = items.iter().filter(|i| i.page == page).count();
if page_item_count > 0 {
let capture_ratio = item_indices.len() as f32 / page_item_count as f32;
// If the grid captures less than 20% of items, it's not a real table
if capture_ratio < 0.20 {
return Vec::new();
}
}
// Reject grids with very uniform row spacing — likely chart gridlines.
// Real tables have variable row heights; chart Y-axes have equal spacing.
if row_edges_desc.len() >= 5 {
let spacings: Vec<f32> = row_edges_desc
.windows(2)
.map(|w| (w[0] - w[1]).abs())
.collect();
let mean_spacing = spacings.iter().sum::<f32>() / spacings.len() as f32;
if mean_spacing > 0.1 {
let variance = spacings
.iter()
.map(|s| (s - mean_spacing).powi(2))
.sum::<f32>()
/ spacings.len() as f32;
let cv = variance.sqrt() / mean_spacing; // coefficient of variation
// CV < 0.05 means nearly identical spacing — chart grid
if cv < 0.05 {
return Vec::new();
}
}
}
let num_cols = col_edges.len() - 1;
let num_rows = row_edges_desc.len() - 1;
if num_rows < 2 || num_cols < 2 {
return Vec::new();
}
log::debug!(
"detect_lines p{}: ACCEPTED {}x{} grid, {} items captured of {} on page, non_empty_rows={}, cols_with_content={}",
page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
);
vec![Table {
columns: col_edges,
rows: row_edges_desc[..num_rows].to_vec(),
cells,
item_indices,
}]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ItemType;
fn make_item(text: &str, x: f32, y: f32, page: u32) -> TextItem {
TextItem {
text: text.into(),
x,
y,
width: 30.0,
height: 10.0,
font: "F1".into(),
font_size: 10.0,
page,
is_bold: false,
is_italic: false,
item_type: ItemType::Text,
}
}
fn make_hline(y: f32, x1: f32, x2: f32, page: u32) -> PdfLine {
PdfLine {
x1,
y1: y,
x2,
y2: y,
page,
}
}
fn make_vline(x: f32, y1: f32, y2: f32, page: u32) -> PdfLine {
PdfLine {
x1: x,
y1,
x2: x,
y2,
page,
}
}
#[test]
fn test_basic_grid_detection() {
// 3x2 grid with horizontal lines at y=500, 480, 460 and vertical at x=100, 200, 300
let lines = vec![
make_hline(500.0, 100.0, 300.0, 1),
make_hline(480.0, 100.0, 300.0, 1),
make_hline(460.0, 100.0, 300.0, 1),
make_vline(100.0, 460.0, 500.0, 1),
make_vline(200.0, 460.0, 500.0, 1),
make_vline(300.0, 460.0, 500.0, 1),
];
let items = vec![
make_item("Col A", 110.0, 490.0, 1),
make_item("Col B", 210.0, 490.0, 1),
make_item("val 1", 110.0, 470.0, 1),
make_item("val 2", 210.0, 470.0, 1),
];
let tables = detect_tables_from_lines(&items, &lines, 1);
assert_eq!(tables.len(), 1);
assert_eq!(tables[0].cells.len(), 2); // 2 data rows
assert_eq!(tables[0].cells[0].len(), 2); // 2 columns
}
#[test]
fn test_short_lines_ignored() {
// Lines shorter than 20pt should be ignored
let lines = vec![
make_hline(500.0, 100.0, 110.0, 1), // 10pt - too short
make_hline(480.0, 100.0, 115.0, 1), // 15pt - too short
make_hline(460.0, 100.0, 112.0, 1), // 12pt - too short
];
let items = vec![make_item("text", 105.0, 490.0, 1)];
let tables = detect_tables_from_lines(&items, &lines, 1);
assert!(tables.is_empty());
}
#[test]
fn test_wrong_page_ignored() {
let lines = vec![
make_hline(500.0, 100.0, 300.0, 2),
make_hline(480.0, 100.0, 300.0, 2),
make_hline(460.0, 100.0, 300.0, 2),
make_vline(100.0, 460.0, 500.0, 2),
make_vline(200.0, 460.0, 500.0, 2),
make_vline(300.0, 460.0, 500.0, 2),
];
let items = vec![make_item("text", 110.0, 490.0, 1)];
// Request page 1, but lines are on page 2
let tables = detect_tables_from_lines(&items, &lines, 1);
assert!(tables.is_empty());
}
#[test]
fn test_empty_grid_rejected() {
// Grid with no text items inside
let lines = vec![
make_hline(500.0, 100.0, 300.0, 1),
make_hline(480.0, 100.0, 300.0, 1),
make_hline(460.0, 100.0, 300.0, 1),
make_vline(100.0, 460.0, 500.0, 1),
make_vline(200.0, 460.0, 500.0, 1),
make_vline(300.0, 460.0, 500.0, 1),
];
let items: Vec<TextItem> = Vec::new();
let tables = detect_tables_from_lines(&items, &lines, 1);
assert!(tables.is_empty());
}
#[test]
fn test_horizontal_rules_not_table() {
// Only horizontal lines with no verticals — separator rules, not a table
let lines = vec![
make_hline(500.0, 100.0, 500.0, 1),
make_hline(480.0, 100.0, 500.0, 1),
make_hline(460.0, 100.0, 500.0, 1),
make_hline(440.0, 100.0, 500.0, 1),
];
let items = vec![
make_item("text1", 110.0, 490.0, 1),
make_item("text2", 110.0, 470.0, 1),
];
let tables = detect_tables_from_lines(&items, &lines, 1);
assert!(tables.is_empty());
}
#[test]
fn test_single_column_rejected() {
// Only 2 col edges (1 column) — not a table even with verticals
let lines = vec![
make_hline(500.0, 100.0, 200.0, 1),
make_hline(480.0, 100.0, 200.0, 1),
make_hline(460.0, 100.0, 200.0, 1),
make_vline(100.0, 460.0, 500.0, 1),
make_vline(200.0, 460.0, 500.0, 1),
];
let items = vec![
make_item("a", 110.0, 490.0, 1),
make_item("b", 110.0, 470.0, 1),
];
let tables = detect_tables_from_lines(&items, &lines, 1);
assert!(
tables.is_empty(),
"Single-column grid should not be a table"
);
}
}
+2
View File
@@ -3,12 +3,14 @@
//! Detects tabular data in PDF text items and converts to markdown tables. //! Detects tabular data in PDF text items and converts to markdown tables.
mod detect_heuristic; mod detect_heuristic;
mod detect_lines;
mod detect_rects; mod detect_rects;
mod financial; mod financial;
mod format; mod format;
mod grid; mod grid;
pub use detect_heuristic::detect_tables; pub use detect_heuristic::detect_tables;
pub use detect_lines::detect_tables_from_lines;
pub use detect_rects::{detect_tables_from_rects, RectHintRegion}; pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
pub use format::table_to_markdown; pub use format::table_to_markdown;
+10
View File
@@ -69,6 +69,16 @@ pub struct LayoutComplexity {
pub pages_with_columns: Vec<u32>, pub pages_with_columns: Vec<u32>,
} }
/// A line segment from PDF path operators (`m`/`l`/`S`).
#[derive(Debug, Clone)]
pub struct PdfLine {
pub x1: f32,
pub y1: f32,
pub x2: f32,
pub y2: f32,
pub page: u32,
}
/// A rectangle from a PDF `re` operator (cell boundary, border, etc.) /// A rectangle from a PDF `re` operator (cell boundary, border, etc.)
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct PdfRect { pub struct PdfRect {