From fb6d98fde4d3398bfd6e671c059bae884dadfad3 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 16 Feb 2026 11:40:50 -0800 Subject: [PATCH] fix(columns): Improve multi-column detection for interleaved reading order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Raise NOISE_FRACTION 0.05→0.15 so width bleed into gutter bins doesn't prevent column detection - Lower table bypass threshold 150→120pt to protect 3-column layouts - Make has_template_image a sufficient condition for OCR routing - Add unit tests for 2/3-column detection, width bleed tolerance, and single-column regression guard Co-Authored-By: Claude Opus 4.6 --- src/detector.rs | 5 +-- src/extractor.rs | 89 +++++++++++++++++++++++++++++++++++++++++++++++- src/markdown.rs | 4 +-- 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/detector.rs b/src/detector.rs index daf503a..3cac0a0 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -236,8 +236,9 @@ fn detect_from_document( } else { continue; }; - if analysis.text_operator_count < config.min_text_ops_per_page - && (analysis.has_images || analysis.has_template_image) + if analysis.has_template_image + || (analysis.text_operator_count < config.min_text_ops_per_page + && analysis.has_images) { ocr_pages.push(page_num); } diff --git a/src/extractor.rs b/src/extractor.rs index c85f594..afa7b2c 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -2124,7 +2124,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec const MIN_GUTTER_WIDTH: f32 = 8.0; const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30; const MIN_ITEMS_PER_COLUMN: usize = 10; - const NOISE_FRACTION: f32 = 0.05; + const NOISE_FRACTION: f32 = 0.15; // Get items for this page let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect(); @@ -2792,4 +2792,91 @@ mod tests { assert_eq!(lines.len(), 1); assert_eq!(lines[0].text(), "である履行義務を識別す"); } + + fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + #[test] + fn test_detect_two_columns() { + let mut items = Vec::new(); + // Left column at x=72, right column at x=350, gutter ~278-350 + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Left text here", 72.0, y, 200.0)); + items.push(make_item("Right text here", 350.0, y, 200.0)); + } + let cols = detect_columns(&items, 1); + assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols); + assert!(cols[0].x_min < cols[1].x_min); + } + + #[test] + fn test_detect_three_columns() { + let mut items = Vec::new(); + // Three columns at x=50, x=220, x=390 + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Col one", 50.0, y, 140.0)); + items.push(make_item("Col two", 220.0, y, 140.0)); + items.push(make_item("Col three", 390.0, y, 140.0)); + } + let cols = detect_columns(&items, 1); + assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols); + } + + #[test] + fn test_width_bleed_tolerance() { + let mut items = Vec::new(); + // Two columns with a clear gutter + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("Left text", 72.0, y, 200.0)); + items.push(make_item("Right text", 350.0, y, 200.0)); + } + // Add a few items that bleed across the gutter + for i in 0..3 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item("wide", 72.0, y, 320.0)); + } + let cols = detect_columns(&items, 1); + assert!( + cols.len() >= 2, + "Width bleed should not prevent column detection, got {:?}", + cols + ); + } + + #[test] + fn test_single_column_no_false_split() { + let mut items = Vec::new(); + // Single column: items spanning full width + for i in 0..30 { + let y = 700.0 - (i as f32) * 14.0; + items.push(make_item( + "This is a full-width paragraph of text", + 72.0, + y, + 468.0, + )); + } + let cols = detect_columns(&items, 1); + assert!( + cols.len() <= 1, + "Full-width text should not be split into columns, got {:?}", + cols + ); + } } diff --git a/src/markdown.rs b/src/markdown.rs index 70926c4..98df9c4 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -193,7 +193,7 @@ pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> // Skip table detection on pages with clear multi-column text layout. // Column detection in group_into_lines handles these correctly, and // table detection would incorrectly claim columnar text as table rows. - // Only skip when all detected columns are wide (>150pt ~ 2 inches), + // Only skip when all detected columns are wide (>120pt), // indicating true text columns rather than narrow table columns. let columns = crate::extractor::detect_columns(&page_items, page); if columns.len() >= 2 { @@ -201,7 +201,7 @@ pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> .iter() .map(|c| c.x_max - c.x_min) .fold(f32::INFINITY, f32::min); - if min_col_width > 150.0 { + if min_col_width > 120.0 { continue; } }