fix(columns): Improve multi-column detection for interleaved reading order
- Raise NOISE_FRACTION 0.05→0.15 so width bleed into gutter bins doesn't prevent column detection - Lower table bypass threshold 150→120pt to protect 3-column layouts - Make has_template_image a sufficient condition for OCR routing - Add unit tests for 2/3-column detection, width bleed tolerance, and single-column regression guard Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
c815de9844
commit
fb6d98fde4
+3
-2
@@ -236,8 +236,9 @@ fn detect_from_document(
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
if analysis.text_operator_count < config.min_text_ops_per_page
|
||||
&& (analysis.has_images || analysis.has_template_image)
|
||||
if analysis.has_template_image
|
||||
|| (analysis.text_operator_count < config.min_text_ops_per_page
|
||||
&& analysis.has_images)
|
||||
{
|
||||
ocr_pages.push(page_num);
|
||||
}
|
||||
|
||||
+88
-1
@@ -2124,7 +2124,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec<ColumnRegion>
|
||||
const MIN_GUTTER_WIDTH: f32 = 8.0;
|
||||
const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30;
|
||||
const MIN_ITEMS_PER_COLUMN: usize = 10;
|
||||
const NOISE_FRACTION: f32 = 0.05;
|
||||
const NOISE_FRACTION: f32 = 0.15;
|
||||
|
||||
// Get items for this page
|
||||
let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect();
|
||||
@@ -2792,4 +2792,91 @@ mod tests {
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "である履行義務を識別す");
|
||||
}
|
||||
|
||||
fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.into(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_two_columns() {
|
||||
let mut items = Vec::new();
|
||||
// Left column at x=72, right column at x=350, gutter ~278-350
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Left text here", 72.0, y, 200.0));
|
||||
items.push(make_item("Right text here", 350.0, y, 200.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols);
|
||||
assert!(cols[0].x_min < cols[1].x_min);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_three_columns() {
|
||||
let mut items = Vec::new();
|
||||
// Three columns at x=50, x=220, x=390
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Col one", 50.0, y, 140.0));
|
||||
items.push(make_item("Col two", 220.0, y, 140.0));
|
||||
items.push(make_item("Col three", 390.0, y, 140.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_width_bleed_tolerance() {
|
||||
let mut items = Vec::new();
|
||||
// Two columns with a clear gutter
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Left text", 72.0, y, 200.0));
|
||||
items.push(make_item("Right text", 350.0, y, 200.0));
|
||||
}
|
||||
// Add a few items that bleed across the gutter
|
||||
for i in 0..3 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("wide", 72.0, y, 320.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert!(
|
||||
cols.len() >= 2,
|
||||
"Width bleed should not prevent column detection, got {:?}",
|
||||
cols
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_no_false_split() {
|
||||
let mut items = Vec::new();
|
||||
// Single column: items spanning full width
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item(
|
||||
"This is a full-width paragraph of text",
|
||||
72.0,
|
||||
y,
|
||||
468.0,
|
||||
));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert!(
|
||||
cols.len() <= 1,
|
||||
"Full-width text should not be split into columns, got {:?}",
|
||||
cols
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -193,7 +193,7 @@ pub fn to_markdown_from_items(items: Vec<TextItem>, options: MarkdownOptions) ->
|
||||
// Skip table detection on pages with clear multi-column text layout.
|
||||
// Column detection in group_into_lines handles these correctly, and
|
||||
// table detection would incorrectly claim columnar text as table rows.
|
||||
// Only skip when all detected columns are wide (>150pt ~ 2 inches),
|
||||
// Only skip when all detected columns are wide (>120pt),
|
||||
// indicating true text columns rather than narrow table columns.
|
||||
let columns = crate::extractor::detect_columns(&page_items, page);
|
||||
if columns.len() >= 2 {
|
||||
@@ -201,7 +201,7 @@ pub fn to_markdown_from_items(items: Vec<TextItem>, options: MarkdownOptions) ->
|
||||
.iter()
|
||||
.map(|c| c.x_max - c.x_min)
|
||||
.fold(f32::INFINITY, f32::min);
|
||||
if min_col_width > 150.0 {
|
||||
if min_col_width > 120.0 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user