fix: prevent multi-column text from being misdetected as tables
On pages where column detection finds 2+ columns, skip body-font heuristic table detection in the merged-band retry path. This prevents sidebar/two-column prose from being formatted as markdown tables. The fix is targeted: per-band heuristic detection still runs (bands are scoped to single columns), so real tables within columns are still detected. Only the merged-band retry (which sees all items across columns) is gated. Also relaxes column validation to accept asymmetric layouts (sidebars) where one side has fewer items, and tries center-based item assignment before edge-based to improve column splitting for asymmetric layouts. Benchmark: NID 0.865→0.869, NID-S 0.798→0.805, overall +0.002. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
1c48a014f7
commit
9c463b9c95
+28
-2
@@ -35,6 +35,7 @@ pub(crate) fn detect_columns(
|
|||||||
if page_items.is_empty() {
|
if page_items.is_empty() {
|
||||||
return vec![];
|
return vec![];
|
||||||
}
|
}
|
||||||
|
debug!("page {}: detect_columns: {} items", page, page_items.len());
|
||||||
|
|
||||||
// Find page bounds
|
// Find page bounds
|
||||||
let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
|
let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
|
||||||
@@ -166,6 +167,23 @@ pub(crate) fn detect_columns(
|
|||||||
return vec![ColumnRegion { x_min, x_max }];
|
return vec![ColumnRegion { x_min, x_max }];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Try center-based assignment first (handles asymmetric layouts / sidebars
|
||||||
|
// better than edge-based). Fall back to edge-based if center produces
|
||||||
|
// a degenerate split (one side empty).
|
||||||
|
let result = validate_and_build_columns(
|
||||||
|
&valleys,
|
||||||
|
&page_items,
|
||||||
|
x_min,
|
||||||
|
BIN_WIDTH,
|
||||||
|
x_max,
|
||||||
|
MIN_ITEMS_PER_COLUMN,
|
||||||
|
MIN_VERTICAL_SPAN_RATIO,
|
||||||
|
page,
|
||||||
|
true, // center-based assignment
|
||||||
|
);
|
||||||
|
if result.len() > 1 {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
return validate_and_build_columns(
|
return validate_and_build_columns(
|
||||||
&valleys,
|
&valleys,
|
||||||
&page_items,
|
&page_items,
|
||||||
@@ -175,7 +193,7 @@ pub(crate) fn detect_columns(
|
|||||||
MIN_ITEMS_PER_COLUMN,
|
MIN_ITEMS_PER_COLUMN,
|
||||||
MIN_VERTICAL_SPAN_RATIO,
|
MIN_VERTICAL_SPAN_RATIO,
|
||||||
page,
|
page,
|
||||||
false, // edge-based assignment for absolute valleys
|
false, // edge-based fallback
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -505,7 +523,15 @@ fn validate_and_build_columns(
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if left_items.len() < min_items || right_items.len() < min_items {
|
// Require both sides to have items. Symmetric layout needs min_items
|
||||||
|
// on each side. Asymmetric layouts (sidebars) are accepted when the
|
||||||
|
// dominant side has ≥ min_items and the smaller side has ≥ 3 items.
|
||||||
|
let (smaller, larger) = if left_items.len() <= right_items.len() {
|
||||||
|
(left_items.len(), right_items.len())
|
||||||
|
} else {
|
||||||
|
(right_items.len(), left_items.len())
|
||||||
|
};
|
||||||
|
if larger < min_items || smaller < 3 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+10
-1
@@ -601,6 +601,15 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
|||||||
let group = page_groups.get(&page).unwrap();
|
let group = page_groups.get(&page).unwrap();
|
||||||
let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();
|
let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();
|
||||||
|
|
||||||
|
// Detect columns early — on multi-column pages, the merged-band retry
|
||||||
|
// should skip body-font heuristic table detection (which mistakes column
|
||||||
|
// text for tables). Individual band heuristic detection is left enabled
|
||||||
|
// because bands are scoped to single columns.
|
||||||
|
let page_has_columns = {
|
||||||
|
let cols = crate::extractor::detect_columns(&page_items, page, false);
|
||||||
|
cols.len() >= 2
|
||||||
|
};
|
||||||
|
|
||||||
// Check for side-by-side layout (e.g. two tables placed left and right)
|
// Check for side-by-side layout (e.g. two tables placed left and right)
|
||||||
let mut bands = split_side_by_side(&page_items);
|
let mut bands = split_side_by_side(&page_items);
|
||||||
// Fallback: use rect hint regions to detect side-by-side layout
|
// Fallback: use rect hint regions to detect side-by-side layout
|
||||||
@@ -915,7 +924,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
|
|||||||
band_items.len(),
|
band_items.len(),
|
||||||
was_split
|
was_split
|
||||||
);
|
);
|
||||||
let heuristic_tables = detect_tables(band_items, base_size, false);
|
let heuristic_tables = detect_tables(band_items, base_size, page_has_columns);
|
||||||
for table in &heuristic_tables {
|
for table in &heuristic_tables {
|
||||||
for &idx in &table.item_indices {
|
for &idx in &table.item_indices {
|
||||||
if let Some(&page_idx) = band_index_map.get(idx) {
|
if let Some(&page_idx) = band_index_map.get(idx) {
|
||||||
|
|||||||
Reference in New Issue
Block a user