feat(tables): recover label columns for numeric-only tables
Two changes to improve balance sheet / financial table detection: 1. split_side_by_side: Don't split when one side is text labels and the other is numeric data at matching Y positions. This prevents splitting a single label+number table into two independent regions. 2. try_add_label_column: After detecting a numeric-only table, look for unclaimed text items to the left at matching Y positions and prepend them as column 0 (row labels). Tested on IN_Annual_Report_2017 balance sheet which now produces proper 3-column tables (Label|2016|2017) instead of separated number tables and paragraph text. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
e6adb29eb8
commit
e5a048f674
@@ -124,6 +124,51 @@ pub(crate) fn split_side_by_side(items: &[TextItem]) -> Vec<(f32, f32)> {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
// Don't split when the left side is text labels and the right side is numeric
|
||||
// data at matching Y positions — this is a single table (labels + numbers),
|
||||
// not two independent side-by-side regions.
|
||||
// Requires ALL THREE: left side is mostly non-numeric, right side is mostly
|
||||
// numeric, AND high Y-correlation between the two sides.
|
||||
let is_numeric_item = |item: &&&TextItem| -> bool {
|
||||
let text = item.text.trim();
|
||||
if text.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let data_chars = text
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c))
|
||||
.count();
|
||||
data_chars as f32 / text.chars().count() as f32 >= 0.6
|
||||
};
|
||||
|
||||
let left_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.x + i.width / 2.0 < best_split)
|
||||
.collect();
|
||||
let right_items: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|i| i.x + i.width / 2.0 >= best_split)
|
||||
.collect();
|
||||
|
||||
if !left_items.is_empty() && !right_items.is_empty() {
|
||||
let left_numeric_ratio =
|
||||
left_items.iter().filter(is_numeric_item).count() as f32 / left_items.len() as f32;
|
||||
let right_numeric_ratio =
|
||||
right_items.iter().filter(is_numeric_item).count() as f32 / right_items.len() as f32;
|
||||
|
||||
// Left side is mostly text (< 30% numeric) AND right side is mostly numbers (≥ 70%)
|
||||
if left_numeric_ratio < 0.30 && right_numeric_ratio >= 0.70 {
|
||||
let y_tol = 5.0;
|
||||
let y_matches = right_items
|
||||
.iter()
|
||||
.filter(|ri| left_items.iter().any(|li| (li.y - ri.y).abs() < y_tol))
|
||||
.count();
|
||||
if y_matches as f32 / right_items.len() as f32 >= 0.5 {
|
||||
return vec![];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vec![(x_min, best_split), (best_split, x_max)]
|
||||
}
|
||||
|
||||
@@ -998,4 +1043,33 @@ mod tests {
|
||||
.collect();
|
||||
assert!(split_from_hint_regions(&items, &rects, 1).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_split_label_plus_number_table() {
|
||||
// Balance sheet layout: text labels on left, numbers on right.
|
||||
// Should NOT split because it's one table, not side-by-side regions.
|
||||
let mut items = Vec::new();
|
||||
for row in 0..30 {
|
||||
// Label at x=50
|
||||
let mut label = make_item(50.0, 700.0 - row as f32 * 15.0, 1);
|
||||
label.text = format!("Row label {}", row);
|
||||
label.width = 100.0;
|
||||
items.push(label);
|
||||
// Number at x=400
|
||||
let mut num1 = make_item(400.0, 700.0 - row as f32 * 15.0, 1);
|
||||
num1.text = format!("{},000.0", 100 + row);
|
||||
num1.width = 50.0;
|
||||
items.push(num1);
|
||||
// Number at x=470
|
||||
let mut num2 = make_item(470.0, 700.0 - row as f32 * 15.0, 1);
|
||||
num2.text = format!("{},500.0", 200 + row);
|
||||
num2.width = 50.0;
|
||||
items.push(num2);
|
||||
}
|
||||
let split = split_side_by_side(&items);
|
||||
assert!(
|
||||
split.is_empty(),
|
||||
"label+number table should not be split side-by-side"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,6 +179,14 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo
|
||||
{
|
||||
// Try to recover body-font header row above the small-font table
|
||||
recover_header_row(&mut table, items, table_font_threshold);
|
||||
// Try to recover a label column from unclaimed items to the left
|
||||
try_add_label_column(
|
||||
&mut table,
|
||||
&table_candidates,
|
||||
&claimed_indices,
|
||||
y_min,
|
||||
y_max,
|
||||
);
|
||||
for &idx in &table.item_indices {
|
||||
claimed_indices.insert(idx);
|
||||
}
|
||||
@@ -1073,3 +1081,111 @@ pub(crate) fn find_first_table_row(
|
||||
|
||||
(first_table_row, excluded_items)
|
||||
}
|
||||
|
||||
/// Try to recover a label column for numeric-only tables.
|
||||
///
|
||||
/// Financial balance sheets often have text labels (row descriptions) to the
|
||||
/// left of numeric columns. The label X-positions vary due to indentation,
|
||||
/// so they don't form a consistent column cluster and are excluded from the
|
||||
/// initial table detection. This function finds unclaimed items at matching
|
||||
/// Y-positions to the left of the table and prepends them as column 0.
|
||||
fn try_add_label_column(
|
||||
table: &mut Table,
|
||||
all_candidates: &[(usize, &TextItem)],
|
||||
claimed_indices: &std::collections::HashSet<usize>,
|
||||
y_min: f32,
|
||||
y_max: f32,
|
||||
) {
|
||||
// Only apply to tables with 2-3 numeric columns and ≥5 rows
|
||||
if table.columns.len() < 2 || table.columns.len() > 3 || table.rows.len() < 5 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if the table is predominantly numeric (no text labels in any column)
|
||||
let numeric_cells = table
|
||||
.cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.filter(|cell| {
|
||||
let text = cell.trim();
|
||||
if text.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let data_chars = text
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c))
|
||||
.count();
|
||||
let total_chars = text.chars().count();
|
||||
total_chars > 0 && data_chars as f32 / total_chars as f32 >= 0.6
|
||||
})
|
||||
.count();
|
||||
let total_non_empty = table
|
||||
.cells
|
||||
.iter()
|
||||
.flat_map(|row| row.iter())
|
||||
.filter(|c| !c.trim().is_empty())
|
||||
.count();
|
||||
if total_non_empty == 0 || (numeric_cells as f32 / total_non_empty as f32) < 0.7 {
|
||||
return;
|
||||
}
|
||||
|
||||
let table_x_min = table.columns.first().copied().unwrap_or(f32::MAX);
|
||||
let y_tol = 5.0;
|
||||
|
||||
// For each table row, find unclaimed items to the left at the same Y
|
||||
let mut label_items_per_row: Vec<Vec<(usize, &TextItem)>> = Vec::new();
|
||||
let mut found_count = 0;
|
||||
for &row_y in &table.rows {
|
||||
let mut row_labels: Vec<(usize, &TextItem)> = all_candidates
|
||||
.iter()
|
||||
.filter(|(idx, item)| {
|
||||
!claimed_indices.contains(idx)
|
||||
&& !table.item_indices.contains(idx)
|
||||
&& (item.y - row_y).abs() < y_tol
|
||||
&& item.x < table_x_min - 10.0
|
||||
&& item.y >= y_min
|
||||
&& item.y <= y_max
|
||||
})
|
||||
.map(|(idx, item)| (*idx, *item))
|
||||
.collect();
|
||||
row_labels.sort_by(|a, b| {
|
||||
a.1.x
|
||||
.partial_cmp(&b.1.x)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
if !row_labels.is_empty() {
|
||||
found_count += 1;
|
||||
}
|
||||
label_items_per_row.push(row_labels);
|
||||
}
|
||||
|
||||
// Require labels for at least 40% of rows
|
||||
if found_count < table.rows.len() * 2 / 5 {
|
||||
return;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"recovering label column: {}/{} rows have labels to the left",
|
||||
found_count,
|
||||
table.rows.len()
|
||||
);
|
||||
|
||||
// Prepend label column
|
||||
let label_col_x = label_items_per_row
|
||||
.iter()
|
||||
.flat_map(|items| items.iter().map(|(_, i)| i.x))
|
||||
.fold(f32::INFINITY, f32::min);
|
||||
|
||||
table.columns.insert(0, label_col_x);
|
||||
for (row_idx, row_labels) in label_items_per_row.iter().enumerate() {
|
||||
let label_text = row_labels
|
||||
.iter()
|
||||
.map(|(_, item)| item.text.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
table.cells[row_idx].insert(0, label_text);
|
||||
for (idx, _) in row_labels {
|
||||
table.item_indices.push(*idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user