Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
0b8cb9165a |
+847
-63
@@ -4,15 +4,385 @@
|
||||
//! elements linked to MCIDs, this module builds `Table` structs directly from
|
||||
//! the semantic hierarchy — no geometry heuristics needed.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use log::debug;
|
||||
|
||||
use crate::structure_tree::StructTable;
|
||||
use crate::structure_tree::{StructTable, StructTableRow};
|
||||
use crate::types::TextItem;
|
||||
|
||||
use super::Table;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct MatchedCell {
|
||||
text: String,
|
||||
item_indices: Vec<usize>,
|
||||
x: Option<f32>,
|
||||
y: Option<f32>,
|
||||
}
|
||||
|
||||
fn legacy_column_positions(
|
||||
page_rows: &[&StructTableRow],
|
||||
mcid_to_items: &HashMap<i64, Vec<usize>>,
|
||||
items: &[TextItem],
|
||||
page: u32,
|
||||
num_cols: usize,
|
||||
) -> Vec<f32> {
|
||||
let mut col_positions: Vec<f32> = vec![0.0; num_cols];
|
||||
for (col, col_pos) in col_positions.iter_mut().enumerate() {
|
||||
for row in page_rows {
|
||||
if col < row.cells.len() {
|
||||
if let Some(x) = row.cells[col]
|
||||
.mcids
|
||||
.iter()
|
||||
.filter(|(_, p)| *p == page)
|
||||
.filter_map(|(mcid, _)| mcid_to_items.get(mcid))
|
||||
.flatten()
|
||||
.map(|&idx| items[idx].x)
|
||||
.reduce(f32::min)
|
||||
{
|
||||
*col_pos = x;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
col_positions
|
||||
}
|
||||
|
||||
fn infer_column_positions(
|
||||
raw_rows: &[Vec<MatchedCell>],
|
||||
fallback_positions: &[f32],
|
||||
num_cols: usize,
|
||||
) -> Vec<f32> {
|
||||
const SAME_COLUMN_TOLERANCE: f32 = 18.0;
|
||||
|
||||
let mut anchors = raw_rows
|
||||
.iter()
|
||||
.max_by_key(|row| row.iter().filter(|cell| cell.x.is_some()).count())
|
||||
.map(|row| row.iter().filter_map(|cell| cell.x).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
if anchors.len() > num_cols {
|
||||
anchors.truncate(num_cols);
|
||||
}
|
||||
|
||||
let mut additional_positions: Vec<f32> = raw_rows
|
||||
.iter()
|
||||
.flat_map(|row| row.iter().filter_map(|cell| cell.x))
|
||||
.collect();
|
||||
additional_positions.sort_by(|a, b| a.total_cmp(b));
|
||||
|
||||
for x in additional_positions {
|
||||
if anchors.len() >= num_cols {
|
||||
break;
|
||||
}
|
||||
if anchors
|
||||
.iter()
|
||||
.all(|existing| (x - *existing).abs() > SAME_COLUMN_TOLERANCE)
|
||||
{
|
||||
anchors.push(x);
|
||||
anchors.sort_by(|a, b| a.total_cmp(b));
|
||||
}
|
||||
}
|
||||
|
||||
if anchors.len() < num_cols {
|
||||
for &x in fallback_positions {
|
||||
if anchors.len() >= num_cols {
|
||||
break;
|
||||
}
|
||||
if anchors
|
||||
.iter()
|
||||
.all(|existing| (x - *existing).abs() > SAME_COLUMN_TOLERANCE)
|
||||
{
|
||||
anchors.push(x);
|
||||
anchors.sort_by(|a, b| a.total_cmp(b));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if anchors.is_empty() {
|
||||
return fallback_positions.to_vec();
|
||||
}
|
||||
|
||||
while anchors.len() < num_cols {
|
||||
anchors.push(*anchors.last().unwrap());
|
||||
}
|
||||
|
||||
anchors
|
||||
}
|
||||
|
||||
fn align_positions_to_columns(cell_xs: &[f32], columns: &[f32]) -> Vec<usize> {
|
||||
if cell_xs.is_empty() || columns.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
if cell_xs.len() >= columns.len() {
|
||||
return (0..cell_xs.len().min(columns.len())).collect();
|
||||
}
|
||||
|
||||
let mut dp = vec![vec![f32::INFINITY; columns.len() + 1]; cell_xs.len() + 1];
|
||||
let mut take = vec![vec![false; columns.len() + 1]; cell_xs.len() + 1];
|
||||
|
||||
for value in &mut dp[0] {
|
||||
*value = 0.0;
|
||||
}
|
||||
|
||||
for i in 1..=cell_xs.len() {
|
||||
for j in 1..=columns.len() {
|
||||
let skip_cost = dp[i][j - 1];
|
||||
let take_cost = dp[i - 1][j - 1] + (cell_xs[i - 1] - columns[j - 1]).abs();
|
||||
if take_cost <= skip_cost {
|
||||
dp[i][j] = take_cost;
|
||||
take[i][j] = true;
|
||||
} else {
|
||||
dp[i][j] = skip_cost;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut assignments_rev = Vec::with_capacity(cell_xs.len());
|
||||
let mut i = cell_xs.len();
|
||||
let mut j = columns.len();
|
||||
while i > 0 && j > 0 {
|
||||
if take[i][j] {
|
||||
assignments_rev.push(j - 1);
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else {
|
||||
j -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
assignments_rev.reverse();
|
||||
assignments_rev
|
||||
}
|
||||
|
||||
fn align_struct_rows(
|
||||
raw_rows: &[Vec<MatchedCell>],
|
||||
col_positions: &[f32],
|
||||
) -> (Vec<Vec<String>>, Vec<f32>, Vec<usize>) {
|
||||
let mut cells: Vec<Vec<String>> = Vec::with_capacity(raw_rows.len());
|
||||
let mut row_positions: Vec<f32> = Vec::with_capacity(raw_rows.len());
|
||||
let mut all_item_indices: Vec<usize> = Vec::new();
|
||||
|
||||
for row in raw_rows {
|
||||
let present_cells: Vec<&MatchedCell> = row
|
||||
.iter()
|
||||
.filter(|cell| {
|
||||
!cell.item_indices.is_empty() || !cell.text.is_empty() || cell.x.is_some()
|
||||
})
|
||||
.collect();
|
||||
let cell_xs: Vec<f32> = present_cells.iter().filter_map(|cell| cell.x).collect();
|
||||
let assignments = if cell_xs.len() == present_cells.len() {
|
||||
align_positions_to_columns(&cell_xs, col_positions)
|
||||
} else {
|
||||
(0..present_cells.len().min(col_positions.len())).collect()
|
||||
};
|
||||
|
||||
let mut row_cells = vec![String::new(); col_positions.len()];
|
||||
for (cell, &col_idx) in present_cells.iter().zip(assignments.iter()) {
|
||||
if !cell.text.is_empty() {
|
||||
if !row_cells[col_idx].is_empty() {
|
||||
row_cells[col_idx].push(' ');
|
||||
}
|
||||
row_cells[col_idx].push_str(&cell.text);
|
||||
}
|
||||
all_item_indices.extend(cell.item_indices.iter().copied());
|
||||
}
|
||||
|
||||
let row_y = row
|
||||
.iter()
|
||||
.filter_map(|cell| cell.y)
|
||||
.reduce(f32::max)
|
||||
.unwrap_or(0.0);
|
||||
cells.push(row_cells);
|
||||
row_positions.push(row_y);
|
||||
}
|
||||
|
||||
(cells, row_positions, all_item_indices)
|
||||
}
|
||||
|
||||
fn left_align_struct_rows(
|
||||
raw_rows: &[Vec<MatchedCell>],
|
||||
num_cols: usize,
|
||||
) -> (Vec<Vec<String>>, Vec<f32>, Vec<usize>) {
|
||||
let mut cells: Vec<Vec<String>> = Vec::with_capacity(raw_rows.len());
|
||||
let mut row_positions: Vec<f32> = Vec::with_capacity(raw_rows.len());
|
||||
let mut all_item_indices: Vec<usize> = Vec::new();
|
||||
|
||||
for row in raw_rows {
|
||||
let mut row_cells: Vec<String> = row.iter().map(|cell| cell.text.clone()).collect();
|
||||
row_cells.truncate(num_cols);
|
||||
while row_cells.len() < num_cols {
|
||||
row_cells.push(String::new());
|
||||
}
|
||||
cells.push(row_cells);
|
||||
|
||||
all_item_indices.extend(
|
||||
row.iter()
|
||||
.flat_map(|cell| cell.item_indices.iter().copied()),
|
||||
);
|
||||
row_positions.push(
|
||||
row.iter()
|
||||
.filter_map(|cell| cell.y)
|
||||
.reduce(f32::max)
|
||||
.unwrap_or(0.0),
|
||||
);
|
||||
}
|
||||
|
||||
(cells, row_positions, all_item_indices)
|
||||
}
|
||||
|
||||
fn recover_unclaimed_header_row(table: &mut Table, items: &[TextItem], has_ragged_rows: bool) {
|
||||
if !has_ragged_rows || table.rows.is_empty() || table.columns.len() < 3 {
|
||||
return;
|
||||
}
|
||||
|
||||
const MAX_HEADER_DISTANCE: f32 = 90.0;
|
||||
const MAX_GAP_TO_TABLE: f32 = 35.0;
|
||||
const MAX_INTER_HEADER_GAP: f32 = 25.0;
|
||||
const MAX_HEADER_ROWS: usize = 3;
|
||||
const Y_TOLERANCE: f32 = 5.0;
|
||||
|
||||
let top_row_y = table.rows[0];
|
||||
let x_min = table.columns.first().copied().unwrap_or(0.0) - 25.0;
|
||||
let x_max = table.columns.last().copied().unwrap_or(0.0) + 120.0;
|
||||
let claimed: HashSet<usize> = table.item_indices.iter().copied().collect();
|
||||
|
||||
let mut candidate_rows: Vec<(f32, Vec<(usize, &TextItem)>)> = Vec::new();
|
||||
for (idx, item) in items.iter().enumerate() {
|
||||
if claimed.contains(&idx)
|
||||
|| item.text.trim().is_empty()
|
||||
|| item.y <= top_row_y
|
||||
|| item.y - top_row_y > MAX_HEADER_DISTANCE
|
||||
|| item.x < x_min
|
||||
|| item.x > x_max
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((_, row_items)) = candidate_rows
|
||||
.iter_mut()
|
||||
.find(|(row_y, _)| (item.y - *row_y).abs() < Y_TOLERANCE)
|
||||
{
|
||||
row_items.push((idx, item));
|
||||
} else {
|
||||
candidate_rows.push((item.y, vec![(idx, item)]));
|
||||
}
|
||||
}
|
||||
|
||||
if candidate_rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
for (_, row_items) in &mut candidate_rows {
|
||||
row_items.sort_by(|a, b| a.1.x.total_cmp(&b.1.x));
|
||||
}
|
||||
candidate_rows.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
|
||||
if candidate_rows[0].0 - top_row_y > MAX_GAP_TO_TABLE {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut candidate_iter = candidate_rows.into_iter();
|
||||
let Some(first_row) = candidate_iter.next() else {
|
||||
return;
|
||||
};
|
||||
let mut selected_rows: Vec<(f32, Vec<(usize, &TextItem)>)> = vec![first_row];
|
||||
let mut prev_y = selected_rows[0].0;
|
||||
for (row_y, row_items) in candidate_iter {
|
||||
if selected_rows.len() >= MAX_HEADER_ROWS {
|
||||
break;
|
||||
}
|
||||
if row_y - prev_y > MAX_INTER_HEADER_GAP {
|
||||
break;
|
||||
}
|
||||
prev_y = row_y;
|
||||
selected_rows.push((row_y, row_items));
|
||||
}
|
||||
|
||||
if selected_rows.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut assigned_rows: Vec<(f32, Vec<String>, Vec<usize>)> = Vec::new();
|
||||
let mut closest_row_populated = 0usize;
|
||||
let mut combined_cols: HashSet<usize> = HashSet::new();
|
||||
|
||||
for (row_idx, (row_y, row_items)) in selected_rows.iter().enumerate() {
|
||||
if row_items.len() > table.columns.len() {
|
||||
return;
|
||||
}
|
||||
|
||||
let row_xs: Vec<f32> = row_items.iter().map(|(_, item)| item.x).collect();
|
||||
let assignments = align_positions_to_columns(&row_xs, &table.columns);
|
||||
if assignments.len() != row_items.len() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut row_cells = vec![String::new(); table.columns.len()];
|
||||
let mut row_indices = Vec::with_capacity(row_items.len());
|
||||
let mut populated_cols: HashSet<usize> = HashSet::new();
|
||||
|
||||
for ((idx, item), &col_idx) in row_items.iter().zip(assignments.iter()) {
|
||||
let text = item.text.trim();
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !row_cells[col_idx].is_empty() {
|
||||
row_cells[col_idx].push(' ');
|
||||
}
|
||||
row_cells[col_idx].push_str(text);
|
||||
row_indices.push(*idx);
|
||||
populated_cols.insert(col_idx);
|
||||
}
|
||||
|
||||
if row_idx == 0 {
|
||||
closest_row_populated = populated_cols.len();
|
||||
}
|
||||
|
||||
combined_cols.extend(populated_cols.iter().copied());
|
||||
assigned_rows.push((*row_y, row_cells, row_indices));
|
||||
}
|
||||
|
||||
let required_cols = if table.columns.len() <= 4 {
|
||||
table.columns.len()
|
||||
} else {
|
||||
table.columns.len() - 1
|
||||
};
|
||||
if closest_row_populated < 2 || combined_cols.len() < required_cols {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut header_cells = vec![String::new(); table.columns.len()];
|
||||
let mut header_indices = Vec::new();
|
||||
for (_, row_cells, row_indices) in assigned_rows.iter().rev() {
|
||||
for (col_idx, cell_text) in row_cells.iter().enumerate() {
|
||||
if cell_text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !header_cells[col_idx].is_empty() {
|
||||
header_cells[col_idx].push(' ');
|
||||
}
|
||||
header_cells[col_idx].push_str(cell_text);
|
||||
}
|
||||
header_indices.extend(row_indices.iter().copied());
|
||||
}
|
||||
|
||||
table.rows.insert(
|
||||
0,
|
||||
assigned_rows
|
||||
.iter()
|
||||
.map(|(row_y, _, _)| *row_y)
|
||||
.reduce(f32::max)
|
||||
.unwrap_or(top_row_y),
|
||||
);
|
||||
table.cells.insert(0, header_cells);
|
||||
table.item_indices.extend(header_indices);
|
||||
table.item_indices.sort_unstable();
|
||||
table.item_indices.dedup();
|
||||
}
|
||||
|
||||
/// Build tables from structure-tree table descriptors by matching MCIDs to TextItems.
|
||||
///
|
||||
/// Returns tables for the given page. Tables where fewer than 50% of cells
|
||||
@@ -67,18 +437,14 @@ pub fn detect_tables_from_struct_tree(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Build cell text and collect item indices
|
||||
let mut cells: Vec<Vec<String>> = Vec::new();
|
||||
let mut all_item_indices: Vec<usize> = Vec::new();
|
||||
// Build cell text and geometry for alignment and header recovery.
|
||||
let mut raw_rows: Vec<Vec<MatchedCell>> = Vec::new();
|
||||
let mut total_cells = 0u32;
|
||||
let mut matched_cells = 0u32;
|
||||
|
||||
for row in &page_rows {
|
||||
let mut row_cells = Vec::with_capacity(num_cols);
|
||||
for (col_idx, cell) in row.cells.iter().enumerate() {
|
||||
if col_idx >= num_cols {
|
||||
break;
|
||||
}
|
||||
let mut row_cells = Vec::with_capacity(row.cells.len());
|
||||
for cell in &row.cells {
|
||||
total_cells += 1;
|
||||
|
||||
// Collect all items for this cell's MCIDs
|
||||
@@ -115,18 +481,18 @@ pub fn detect_tables_from_struct_tree(
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
|
||||
for (idx, _) in &cell_items {
|
||||
all_item_indices.push(*idx);
|
||||
}
|
||||
let item_indices = cell_items.iter().map(|(idx, _)| *idx).collect::<Vec<_>>();
|
||||
let x = cell_items.iter().map(|(_, item)| item.x).reduce(f32::min);
|
||||
let y = cell_items.iter().map(|(_, item)| item.y).reduce(f32::max);
|
||||
|
||||
row_cells.push(text);
|
||||
row_cells.push(MatchedCell {
|
||||
text,
|
||||
item_indices,
|
||||
x,
|
||||
y,
|
||||
});
|
||||
}
|
||||
|
||||
// Pad to num_cols
|
||||
while row_cells.len() < num_cols {
|
||||
row_cells.push(String::new());
|
||||
}
|
||||
cells.push(row_cells);
|
||||
raw_rows.push(row_cells);
|
||||
}
|
||||
|
||||
// Reject if too few cells matched (stale structure tree)
|
||||
@@ -148,52 +514,55 @@ pub fn detect_tables_from_struct_tree(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Derive row/column positions from item geometry
|
||||
let mut row_positions: Vec<f32> = Vec::new();
|
||||
for row in &page_rows {
|
||||
let y = row
|
||||
.cells
|
||||
.iter()
|
||||
.flat_map(|c| c.mcids.iter())
|
||||
.filter(|(_, p)| *p == page)
|
||||
.filter_map(|(mcid, _)| mcid_to_items.get(mcid))
|
||||
.flatten()
|
||||
.map(|&idx| items[idx].y)
|
||||
.reduce(f32::max)
|
||||
.unwrap_or(0.0);
|
||||
row_positions.push(y);
|
||||
}
|
||||
let has_ragged_rows = raw_rows
|
||||
.iter()
|
||||
.any(|row| row.iter().filter(|cell| cell.x.is_some()).count() < num_cols);
|
||||
let first_row_has_tagged_header = page_rows.first().is_some_and(|row| {
|
||||
let header_cells = row.cells.iter().filter(|cell| cell.is_header).count();
|
||||
header_cells * 2 >= row.cells.len()
|
||||
});
|
||||
let fallback_col_positions =
|
||||
legacy_column_positions(&page_rows, &mcid_to_items, items, page, num_cols);
|
||||
let (legacy_cells, legacy_row_positions, mut legacy_item_indices) =
|
||||
left_align_struct_rows(&raw_rows, num_cols);
|
||||
legacy_item_indices.sort_unstable();
|
||||
legacy_item_indices.dedup();
|
||||
let legacy_table = Table::new(
|
||||
fallback_col_positions.clone(),
|
||||
legacy_row_positions,
|
||||
legacy_cells,
|
||||
legacy_item_indices,
|
||||
);
|
||||
|
||||
// Column positions: use X positions of first non-empty cell in each column
|
||||
let mut col_positions: Vec<f32> = vec![0.0; num_cols];
|
||||
for (col, col_pos) in col_positions.iter_mut().enumerate() {
|
||||
for row in &page_rows {
|
||||
if col < row.cells.len() {
|
||||
if let Some(x) = row.cells[col]
|
||||
.mcids
|
||||
.iter()
|
||||
.filter(|(_, p)| *p == page)
|
||||
.filter_map(|(mcid, _)| mcid_to_items.get(mcid))
|
||||
.flatten()
|
||||
.map(|&idx| items[idx].x)
|
||||
.reduce(f32::min)
|
||||
{
|
||||
*col_pos = x;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let col_positions = infer_column_positions(&raw_rows, &fallback_col_positions, num_cols);
|
||||
let (aligned_cells, aligned_row_positions, mut aligned_item_indices) =
|
||||
align_struct_rows(&raw_rows, &col_positions);
|
||||
aligned_item_indices.sort_unstable();
|
||||
aligned_item_indices.dedup();
|
||||
|
||||
all_item_indices.sort_unstable();
|
||||
all_item_indices.dedup();
|
||||
|
||||
tables.push(Table::new(
|
||||
let mut aligned_table = Table::new(
|
||||
col_positions,
|
||||
row_positions,
|
||||
cells,
|
||||
all_item_indices,
|
||||
));
|
||||
aligned_row_positions,
|
||||
aligned_cells,
|
||||
aligned_item_indices,
|
||||
);
|
||||
let item_count_before_header = aligned_table.item_indices.len();
|
||||
let row_count_before_header = aligned_table.cells.len();
|
||||
recover_unclaimed_header_row(
|
||||
&mut aligned_table,
|
||||
items,
|
||||
has_ragged_rows && !first_row_has_tagged_header,
|
||||
);
|
||||
|
||||
let recovered_header = aligned_table.item_indices.len() > item_count_before_header
|
||||
|| aligned_table.cells.len() > row_count_before_header;
|
||||
let prefer_aligned = recovered_header;
|
||||
|
||||
tables.push(if prefer_aligned {
|
||||
aligned_table
|
||||
} else {
|
||||
legacy_table
|
||||
});
|
||||
}
|
||||
|
||||
tables
|
||||
@@ -374,4 +743,419 @@ mod tests {
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 2);
|
||||
assert_eq!(tables.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn realigns_ragged_rows_and_recovers_untagged_header() {
|
||||
let items = vec![
|
||||
make_item("Category", 50.0, 120.0, 1, None),
|
||||
make_item("Potentially", 150.0, 120.0, 1, None),
|
||||
make_item("Summary", 250.0, 120.0, 1, None),
|
||||
make_item("Most commonly", 350.0, 120.0, 1, None),
|
||||
make_item("concerning aspect", 150.0, 110.0, 1, None),
|
||||
make_item("suggested", 350.0, 110.0, 1, None),
|
||||
make_item("of circumstances", 150.0, 100.0, 1, None),
|
||||
make_item("intervention", 350.0, 100.0, 1, None),
|
||||
make_item("Existence of red-teaming", 150.0, 80.0, 1, Some(10)),
|
||||
make_item("Important for safety", 250.0, 80.0, 1, Some(11)),
|
||||
make_item("Ensure welfare interviews", 350.0, 80.0, 1, Some(12)),
|
||||
make_item("Identity & self-knowledge", 50.0, 60.0, 1, Some(20)),
|
||||
make_item("Lack of knowledge", 150.0, 60.0, 1, Some(21)),
|
||||
make_item("Overall negative", 250.0, 60.0, 1, Some(22)),
|
||||
make_item("Describe training process", 350.0, 60.0, 1, Some(23)),
|
||||
make_item("Uncertainty around other copies", 150.0, 40.0, 1, Some(30)),
|
||||
make_item("High uncertainty", 250.0, 40.0, 1, Some(31)),
|
||||
make_item("No intervention suggested", 350.0, 40.0, 1, Some(32)),
|
||||
];
|
||||
|
||||
let struct_tables = vec![StructTable {
|
||||
rows: vec![
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(10, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(11, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(12, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(20, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(21, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(22, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(23, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(30, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(31, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(32, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1);
|
||||
assert_eq!(tables.len(), 1);
|
||||
let table = &tables[0];
|
||||
assert_eq!(table.cells.len(), 4);
|
||||
assert_eq!(
|
||||
table.cells[0],
|
||||
vec![
|
||||
"Category",
|
||||
"Potentially concerning aspect of circumstances",
|
||||
"Summary",
|
||||
"Most commonly suggested intervention",
|
||||
]
|
||||
);
|
||||
assert_eq!(table.cells[1][0], "");
|
||||
assert_eq!(table.cells[1][1], "Existence of red-teaming");
|
||||
assert_eq!(table.cells[2][0], "Identity & self-knowledge");
|
||||
assert_eq!(table.cells[3][0], "");
|
||||
assert_eq!(table.columns.len(), 4);
|
||||
assert!(table.columns.windows(2).all(|w| w[0] < w[1]));
|
||||
assert_eq!(table.item_indices.len(), items.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_absorb_caption_above_ragged_struct_table() {
|
||||
let items = vec![
|
||||
make_item("Table 5-7: Summary of responses", 50.0, 120.0, 1, None),
|
||||
make_item("Aspect one", 150.0, 80.0, 1, Some(10)),
|
||||
make_item("Summary one", 250.0, 80.0, 1, Some(11)),
|
||||
make_item("Category", 50.0, 60.0, 1, Some(20)),
|
||||
make_item("Aspect two", 150.0, 60.0, 1, Some(21)),
|
||||
make_item("Summary two", 250.0, 60.0, 1, Some(22)),
|
||||
make_item("Aspect three", 150.0, 40.0, 1, Some(30)),
|
||||
make_item("Summary three", 250.0, 40.0, 1, Some(31)),
|
||||
];
|
||||
|
||||
let struct_tables = vec![StructTable {
|
||||
rows: vec![
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(10, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(11, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(20, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(21, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(22, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(30, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(31, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1);
|
||||
assert_eq!(tables.len(), 1);
|
||||
let table = &tables[0];
|
||||
assert_eq!(table.cells.len(), 3);
|
||||
assert!(
|
||||
table
|
||||
.cells
|
||||
.iter()
|
||||
.flatten()
|
||||
.all(|cell| !cell.contains("Table 5-7")),
|
||||
"caption must stay outside the table"
|
||||
);
|
||||
assert!(!table.item_indices.contains(&0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_existing_tagged_header_without_absorbing_intro_or_caption() {
|
||||
let items = vec![
|
||||
make_item(
|
||||
"Eighteen people left other comments regarding the Project.",
|
||||
50.0,
|
||||
130.0,
|
||||
1,
|
||||
None,
|
||||
),
|
||||
make_item("Table 5-1:", 220.0, 130.0, 1, None),
|
||||
make_item("Other Comments", 350.0, 130.0, 1, None),
|
||||
make_item("Theme", 50.0, 110.0, 1, Some(10)),
|
||||
make_item("Specific Concern/Inquiry", 200.0, 110.0, 1, Some(11)),
|
||||
make_item("Response", 420.0, 110.0, 1, Some(12)),
|
||||
make_item("Traffic", 50.0, 90.0, 1, Some(20)),
|
||||
make_item("Road conditions", 200.0, 90.0, 1, Some(21)),
|
||||
make_item("Maintenance response", 420.0, 90.0, 1, Some(22)),
|
||||
make_item("Noise", 50.0, 70.0, 1, Some(30)),
|
||||
make_item("Dust concerns", 200.0, 70.0, 1, Some(31)),
|
||||
make_item("Mitigation response", 420.0, 70.0, 1, Some(32)),
|
||||
make_item("Resource Use", 50.0, 50.0, 1, Some(40)),
|
||||
make_item("Snowmobile trails", 200.0, 50.0, 1, Some(41)),
|
||||
make_item("Access response", 420.0, 50.0, 1, Some(42)),
|
||||
];
|
||||
|
||||
let struct_tables = vec![StructTable {
|
||||
rows: vec![
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(10, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(11, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(12, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(20, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(21, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(22, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(30, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(31, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(32, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(40, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(41, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(42, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1);
|
||||
assert_eq!(tables.len(), 1);
|
||||
let table = &tables[0];
|
||||
assert_eq!(table.cells.len(), 4);
|
||||
assert_eq!(
|
||||
table.cells[0],
|
||||
vec!["Theme", "Specific Concern/Inquiry", "Response"]
|
||||
);
|
||||
assert!(!table.item_indices.contains(&0));
|
||||
assert!(!table.item_indices.contains(&1));
|
||||
assert!(!table.item_indices.contains(&2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_recover_header_for_narrow_two_column_table() {
|
||||
let items = vec![
|
||||
make_item("Alpha", 50.0, 120.0, 1, None),
|
||||
make_item("Beta", 200.0, 120.0, 1, None),
|
||||
make_item("First value", 200.0, 80.0, 1, Some(10)),
|
||||
make_item("Only labeled row", 50.0, 60.0, 1, Some(20)),
|
||||
make_item("Second value", 200.0, 60.0, 1, Some(21)),
|
||||
make_item("Third value", 200.0, 40.0, 1, Some(30)),
|
||||
];
|
||||
|
||||
let struct_tables = vec![StructTable {
|
||||
rows: vec![
|
||||
StructTableRow {
|
||||
cells: vec![StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(10, 1)],
|
||||
}],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(20, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(21, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(30, 1)],
|
||||
}],
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1);
|
||||
assert_eq!(tables.len(), 1);
|
||||
let table = &tables[0];
|
||||
assert_eq!(table.cells.len(), 3);
|
||||
assert_eq!(table.cells[0], vec!["First value", ""]);
|
||||
assert_eq!(table.cells[1], vec!["Only labeled row", "Second value"]);
|
||||
assert_eq!(table.cells[2], vec!["Third value", ""]);
|
||||
assert!(!table.item_indices.contains(&0));
|
||||
assert!(!table.item_indices.contains(&1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_rows_without_recovered_header_keep_legacy_alignment() {
|
||||
let items = vec![
|
||||
make_item("Date", 150.0, 120.0, 1, Some(10)),
|
||||
make_item("Title", 250.0, 120.0, 1, Some(11)),
|
||||
make_item("PE", 350.0, 120.0, 1, Some(12)),
|
||||
make_item("Bidder", 450.0, 120.0, 1, Some(13)),
|
||||
make_item("Amount", 550.0, 120.0, 1, Some(14)),
|
||||
make_item("1", 50.0, 100.0, 1, Some(20)),
|
||||
make_item("8/1", 150.0, 100.0, 1, Some(21)),
|
||||
make_item("Procurement", 250.0, 100.0, 1, Some(22)),
|
||||
make_item("PUC", 350.0, 100.0, 1, Some(23)),
|
||||
make_item("Vendor", 450.0, 100.0, 1, Some(24)),
|
||||
make_item("SR1", 550.0, 100.0, 1, Some(25)),
|
||||
];
|
||||
|
||||
let struct_tables = vec![StructTable {
|
||||
rows: vec![
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(10, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(11, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(12, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(13, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: true,
|
||||
mcids: vec![(14, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
StructTableRow {
|
||||
cells: vec![
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(20, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(21, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(22, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(23, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(24, 1)],
|
||||
},
|
||||
StructTableCell {
|
||||
is_header: false,
|
||||
mcids: vec![(25, 1)],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}];
|
||||
|
||||
let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1);
|
||||
assert_eq!(tables.len(), 1);
|
||||
let table = &tables[0];
|
||||
assert_eq!(table.cells[0][0], "Date");
|
||||
assert_eq!(table.cells[0][4], "Amount");
|
||||
assert_eq!(table.cells[0][5], "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,6 +154,12 @@ fn is_dots_only(cell: &str) -> bool {
|
||||
dots >= 3 && t.chars().all(|c| c == '.' || c.is_whitespace())
|
||||
}
|
||||
|
||||
fn starts_with_uppercase_word(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|c| c.is_alphanumeric())
|
||||
.is_some_and(|c| c.is_uppercase())
|
||||
}
|
||||
|
||||
/// Clean up table cells: merge continuation rows, extract footnotes, remove empty rows
|
||||
fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
let mut cleaned: Vec<Vec<String>> = Vec::new();
|
||||
@@ -212,11 +218,20 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
let looks_like_data_row = non_first_cells.len() >= 2
|
||||
&& avg_cell_len <= 10.0
|
||||
&& numeric_cells > non_first_cells.len() / 2;
|
||||
let uppercase_leading_cells = non_first_cells
|
||||
.iter()
|
||||
.filter(|cell| starts_with_uppercase_word(cell))
|
||||
.count();
|
||||
let looks_like_spanning_first_column_row = first_cell.is_empty()
|
||||
&& row.len() >= 4
|
||||
&& non_first_cells.len() == row.len().saturating_sub(1)
|
||||
&& uppercase_leading_cells >= non_first_cells.len().saturating_sub(1);
|
||||
// Classic continuation: first cell empty, content in other cells
|
||||
let is_classic_continuation = first_cell.is_empty()
|
||||
&& !non_first_cells.is_empty()
|
||||
&& !is_short_subheader
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& cleaned.len() > 1;
|
||||
|
||||
// Wrapped-cell continuation: row has fewer filled cells than the header
|
||||
@@ -246,6 +261,7 @@ fn clean_table_cells(cells: &[Vec<String>]) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
&& filled_cells <= max_filled_for_merge
|
||||
&& prev_filled > filled_cells
|
||||
&& !looks_like_data_row
|
||||
&& !looks_like_spanning_first_column_row
|
||||
&& !is_short_subheader;
|
||||
|
||||
let is_continuation = is_classic_continuation || is_wrapped_continuation;
|
||||
@@ -415,6 +431,65 @@ mod tests {
|
||||
assert_eq!(cleaned.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_spanning_first_column_row_not_merged() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Category".into(),
|
||||
"Potentially concerning aspect".into(),
|
||||
"Summary".into(),
|
||||
"Intervention".into(),
|
||||
],
|
||||
vec![
|
||||
"Identity & self-knowledge".into(),
|
||||
"Lack of knowledge".into(),
|
||||
"Overall negative".into(),
|
||||
"Describe training".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"Uncertainty around other copies".into(),
|
||||
"High uncertainty".into(),
|
||||
"No intervention suggested".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
assert_eq!(cleaned.len(), 3);
|
||||
assert_eq!(cleaned[2][0], "");
|
||||
assert_eq!(cleaned[2][1], "Uncertainty around other copies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_full_width_continuation_row_still_merges_when_lowercase() {
|
||||
let cells = vec![
|
||||
vec![
|
||||
"Classification".into(),
|
||||
"Before tax".into(),
|
||||
"After tax".into(),
|
||||
"Standard equipment".into(),
|
||||
"Options".into(),
|
||||
],
|
||||
vec![
|
||||
"Exclusive Special".into(),
|
||||
"83,500,000".into(),
|
||||
"79,275,000".into(),
|
||||
"Standard equipment".into(),
|
||||
"Option A".into(),
|
||||
],
|
||||
vec![
|
||||
"".into(),
|
||||
"with 3.5% individual consumption tax applied".into(),
|
||||
"with 3.5% individual consumption tax applied".into(),
|
||||
"lighting(crash pad)".into(),
|
||||
"sound system".into(),
|
||||
],
|
||||
];
|
||||
let (cleaned, _) = clean_table_cells(&cells);
|
||||
assert_eq!(cleaned.len(), 2);
|
||||
assert!(cleaned[1][1].contains("83,500,000"));
|
||||
assert!(cleaned[1][1].contains("with 3.5%"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_table_cells_header_row_not_merged() {
|
||||
// Continuation requires cleaned.len() > 1 (don't merge into header)
|
||||
|
||||
Reference in New Issue
Block a user