Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57bf8b89e2 | ||
|
|
462a7fdfde | ||
|
|
9abcbbb359 | ||
|
|
0b3ba8784c |
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@firecrawl/pdf-inspector",
|
"name": "@firecrawl/pdf-inspector",
|
||||||
"version": "1.2.0",
|
"version": "1.3.0",
|
||||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"types": "index.d.ts",
|
"types": "index.d.ts",
|
||||||
|
|||||||
+9
-173
@@ -315,78 +315,20 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
|||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Group items by (page, Y position) with 5pt tolerance, walking in
|
// Group items by (page, Y position) with 5pt tolerance
|
||||||
// stream order. A new group also starts when the incoming item's X
|
|
||||||
// would be a significant backtrack — signature of a new PDF text
|
|
||||||
// block (`BT ... Tm`) starting at the left of the same Y band.
|
|
||||||
// Without this split, slide-deck PDFs that stack several copy-paste
|
|
||||||
// layers at the same tiny-text Y (e.g. a color-palette row and a
|
|
||||||
// disclaimer body) later sort-by-X into character-interleaved
|
|
||||||
// gibberish.
|
|
||||||
let y_tolerance = 5.0;
|
let y_tolerance = 5.0;
|
||||||
struct LineGroup<'a> {
|
let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new();
|
||||||
page: u32,
|
|
||||||
y: f32,
|
|
||||||
end_x: f32,
|
|
||||||
font_size: f32,
|
|
||||||
items: Vec<&'a TextItem>,
|
|
||||||
}
|
|
||||||
let mut line_groups: Vec<LineGroup> = Vec::new();
|
|
||||||
|
|
||||||
for item in &items {
|
for item in &items {
|
||||||
// Find the most recently touched matching (page, y) bucket that
|
let found = line_groups
|
||||||
// the item can legitimately continue (no sharp X backtrack).
|
.iter_mut()
|
||||||
// Scanning from the end means the last-updated bucket wins, so a
|
.find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance);
|
||||||
// new back-tracked block reliably opens a fresh group instead of
|
if let Some((_, _, group)) = found {
|
||||||
// re-joining the old one.
|
group.push(item);
|
||||||
let item_right_limit = item.x + effective_merge_width(item);
|
} else {
|
||||||
let mut target: Option<usize> = None;
|
line_groups.push((item.page, item.y, vec![item]));
|
||||||
for (i, g) in line_groups.iter().enumerate().rev() {
|
|
||||||
if g.page != item.page {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (g.y - item.y).abs() >= y_tolerance {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Detect a sharp X backtrack that can only be a new PDF text
|
|
||||||
// block (`BT ... Tm`) starting at the same Y band. Only
|
|
||||||
// trigger for very small fonts: in slide-deck PDFs, multiple
|
|
||||||
// text blocks get stacked at the same Y in a tiny copy-paste
|
|
||||||
// layer, and sort-by-X later interleaves them. Normal-sized
|
|
||||||
// fonts (≥ 3pt) stay on the original behaviour — splitting
|
|
||||||
// there tends to break tables and other valid layouts.
|
|
||||||
if item.font_size < 3.0 && g.font_size < 3.0 {
|
|
||||||
let backtrack = g.end_x - item.x;
|
|
||||||
let min_backtrack = (item.font_size * 5.0).max(5.0);
|
|
||||||
if backtrack > min_backtrack {
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
target = Some(i);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
match target {
|
|
||||||
Some(i) => {
|
|
||||||
line_groups[i].end_x = line_groups[i].end_x.max(item_right_limit);
|
|
||||||
line_groups[i].items.push(item);
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
line_groups.push(LineGroup {
|
|
||||||
page: item.page,
|
|
||||||
y: item.y,
|
|
||||||
end_x: item_right_limit,
|
|
||||||
font_size: item.font_size,
|
|
||||||
items: vec![item],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Re-shape into the tuple layout the rest of this function expects.
|
|
||||||
let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = line_groups
|
|
||||||
.into_iter()
|
|
||||||
.map(|g| (g.page, g.y, g.items))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Sort each group by X position (direction-aware)
|
// Sort each group by X position (direction-aware)
|
||||||
for (_, _, group) in &mut line_groups {
|
for (_, _, group) in &mut line_groups {
|
||||||
@@ -631,112 +573,6 @@ mod tests {
|
|||||||
assert_eq!(merged[0].text, "hello world");
|
assert_eq!(merged[0].text, "hello world");
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_tiny_item(text: &str, x: f32, width: f32) -> TextItem {
|
|
||||||
TextItem {
|
|
||||||
text: text.into(),
|
|
||||||
x,
|
|
||||||
y: 700.0,
|
|
||||||
width,
|
|
||||||
height: 1.3,
|
|
||||||
font: "F1".into(),
|
|
||||||
font_size: 1.3, // tiny copy-paste / accessibility layer font
|
|
||||||
page: 1,
|
|
||||||
is_bold: false,
|
|
||||||
is_italic: false,
|
|
||||||
item_type: ItemType::Text,
|
|
||||||
mcid: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn merge_items_splits_tiny_interleaved_text_blocks_at_same_y() {
|
|
||||||
// Slide-deck copy-paste layers render several `BT` blocks into a
|
|
||||||
// tiny (<3pt) text layer all at the same Y. Before the fix,
|
|
||||||
// grouping by Y alone would fuse them into one bucket, and the
|
|
||||||
// subsequent sort-by-X would character-interleave them:
|
|
||||||
// "A O B S G T..." instead of "Alpha...Omega...".
|
|
||||||
//
|
|
||||||
// The sharp X backtrack between blocks must open a new group so
|
|
||||||
// block-A items stay before block-B items in the output, even
|
|
||||||
// after their shared Y bucket would be X-sorted.
|
|
||||||
let items = vec![
|
|
||||||
make_tiny_item("Alpha", 10.0, 2.0),
|
|
||||||
make_tiny_item("Gamma", 40.0, 2.0),
|
|
||||||
// Block B: Tm resets X back to the left. With the fix, this
|
|
||||||
// opens a new group; without, X-sort would put "Omega"
|
|
||||||
// between "Alpha" and "Gamma" (interleaving).
|
|
||||||
make_tiny_item("Omega", 20.0, 2.0),
|
|
||||||
make_tiny_item("Theta", 50.0, 2.0),
|
|
||||||
];
|
|
||||||
let merged = merge_text_items(items);
|
|
||||||
let texts: Vec<String> = merged.iter().map(|m| m.text.clone()).collect();
|
|
||||||
// The critical property is that "Alpha" + "Gamma" appear
|
|
||||||
// together in stream order before "Omega" + "Theta" — not
|
|
||||||
// interleaved. Individual merges within each block depend on
|
|
||||||
// gap thresholds that don't matter for this invariant.
|
|
||||||
let alpha_idx = texts.iter().position(|t| t.contains("Alpha")).unwrap();
|
|
||||||
let gamma_idx = texts.iter().position(|t| t.contains("Gamma")).unwrap();
|
|
||||||
let omega_idx = texts.iter().position(|t| t.contains("Omega")).unwrap();
|
|
||||||
let theta_idx = texts.iter().position(|t| t.contains("Theta")).unwrap();
|
|
||||||
assert!(
|
|
||||||
alpha_idx < omega_idx && gamma_idx < omega_idx,
|
|
||||||
"block-A items (Alpha, Gamma) must precede block-B items (Omega, Theta); got {texts:?}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
omega_idx < theta_idx,
|
|
||||||
"Omega must come before Theta within block B; got {texts:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn merge_items_does_not_split_normal_font_table_rows() {
|
|
||||||
// Normal body/table text at 12pt must not be split by the
|
|
||||||
// X-backtrack rule — that rule applies only to tiny (<3pt)
|
|
||||||
// copy-paste layers. A single 12pt group here means the rule
|
|
||||||
// didn't fire: the items all end up X-sorted together, so
|
|
||||||
// "extra" at X=50 sits between "Row1-ColA" (10) and "Row1-ColB"
|
|
||||||
// (100) rather than staying at the end.
|
|
||||||
let items = vec![
|
|
||||||
make_merge_item("Row1-ColA", 10.0, 50.0),
|
|
||||||
make_merge_item("Row1-ColB", 100.0, 50.0),
|
|
||||||
make_merge_item("Row1-ColC", 200.0, 50.0),
|
|
||||||
make_merge_item("extra", 50.0, 30.0),
|
|
||||||
];
|
|
||||||
let merged = merge_text_items(items);
|
|
||||||
let texts: Vec<String> = merged.iter().map(|m| m.text.clone()).collect();
|
|
||||||
// Find positions of the non-merged anchors.
|
|
||||||
let a_idx = texts
|
|
||||||
.iter()
|
|
||||||
.position(|t| t.contains("Row1-ColA"))
|
|
||||||
.expect("ColA in output");
|
|
||||||
let extra_idx = texts
|
|
||||||
.iter()
|
|
||||||
.position(|t| t.contains("extra"))
|
|
||||||
.expect("extra in output");
|
|
||||||
let b_idx = texts
|
|
||||||
.iter()
|
|
||||||
.position(|t| t.contains("Row1-ColB"))
|
|
||||||
.expect("ColB in output");
|
|
||||||
assert!(
|
|
||||||
a_idx < extra_idx && extra_idx < b_idx,
|
|
||||||
"12pt items must be X-sorted together (Row1-ColA, extra, Row1-ColB); got {texts:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn merge_items_tolerates_small_kerning_overlap() {
|
|
||||||
// Small negative gaps (ligature/kerning) must NOT split a word.
|
|
||||||
// A ~1pt overlap between "Wo" and "rld" is well within normal
|
|
||||||
// kerning and should stay in one group.
|
|
||||||
let items = vec![
|
|
||||||
make_merge_item("Wo", 100.0, 20.0), // end = 120
|
|
||||||
make_merge_item("rld", 119.0, 25.0), // 1pt overlap: kerning, not a reset
|
|
||||||
];
|
|
||||||
let merged = merge_text_items(items);
|
|
||||||
assert_eq!(merged.len(), 1);
|
|
||||||
assert_eq!(merged[0].text, "World");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_group_into_lines() {
|
fn test_group_into_lines() {
|
||||||
let items = vec![
|
let items = vec![
|
||||||
|
|||||||
+10
-3
@@ -1782,19 +1782,26 @@ fn compute_layout_complexity(
|
|||||||
markdown::filter_lines_to_band(lines, page, x_lo, x_hi)
|
markdown::filter_lines_to_band(lines, page, x_lo, x_hi)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// TOC pages route through the table detector but render as flat
|
||||||
|
// lists. They aren't tables in any user-facing sense, so don't
|
||||||
|
// count them toward LayoutComplexity (would also trip the
|
||||||
|
// table-page guard in column detection below).
|
||||||
|
let has_data_table =
|
||||||
|
|tables: &[tables::Table]| tables.iter().any(|t| t.kind == tables::TableKind::Data);
|
||||||
|
|
||||||
let (rect_tables, _) = tables::detect_tables_from_rects(&band_items, &band_rects, page);
|
let (rect_tables, _) = tables::detect_tables_from_rects(&band_items, &band_rects, page);
|
||||||
if !rect_tables.is_empty() {
|
if has_data_table(&rect_tables) {
|
||||||
found_table = true;
|
found_table = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let line_tables = tables::detect_tables_from_lines(&band_items, &band_lines, page);
|
let line_tables = tables::detect_tables_from_lines(&band_items, &band_lines, page);
|
||||||
if !line_tables.is_empty() {
|
if has_data_table(&line_tables) {
|
||||||
found_table = true;
|
found_table = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// Heuristic fallback for borderless tables
|
// Heuristic fallback for borderless tables
|
||||||
let heuristic_tables = tables::detect_tables(&band_items, base_size, false);
|
let heuristic_tables = tables::detect_tables(&band_items, base_size, false);
|
||||||
if !heuristic_tables.is_empty() {
|
if has_data_table(&heuristic_tables) {
|
||||||
found_table = true;
|
found_table = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -581,8 +581,14 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
|||||||
// Validation 1: some rows should have content in first column.
|
// Validation 1: some rows should have content in first column.
|
||||||
// Use a lower threshold (25%) for tables with wrapped cells where
|
// Use a lower threshold (25%) for tables with wrapped cells where
|
||||||
// continuation lines leave the first column empty.
|
// continuation lines leave the first column empty.
|
||||||
|
// Skip when cells form a narrow TOC pattern: hierarchical entries indented
|
||||||
|
// across multiple X levels leave the leftmost column sparse (only top-level
|
||||||
|
// chapters land there) but the structure is still a valid TOC. Narrow only
|
||||||
|
// (<=5 cols) — wide multi-column TOCs (e.g. 2-up indices) would render
|
||||||
|
// poorly through format_toc_as_list, which assumes one entry per row.
|
||||||
let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
|
let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
|
||||||
if rows_with_first_col < rows.len() / 4 {
|
let is_narrow_toc = columns.len() <= 5 && is_table_of_contents(&cells);
|
||||||
|
if rows_with_first_col < rows.len() / 4 && !is_narrow_toc {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
" validation 1 fail: {}/{} rows have first col",
|
" validation 1 fail: {}/{} rows have first col",
|
||||||
rows_with_first_col,
|
rows_with_first_col,
|
||||||
@@ -653,8 +659,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validation 8: Reject paragraph-like content falsely detected as tables
|
// Validation 8: Reject paragraph-like content falsely detected as tables.
|
||||||
if is_paragraph_content(&cells) {
|
// TOC pages with deep indentation (top-level chapters in col 0, subsections
|
||||||
|
// in cols 1-3, page numbers in last col) leave most cells empty and trip
|
||||||
|
// the paragraph heuristic; TOC shape is a safer signal here. Narrow only
|
||||||
|
// — see narrow-TOC rationale at validation 1.
|
||||||
|
if is_paragraph_content(&cells) && !is_narrow_toc {
|
||||||
log::debug!(" validation 9 fail: paragraph content");
|
log::debug!(" validation 9 fail: paragraph content");
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -677,12 +687,7 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
|||||||
item_indices.len()
|
item_indices.len()
|
||||||
);
|
);
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(columns, rows, cells, item_indices))
|
||||||
columns,
|
|
||||||
rows,
|
|
||||||
cells,
|
|
||||||
item_indices,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if this looks like a key-value pair layout rather than a table
|
/// Check if this looks like a key-value pair layout rather than a table
|
||||||
@@ -906,7 +911,7 @@ fn looks_like_number(s: &str) -> bool {
|
|||||||
/// Check if this looks like a Table of Contents (either style).
|
/// Check if this looks like a Table of Contents (either style).
|
||||||
///
|
///
|
||||||
/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
|
/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
|
||||||
pub(super) fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
pub fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||||
is_dot_leader_toc(cells) || is_tabular_toc(cells)
|
is_dot_leader_toc(cells) || is_tabular_toc(cells)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1601,6 +1606,62 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_table_of_contents_accepts_hierarchical_indented_toc() {
|
||||||
|
// Mythos system card pages 4-5: top-level chapters indent at col 0,
|
||||||
|
// subsections at cols 1-2, leaving col 0 mostly empty (only ~10% of
|
||||||
|
// rows). Validation 1 was rejecting these even though the structure
|
||||||
|
// is unambiguously a TOC.
|
||||||
|
let cells = vec![
|
||||||
|
vec!["Abstract".to_string(), String::new(), "3".to_string()],
|
||||||
|
vec![
|
||||||
|
"1 Introduction".to_string(),
|
||||||
|
String::new(),
|
||||||
|
"10".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"1.1 Model training".to_string(),
|
||||||
|
"11".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"1.1.1 Training data".to_string(),
|
||||||
|
"11".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"1.1.2 Crowd workers".to_string(),
|
||||||
|
"12".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"1.2 Release decision".to_string(),
|
||||||
|
"13".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
"2 RSP evaluations".to_string(),
|
||||||
|
String::new(),
|
||||||
|
"16".to_string(),
|
||||||
|
],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"2.1 RSP risk assessment".to_string(),
|
||||||
|
"16".to_string(),
|
||||||
|
],
|
||||||
|
vec![String::new(), "2.1.1 Context".to_string(), "16".to_string()],
|
||||||
|
vec![
|
||||||
|
String::new(),
|
||||||
|
"2.2 CB evaluations".to_string(),
|
||||||
|
"20".to_string(),
|
||||||
|
],
|
||||||
|
];
|
||||||
|
assert!(
|
||||||
|
is_table_of_contents(&cells),
|
||||||
|
"hierarchical TOC with sparse col 0 should still be detected"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn is_table_of_contents_rejects_dotless_toc() {
|
fn is_table_of_contents_rejects_dotless_toc() {
|
||||||
// Tabular TOC without leader dots: first column starts with dotted
|
// Tabular TOC without leader dots: first column starts with dotted
|
||||||
|
|||||||
@@ -265,12 +265,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
|||||||
page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
|
page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
|
||||||
);
|
);
|
||||||
|
|
||||||
vec![Table {
|
vec![Table::new(
|
||||||
columns: col_edges,
|
col_edges,
|
||||||
rows: row_edges_desc[..num_rows].to_vec(),
|
row_edges_desc[..num_rows].to_vec(),
|
||||||
cells,
|
cells,
|
||||||
item_indices,
|
item_indices,
|
||||||
}]
|
)]
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -1037,12 +1037,7 @@ fn try_build_grid(
|
|||||||
(columns, cells)
|
(columns, cells)
|
||||||
};
|
};
|
||||||
|
|
||||||
GridResult::Ok(Table {
|
GridResult::Ok(Table::new(columns, rows, cells, item_indices))
|
||||||
columns,
|
|
||||||
rows,
|
|
||||||
cells,
|
|
||||||
item_indices,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges.
|
/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges.
|
||||||
@@ -1442,12 +1437,7 @@ fn detect_row_stripe_table(
|
|||||||
content_ratio * 100.0
|
content_ratio * 100.0
|
||||||
);
|
);
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||||
columns: column_centers,
|
|
||||||
rows: row_centers,
|
|
||||||
cells,
|
|
||||||
item_indices,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detect a table from cell-background rects that failed grid detection.
|
/// Detect a table from cell-background rects that failed grid detection.
|
||||||
@@ -1691,12 +1681,7 @@ fn detect_row_stripe_table_from_cell_rects(
|
|||||||
non_empty_cells as f32 / total_cells * 100.0
|
non_empty_cells as f32 / total_cells * 100.0
|
||||||
);
|
);
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||||
columns: column_centers,
|
|
||||||
rows: row_centers,
|
|
||||||
cells,
|
|
||||||
item_indices,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detect a table by merging all cluster rects into one group.
|
/// Detect a table by merging all cluster rects into one group.
|
||||||
@@ -1875,12 +1860,7 @@ fn detect_merged_cluster_table(
|
|||||||
content_ratio * 100.0
|
content_ratio * 100.0
|
||||||
);
|
);
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||||
columns: column_centers,
|
|
||||||
rows: row_centers,
|
|
||||||
cells,
|
|
||||||
item_indices,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Cluster text item X positions into column centers with a given minimum threshold.
|
/// Cluster text item X positions into column centers with a given minimum threshold.
|
||||||
|
|||||||
@@ -188,12 +188,12 @@ pub fn detect_tables_from_struct_tree(
|
|||||||
all_item_indices.sort_unstable();
|
all_item_indices.sort_unstable();
|
||||||
all_item_indices.dedup();
|
all_item_indices.dedup();
|
||||||
|
|
||||||
tables.push(Table {
|
tables.push(Table::new(
|
||||||
columns: col_positions,
|
col_positions,
|
||||||
rows: row_positions,
|
row_positions,
|
||||||
cells,
|
cells,
|
||||||
item_indices: all_item_indices,
|
all_item_indices,
|
||||||
});
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
tables
|
tables
|
||||||
|
|||||||
+21
-21
@@ -1,26 +1,19 @@
|
|||||||
//! Table-to-markdown formatting and cell cleanup.
|
//! Table-to-markdown formatting and cell cleanup.
|
||||||
|
|
||||||
use super::detect_heuristic::is_table_of_contents;
|
use super::{Table, TableKind};
|
||||||
use super::Table;
|
|
||||||
|
|
||||||
pub fn table_to_markdown(table: &Table) -> String {
|
pub fn table_to_markdown(table: &Table) -> String {
|
||||||
if table.cells.is_empty() || table.cells[0].is_empty() {
|
if table.cells.is_empty() || table.cells[0].is_empty() {
|
||||||
return String::new();
|
return String::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Detect TOC on the raw cells: clean_table_cells merges rows in ways
|
// TOCs render poorly as markdown tables — emit a flat per-row text list
|
||||||
// that can make genuine data tables superficially resemble a TOC
|
// instead so the page numbers stay aligned with their section titles
|
||||||
// (short numeric cells, few columns) — but the raw detection here
|
// rather than drifting to a separate column. Format from raw cells
|
||||||
// preserves the original multi-column structure and only matches the
|
// because continuation-row merging in clean_table_cells collapses
|
||||||
// true TOC pattern.
|
// separate TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench")
|
||||||
//
|
// into one line where sub-entries leave column 0 empty.
|
||||||
// Tables of contents render poorly as markdown tables — emit a flat
|
if table.kind == TableKind::Toc {
|
||||||
// per-row text list instead so the page numbers stay aligned with
|
|
||||||
// their section titles rather than drifting to a separate column.
|
|
||||||
// Format from raw cells: continuation-row merging collapses separate
|
|
||||||
// TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench") into a
|
|
||||||
// single line because sub-entries leave column 0 empty.
|
|
||||||
if is_table_of_contents(&table.cells) {
|
|
||||||
return format_toc_as_list(&table.cells, &[]);
|
return format_toc_as_list(&table.cells, &[]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -470,6 +463,7 @@ mod tests {
|
|||||||
vec!["Bob".into(), "25".into()],
|
vec!["Bob".into(), "25".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
assert!(md.contains("|Name|"));
|
assert!(md.contains("|Name|"));
|
||||||
@@ -485,6 +479,7 @@ mod tests {
|
|||||||
rows: vec![500.0],
|
rows: vec![500.0],
|
||||||
cells: vec![vec!["Only".into(), "Row".into()]],
|
cells: vec![vec!["Only".into(), "Row".into()]],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
assert!(md.contains("|Only|"));
|
assert!(md.contains("|Only|"));
|
||||||
@@ -498,6 +493,7 @@ mod tests {
|
|||||||
rows: vec![],
|
rows: vec![],
|
||||||
cells: vec![],
|
cells: vec![],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
assert_eq!(table_to_markdown(&table), "");
|
assert_eq!(table_to_markdown(&table), "");
|
||||||
}
|
}
|
||||||
@@ -513,6 +509,7 @@ mod tests {
|
|||||||
vec!["(1)".into(), "Footnote text".into()],
|
vec!["(1)".into(), "Footnote text".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
assert!(md.contains("(1) Footnote text"));
|
assert!(md.contains("(1) Footnote text"));
|
||||||
@@ -528,6 +525,7 @@ mod tests {
|
|||||||
vec!["太郎".into(), "25".into()],
|
vec!["太郎".into(), "25".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
assert!(md.contains("名前"));
|
assert!(md.contains("名前"));
|
||||||
@@ -541,6 +539,7 @@ mod tests {
|
|||||||
rows: vec![500.0],
|
rows: vec![500.0],
|
||||||
cells: vec![vec![]],
|
cells: vec![vec![]],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
assert_eq!(table_to_markdown(&table), "");
|
assert_eq!(table_to_markdown(&table), "");
|
||||||
}
|
}
|
||||||
@@ -550,10 +549,10 @@ mod tests {
|
|||||||
// A TOC-shaped table with section numbers in col 0 and page numbers
|
// A TOC-shaped table with section numbers in col 0 and page numbers
|
||||||
// in the last column should render as a flat list, not a markdown
|
// in the last column should render as a flat list, not a markdown
|
||||||
// table, so the page numbers stay on the same line as their titles.
|
// table, so the page numbers stay on the same line as their titles.
|
||||||
let table = Table {
|
let table = Table::new(
|
||||||
columns: vec![50.0, 80.0, 300.0],
|
vec![50.0, 80.0, 300.0],
|
||||||
rows: vec![500.0; 5],
|
vec![500.0; 5],
|
||||||
cells: vec![
|
vec![
|
||||||
vec![
|
vec![
|
||||||
"4.3".into(),
|
"4.3".into(),
|
||||||
"Case studies and targeted evaluations".into(),
|
"Case studies and targeted evaluations".into(),
|
||||||
@@ -572,8 +571,9 @@ mod tests {
|
|||||||
vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
|
vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
|
||||||
vec!["4.5".into(), "White-box analyses".into(), "113".into()],
|
vec!["4.5".into(), "White-box analyses".into(), "113".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
vec![],
|
||||||
};
|
);
|
||||||
|
assert_eq!(table.kind, TableKind::Toc);
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
assert!(
|
assert!(
|
||||||
!md.contains("|---|"),
|
!md.contains("|---|"),
|
||||||
|
|||||||
@@ -499,6 +499,7 @@ pub(crate) fn recover_header_row(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::tables::TableKind;
|
||||||
use crate::types::ItemType;
|
use crate::types::ItemType;
|
||||||
|
|
||||||
fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem {
|
fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem {
|
||||||
@@ -756,6 +757,7 @@ mod tests {
|
|||||||
rows: vec![500.0, 480.0],
|
rows: vec![500.0, 480.0],
|
||||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||||
item_indices: vec![2, 3],
|
item_indices: vec![2, 3],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
recover_header_row(&mut table, &all_items, 9.0);
|
recover_header_row(&mut table, &all_items, 9.0);
|
||||||
@@ -774,6 +776,7 @@ mod tests {
|
|||||||
rows: vec![500.0],
|
rows: vec![500.0],
|
||||||
cells: vec![vec!["A".into(), "B".into()]],
|
cells: vec![vec!["A".into(), "B".into()]],
|
||||||
item_indices: vec![0, 1],
|
item_indices: vec![0, 1],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
let rows_before = table.rows.len();
|
let rows_before = table.rows.len();
|
||||||
@@ -794,6 +797,7 @@ mod tests {
|
|||||||
rows: vec![500.0, 480.0],
|
rows: vec![500.0, 480.0],
|
||||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||||
item_indices: vec![2, 3],
|
item_indices: vec![2, 3],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
let rows_before = table.rows.len();
|
let rows_before = table.rows.len();
|
||||||
@@ -814,6 +818,7 @@ mod tests {
|
|||||||
rows: vec![500.0],
|
rows: vec![500.0],
|
||||||
cells: vec![vec!["A".into(), "B".into()]],
|
cells: vec![vec!["A".into(), "B".into()]],
|
||||||
item_indices: vec![1, 2],
|
item_indices: vec![1, 2],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
let rows_before = table.rows.len();
|
let rows_before = table.rows.len();
|
||||||
@@ -829,6 +834,7 @@ mod tests {
|
|||||||
rows: vec![],
|
rows: vec![],
|
||||||
cells: vec![],
|
cells: vec![],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
recover_header_row(&mut table, &all_items, 9.0);
|
recover_header_row(&mut table, &all_items, 9.0);
|
||||||
|
|||||||
+49
-11
@@ -11,6 +11,7 @@ mod format;
|
|||||||
mod grid;
|
mod grid;
|
||||||
|
|
||||||
pub use detect_heuristic::detect_tables;
|
pub use detect_heuristic::detect_tables;
|
||||||
|
pub(crate) use detect_heuristic::is_table_of_contents;
|
||||||
pub use detect_lines::detect_tables_from_lines;
|
pub use detect_lines::detect_tables_from_lines;
|
||||||
pub(crate) use detect_rects::cluster_rects;
|
pub(crate) use detect_rects::cluster_rects;
|
||||||
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
||||||
@@ -166,12 +167,12 @@ pub(crate) fn try_build_rect_guided_table(
|
|||||||
used_indices.sort_unstable();
|
used_indices.sort_unstable();
|
||||||
used_indices.dedup();
|
used_indices.dedup();
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(
|
||||||
columns: col_boundaries,
|
col_boundaries,
|
||||||
rows: row_boundaries,
|
row_boundaries,
|
||||||
cells,
|
cells,
|
||||||
item_indices: used_indices,
|
used_indices,
|
||||||
})
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Split a TextItem whose text contains multiple whitespace-separated tokens
|
/// Split a TextItem whose text contains multiple whitespace-separated tokens
|
||||||
@@ -541,12 +542,22 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
|||||||
multi_col_rows
|
multi_col_rows
|
||||||
);
|
);
|
||||||
|
|
||||||
Some(Table {
|
Some(Table::new(col_xs, row_ys, cells, item_indices))
|
||||||
columns: col_xs,
|
}
|
||||||
rows: row_ys,
|
|
||||||
cells,
|
/// What kind of structure a detected `Table` represents. Classification is
|
||||||
item_indices,
|
/// computed once at construction so consumers don't have to re-analyze the
|
||||||
})
|
/// cells (and stay consistent across detection backends).
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
|
pub enum TableKind {
|
||||||
|
/// A real data table — renders as markdown table syntax.
|
||||||
|
#[default]
|
||||||
|
Data,
|
||||||
|
/// A table of contents — renders as a flat list with tab-aligned page
|
||||||
|
/// numbers via `format_toc_as_list`. Detected through the table pipeline
|
||||||
|
/// because TOCs share row/column structure with tables, but they are not
|
||||||
|
/// data tables and shouldn't appear in `pages_with_tables` etc.
|
||||||
|
Toc,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A detected table.
|
/// A detected table.
|
||||||
@@ -560,6 +571,31 @@ pub struct Table {
|
|||||||
pub cells: Vec<Vec<String>>,
|
pub cells: Vec<Vec<String>>,
|
||||||
/// Items that belong to this table
|
/// Items that belong to this table
|
||||||
pub item_indices: Vec<usize>,
|
pub item_indices: Vec<usize>,
|
||||||
|
/// Data table vs TOC. Set by `Table::new` from `cells`.
|
||||||
|
pub kind: TableKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Table {
|
||||||
|
/// Build a table and classify it (data vs TOC) from its cells.
|
||||||
|
pub fn new(
|
||||||
|
columns: Vec<f32>,
|
||||||
|
rows: Vec<f32>,
|
||||||
|
cells: Vec<Vec<String>>,
|
||||||
|
item_indices: Vec<usize>,
|
||||||
|
) -> Self {
|
||||||
|
let kind = if is_table_of_contents(&cells) {
|
||||||
|
TableKind::Toc
|
||||||
|
} else {
|
||||||
|
TableKind::Data
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
columns,
|
||||||
|
rows,
|
||||||
|
cells,
|
||||||
|
item_indices,
|
||||||
|
kind,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -642,6 +678,7 @@ mod tests {
|
|||||||
vec!["Cell 1".into(), "Cell 2".into()],
|
vec!["Cell 1".into(), "Cell 2".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
@@ -1002,6 +1039,7 @@ mod tests {
|
|||||||
vec!["3".into(), "5/2".into(), "Item C".into(), "300".into()],
|
vec!["3".into(), "5/2".into(), "Item C".into(), "300".into()],
|
||||||
],
|
],
|
||||||
item_indices: vec![],
|
item_indices: vec![],
|
||||||
|
kind: TableKind::Data,
|
||||||
};
|
};
|
||||||
|
|
||||||
let md = table_to_markdown(&table);
|
let md = table_to_markdown(&table);
|
||||||
|
|||||||
Reference in New Issue
Block a user