Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57bf8b89e2 | ||
|
|
462a7fdfde | ||
|
|
9abcbbb359 | ||
|
|
0b3ba8784c |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+9
-173
@@ -315,79 +315,21 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
return items;
|
||||
}
|
||||
|
||||
// Group items by (page, Y position) with 5pt tolerance, walking in
|
||||
// stream order. A new group also starts when the incoming item's X
|
||||
// would be a significant backtrack — signature of a new PDF text
|
||||
// block (`BT ... Tm`) starting at the left of the same Y band.
|
||||
// Without this split, slide-deck PDFs that stack several copy-paste
|
||||
// layers at the same tiny-text Y (e.g. a color-palette row and a
|
||||
// disclaimer body) later sort-by-X into character-interleaved
|
||||
// gibberish.
|
||||
// Group items by (page, Y position) with 5pt tolerance
|
||||
let y_tolerance = 5.0;
|
||||
struct LineGroup<'a> {
|
||||
page: u32,
|
||||
y: f32,
|
||||
end_x: f32,
|
||||
font_size: f32,
|
||||
items: Vec<&'a TextItem>,
|
||||
}
|
||||
let mut line_groups: Vec<LineGroup> = Vec::new();
|
||||
let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new();
|
||||
|
||||
for item in &items {
|
||||
// Find the most recently touched matching (page, y) bucket that
|
||||
// the item can legitimately continue (no sharp X backtrack).
|
||||
// Scanning from the end means the last-updated bucket wins, so a
|
||||
// new back-tracked block reliably opens a fresh group instead of
|
||||
// re-joining the old one.
|
||||
let item_right_limit = item.x + effective_merge_width(item);
|
||||
let mut target: Option<usize> = None;
|
||||
for (i, g) in line_groups.iter().enumerate().rev() {
|
||||
if g.page != item.page {
|
||||
continue;
|
||||
}
|
||||
if (g.y - item.y).abs() >= y_tolerance {
|
||||
continue;
|
||||
}
|
||||
// Detect a sharp X backtrack that can only be a new PDF text
|
||||
// block (`BT ... Tm`) starting at the same Y band. Only
|
||||
// trigger for very small fonts: in slide-deck PDFs, multiple
|
||||
// text blocks get stacked at the same Y in a tiny copy-paste
|
||||
// layer, and sort-by-X later interleaves them. Normal-sized
|
||||
// fonts (≥ 3pt) stay on the original behaviour — splitting
|
||||
// there tends to break tables and other valid layouts.
|
||||
if item.font_size < 3.0 && g.font_size < 3.0 {
|
||||
let backtrack = g.end_x - item.x;
|
||||
let min_backtrack = (item.font_size * 5.0).max(5.0);
|
||||
if backtrack > min_backtrack {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
target = Some(i);
|
||||
break;
|
||||
}
|
||||
match target {
|
||||
Some(i) => {
|
||||
line_groups[i].end_x = line_groups[i].end_x.max(item_right_limit);
|
||||
line_groups[i].items.push(item);
|
||||
}
|
||||
None => {
|
||||
line_groups.push(LineGroup {
|
||||
page: item.page,
|
||||
y: item.y,
|
||||
end_x: item_right_limit,
|
||||
font_size: item.font_size,
|
||||
items: vec![item],
|
||||
});
|
||||
}
|
||||
let found = line_groups
|
||||
.iter_mut()
|
||||
.find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance);
|
||||
if let Some((_, _, group)) = found {
|
||||
group.push(item);
|
||||
} else {
|
||||
line_groups.push((item.page, item.y, vec![item]));
|
||||
}
|
||||
}
|
||||
|
||||
// Re-shape into the tuple layout the rest of this function expects.
|
||||
let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = line_groups
|
||||
.into_iter()
|
||||
.map(|g| (g.page, g.y, g.items))
|
||||
.collect();
|
||||
|
||||
// Sort each group by X position (direction-aware)
|
||||
for (_, _, group) in &mut line_groups {
|
||||
let rtl = is_rtl_text(group.iter().map(|i| &i.text));
|
||||
@@ -631,112 +573,6 @@ mod tests {
|
||||
assert_eq!(merged[0].text, "hello world");
|
||||
}
|
||||
|
||||
fn make_tiny_item(text: &str, x: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.into(),
|
||||
x,
|
||||
y: 700.0,
|
||||
width,
|
||||
height: 1.3,
|
||||
font: "F1".into(),
|
||||
font_size: 1.3, // tiny copy-paste / accessibility layer font
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_splits_tiny_interleaved_text_blocks_at_same_y() {
|
||||
// Slide-deck copy-paste layers render several `BT` blocks into a
|
||||
// tiny (<3pt) text layer all at the same Y. Before the fix,
|
||||
// grouping by Y alone would fuse them into one bucket, and the
|
||||
// subsequent sort-by-X would character-interleave them:
|
||||
// "A O B S G T..." instead of "Alpha...Omega...".
|
||||
//
|
||||
// The sharp X backtrack between blocks must open a new group so
|
||||
// block-A items stay before block-B items in the output, even
|
||||
// after their shared Y bucket would be X-sorted.
|
||||
let items = vec![
|
||||
make_tiny_item("Alpha", 10.0, 2.0),
|
||||
make_tiny_item("Gamma", 40.0, 2.0),
|
||||
// Block B: Tm resets X back to the left. With the fix, this
|
||||
// opens a new group; without, X-sort would put "Omega"
|
||||
// between "Alpha" and "Gamma" (interleaving).
|
||||
make_tiny_item("Omega", 20.0, 2.0),
|
||||
make_tiny_item("Theta", 50.0, 2.0),
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
let texts: Vec<String> = merged.iter().map(|m| m.text.clone()).collect();
|
||||
// The critical property is that "Alpha" + "Gamma" appear
|
||||
// together in stream order before "Omega" + "Theta" — not
|
||||
// interleaved. Individual merges within each block depend on
|
||||
// gap thresholds that don't matter for this invariant.
|
||||
let alpha_idx = texts.iter().position(|t| t.contains("Alpha")).unwrap();
|
||||
let gamma_idx = texts.iter().position(|t| t.contains("Gamma")).unwrap();
|
||||
let omega_idx = texts.iter().position(|t| t.contains("Omega")).unwrap();
|
||||
let theta_idx = texts.iter().position(|t| t.contains("Theta")).unwrap();
|
||||
assert!(
|
||||
alpha_idx < omega_idx && gamma_idx < omega_idx,
|
||||
"block-A items (Alpha, Gamma) must precede block-B items (Omega, Theta); got {texts:?}"
|
||||
);
|
||||
assert!(
|
||||
omega_idx < theta_idx,
|
||||
"Omega must come before Theta within block B; got {texts:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_does_not_split_normal_font_table_rows() {
|
||||
// Normal body/table text at 12pt must not be split by the
|
||||
// X-backtrack rule — that rule applies only to tiny (<3pt)
|
||||
// copy-paste layers. A single 12pt group here means the rule
|
||||
// didn't fire: the items all end up X-sorted together, so
|
||||
// "extra" at X=50 sits between "Row1-ColA" (10) and "Row1-ColB"
|
||||
// (100) rather than staying at the end.
|
||||
let items = vec![
|
||||
make_merge_item("Row1-ColA", 10.0, 50.0),
|
||||
make_merge_item("Row1-ColB", 100.0, 50.0),
|
||||
make_merge_item("Row1-ColC", 200.0, 50.0),
|
||||
make_merge_item("extra", 50.0, 30.0),
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
let texts: Vec<String> = merged.iter().map(|m| m.text.clone()).collect();
|
||||
// Find positions of the non-merged anchors.
|
||||
let a_idx = texts
|
||||
.iter()
|
||||
.position(|t| t.contains("Row1-ColA"))
|
||||
.expect("ColA in output");
|
||||
let extra_idx = texts
|
||||
.iter()
|
||||
.position(|t| t.contains("extra"))
|
||||
.expect("extra in output");
|
||||
let b_idx = texts
|
||||
.iter()
|
||||
.position(|t| t.contains("Row1-ColB"))
|
||||
.expect("ColB in output");
|
||||
assert!(
|
||||
a_idx < extra_idx && extra_idx < b_idx,
|
||||
"12pt items must be X-sorted together (Row1-ColA, extra, Row1-ColB); got {texts:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_items_tolerates_small_kerning_overlap() {
|
||||
// Small negative gaps (ligature/kerning) must NOT split a word.
|
||||
// A ~1pt overlap between "Wo" and "rld" is well within normal
|
||||
// kerning and should stay in one group.
|
||||
let items = vec![
|
||||
make_merge_item("Wo", 100.0, 20.0), // end = 120
|
||||
make_merge_item("rld", 119.0, 25.0), // 1pt overlap: kerning, not a reset
|
||||
];
|
||||
let merged = merge_text_items(items);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].text, "World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_into_lines() {
|
||||
let items = vec![
|
||||
|
||||
+10
-3
@@ -1782,19 +1782,26 @@ fn compute_layout_complexity(
|
||||
markdown::filter_lines_to_band(lines, page, x_lo, x_hi)
|
||||
};
|
||||
|
||||
// TOC pages route through the table detector but render as flat
|
||||
// lists. They aren't tables in any user-facing sense, so don't
|
||||
// count them toward LayoutComplexity (would also trip the
|
||||
// table-page guard in column detection below).
|
||||
let has_data_table =
|
||||
|tables: &[tables::Table]| tables.iter().any(|t| t.kind == tables::TableKind::Data);
|
||||
|
||||
let (rect_tables, _) = tables::detect_tables_from_rects(&band_items, &band_rects, page);
|
||||
if !rect_tables.is_empty() {
|
||||
if has_data_table(&rect_tables) {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
let line_tables = tables::detect_tables_from_lines(&band_items, &band_lines, page);
|
||||
if !line_tables.is_empty() {
|
||||
if has_data_table(&line_tables) {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
// Heuristic fallback for borderless tables
|
||||
let heuristic_tables = tables::detect_tables(&band_items, base_size, false);
|
||||
if !heuristic_tables.is_empty() {
|
||||
if has_data_table(&heuristic_tables) {
|
||||
found_table = true;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -581,8 +581,14 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
// Validation 1: some rows should have content in first column.
|
||||
// Use a lower threshold (25%) for tables with wrapped cells where
|
||||
// continuation lines leave the first column empty.
|
||||
// Skip when cells form a narrow TOC pattern: hierarchical entries indented
|
||||
// across multiple X levels leave the leftmost column sparse (only top-level
|
||||
// chapters land there) but the structure is still a valid TOC. Narrow only
|
||||
// (<=5 cols) — wide multi-column TOCs (e.g. 2-up indices) would render
|
||||
// poorly through format_toc_as_list, which assumes one entry per row.
|
||||
let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count();
|
||||
if rows_with_first_col < rows.len() / 4 {
|
||||
let is_narrow_toc = columns.len() <= 5 && is_table_of_contents(&cells);
|
||||
if rows_with_first_col < rows.len() / 4 && !is_narrow_toc {
|
||||
log::debug!(
|
||||
" validation 1 fail: {}/{} rows have first col",
|
||||
rows_with_first_col,
|
||||
@@ -653,8 +659,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
return None;
|
||||
}
|
||||
|
||||
// Validation 8: Reject paragraph-like content falsely detected as tables
|
||||
if is_paragraph_content(&cells) {
|
||||
// Validation 8: Reject paragraph-like content falsely detected as tables.
|
||||
// TOC pages with deep indentation (top-level chapters in col 0, subsections
|
||||
// in cols 1-3, page numbers in last col) leave most cells empty and trip
|
||||
// the paragraph heuristic; TOC shape is a safer signal here. Narrow only
|
||||
// — see narrow-TOC rationale at validation 1.
|
||||
if is_paragraph_content(&cells) && !is_narrow_toc {
|
||||
log::debug!(" validation 9 fail: paragraph content");
|
||||
return None;
|
||||
}
|
||||
@@ -677,12 +687,7 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode
|
||||
item_indices.len()
|
||||
);
|
||||
|
||||
Some(Table {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
Some(Table::new(columns, rows, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Check if this looks like a key-value pair layout rather than a table
|
||||
@@ -906,7 +911,7 @@ fn looks_like_number(s: &str) -> bool {
|
||||
/// Check if this looks like a Table of Contents (either style).
|
||||
///
|
||||
/// Used by format.rs to render TOCs as flat lists instead of markdown tables.
|
||||
pub(super) fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||
pub fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
|
||||
is_dot_leader_toc(cells) || is_tabular_toc(cells)
|
||||
}
|
||||
|
||||
@@ -1601,6 +1606,62 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_table_of_contents_accepts_hierarchical_indented_toc() {
|
||||
// Mythos system card pages 4-5: top-level chapters indent at col 0,
|
||||
// subsections at cols 1-2, leaving col 0 mostly empty (only ~10% of
|
||||
// rows). Validation 1 was rejecting these even though the structure
|
||||
// is unambiguously a TOC.
|
||||
let cells = vec![
|
||||
vec!["Abstract".to_string(), String::new(), "3".to_string()],
|
||||
vec![
|
||||
"1 Introduction".to_string(),
|
||||
String::new(),
|
||||
"10".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1 Model training".to_string(),
|
||||
"11".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1.1 Training data".to_string(),
|
||||
"11".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.1.2 Crowd workers".to_string(),
|
||||
"12".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"1.2 Release decision".to_string(),
|
||||
"13".to_string(),
|
||||
],
|
||||
vec![
|
||||
"2 RSP evaluations".to_string(),
|
||||
String::new(),
|
||||
"16".to_string(),
|
||||
],
|
||||
vec![
|
||||
String::new(),
|
||||
"2.1 RSP risk assessment".to_string(),
|
||||
"16".to_string(),
|
||||
],
|
||||
vec![String::new(), "2.1.1 Context".to_string(), "16".to_string()],
|
||||
vec![
|
||||
String::new(),
|
||||
"2.2 CB evaluations".to_string(),
|
||||
"20".to_string(),
|
||||
],
|
||||
];
|
||||
assert!(
|
||||
is_table_of_contents(&cells),
|
||||
"hierarchical TOC with sparse col 0 should still be detected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_table_of_contents_rejects_dotless_toc() {
|
||||
// Tabular TOC without leader dots: first column starts with dotted
|
||||
|
||||
@@ -265,12 +265,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
|
||||
page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content
|
||||
);
|
||||
|
||||
vec![Table {
|
||||
columns: col_edges,
|
||||
rows: row_edges_desc[..num_rows].to_vec(),
|
||||
vec![Table::new(
|
||||
col_edges,
|
||||
row_edges_desc[..num_rows].to_vec(),
|
||||
cells,
|
||||
item_indices,
|
||||
}]
|
||||
)]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1037,12 +1037,7 @@ fn try_build_grid(
|
||||
(columns, cells)
|
||||
};
|
||||
|
||||
GridResult::Ok(Table {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
GridResult::Ok(Table::new(columns, rows, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges.
|
||||
@@ -1442,12 +1437,7 @@ fn detect_row_stripe_table(
|
||||
content_ratio * 100.0
|
||||
);
|
||||
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Detect a table from cell-background rects that failed grid detection.
|
||||
@@ -1691,12 +1681,7 @@ fn detect_row_stripe_table_from_cell_rects(
|
||||
non_empty_cells as f32 / total_cells * 100.0
|
||||
);
|
||||
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Detect a table by merging all cluster rects into one group.
|
||||
@@ -1875,12 +1860,7 @@ fn detect_merged_cluster_table(
|
||||
content_ratio * 100.0
|
||||
);
|
||||
|
||||
Some(Table {
|
||||
columns: column_centers,
|
||||
rows: row_centers,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Cluster text item X positions into column centers with a given minimum threshold.
|
||||
|
||||
@@ -188,12 +188,12 @@ pub fn detect_tables_from_struct_tree(
|
||||
all_item_indices.sort_unstable();
|
||||
all_item_indices.dedup();
|
||||
|
||||
tables.push(Table {
|
||||
columns: col_positions,
|
||||
rows: row_positions,
|
||||
tables.push(Table::new(
|
||||
col_positions,
|
||||
row_positions,
|
||||
cells,
|
||||
item_indices: all_item_indices,
|
||||
});
|
||||
all_item_indices,
|
||||
));
|
||||
}
|
||||
|
||||
tables
|
||||
|
||||
+21
-21
@@ -1,26 +1,19 @@
|
||||
//! Table-to-markdown formatting and cell cleanup.
|
||||
|
||||
use super::detect_heuristic::is_table_of_contents;
|
||||
use super::Table;
|
||||
use super::{Table, TableKind};
|
||||
|
||||
pub fn table_to_markdown(table: &Table) -> String {
|
||||
if table.cells.is_empty() || table.cells[0].is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Detect TOC on the raw cells: clean_table_cells merges rows in ways
|
||||
// that can make genuine data tables superficially resemble a TOC
|
||||
// (short numeric cells, few columns) — but the raw detection here
|
||||
// preserves the original multi-column structure and only matches the
|
||||
// true TOC pattern.
|
||||
//
|
||||
// Tables of contents render poorly as markdown tables — emit a flat
|
||||
// per-row text list instead so the page numbers stay aligned with
|
||||
// their section titles rather than drifting to a separate column.
|
||||
// Format from raw cells: continuation-row merging collapses separate
|
||||
// TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench") into a
|
||||
// single line because sub-entries leave column 0 empty.
|
||||
if is_table_of_contents(&table.cells) {
|
||||
// TOCs render poorly as markdown tables — emit a flat per-row text list
|
||||
// instead so the page numbers stay aligned with their section titles
|
||||
// rather than drifting to a separate column. Format from raw cells
|
||||
// because continuation-row merging in clean_table_cells collapses
|
||||
// separate TOC entries (e.g. "6.2 Contamination" + "6.2.1 SWE-bench")
|
||||
// into one line where sub-entries leave column 0 empty.
|
||||
if table.kind == TableKind::Toc {
|
||||
return format_toc_as_list(&table.cells, &[]);
|
||||
}
|
||||
|
||||
@@ -470,6 +463,7 @@ mod tests {
|
||||
vec!["Bob".into(), "25".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("|Name|"));
|
||||
@@ -485,6 +479,7 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["Only".into(), "Row".into()]],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("|Only|"));
|
||||
@@ -498,6 +493,7 @@ mod tests {
|
||||
rows: vec![],
|
||||
cells: vec![],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
assert_eq!(table_to_markdown(&table), "");
|
||||
}
|
||||
@@ -513,6 +509,7 @@ mod tests {
|
||||
vec!["(1)".into(), "Footnote text".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("(1) Footnote text"));
|
||||
@@ -528,6 +525,7 @@ mod tests {
|
||||
vec!["太郎".into(), "25".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(md.contains("名前"));
|
||||
@@ -541,6 +539,7 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec![]],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
assert_eq!(table_to_markdown(&table), "");
|
||||
}
|
||||
@@ -550,10 +549,10 @@ mod tests {
|
||||
// A TOC-shaped table with section numbers in col 0 and page numbers
|
||||
// in the last column should render as a flat list, not a markdown
|
||||
// table, so the page numbers stay on the same line as their titles.
|
||||
let table = Table {
|
||||
columns: vec![50.0, 80.0, 300.0],
|
||||
rows: vec![500.0; 5],
|
||||
cells: vec![
|
||||
let table = Table::new(
|
||||
vec![50.0, 80.0, 300.0],
|
||||
vec![500.0; 5],
|
||||
vec![
|
||||
vec![
|
||||
"4.3".into(),
|
||||
"Case studies and targeted evaluations".into(),
|
||||
@@ -572,8 +571,9 @@ mod tests {
|
||||
vec!["4.4".into(), "Capability evaluations".into(), "101".into()],
|
||||
vec!["4.5".into(), "White-box analyses".into(), "113".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
};
|
||||
vec![],
|
||||
);
|
||||
assert_eq!(table.kind, TableKind::Toc);
|
||||
let md = table_to_markdown(&table);
|
||||
assert!(
|
||||
!md.contains("|---|"),
|
||||
|
||||
@@ -499,6 +499,7 @@ pub(crate) fn recover_header_row(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tables::TableKind;
|
||||
use crate::types::ItemType;
|
||||
|
||||
fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem {
|
||||
@@ -756,6 +757,7 @@ mod tests {
|
||||
rows: vec![500.0, 480.0],
|
||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||
item_indices: vec![2, 3],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
recover_header_row(&mut table, &all_items, 9.0);
|
||||
@@ -774,6 +776,7 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["A".into(), "B".into()]],
|
||||
item_indices: vec![0, 1],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -794,6 +797,7 @@ mod tests {
|
||||
rows: vec![500.0, 480.0],
|
||||
cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]],
|
||||
item_indices: vec![2, 3],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -814,6 +818,7 @@ mod tests {
|
||||
rows: vec![500.0],
|
||||
cells: vec![vec!["A".into(), "B".into()]],
|
||||
item_indices: vec![1, 2],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let rows_before = table.rows.len();
|
||||
@@ -829,6 +834,7 @@ mod tests {
|
||||
rows: vec![],
|
||||
cells: vec![],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
recover_header_row(&mut table, &all_items, 9.0);
|
||||
|
||||
+49
-11
@@ -11,6 +11,7 @@ mod format;
|
||||
mod grid;
|
||||
|
||||
pub use detect_heuristic::detect_tables;
|
||||
pub(crate) use detect_heuristic::is_table_of_contents;
|
||||
pub use detect_lines::detect_tables_from_lines;
|
||||
pub(crate) use detect_rects::cluster_rects;
|
||||
pub use detect_rects::{detect_tables_from_rects, RectHintRegion};
|
||||
@@ -166,12 +167,12 @@ pub(crate) fn try_build_rect_guided_table(
|
||||
used_indices.sort_unstable();
|
||||
used_indices.dedup();
|
||||
|
||||
Some(Table {
|
||||
columns: col_boundaries,
|
||||
rows: row_boundaries,
|
||||
Some(Table::new(
|
||||
col_boundaries,
|
||||
row_boundaries,
|
||||
cells,
|
||||
item_indices: used_indices,
|
||||
})
|
||||
used_indices,
|
||||
))
|
||||
}
|
||||
|
||||
/// Split a TextItem whose text contains multiple whitespace-separated tokens
|
||||
@@ -541,12 +542,22 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt
|
||||
multi_col_rows
|
||||
);
|
||||
|
||||
Some(Table {
|
||||
columns: col_xs,
|
||||
rows: row_ys,
|
||||
cells,
|
||||
item_indices,
|
||||
})
|
||||
Some(Table::new(col_xs, row_ys, cells, item_indices))
|
||||
}
|
||||
|
||||
/// What kind of structure a detected `Table` represents. Classification is
|
||||
/// computed once at construction so consumers don't have to re-analyze the
|
||||
/// cells (and stay consistent across detection backends).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum TableKind {
|
||||
/// A real data table — renders as markdown table syntax.
|
||||
#[default]
|
||||
Data,
|
||||
/// A table of contents — renders as a flat list with tab-aligned page
|
||||
/// numbers via `format_toc_as_list`. Detected through the table pipeline
|
||||
/// because TOCs share row/column structure with tables, but they are not
|
||||
/// data tables and shouldn't appear in `pages_with_tables` etc.
|
||||
Toc,
|
||||
}
|
||||
|
||||
/// A detected table.
|
||||
@@ -560,6 +571,31 @@ pub struct Table {
|
||||
pub cells: Vec<Vec<String>>,
|
||||
/// Items that belong to this table
|
||||
pub item_indices: Vec<usize>,
|
||||
/// Data table vs TOC. Set by `Table::new` from `cells`.
|
||||
pub kind: TableKind,
|
||||
}
|
||||
|
||||
impl Table {
|
||||
/// Build a table and classify it (data vs TOC) from its cells.
|
||||
pub fn new(
|
||||
columns: Vec<f32>,
|
||||
rows: Vec<f32>,
|
||||
cells: Vec<Vec<String>>,
|
||||
item_indices: Vec<usize>,
|
||||
) -> Self {
|
||||
let kind = if is_table_of_contents(&cells) {
|
||||
TableKind::Toc
|
||||
} else {
|
||||
TableKind::Data
|
||||
};
|
||||
Self {
|
||||
columns,
|
||||
rows,
|
||||
cells,
|
||||
item_indices,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -642,6 +678,7 @@ mod tests {
|
||||
vec!["Cell 1".into(), "Cell 2".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let md = table_to_markdown(&table);
|
||||
@@ -1002,6 +1039,7 @@ mod tests {
|
||||
vec!["3".into(), "5/2".into(), "Item C".into(), "300".into()],
|
||||
],
|
||||
item_indices: vec![],
|
||||
kind: TableKind::Data,
|
||||
};
|
||||
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
Reference in New Issue
Block a user