From 28823bd849a2481f1e73d5ddfa01f9803af044bb Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Sat, 4 Apr 2026 01:07:08 -0700 Subject: [PATCH] feat: improve heuristic table detection for small body-font tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Lower minimum item count for body-font table candidates from 9 to 6, allowing small 2-3 row tables to be detected. - Allow 2-column body-font tables with short cells (avg ≤25 chars) to bypass the "table-like content" validation. This catches text-only definition/category tables (e.g., species lists) without false-positiving on 2-column paragraph text (which has longer cells). Benchmark improvement: TEDS 0.498→0.519, overall 0.750→0.754. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/tables/detect_heuristic.rs | 24 +++++++++++++++++++++--- tests/snapshots/thermo-freon12.md | 21 +++++++++------------ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 1dc076e..39a3cd7 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -219,7 +219,7 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo body_font_low, body_font_high, ); - if body_candidates.len() >= 9 { + if body_candidates.len() >= 6 { let regions = find_table_regions_strict(&body_candidates); log::debug!("body-font: {} strict regions found", regions.len()); @@ -241,7 +241,7 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo body_candidates.len() ); - if region_items.len() < 9 { + if region_items.len() < 6 { continue; } @@ -801,7 +801,25 @@ fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bo // Bypass content check for wide tables (3+ columns) — text-only tables // (category lists, program descriptions) are legitimate if they passed // all structural validations (alignment, consistency, not key-value). - pct_data > min_pct || num_cols >= 3 + // Also bypass for 2-column body-font tables with short cells (avg ≤40 chars), + // which are likely definition/category lists, not paragraph text. + if pct_data > min_pct || num_cols >= 3 { + return true; + } + if num_cols == 2 && matches!(mode, TableDetectionMode::BodyFont) { + let non_empty: Vec = cells + .iter() + .skip(1) + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .map(|c| c.trim().len()) + .collect(); + if !non_empty.is_empty() { + let avg_len = non_empty.iter().sum::() / non_empty.len(); + return avg_len <= 25; + } + } + false } /// Check if a cell value looks like table data diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 25ea607..a575f0b 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -26,19 +26,16 @@ S = Entropy (kJ/kg.K) **Physical Properties** -Chemical Formula CCl2F2 +|Chemical Formula|CCl2F2| +|---|---| +|Molecular mass|120.91| +|Boiling Point At one atmosphere|-29.75°C| +|Critical Temperature|111.97°C| +|Critical Pressure|4136 kPa| +|Critical Density|565.0 kg/m| +|Critical Volume|0.0018 m| -Molecular mass 120.91 - -Boiling Point-29.75°C At one atmosphere - -Critical Temperature 111.97°C - -Critical Pressure 4136 kPa - -3 Critical Density 565.0 kg/m - -Critical Volume 0.0018 m /kg +/kg l