Compare commits

..
Author SHA1 Message Date
Abimael MartellandClaude Fable 5 1470dc162a fix(markdown): one-word bold headings, block ToC entries from headings
Two heading-classification fixes:

- Accept single-word headings ("IMPLEMENTATION", "CONTENTS") when the
  line is all-bold and isolated; the word_count >= 2 gate rejected them
  unconditionally.
- Add is_toc_entry_line: a line ending in a dot-leader group plus page
  number ("Measurement Lab worksheet ... 3") is a table-of-contents
  entry, never a heading. has_dot_leaders misses single-group leaders,
  so entire ToC pages were being promoted to ## headings.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 18:14:32 -07:00
2 changed files with 0 additions and 135 deletions
-121
View File
@@ -1477,10 +1477,6 @@ fn detect_row_stripe_table(
debug!(" row-stripe rejected: sparse outline/prose continuation shape");
return None;
}
if has_dominant_prose_cell(&cells) {
debug!(" row-stripe rejected: dominant prose cell (chart/figure region over body text)");
return None;
}
let column_centers: Vec<f32> = (0..num_cols)
.map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0)
@@ -1499,34 +1495,6 @@ fn detect_row_stripe_table(
Some(Table::new(column_centers, row_centers, cells, item_indices))
}
/// Detect a grid that swallowed body text instead of tabular data.
///
/// Charts (bar graphs, axis gridlines) emit fields of drawing rects that can
/// pass the row-stripe shape test; the resulting "table" then captures the
/// page's prose. The signature: one cell holds an entire paragraph — ≥60 words
/// AND at least a third of all words in the table.
///
/// There is deliberately no row-count exemption. A small table whose single
/// long cell dominates its word count is indistinguishable by content from a
/// phantom grid over body text, and across the regression corpora every such
/// grid observed has been swallowed prose, never a real note table. The costs
/// are also asymmetric: rejecting a real table degrades it to readable prose,
/// while accepting a phantom scrambles the page into Y-interleaved cells.
/// Larger legitimate tables are safe because the one-third-of-total threshold
/// scales with table size.
fn has_dominant_prose_cell(cells: &[Vec<String>]) -> bool {
let mut total_words = 0usize;
let mut max_cell_words = 0usize;
for row in cells {
for cell in row {
let words = cell.split_whitespace().count();
total_words += words;
max_cell_words = max_cell_words.max(words);
}
}
max_cell_words >= 60 && max_cell_words * 3 >= total_words
}
fn row_stripe_is_sparse_prose_outline(cells: &[Vec<String>]) -> bool {
let Some(num_cols) = cells.first().map(|row| row.len()) else {
return false;
@@ -2326,12 +2294,6 @@ fn detect_merged_cluster_table(
);
return None;
}
if has_dominant_prose_cell(&cells) {
debug!(
" merged-cluster rejected: dominant prose cell (chart/figure region over body text)"
);
return None;
}
// No empty columns
for col in 0..num_cols {
@@ -2436,89 +2398,6 @@ mod tests {
}
}
// --- has_dominant_prose_cell ---
fn cells_of(rows: &[&[&str]]) -> Vec<Vec<String>> {
rows.iter()
.map(|r| r.iter().map(|c| c.to_string()).collect())
.collect()
}
#[test]
fn dominant_prose_cell_rejects_swallowed_paragraph() {
// Two cells hold paragraphs (the shape every observed phantom grid
// has: swallowed body text spans multiple cells), rest are chart labels
let para = ["word"; 70].join(" ");
let para2 = ["word"; 35].join(" ");
let cells = cells_of(&[
&[para.as_str(), "81", "76"],
&[para2.as_str(), "56", "9"],
&["2019", "2020", ""],
]);
assert!(has_dominant_prose_cell(&cells));
}
#[test]
fn dominant_prose_cell_rejects_small_table_dominated_by_one_cell() {
// Boundary case, documented as INTENDED: a small grid whose single
// long cell dominates the word count is rejected even at 4+ rows.
// By content alone this shape is indistinguishable from a phantom
// grid over body text, and every observed instance in the regression
// corpora was swallowed prose (chart/figure regions), not a real
// note table. Rejection degrades gracefully — the text is still
// extracted as prose — while accepting a phantom scrambles reading
// order.
let note = ["word"; 70].join(" ");
let cells = cells_of(&[
&["Purpose", note.as_str()],
&["Owner", "Facilities team"],
&["Date", "2024-06-01"],
&["Status", "Active"],
]);
assert!(has_dominant_prose_cell(&cells));
}
#[test]
fn dominant_prose_cell_allows_description_column() {
// Long-ish description cells, but text is spread across the table
let desc = ["word"; 25].join(" ");
let cells = cells_of(&[
&["Item A", desc.as_str(), "100"],
&["Item B", desc.as_str(), "200"],
&["Item C", desc.as_str(), "300"],
&["Item D", desc.as_str(), "400"],
]);
assert!(!has_dominant_prose_cell(&cells));
}
#[test]
fn dominant_prose_cell_allows_short_tables() {
let cells = cells_of(&[&["Name", "Value"], &["Total", "42"]]);
assert!(!has_dominant_prose_cell(&cells));
}
#[test]
fn dominant_prose_cell_allows_data_table_with_long_note() {
// A real 4+ row table with one verbose remark cell: the note is ≥60
// words but the table's other content carries more than 2× its word
// count, so concentration stays below the 1/3 threshold. The
// denominator scales with table size — this is what keeps large
// legitimate tables safe where a bare length cap would not.
let note = ["word"; 60].join(" ");
let row_text = ["data"; 12].join(" ");
let mut rows: Vec<Vec<String>> = (0..11)
.map(|i| {
vec![
format!("Item {i}"),
row_text.clone(),
format!("{}", i * 100),
]
})
.collect();
rows.push(vec!["Note".into(), note, String::new()]);
assert!(!has_dominant_prose_cell(&rows));
}
// --- rects_overlap ---
#[test]
-14
View File
@@ -93,9 +93,6 @@ pub fn is_bold_font(font_name: &str) -> bool {
|| lower.contains("extrabold")
|| lower.contains("ultrabold")
|| lower.contains("medium") && !lower.contains("mediumitalic") // Some fonts use Medium for semi-bold
// URW Type 1 fonts abbreviate Medium as "Medi" (e.g. NimbusRomNo9L-Medi,
// the Times-Bold substitute in LaTeX documents; -MediItal is bold italic).
|| lower.contains("-medi") && !lower.contains("mediumital")
}
/// Detect if a font name indicates italic/oblique style
@@ -765,17 +762,6 @@ mod tests {
use super::*;
use crate::types::ItemType;
#[test]
fn bold_font_urw_medi_abbreviation() {
// URW Type 1 fonts (LaTeX default Times) abbreviate Medium as "Medi"
assert!(is_bold_font("NROFIU+NimbusRomNo9L-Medi"));
assert!(is_bold_font("NimbusRomNo9L-MediItal"));
assert!(!is_bold_font("DSSZWN+NimbusRomNo9L-Regu"));
assert!(!is_bold_font("NimbusRomNo9L-ReguItal"));
// Medium-Italic exclusion still holds
assert!(!is_bold_font("Foo-MediumItalic"));
}
#[test]
fn strip_soft_hyphen() {
assert_eq!(expand_ligatures("con\u{00AD}tent"), "content");