From 4c3330f93acf99af16291d68b27718a26b820733 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:14:37 -0700 Subject: [PATCH] fix(headings): bold-size fallback tiers when nothing clears the ratio gate (#162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(headings): bold-size fallback tiers when nothing clears the ratio gate Books often set section headings barely above body size (11pt bold over 10pt text). Nothing cleared the 1.2x heading-tier ratio gate, so tiers stayed empty and every bold heading defaulted to H2 — H1 was unreachable for the whole document. When no size clears the gate, build tiers from bold line sizes >=1.05x body, and let tier matches through detect_header_level down to that ratio. Documents with real (>=1.2x) tiers are untouched. Bench-neutral by construction (the MHS metric scores relative hierarchy, not absolute levels); pdf-evals semantic composite +0.004 on the 11 affected docs with all percentiles up. Co-Authored-By: Claude Fable 5 * fix(headings): require boldness for sub-gate tier matches Review follow-up: fallback tiers come from bold lines, so honoring them for non-bold text at the same size would promote captions. detect_header_level now takes is_bold and only matches tiers below the 1.2x gate for bold lines; >=1.2x matches stay bold-agnostic. Also restores the >=1.2x tier-match loop the refactor dropped. Co-Authored-By: Claude Fable 5 * fix(headings): judge line boldness by character mass Review follow-up: a heading with an unbold section-number prefix ('4. ' + bold title) failed the first-item boldness test. Judge the line by bold character mass instead, at all three call sites. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- src/markdown/analysis.rs | 97 ++++++++++++++++++++++++++++++++++++++ src/markdown/convert.rs | 68 ++++++++++++++------------ src/markdown/mod.rs | 28 +++++------ src/markdown/preprocess.rs | 7 ++- 4 files changed, 156 insertions(+), 44 deletions(-) diff --git a/src/markdown/analysis.rs b/src/markdown/analysis.rs index 5ce2823..1ffdfca 100644 --- a/src/markdown/analysis.rs +++ b/src/markdown/analysis.rs @@ -391,11 +391,40 @@ pub(crate) fn compute_heading_tiers(lines: &[TextLine], base_size: f32) -> Vec = lines + .iter() + .filter_map(|line| line.items.first()) + .filter(|it| it.is_bold && it.font_size / base_size >= 1.05) + .map(|it| it.font_size) + .collect(); + bold_sizes.sort_by(|a, b| b.total_cmp(a)); + for size in bold_sizes { + if !tiers.iter().any(|&t| (t - size).abs() < 0.5) { + tiers.push(size); + } + } + } + // Cap at 4 tiers tiers.truncate(4); tiers } +/// Boldness of a line judged by character mass, so a heading with an +/// unbold section-number prefix ("4. " + bold title) still counts as bold. +pub(crate) fn line_is_mostly_bold(line: &TextLine) -> bool { + let (bold, total) = line.items.iter().fold((0usize, 0usize), |(b, t), it| { + let n = it.text.trim().chars().count(); + (b + if it.is_bold { n } else { 0 }, t + n) + }); + total > 0 && bold * 2 >= total +} + /// Detect header level from font size using document-specific heading tiers. /// When tiers are available, maps tier 0→H1, tier 1→H2, etc. /// Falls back to ratio-based thresholds when no tiers exist. @@ -403,9 +432,21 @@ pub(crate) fn detect_header_level( font_size: f32, base_size: f32, heading_tiers: &[f32], + is_bold: bool, ) -> Option { let ratio = font_size / base_size; + // Tier matches are trusted below the 1.2x gate (down to 1.05x) only for + // bold lines: sub-gate tiers come from the bold fallback, and honoring + // them for non-bold text at the same size would promote captions. + if (1.05..1.2).contains(&ratio) && is_bold && !heading_tiers.is_empty() { + for (i, &tier_size) in heading_tiers.iter().enumerate() { + if (font_size - tier_size).abs() < 0.5 { + return Some(i + 1); // tier 0 → H1, tier 1 → H2, etc. + } + } + } + if ratio < 1.2 { return None; // Regular text } @@ -442,6 +483,62 @@ pub(crate) fn detect_header_level( mod tests { use super::*; + fn line_of(text: &str, font_size: f32, bold: bool, y: f32) -> crate::types::TextLine { + let item = crate::types::TextItem { + text: text.into(), + x: 72.0, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: "Test".into(), + font_size, + page: 1, + is_bold: bold, + is_italic: false, + is_underline: false, + is_strikeout: false, + item_type: crate::types::ItemType::Text, + mcid: None, + }; + crate::types::TextLine { + items: vec![item], + y, + page: 1, + adaptive_threshold: 0.10, + } + } + + #[test] + fn bold_fallback_tiers_when_nothing_clears_ratio_gate() { + // 10pt body, 11pt bold section headings (book-style): no size clears + // 1.2x, so bold sizes modestly above body form the tiers. + let lines = vec![ + line_of("4. Entropy", 11.0, true, 700.0), + line_of("body text about entropy", 10.0, false, 680.0), + line_of("5. The dynamics", 11.0, true, 500.0), + ]; + let tiers = compute_heading_tiers(&lines, 10.0); + assert_eq!(tiers, vec![11.0]); + assert_eq!(detect_header_level(11.0, 10.0, &tiers, true), Some(1)); + // Non-bold text at the fallback size must not become a heading. + assert_eq!(detect_header_level(11.0, 10.0, &tiers, false), None); + // Non-tier body text stays regular. + assert_eq!(detect_header_level(10.0, 10.0, &tiers, true), None); + } + + #[test] + fn bold_fallback_skipped_when_real_tiers_exist() { + let lines = vec![ + line_of("Chapter One", 18.0, false, 700.0), + line_of("bold label", 11.0, true, 600.0), + line_of("body", 10.0, false, 580.0), + ]; + let tiers = compute_heading_tiers(&lines, 10.0); + assert_eq!(tiers, vec![18.0]); + // The 11pt bold label does not match any tier and stays non-heading. + assert_eq!(detect_header_level(11.0, 10.0, &tiers, true), None); + } + #[test] fn toc_entry_with_single_dot_group() { assert!(is_toc_entry_line("Measurement Lab worksheet ... 3")); diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index c968353..60fdd5c 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -721,7 +721,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( && toc_suppress_page != Some(line.page) { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| { + detect_header_level( + line_font_size, + base_size, + &heading_tiers, + crate::markdown::analysis::line_is_mostly_bold(line), + ) + .or_else(|| { // Rarity-based heading detection (inspired by opendataloader). // Heading probability scoring with lookahead context. // Score = rarity * 0.5 + bold * 0.3 + standalone * 0.2 @@ -1067,34 +1073,38 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> && !(options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))) { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| { - if line_font_size < base_size * 0.95 { - return None; - } - let word_count = plain_trimmed.split_whitespace().count(); - if !(1..=15).contains(&word_count) { - return None; - } - if wrapped_bold_paragraph_lines.contains(&line_idx) { - return None; - } - let rarity = font_size_rarity(line_font_size, &font_stats); - let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold); - let standalone = !in_paragraph; - let isolated = isolated_lines.contains(&line_idx); - let score = rarity * 0.5 - + if all_bold { 0.3 } else { 0.0 } - + if standalone { 0.2 } else { 0.0 } - + if isolated { 0.3 } else { 0.0 }; - let enough_words = - word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4); - if score >= 0.5 && standalone && enough_words { - return Some(bold_heading_level(&heading_tiers)); - } - None - }) - { + if let Some(header_level) = detect_header_level( + line_font_size, + base_size, + &heading_tiers, + crate::markdown::analysis::line_is_mostly_bold(line), + ) + .or_else(|| { + if line_font_size < base_size * 0.95 { + return None; + } + let word_count = plain_trimmed.split_whitespace().count(); + if !(1..=15).contains(&word_count) { + return None; + } + if wrapped_bold_paragraph_lines.contains(&line_idx) { + return None; + } + let rarity = font_size_rarity(line_font_size, &font_stats); + let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold); + let standalone = !in_paragraph; + let isolated = isolated_lines.contains(&line_idx); + let score = rarity * 0.5 + + if all_bold { 0.3 } else { 0.0 } + + if standalone { 0.2 } else { 0.0 } + + if isolated { 0.3 } else { 0.0 }; + let enough_words = + word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4); + if score >= 0.5 && standalone && enough_words { + return Some(bold_heading_level(&heading_tiers)); + } + None + }) { if in_paragraph { output.push_str("\n\n"); in_paragraph = false; diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 5847fc2..63b4747 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -1360,29 +1360,29 @@ mod tests { fn test_detect_header_level() { // With three tiers: 24→H1, 18→H2, 15→H3, 12→None let tiers = vec![24.0, 18.0, 15.0]; - assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1)); + assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2)); + assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3)); + assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None); // Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2) let tiers = vec![15.0]; - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(1)); + assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None); + assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None); // No tiers (empty): falls back to ratio thresholds let tiers: Vec = vec![]; - assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1)); - assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2)); - assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3)); - assert_eq!(detect_header_level(14.5, 12.0, &tiers), Some(4)); - assert_eq!(detect_header_level(14.0, 12.0, &tiers), None); - assert_eq!(detect_header_level(12.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1)); + assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2)); + assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3)); + assert_eq!(detect_header_level(14.5, 12.0, &tiers, false), Some(4)); + assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None); + assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None); // Body text excluded when tiers exist: 13pt (ratio 1.08) → None let tiers = vec![20.0]; - assert_eq!(detect_header_level(13.0, 12.0, &tiers), None); + assert_eq!(detect_header_level(13.0, 12.0, &tiers, false), None); } #[test] diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index 5031ff8..16866c9 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -42,7 +42,12 @@ fn effective_heading_level( // Fall back to font-size heuristic let font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - detect_header_level(font, base_size, heading_tiers) + detect_header_level( + font, + base_size, + heading_tiers, + crate::markdown::analysis::line_is_mostly_bold(line), + ) } /// Merge consecutive heading lines at the same level into a single line.