fix(headings): bold-size fallback tiers when nothing clears the ratio gate (#162)

* fix(headings): bold-size fallback tiers when nothing clears the ratio gate

Books often set section headings barely above body size (11pt bold
over 10pt text). Nothing cleared the 1.2x heading-tier ratio gate, so
tiers stayed empty and every bold heading defaulted to H2 — H1 was
unreachable for the whole document.

When no size clears the gate, build tiers from bold line sizes >=1.05x
body, and let tier matches through detect_header_level down to that
ratio. Documents with real (>=1.2x) tiers are untouched.

Bench-neutral by construction (the MHS metric scores relative
hierarchy, not absolute levels); pdf-evals semantic composite +0.004
on the 11 affected docs with all percentiles up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(headings): require boldness for sub-gate tier matches

Review follow-up: fallback tiers come from bold lines, so honoring
them for non-bold text at the same size would promote captions.
detect_header_level now takes is_bold and only matches tiers below
the 1.2x gate for bold lines; >=1.2x matches stay bold-agnostic.
Also restores the >=1.2x tier-match loop the refactor dropped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(headings): judge line boldness by character mass

Review follow-up: a heading with an unbold section-number prefix
('4. ' + bold title) failed the first-item boldness test. Judge the
line by bold character mass instead, at all three call sites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/markdown/analysis.rs

@@ -391,11 +391,40 @@ pub(crate) fn compute_heading_tiers(lines: &[TextLine], base_size: f32) -> Vec<f
         }
     }
 
+    // Books often set section headings barely above body size (e.g. 11pt
+    // bold over 10pt text). When nothing clears the 1.2x ratio gate, fall
+    // back to bold lines modestly larger than body so those documents still
+    // get an H1 instead of every bold heading defaulting to H2.
+    if tiers.is_empty() {
+        let mut bold_sizes: Vec<f32> = lines
+            .iter()
+            .filter_map(|line| line.items.first())
+            .filter(|it| it.is_bold && it.font_size / base_size >= 1.05)
+            .map(|it| it.font_size)
+            .collect();
+        bold_sizes.sort_by(|a, b| b.total_cmp(a));
+        for size in bold_sizes {
+            if !tiers.iter().any(|&t| (t - size).abs() < 0.5) {
+                tiers.push(size);
+            }
+        }
+    }
+
     // Cap at 4 tiers
     tiers.truncate(4);
     tiers
 }
 
+/// Boldness of a line judged by character mass, so a heading with an
+/// unbold section-number prefix ("4. " + bold title) still counts as bold.
+pub(crate) fn line_is_mostly_bold(line: &TextLine) -> bool {
+    let (bold, total) = line.items.iter().fold((0usize, 0usize), |(b, t), it| {
+        let n = it.text.trim().chars().count();
+        (b + if it.is_bold { n } else { 0 }, t + n)
+    });
+    total > 0 && bold * 2 >= total
+}
+
 /// Detect header level from font size using document-specific heading tiers.
 /// When tiers are available, maps tier 0→H1, tier 1→H2, etc.
 /// Falls back to ratio-based thresholds when no tiers exist.
@@ -403,9 +432,21 @@ pub(crate) fn detect_header_level(
     font_size: f32,
     base_size: f32,
     heading_tiers: &[f32],
+    is_bold: bool,
 ) -> Option<usize> {
     let ratio = font_size / base_size;
 
+    // Tier matches are trusted below the 1.2x gate (down to 1.05x) only for
+    // bold lines: sub-gate tiers come from the bold fallback, and honoring
+    // them for non-bold text at the same size would promote captions.
+    if (1.05..1.2).contains(&ratio) && is_bold && !heading_tiers.is_empty() {
+        for (i, &tier_size) in heading_tiers.iter().enumerate() {
+            if (font_size - tier_size).abs() < 0.5 {
+                return Some(i + 1); // tier 0 → H1, tier 1 → H2, etc.
+            }
+        }
+    }
+
     if ratio < 1.2 {
         return None; // Regular text
     }
@@ -442,6 +483,62 @@ pub(crate) fn detect_header_level(
 mod tests {
     use super::*;
 
+    fn line_of(text: &str, font_size: f32, bold: bool, y: f32) -> crate::types::TextLine {
+        let item = crate::types::TextItem {
+            text: text.into(),
+            x: 72.0,
+            y,
+            width: text.len() as f32 * font_size * 0.5,
+            height: font_size,
+            font: "Test".into(),
+            font_size,
+            page: 1,
+            is_bold: bold,
+            is_italic: false,
+            is_underline: false,
+            is_strikeout: false,
+            item_type: crate::types::ItemType::Text,
+            mcid: None,
+        };
+        crate::types::TextLine {
+            items: vec![item],
+            y,
+            page: 1,
+            adaptive_threshold: 0.10,
+        }
+    }
+
+    #[test]
+    fn bold_fallback_tiers_when_nothing_clears_ratio_gate() {
+        // 10pt body, 11pt bold section headings (book-style): no size clears
+        // 1.2x, so bold sizes modestly above body form the tiers.
+        let lines = vec![
+            line_of("4. Entropy", 11.0, true, 700.0),
+            line_of("body text about entropy", 10.0, false, 680.0),
+            line_of("5. The dynamics", 11.0, true, 500.0),
+        ];
+        let tiers = compute_heading_tiers(&lines, 10.0);
+        assert_eq!(tiers, vec![11.0]);
+        assert_eq!(detect_header_level(11.0, 10.0, &tiers, true), Some(1));
+        // Non-bold text at the fallback size must not become a heading.
+        assert_eq!(detect_header_level(11.0, 10.0, &tiers, false), None);
+        // Non-tier body text stays regular.
+        assert_eq!(detect_header_level(10.0, 10.0, &tiers, true), None);
+    }
+
+    #[test]
+    fn bold_fallback_skipped_when_real_tiers_exist() {
+        let lines = vec![
+            line_of("Chapter One", 18.0, false, 700.0),
+            line_of("bold label", 11.0, true, 600.0),
+            line_of("body", 10.0, false, 580.0),
+        ];
+        let tiers = compute_heading_tiers(&lines, 10.0);
+        assert_eq!(tiers, vec![18.0]);
+        // The 11pt bold label does not match any tier and stays non-heading.
+        assert_eq!(detect_header_level(11.0, 10.0, &tiers, true), None);
+    }
+
     #[test]
     fn toc_entry_with_single_dot_group() {
         assert!(is_toc_entry_line("Measurement Lab worksheet ... 3"));

src/markdown/convert.rs

@@ -721,7 +721,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
             && toc_suppress_page != Some(line.page)
         {
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
-            detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
+            detect_header_level(
+                line_font_size,
+                base_size,
+                &heading_tiers,
+                crate::markdown::analysis::line_is_mostly_bold(line),
+            )
+            .or_else(|| {
                 // Rarity-based heading detection (inspired by opendataloader).
                 // Heading probability scoring with lookahead context.
                 // Score = rarity * 0.5 + bold * 0.3 + standalone * 0.2
@@ -1067,34 +1073,38 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
             && !(options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font)))
         {
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
-            if let Some(header_level) =
-                detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
-                    if line_font_size < base_size * 0.95 {
-                        return None;
-                    }
-                    let word_count = plain_trimmed.split_whitespace().count();
-                    if !(1..=15).contains(&word_count) {
-                        return None;
-                    }
-                    if wrapped_bold_paragraph_lines.contains(&line_idx) {
-                        return None;
-                    }
-                    let rarity = font_size_rarity(line_font_size, &font_stats);
-                    let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
-                    let standalone = !in_paragraph;
-                    let isolated = isolated_lines.contains(&line_idx);
-                    let score = rarity * 0.5
-                        + if all_bold { 0.3 } else { 0.0 }
-                        + if standalone { 0.2 } else { 0.0 }
-                        + if isolated { 0.3 } else { 0.0 };
-                    let enough_words =
-                        word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
-                    if score >= 0.5 && standalone && enough_words {
-                        return Some(bold_heading_level(&heading_tiers));
-                    }
-                    None
-                })
-            {
+            if let Some(header_level) = detect_header_level(
+                line_font_size,
+                base_size,
+                &heading_tiers,
+                crate::markdown::analysis::line_is_mostly_bold(line),
+            )
+            .or_else(|| {
+                if line_font_size < base_size * 0.95 {
+                    return None;
+                }
+                let word_count = plain_trimmed.split_whitespace().count();
+                if !(1..=15).contains(&word_count) {
+                    return None;
+                }
+                if wrapped_bold_paragraph_lines.contains(&line_idx) {
+                    return None;
+                }
+                let rarity = font_size_rarity(line_font_size, &font_stats);
+                let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
+                let standalone = !in_paragraph;
+                let isolated = isolated_lines.contains(&line_idx);
+                let score = rarity * 0.5
+                    + if all_bold { 0.3 } else { 0.0 }
+                    + if standalone { 0.2 } else { 0.0 }
+                    + if isolated { 0.3 } else { 0.0 };
+                let enough_words =
+                    word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
+                if score >= 0.5 && standalone && enough_words {
+                    return Some(bold_heading_level(&heading_tiers));
+                }
+                None
+            }) {
                 if in_paragraph {
                     output.push_str("\n\n");
                     in_paragraph = false;

src/markdown/mod.rs

@@ -1360,29 +1360,29 @@ mod tests {
     fn test_detect_header_level() {
         // With three tiers: 24→H1, 18→H2, 15→H3, 12→None
         let tiers = vec![24.0, 18.0, 15.0];
-        assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1));
-        assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2));
-        assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3));
-        assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
+        assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1));
+        assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2));
+        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3));
+        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);
 
         // Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2)
         let tiers = vec![15.0];
-        assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(1));
-        assert_eq!(detect_header_level(14.0, 12.0, &tiers), None);
-        assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
+        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(1));
+        assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None);
+        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);
 
         // No tiers (empty): falls back to ratio thresholds
         let tiers: Vec<f32> = vec![];
-        assert_eq!(detect_header_level(24.0, 12.0, &tiers), Some(1));
-        assert_eq!(detect_header_level(18.0, 12.0, &tiers), Some(2));
-        assert_eq!(detect_header_level(15.0, 12.0, &tiers), Some(3));
-        assert_eq!(detect_header_level(14.5, 12.0, &tiers), Some(4));
-        assert_eq!(detect_header_level(14.0, 12.0, &tiers), None);
-        assert_eq!(detect_header_level(12.0, 12.0, &tiers), None);
+        assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1));
+        assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2));
+        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3));
+        assert_eq!(detect_header_level(14.5, 12.0, &tiers, false), Some(4));
+        assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None);
+        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);
 
         // Body text excluded when tiers exist: 13pt (ratio 1.08) → None
         let tiers = vec![20.0];
-        assert_eq!(detect_header_level(13.0, 12.0, &tiers), None);
+        assert_eq!(detect_header_level(13.0, 12.0, &tiers, false), None);
     }
 
     #[test]

src/markdown/preprocess.rs

@@ -42,7 +42,12 @@ fn effective_heading_level(
 
     // Fall back to font-size heuristic
     let font = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
-    detect_header_level(font, base_size, heading_tiers)
+    detect_header_level(
+        font,
+        base_size,
+        heading_tiers,
+        crate::markdown::analysis::line_is_mostly_bold(line),
+    )
 }
 
 /// Merge consecutive heading lines at the same level into a single line.