review: guard tier path, protect prev headings in merge, narrow (N) rule

All three review findings applied, calibrated against the corpora:

- is_heading_fragment now gates the font-size-tier path too, not just
  the rarity heuristic.
- The bold wrap-merge requires the previous line to be tier-less as
  suggested; corpus diff confirmed the old behavior was absorbing a
  wrapped list-item fragment into a real heading.
- The bare "(N)" suffix rule suppressed real headings ("Nicaea (325)",
  appendix numbering). It now requires math evidence: an operator
  (=, <=, <<, ...) in the line or ,/: immediately before the number.
  Page-of-total running headers ("PM 2 (10)") get an explicit rule
  since the old blanket suffix check had been catching them only by
  accident.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/markdown/analysis.rs

@@ -153,6 +153,85 @@ pub(crate) fn is_toc_entry_line(text: &str) -> bool {
     dots >= 3
 }
 
+/// A heading that announces a table of contents ("Contents", "Table of
+/// Contents"). Lines after it on the same page are ToC entries — section
+/// titles that look exactly like headings but must not be promoted.
+pub(crate) fn is_toc_marker_heading(text: &str) -> bool {
+    let t = text.trim().trim_end_matches(':').trim().to_lowercase();
+    matches!(t.as_str(), "contents" | "table of contents")
+}
+
+/// Lines that resemble headings structurally but are display-math fragments:
+/// equations ending in an equation number ("S = kB ln W, (2)") or equation
+/// lead-ins ("Rearranging Equation (8) gives:"). Both carry an "(N)" equation
+/// reference — but a trailing "(N)" alone is not enough: real headings end
+/// with parenthesized numbers too ("Nicaea (325)", appendix numbering), so
+/// the suffix form additionally requires math evidence — an "=" in the line
+/// or a comma immediately before the number, both present in every display
+/// equation and absent from name-plus-number headings. A bare trailing colon
+/// is NOT a fragment signal either: real headings frequently end with colons
+/// ("Procedure:", "Steps for Using the Microscope:").
+pub(crate) fn is_heading_fragment(text: &str) -> bool {
+    let t = text.trim_end();
+
+    fn is_equation_number(s: &str) -> bool {
+        s.strip_prefix('(')
+            .and_then(|r| r.strip_suffix(')'))
+            .is_some_and(|inner| {
+                !inner.is_empty() && inner.len() <= 3 && inner.chars().all(|c| c.is_ascii_digit())
+            })
+    }
+
+    // Equation-number suffix with math evidence: "S = kB ln W, (2)"
+    let mut rev = t.rsplit(' ');
+    let last = rev.next().unwrap_or("");
+    if is_equation_number(last) {
+        // Page-of-total running headers: "LIVSMEDELSVERKET PM 2 (10)"
+        if let Some(prev_word) = t.rsplit(' ').nth(1) {
+            if let (Ok(page), Some(total)) = (
+                prev_word.parse::<u32>(),
+                last.trim_start_matches('(')
+                    .trim_end_matches(')')
+                    .parse::<u32>()
+                    .ok(),
+            ) {
+                if page <= total {
+                    return true;
+                }
+            }
+        }
+        let punct_before = rev
+            .next()
+            .is_some_and(|w| w.ends_with(',') || w.ends_with(':'));
+        let has_math_op = t.chars().any(|c| {
+            matches!(
+                c,
+                '=' | '<'
+                    | '>'
+                    | '≤'
+                    | '≥'
+                    | '≪'
+                    | '≫'
+                    | '≈'
+                    | '≠'
+                    | '±'
+                    | '∑'
+                    | '∫'
+                    | '√'
+                    | '∝'
+            )
+        });
+        if punct_before || has_math_op {
+            return true;
+        }
+    }
+    // Lead-in: ends with a colon AND references an equation number inline
+    if t.ends_with(':') && t.split_whitespace().any(is_equation_number) {
+        return true;
+    }
+    false
+}
+
 /// Compute the Y-gap threshold for paragraph break detection.
 ///
 /// Instead of using a fixed multiple of base_size (which fails for double-spaced
@@ -367,4 +446,41 @@ mod tests {
         // Long numbers are data, not page refs
         assert!(!is_toc_entry_line("ISBN ... 97814"));
     }
+
+    #[test]
+    fn toc_marker_headings() {
+        assert!(is_toc_marker_heading("Contents"));
+        assert!(is_toc_marker_heading("CONTENTS"));
+        assert!(is_toc_marker_heading("Table of Contents"));
+        assert!(is_toc_marker_heading("Table of contents:"));
+        assert!(!is_toc_marker_heading("Contents of the Shipment"));
+        assert!(!is_toc_marker_heading("Introduction"));
+    }
+
+    #[test]
+    fn heading_fragments() {
+        // Equation lead-ins: colon ending + inline equation reference
+        assert!(is_heading_fragment("Rearranging Equation (8) gives:"));
+        // Display-equation neighbours ending in an equation number
+        assert!(is_heading_fragment("S = kB ln W, (2)"));
+        assert!(is_heading_fragment("E = mc2 (12)"));
+        assert!(is_heading_fragment("x + y = z, (3)"));
+        // Page-of-total running headers
+        assert!(is_heading_fragment("LIVSMEDELSVERKET PM 2 (10)"));
+        // Comparison-operator evidence and colon-before-number
+        assert!(is_heading_fragment(
+            "PLL\u{fe} PHH\u{226a} PLH\u{fe} PHL: (12)"
+        ));
+        // Real headings pass — including name-plus-number and colon-ended ones
+        assert!(!is_heading_fragment("Nicaea (325)"));
+        assert!(!is_heading_fragment(
+            "\u{627}\u{644}\u{645}\u{644}\u{62d}\u{642} \u{631}\u{642}\u{645} (1)"
+        ));
+        assert!(!is_heading_fragment("4. Entropy"));
+        assert!(!is_heading_fragment("Procedure:"));
+        assert!(!is_heading_fragment("Steps for Using the Microscope:"));
+        assert!(!is_heading_fragment("Changing objectives:"));
+        assert!(!is_heading_fragment("Sales by Region (2024)"));
+        assert!(!is_heading_fragment("Results (preliminary)"));
+    }
 }

src/markdown/convert.rs

@@ -7,7 +7,8 @@ use crate::types::TextLine;
 
 use super::analysis::{
     bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
-    detect_header_level, font_size_rarity, has_dot_leaders, is_toc_entry_line,
+    detect_header_level, font_size_rarity, has_dot_leaders, is_heading_fragment, is_toc_entry_line,
+    is_toc_marker_heading,
 };
 use super::classify::{
     format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
@@ -486,6 +487,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
     let mut in_code_block = false;
     let mut prev_had_dot_leaders = false;
     let mut paragraph_in_wrapped_bold_run = false;
+    let mut toc_suppress_page: Option<u32> = None;
     let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
     let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
 
@@ -704,6 +706,8 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
             && plain_trimmed.split_whitespace().count() <= 15
             && !starts_with_bullet_marker(plain_trimmed)
             && !is_toc_entry_line(plain_trimmed)
+            && !is_heading_fragment(plain_trimmed)
+            && toc_suppress_page != Some(line.page)
         {
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
             detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
@@ -768,6 +772,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
                 plain_text.clone()
             };
             output.push_str(&format!("{} {}\n\n", prefix, heading_text.trim()));
+            if is_toc_marker_heading(plain_trimmed) {
+                toc_suppress_page = Some(line.page);
+            }
             in_list = false;
             continue;
         }
@@ -964,6 +971,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
     let mut last_list_x: Option<f32> = None;
     let mut prev_had_dot_leaders = false;
     let mut paragraph_in_wrapped_bold_run = false;
+    let mut toc_suppress_page: Option<u32> = None;
 
     for (line_idx, line) in lines.iter().enumerate() {
         // Page break
@@ -1043,6 +1051,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
             && plain_trimmed.len() > 3
             && plain_trimmed.split_whitespace().count() <= 15
             && !is_toc_entry_line(plain_trimmed)
+            && !is_heading_fragment(plain_trimmed)
+            && toc_suppress_page != Some(line.page)
         {
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
             if let Some(header_level) =
@@ -1086,6 +1096,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
                     plain_text.clone()
                 };
                 output.push_str(&format!("{} {}\n\n", prefix, heading_text.trim()));
+                if is_toc_marker_heading(plain_trimmed) {
+                    toc_suppress_page = Some(line.page);
+                }
                 in_list = false;
                 continue;
             }

src/markdown/preprocess.rs

@@ -3,7 +3,7 @@
 use std::collections::{HashMap, HashSet};
 
 use crate::structure_tree::StructRole;
-use crate::types::TextLine;
+use crate::types::{TextItem, TextLine};
 
 use super::analysis::detect_header_level;
 
@@ -87,6 +87,41 @@ pub(crate) fn merge_heading_lines(
             false
         };
 
+        // Bold headings at body font size never reach a tier, so wrapped ones
+        // split into two output headings ("…of wood pellets and cost" /
+        // "structure in Japan"). Merge a fully-bold line into the previous
+        // fully-bold line when it reads as a wrap continuation: starts
+        // lowercase, tiny Y gap, and the previous line has no terminal
+        // punctuation. Kept deliberately narrow — bold list labels and bold
+        // sentences start with markers or capitals and are unaffected.
+        let should_merge = should_merge
+            || if let Some(prev) = result.last() {
+                let all_bold = |l: &TextLine| {
+                    !l.items.is_empty() && l.items.iter().all(|i: &TextItem| i.is_bold)
+                };
+                let prev_text = prev.text();
+                let prev_trim = prev_text.trim_end();
+                let curr_text = line.text();
+                let curr_trim = curr_text.trim();
+                let y_gap = prev.y - line.y;
+                // Both lines must be tier-less: a tiered/tagged bold heading
+                // followed by bold body text must not absorb it.
+                line_level.is_none()
+                    && effective_heading_level(prev, base_size, heading_tiers, struct_roles)
+                        .is_none()
+                    && prev.page == line.page
+                    && all_bold(prev)
+                    && all_bold(&line)
+                    && y_gap > 0.0
+                    && y_gap < line_font * 1.6
+                    && curr_trim.chars().next().is_some_and(|c| c.is_lowercase())
+                    && !prev_trim.ends_with(['.', ':', ';', '!', '?'])
+                    && prev_trim.split_whitespace().count() + curr_trim.split_whitespace().count()
+                        <= 20
+            } else {
+                false
+            };
+
         if should_merge {
             // Append this line's items to the previous line
             let prev = result.last_mut().unwrap();
@@ -685,4 +720,68 @@ mod tests {
             .unwrap();
         assert_eq!(first_header.page, 1, "first occurrence should be on page 1");
     }
+
+    fn make_bold_line(text: &str, page: u32, y: f32) -> TextLine {
+        let mut item = make_item(text, 12.0, None);
+        item.is_bold = true;
+        TextLine {
+            items: vec![item],
+            y,
+            page,
+            adaptive_threshold: 0.10,
+        }
+    }
+
+    #[test]
+    fn merge_wrapped_bold_heading_lowercase_continuation() {
+        // Bold-at-body-size heading wrapped across two lines: the second line
+        // starts lowercase and must merge into the first.
+        let lines = vec![
+            make_bold_line(
+                "3. Perspective of supply and demand balance and cost",
+                1,
+                700.0,
+            ),
+            make_bold_line("structure in Japan", 1, 686.0),
+            make_line("Body text paragraph follows here.", 12.0, 1, 660.0, None),
+        ];
+        let result = merge_heading_lines(lines, 12.0, &[], None);
+        assert_eq!(result.len(), 2, "wrapped bold heading should merge");
+        assert!(result[0].text().contains("cost structure in Japan"));
+    }
+
+    #[test]
+    fn no_merge_for_bold_sentences_or_new_headings() {
+        // Second bold line starts with a capital — a new heading or label,
+        // not a wrap continuation.
+        let lines = vec![
+            make_bold_line("Replace", 1, 700.0),
+            make_bold_line("Trash", 1, 686.0),
+        ];
+        let result = merge_heading_lines(lines, 12.0, &[], None);
+        assert_eq!(result.len(), 2, "distinct bold lines must not merge");
+
+        // Previous line ends a sentence — continuation must not merge.
+        let lines = vec![
+            make_bold_line("This is a bold sentence.", 1, 700.0),
+            make_bold_line("another bold line", 1, 686.0),
+        ];
+        let result = merge_heading_lines(lines, 12.0, &[], None);
+        assert_eq!(result.len(), 2, "sentence-final bold line must not merge");
+    }
+
+    #[test]
+    fn tiered_bold_heading_does_not_absorb_bold_body() {
+        // Previous line is a tier-level bold heading (16pt vs 12pt body);
+        // a following lowercase bold body line must NOT merge into it.
+        let mut heading = make_bold_line("Section Title", 1, 700.0);
+        heading.items[0].font_size = 16.0;
+        heading.items[0].height = 16.0;
+        let lines = vec![
+            heading,
+            make_bold_line("emphasized body text continues here", 1, 686.0),
+        ];
+        let result = merge_heading_lines(lines, 12.0, &[16.0], None);
+        assert_eq!(result.len(), 2, "tiered heading must not absorb bold body");
+    }
 }

tests/snapshots/p1244-1996.md

@@ -48,7 +48,7 @@ tips of directly from customers received other employees paid tips rec’d. entr
 
 **Page 3**
 
-27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals**
+27 28 29 30 31 **Subtotals from pages** **1, 2, and 3** **Totals**
 
 **1.** Report total cash tips (col. **a**) on Form 4070, line **1.**
 **2.** Report total credit card tips (col. **b**) on Form 4070, line **2.**
@@ -79,4 +79,3 @@ forms simpler, we would be happy to hear from you. You can write to the Tax Form
 **Instructions** *(continued)*
 
 Use this space to total your tips for the year
-