fix: don't treat bullet-marker columns as real text columns (#55)

On PDFs where every list item starts with ● at the left margin and content
at a fixed offset, histogram column detection sees the gap between marker
and content as a gutter and splits each line across two phantom "columns,"
scrambling the reading order (Anthropic's Mythos system card p.73–74).

- layout: reject gutter candidates where the smaller side is ≥80%
  standalone bullet-marker glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇, ■, □)
- markdown/classify: add starts_with_bullet_marker helper (narrower than
  is_list_item — excludes numbered/lettered patterns like 1. and a) so
  numbered section headings stay as headings)
- markdown/convert: skip heuristic heading detection on lines that start
  with a bullet marker
- markdown/classify: strip a leading bullet wrapped in a bold/italic run
  (e.g. "**● Label:**" → "- **Label:**") — some PDFs put the marker inside
  the same bold run as the label

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-04-20 22:47:34 -07:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 2e933ba8c1
commit f44509ac3a
3 changed files with 166 additions and 1 deletions
+97
View File
@@ -626,6 +626,33 @@ fn find_relative_valleys(
valleys
}
/// Detect whether a side of a gutter consists predominantly of list-marker
/// glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇). A column of bullets on the left margin
/// creates a spurious histogram valley between the bullet and the content.
/// Treating it as a real column splits each list item's text across two
/// "columns," so we reject these candidates.
fn is_list_marker_column(items: &[&&TextItem]) -> bool {
const LIST_MARKERS: &[char] = &['•', '●', '○', '◦', '▪', '▫', '◆', '◇', '■', '□'];
if items.is_empty() {
return false;
}
let marker_count = items
.iter()
.filter(|i| {
let t = i.text.trim();
let mut chars = t.chars();
match (chars.next(), chars.next()) {
(Some(c), None) => LIST_MARKERS.contains(&c),
_ => false,
}
})
.count();
// Require ≥80% of items on this side to be standalone markers. A handful
// of non-marker items (stray page numbers, footnote refs) shouldn't
// defeat the check.
marker_count as f32 / items.len() as f32 >= 0.8
}
/// Validate valley candidates with vertical consistency checks and build column regions.
///
/// When `center_assign` is true, items are assigned to columns based on their
@@ -694,6 +721,19 @@ fn validate_and_build_columns(
continue;
}
// Reject valleys where the smaller side is just a column of list
// markers (bullets aligned at the left margin). This is a common
// pattern in PDFs where ● starts each list item: histogram detection
// sees the gap between bullet and content as a gutter.
let smaller_items: &[&&TextItem] = if left_items.len() <= right_items.len() {
&left_items
} else {
&right_items
};
if is_list_marker_column(smaller_items) {
continue;
}
// Check vertical overlap
if y_range > 0.0 {
let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
@@ -1780,6 +1820,63 @@ mod tests {
);
}
#[test]
fn bullet_marker_column_not_detected_as_column() {
// Pattern: every line is `● <content>`, with ● at x=90 and content
// starting at x=104. Histogram detection sees a gutter between them
// and would split the page into a "bullet column" and "content column",
// scrambling every list item.
let mut items = Vec::new();
for i in 0..15 {
let y = 750.0 - i as f32 * 30.0;
items.push(make_item(1, 90.0, y, ""));
items.push(make_item(
1,
104.0,
y,
"FullContentLineTextHere________________",
));
}
// Pad with content to satisfy min item count for column detection.
for i in 0..15 {
let y = 300.0 - i as f32 * 14.0;
items.push(make_item(1, 72.0, y, "FootnoteText_____________________"));
}
let cols = detect_columns(&items, 1, false);
assert_eq!(
cols.len(),
1,
"Bullet markers aligned at left margin should not be treated as their own column"
);
}
#[test]
fn is_list_marker_column_detects_bullets() {
let items = vec![
make_item(1, 90.0, 100.0, ""),
make_item(1, 90.0, 114.0, ""),
make_item(1, 90.0, 128.0, ""),
make_item(1, 90.0, 142.0, ""),
];
let refs: Vec<&TextItem> = items.iter().collect();
let wrapped: Vec<&&TextItem> = refs.iter().collect();
assert!(is_list_marker_column(&wrapped));
}
#[test]
fn is_list_marker_column_rejects_prose() {
let items = vec![
make_item(1, 30.0, 100.0, "Regular prose line"),
make_item(1, 30.0, 114.0, "Another sentence"),
make_item(1, 30.0, 128.0, "Third line"),
make_item(1, 30.0, 142.0, "Fourth line"),
];
let refs: Vec<&TextItem> = items.iter().collect();
let wrapped: Vec<&&TextItem> = refs.iter().collect();
assert!(!is_list_marker_column(&wrapped));
}
#[test]
fn premask_narrow_line_not_masked() {
// Items that form a line spanning only ~40% of column width → not masked