fix: don't treat bullet-marker columns as real text columns (#55)
On PDFs where every list item starts with ● at the left margin and content at a fixed offset, histogram column detection sees the gap between marker and content as a gutter and splits each line across two phantom "columns," scrambling the reading order (Anthropic's Mythos system card p.73–74). - layout: reject gutter candidates where the smaller side is ≥80% standalone bullet-marker glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇, ■, □) - markdown/classify: add starts_with_bullet_marker helper (narrower than is_list_item — excludes numbered/lettered patterns like 1. and a) so numbered section headings stay as headings) - markdown/convert: skip heuristic heading detection on lines that start with a bullet marker - markdown/classify: strip a leading bullet wrapped in a bold/italic run (e.g. "**● Label:**" → "- **Label:**") — some PDFs put the marker inside the same bold run as the label Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
2e933ba8c1
commit
f44509ac3a
@@ -626,6 +626,33 @@ fn find_relative_valleys(
|
||||
valleys
|
||||
}
|
||||
|
||||
/// Detect whether a side of a gutter consists predominantly of list-marker
|
||||
/// glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇). A column of bullets on the left margin
|
||||
/// creates a spurious histogram valley between the bullet and the content.
|
||||
/// Treating it as a real column splits each list item's text across two
|
||||
/// "columns," so we reject these candidates.
|
||||
fn is_list_marker_column(items: &[&&TextItem]) -> bool {
|
||||
const LIST_MARKERS: &[char] = &['•', '●', '○', '◦', '▪', '▫', '◆', '◇', '■', '□'];
|
||||
if items.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let marker_count = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
let t = i.text.trim();
|
||||
let mut chars = t.chars();
|
||||
match (chars.next(), chars.next()) {
|
||||
(Some(c), None) => LIST_MARKERS.contains(&c),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
.count();
|
||||
// Require ≥80% of items on this side to be standalone markers. A handful
|
||||
// of non-marker items (stray page numbers, footnote refs) shouldn't
|
||||
// defeat the check.
|
||||
marker_count as f32 / items.len() as f32 >= 0.8
|
||||
}
|
||||
|
||||
/// Validate valley candidates with vertical consistency checks and build column regions.
|
||||
///
|
||||
/// When `center_assign` is true, items are assigned to columns based on their
|
||||
@@ -694,6 +721,19 @@ fn validate_and_build_columns(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reject valleys where the smaller side is just a column of list
|
||||
// markers (bullets aligned at the left margin). This is a common
|
||||
// pattern in PDFs where ● starts each list item: histogram detection
|
||||
// sees the gap between bullet and content as a gutter.
|
||||
let smaller_items: &[&&TextItem] = if left_items.len() <= right_items.len() {
|
||||
&left_items
|
||||
} else {
|
||||
&right_items
|
||||
};
|
||||
if is_list_marker_column(smaller_items) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check vertical overlap
|
||||
if y_range > 0.0 {
|
||||
let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
|
||||
@@ -1780,6 +1820,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bullet_marker_column_not_detected_as_column() {
|
||||
// Pattern: every line is `● <content>`, with ● at x=90 and content
|
||||
// starting at x=104. Histogram detection sees a gutter between them
|
||||
// and would split the page into a "bullet column" and "content column",
|
||||
// scrambling every list item.
|
||||
let mut items = Vec::new();
|
||||
for i in 0..15 {
|
||||
let y = 750.0 - i as f32 * 30.0;
|
||||
items.push(make_item(1, 90.0, y, "●"));
|
||||
items.push(make_item(
|
||||
1,
|
||||
104.0,
|
||||
y,
|
||||
"FullContentLineTextHere________________",
|
||||
));
|
||||
}
|
||||
// Pad with content to satisfy min item count for column detection.
|
||||
for i in 0..15 {
|
||||
let y = 300.0 - i as f32 * 14.0;
|
||||
items.push(make_item(1, 72.0, y, "FootnoteText_____________________"));
|
||||
}
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(
|
||||
cols.len(),
|
||||
1,
|
||||
"Bullet markers aligned at left margin should not be treated as their own column"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_detects_bullets() {
|
||||
let items = vec![
|
||||
make_item(1, 90.0, 100.0, "●"),
|
||||
make_item(1, 90.0, 114.0, "●"),
|
||||
make_item(1, 90.0, 128.0, "●"),
|
||||
make_item(1, 90.0, 142.0, "●"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_rejects_prose() {
|
||||
let items = vec![
|
||||
make_item(1, 30.0, 100.0, "Regular prose line"),
|
||||
make_item(1, 30.0, 114.0, "Another sentence"),
|
||||
make_item(1, 30.0, 128.0, "Third line"),
|
||||
make_item(1, 30.0, 142.0, "Fourth line"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(!is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn premask_narrow_line_not_masked() {
|
||||
// Items that form a line spanning only ~40% of column width → not masked
|
||||
|
||||
Reference in New Issue
Block a user