Compare commits

...
Author SHA1 Message Date
Abimael MartellandClaude Opus 4.7 65878e86d7 fix: don't treat bullet-marker columns as real text columns
On PDFs where every list item starts with ● at the left margin and content
at a fixed offset, histogram column detection sees the gap between marker
and content as a gutter and splits each line across two phantom "columns,"
scrambling the reading order (Anthropic's Mythos system card p.73–74).

- layout: reject gutter candidates where the smaller side is ≥80%
  standalone bullet-marker glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇, ■, □)
- markdown/classify: add starts_with_bullet_marker helper (narrower than
  is_list_item — excludes numbered/lettered patterns like 1. and a) so
  numbered section headings stay as headings)
- markdown/convert: skip heuristic heading detection on lines that start
  with a bullet marker
- markdown/classify: strip a leading bullet wrapped in a bold/italic run
  (e.g. "**● Label:**" → "- **Label:**") — some PDFs put the marker inside
  the same bold run as the label

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-20 22:22:28 -07:00
3 changed files with 166 additions and 1 deletions
+97
View File
@@ -626,6 +626,33 @@ fn find_relative_valleys(
valleys
}
/// Detect whether a side of a gutter consists predominantly of list-marker
/// glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇). A column of bullets on the left margin
/// creates a spurious histogram valley between the bullet and the content.
/// Treating it as a real column splits each list item's text across two
/// "columns," so we reject these candidates.
fn is_list_marker_column(items: &[&&TextItem]) -> bool {
const LIST_MARKERS: &[char] = &['•', '●', '○', '◦', '▪', '▫', '◆', '◇', '■', '□'];
if items.is_empty() {
return false;
}
let marker_count = items
.iter()
.filter(|i| {
let t = i.text.trim();
let mut chars = t.chars();
match (chars.next(), chars.next()) {
(Some(c), None) => LIST_MARKERS.contains(&c),
_ => false,
}
})
.count();
// Require ≥80% of items on this side to be standalone markers. A handful
// of non-marker items (stray page numbers, footnote refs) shouldn't
// defeat the check.
marker_count as f32 / items.len() as f32 >= 0.8
}
/// Validate valley candidates with vertical consistency checks and build column regions.
///
/// When `center_assign` is true, items are assigned to columns based on their
@@ -694,6 +721,19 @@ fn validate_and_build_columns(
continue;
}
// Reject valleys where the smaller side is just a column of list
// markers (bullets aligned at the left margin). This is a common
// pattern in PDFs where ● starts each list item: histogram detection
// sees the gap between bullet and content as a gutter.
let smaller_items: &[&&TextItem] = if left_items.len() <= right_items.len() {
&left_items
} else {
&right_items
};
if is_list_marker_column(smaller_items) {
continue;
}
// Check vertical overlap
if y_range > 0.0 {
let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
@@ -1780,6 +1820,63 @@ mod tests {
);
}
#[test]
fn bullet_marker_column_not_detected_as_column() {
// Pattern: every line is `● <content>`, with ● at x=90 and content
// starting at x=104. Histogram detection sees a gutter between them
// and would split the page into a "bullet column" and "content column",
// scrambling every list item.
let mut items = Vec::new();
for i in 0..15 {
let y = 750.0 - i as f32 * 30.0;
items.push(make_item(1, 90.0, y, ""));
items.push(make_item(
1,
104.0,
y,
"FullContentLineTextHere________________",
));
}
// Pad with content to satisfy min item count for column detection.
for i in 0..15 {
let y = 300.0 - i as f32 * 14.0;
items.push(make_item(1, 72.0, y, "FootnoteText_____________________"));
}
let cols = detect_columns(&items, 1, false);
assert_eq!(
cols.len(),
1,
"Bullet markers aligned at left margin should not be treated as their own column"
);
}
#[test]
fn is_list_marker_column_detects_bullets() {
let items = vec![
make_item(1, 90.0, 100.0, ""),
make_item(1, 90.0, 114.0, ""),
make_item(1, 90.0, 128.0, ""),
make_item(1, 90.0, 142.0, ""),
];
let refs: Vec<&TextItem> = items.iter().collect();
let wrapped: Vec<&&TextItem> = refs.iter().collect();
assert!(is_list_marker_column(&wrapped));
}
#[test]
fn is_list_marker_column_rejects_prose() {
let items = vec![
make_item(1, 30.0, 100.0, "Regular prose line"),
make_item(1, 30.0, 114.0, "Another sentence"),
make_item(1, 30.0, 128.0, "Third line"),
make_item(1, 30.0, 142.0, "Fourth line"),
];
let refs: Vec<&TextItem> = items.iter().collect();
let wrapped: Vec<&&TextItem> = refs.iter().collect();
assert!(!is_list_marker_column(&wrapped));
}
#[test]
fn premask_narrow_line_not_masked() {
// Items that form a line spanning only ~40% of column width → not masked
+65
View File
@@ -64,6 +64,22 @@ pub(crate) fn is_caption_line(text: &str) -> bool {
false
}
/// Check if text starts with an unambiguous bullet marker (●, •, ○, ◦).
///
/// Narrower than [`is_list_item`]: it excludes numbered/lettered patterns
/// like `1.` or `a)`, which legitimately appear as section headings in many
/// documents. Used by the heading classifier to reject bullet lines without
/// also demoting numbered headings.
pub(crate) fn starts_with_bullet_marker(text: &str) -> bool {
let trimmed = text.trim_start();
trimmed.starts_with("")
|| trimmed.starts_with("")
|| trimmed.starts_with("")
|| trimmed.starts_with("")
|| trimmed.starts_with("- ")
|| trimmed.starts_with("* ")
}
/// Check if text looks like a list item
pub(crate) fn is_list_item(text: &str) -> bool {
let trimmed = text.trim_start();
@@ -115,6 +131,16 @@ pub(crate) fn format_list_item(text: &str) -> String {
if let Some(rest) = trimmed.strip_prefix(*bullet) {
return format!("- {}", rest.trim_start());
}
// Bullet inside a leading bold/italic run (e.g. "**● Label:** rest").
// The run wraps both the marker and the following label because both
// use a bold font in the PDF.
for wrapper in ["**", "*"] {
if let Some(after_open) = trimmed.strip_prefix(wrapper) {
if let Some(rest) = after_open.strip_prefix(*bullet) {
return format!("- {}{}", wrapper, rest.trim_start());
}
}
}
}
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
@@ -198,3 +224,42 @@ pub(crate) fn is_monospace_font(font_name: &str) -> bool {
patterns.iter().any(|p| lower.contains(p))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn format_list_item_plain_bullet() {
assert_eq!(format_list_item("● Item"), "- Item");
assert_eq!(format_list_item("• Item"), "- Item");
}
#[test]
fn format_list_item_bullet_inside_bold() {
// PDF that uses bold font for both the marker and the label produces
// a single bold run like "**● Label:** rest"; the bullet must still
// be stripped and the bold wrapper preserved on the label.
assert_eq!(
format_list_item("**● Fraud: Willing cooperation;**"),
"- **Fraud: Willing cooperation;**"
);
assert_eq!(
format_list_item("**● Label:** rest of line"),
"- **Label:** rest of line"
);
assert_eq!(format_list_item("*● Italic:* rest"), "- *Italic:* rest");
}
#[test]
fn format_list_item_already_dash() {
assert_eq!(format_list_item("- existing"), "- existing");
}
#[test]
fn is_list_item_with_bullet_space() {
assert!(is_list_item("● Item"));
assert!(is_list_item("• Item"));
assert!(is_list_item("- Item"));
}
}
+4 -1
View File
@@ -9,7 +9,9 @@ use super::analysis::{
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
detect_header_level, font_size_rarity, has_dot_leaders,
};
use super::classify::{format_list_item, is_caption_line, is_list_item, is_monospace_font};
use super::classify::{
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
};
use super::postprocess::clean_markdown;
use super::preprocess::{merge_drop_caps, merge_heading_lines};
use super::MarkdownOptions;
@@ -587,6 +589,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
let heuristic_heading = if options.detect_headers
&& plain_trimmed.len() > 3
&& plain_trimmed.split_whitespace().count() <= 15
&& !starts_with_bullet_marker(plain_trimmed)
{
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {