Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65878e86d7 |
@@ -626,6 +626,33 @@ fn find_relative_valleys(
|
||||
valleys
|
||||
}
|
||||
|
||||
/// Detect whether a side of a gutter consists predominantly of list-marker
|
||||
/// glyphs (•, ●, ○, ◦, ▪, ▫, ◆, ◇). A column of bullets on the left margin
|
||||
/// creates a spurious histogram valley between the bullet and the content.
|
||||
/// Treating it as a real column splits each list item's text across two
|
||||
/// "columns," so we reject these candidates.
|
||||
fn is_list_marker_column(items: &[&&TextItem]) -> bool {
|
||||
const LIST_MARKERS: &[char] = &['•', '●', '○', '◦', '▪', '▫', '◆', '◇', '■', '□'];
|
||||
if items.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let marker_count = items
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
let t = i.text.trim();
|
||||
let mut chars = t.chars();
|
||||
match (chars.next(), chars.next()) {
|
||||
(Some(c), None) => LIST_MARKERS.contains(&c),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
.count();
|
||||
// Require ≥80% of items on this side to be standalone markers. A handful
|
||||
// of non-marker items (stray page numbers, footnote refs) shouldn't
|
||||
// defeat the check.
|
||||
marker_count as f32 / items.len() as f32 >= 0.8
|
||||
}
|
||||
|
||||
/// Validate valley candidates with vertical consistency checks and build column regions.
|
||||
///
|
||||
/// When `center_assign` is true, items are assigned to columns based on their
|
||||
@@ -694,6 +721,19 @@ fn validate_and_build_columns(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reject valleys where the smaller side is just a column of list
|
||||
// markers (bullets aligned at the left margin). This is a common
|
||||
// pattern in PDFs where ● starts each list item: histogram detection
|
||||
// sees the gap between bullet and content as a gutter.
|
||||
let smaller_items: &[&&TextItem] = if left_items.len() <= right_items.len() {
|
||||
&left_items
|
||||
} else {
|
||||
&right_items
|
||||
};
|
||||
if is_list_marker_column(smaller_items) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check vertical overlap
|
||||
if y_range > 0.0 {
|
||||
let left_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
|
||||
@@ -1780,6 +1820,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bullet_marker_column_not_detected_as_column() {
|
||||
// Pattern: every line is `● <content>`, with ● at x=90 and content
|
||||
// starting at x=104. Histogram detection sees a gutter between them
|
||||
// and would split the page into a "bullet column" and "content column",
|
||||
// scrambling every list item.
|
||||
let mut items = Vec::new();
|
||||
for i in 0..15 {
|
||||
let y = 750.0 - i as f32 * 30.0;
|
||||
items.push(make_item(1, 90.0, y, "●"));
|
||||
items.push(make_item(
|
||||
1,
|
||||
104.0,
|
||||
y,
|
||||
"FullContentLineTextHere________________",
|
||||
));
|
||||
}
|
||||
// Pad with content to satisfy min item count for column detection.
|
||||
for i in 0..15 {
|
||||
let y = 300.0 - i as f32 * 14.0;
|
||||
items.push(make_item(1, 72.0, y, "FootnoteText_____________________"));
|
||||
}
|
||||
|
||||
let cols = detect_columns(&items, 1, false);
|
||||
assert_eq!(
|
||||
cols.len(),
|
||||
1,
|
||||
"Bullet markers aligned at left margin should not be treated as their own column"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_detects_bullets() {
|
||||
let items = vec![
|
||||
make_item(1, 90.0, 100.0, "●"),
|
||||
make_item(1, 90.0, 114.0, "●"),
|
||||
make_item(1, 90.0, 128.0, "●"),
|
||||
make_item(1, 90.0, 142.0, "●"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_marker_column_rejects_prose() {
|
||||
let items = vec![
|
||||
make_item(1, 30.0, 100.0, "Regular prose line"),
|
||||
make_item(1, 30.0, 114.0, "Another sentence"),
|
||||
make_item(1, 30.0, 128.0, "Third line"),
|
||||
make_item(1, 30.0, 142.0, "Fourth line"),
|
||||
];
|
||||
let refs: Vec<&TextItem> = items.iter().collect();
|
||||
let wrapped: Vec<&&TextItem> = refs.iter().collect();
|
||||
assert!(!is_list_marker_column(&wrapped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn premask_narrow_line_not_masked() {
|
||||
// Items that form a line spanning only ~40% of column width → not masked
|
||||
|
||||
@@ -64,6 +64,22 @@ pub(crate) fn is_caption_line(text: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if text starts with an unambiguous bullet marker (●, •, ○, ◦).
|
||||
///
|
||||
/// Narrower than [`is_list_item`]: it excludes numbered/lettered patterns
|
||||
/// like `1.` or `a)`, which legitimately appear as section headings in many
|
||||
/// documents. Used by the heading classifier to reject bullet lines without
|
||||
/// also demoting numbered headings.
|
||||
pub(crate) fn starts_with_bullet_marker(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
trimmed.starts_with("• ")
|
||||
|| trimmed.starts_with("● ")
|
||||
|| trimmed.starts_with("○ ")
|
||||
|| trimmed.starts_with("◦ ")
|
||||
|| trimmed.starts_with("- ")
|
||||
|| trimmed.starts_with("* ")
|
||||
}
|
||||
|
||||
/// Check if text looks like a list item
|
||||
pub(crate) fn is_list_item(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
@@ -115,6 +131,16 @@ pub(crate) fn format_list_item(text: &str) -> String {
|
||||
if let Some(rest) = trimmed.strip_prefix(*bullet) {
|
||||
return format!("- {}", rest.trim_start());
|
||||
}
|
||||
// Bullet inside a leading bold/italic run (e.g. "**● Label:** rest").
|
||||
// The run wraps both the marker and the following label because both
|
||||
// use a bold font in the PDF.
|
||||
for wrapper in ["**", "*"] {
|
||||
if let Some(after_open) = trimmed.strip_prefix(wrapper) {
|
||||
if let Some(rest) = after_open.strip_prefix(*bullet) {
|
||||
return format!("- {}{}", wrapper, rest.trim_start());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
|
||||
@@ -198,3 +224,42 @@ pub(crate) fn is_monospace_font(font_name: &str) -> bool {
|
||||
|
||||
patterns.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn format_list_item_plain_bullet() {
|
||||
assert_eq!(format_list_item("● Item"), "- Item");
|
||||
assert_eq!(format_list_item("• Item"), "- Item");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_list_item_bullet_inside_bold() {
|
||||
// PDF that uses bold font for both the marker and the label produces
|
||||
// a single bold run like "**● Label:** rest"; the bullet must still
|
||||
// be stripped and the bold wrapper preserved on the label.
|
||||
assert_eq!(
|
||||
format_list_item("**● Fraud: Willing cooperation;**"),
|
||||
"- **Fraud: Willing cooperation;**"
|
||||
);
|
||||
assert_eq!(
|
||||
format_list_item("**● Label:** rest of line"),
|
||||
"- **Label:** rest of line"
|
||||
);
|
||||
assert_eq!(format_list_item("*● Italic:* rest"), "- *Italic:* rest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_list_item_already_dash() {
|
||||
assert_eq!(format_list_item("- existing"), "- existing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_list_item_with_bullet_space() {
|
||||
assert!(is_list_item("● Item"));
|
||||
assert!(is_list_item("• Item"));
|
||||
assert!(is_list_item("- Item"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,9 @@ use super::analysis::{
|
||||
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders,
|
||||
};
|
||||
use super::classify::{format_list_item, is_caption_line, is_list_item, is_monospace_font};
|
||||
use super::classify::{
|
||||
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
||||
};
|
||||
use super::postprocess::clean_markdown;
|
||||
use super::preprocess::{merge_drop_caps, merge_heading_lines};
|
||||
use super::MarkdownOptions;
|
||||
@@ -587,6 +589,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
let heuristic_heading = if options.detect_headers
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !starts_with_bullet_marker(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
||||
|
||||
Reference in New Issue
Block a user