diff --git a/src/extractor.rs b/src/extractor.rs index 728546d..8ac94bd 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -2851,12 +2851,20 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec return vec![ColumnRegion { x_min, x_max }]; } - // Build occupancy histogram + // Build occupancy histogram. + // Exclude items wider than 60% of page width — these are spanning items + // (titles, full-width paragraphs) that would fill the gutter and prevent + // detection of partial-page column layouts (e.g. two-column abstracts on + // a page that also has single-column introduction text). + let wide_threshold = page_width * 0.6; let num_bins = ((page_width / BIN_WIDTH).ceil() as usize).max(1); let mut histogram = vec![0u32; num_bins]; for item in &page_items { let w = effective_width(item); + if w > wide_threshold { + continue; + } let left = ((item.x - x_min) / BIN_WIDTH).floor() as usize; let right = (((item.x + w) - x_min) / BIN_WIDTH).ceil() as usize; let left = left.min(num_bins); @@ -3034,6 +3042,111 @@ fn is_page_number(item: &TextItem) -> bool { } /// Group text items into lines, with multi-column support +/// Detect newspaper-style columns: independent text flows that should be read +/// sequentially (all of col1, then col2) rather than Y-interleaved. +fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { + if per_column_lines.len() < 2 { + return false; + } + + // Each column must independently have substantial content + let min_lines = per_column_lines.iter().map(|c| c.len()).min().unwrap_or(0); + if min_lines < 15 { + return false; + } + + // Check Y-collision: count lines in the smallest column that have a + // Y-match in any other column. High collision with many lines = newspaper. + let y_tol = 3.0; + let (smallest_idx, _) = per_column_lines + .iter() + .enumerate() + .min_by_key(|(_, c)| c.len()) + .unwrap(); + + let smallest = &per_column_lines[smallest_idx]; + let mut collisions = 0u32; + for line in smallest { + for (ci, col) in per_column_lines.iter().enumerate() { + if ci == smallest_idx { + continue; + } + if col.iter().any(|ol| (ol.y - line.y).abs() < y_tol) { + collisions += 1; + break; + } + } + } + + let ratio = collisions as f32 / smallest.len() as f32; + ratio > 0.5 +} + +/// Split column lines into a core cluster and stragglers. +/// The core is the largest group of consecutive lines separated by normal +/// line spacing. Lines in other groups (header remnants, per-word items from +/// full-width lines) are returned as stragglers. +fn split_column_stragglers(lines: Vec) -> (Vec, Vec) { + if lines.len() < 3 { + return (lines, Vec::new()); + } + + // Lines are sorted Y descending (top-first). Compute gaps. + let mut gaps: Vec = Vec::new(); + for i in 0..lines.len() - 1 { + gaps.push(lines[i].y - lines[i + 1].y); + } + + // Median gap = typical line spacing + let mut sorted_gaps = gaps.clone(); + sorted_gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_gap = sorted_gaps[sorted_gaps.len() / 2]; + + // A gap > 3× median (min 30pt) indicates a break between content clusters + let threshold = (median_gap * 3.0).max(30.0); + + // Find all split points + let mut split_indices: Vec = Vec::new(); + for (i, &gap) in gaps.iter().enumerate() { + if gap > threshold { + split_indices.push(i); + } + } + + if split_indices.is_empty() { + return (lines, Vec::new()); + } + + // Build segments: (start_line_idx, end_line_idx_exclusive) + let mut segments: Vec<(usize, usize)> = Vec::new(); + let mut start = 0usize; + for &si in &split_indices { + segments.push((start, si + 1)); + start = si + 1; + } + segments.push((start, lines.len())); + + // Find the largest segment (the core cluster) + let (core_seg, _) = segments + .iter() + .enumerate() + .max_by_key(|(_, (s, e))| e - s) + .unwrap(); + + let (cs, ce) = segments[core_seg]; + let mut core = Vec::with_capacity(ce - cs); + let mut stragglers = Vec::new(); + for (i, line) in lines.into_iter().enumerate() { + if i >= cs && i < ce { + core.push(line); + } else { + stragglers.push(line); + } + } + + (core, stragglers) +} + pub fn group_into_lines(items: Vec) -> Vec { if items.is_empty() { return Vec::new(); @@ -3103,45 +3216,104 @@ pub fn group_into_lines(items: Vec) -> Vec { // Process spanning items as their own group let spanning_lines = group_single_column(spanning_items); - // Y-interleaved merge: combine all column lines and spanning lines, - // sort by Y position, and merge lines at the same Y from different - // columns into a single line. This correctly handles both tabular - // layouts (where rows span columns) and newspaper-style columns. - let mut all_page_lines: Vec = Vec::new(); - all_page_lines.extend(spanning_lines); - for col_lines in per_column_lines { - all_page_lines.extend(col_lines); - } + if is_newspaper_layout(&per_column_lines) { + // Newspaper: columns are independent text flows. + // 1. Split each column into its densest cluster (core) and stragglers + // 2. Use core columns to determine the above/below threshold + // 3. Emit: above items → core columns sequentially → below items + let mut core_columns: Vec> = Vec::new(); + let mut col_stragglers: Vec> = Vec::new(); + for col in per_column_lines { + let (core, stragglers) = split_column_stragglers(col); + core_columns.push(core); + col_stragglers.push(stragglers); + } - // Sort by Y descending (top-first), then by X for same-Y lines - all_page_lines.sort_by(|a, b| { - b.y.partial_cmp(&a.y) - .unwrap_or(std::cmp::Ordering::Equal) - .then( - a.items - .first() - .map(|i| i.x) - .unwrap_or(0.0) - .partial_cmp(&b.items.first().map(|i| i.x).unwrap_or(0.0)) - .unwrap_or(std::cmp::Ordering::Equal), - ) - }); + // col_top = min of max Y across core columns + let col_top = core_columns + .iter() + .filter(|c| !c.is_empty()) + .map(|c| c.iter().map(|l| l.y).fold(f32::NEG_INFINITY, f32::max)) + .fold(f32::INFINITY, f32::min); + let margin = 5.0; - // Merge lines at the same Y (within tolerance) into single lines - let y_tol = 3.0; - let mut merged: Vec = Vec::new(); - for line in all_page_lines { - if let Some(last) = merged.last_mut() { - if last.page == line.page && (last.y - line.y).abs() < y_tol { - last.items.extend(line.items); - sort_line_items(&mut last.items); - continue; + let mut above: Vec = Vec::new(); + let mut below_spanning: Vec = Vec::new(); + + // Spanning items: above or below the column region + for line in spanning_lines { + if line.y > col_top + margin { + above.push(line); + } else { + below_spanning.push(line); } } - merged.push(line); - } - all_lines.extend(merged); + // Column stragglers above col_top go to "above"; + // below col_top they stay with their column to avoid + // re-interleaving when sorted by Y. + let mut col_below: Vec> = vec![Vec::new(); core_columns.len()]; + for (ci, stragglers) in col_stragglers.into_iter().enumerate() { + for line in stragglers { + if line.y > col_top + margin { + above.push(line); + } else { + col_below[ci].push(line); + } + } + } + + above.sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + below_spanning + .sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + + all_lines.extend(above); + for col in core_columns { + all_lines.extend(col); + } + for cb in col_below { + all_lines.extend(cb); + } + all_lines.extend(below_spanning); + } else { + // Tabular: Y-interleaved merge — rows at the same Y from + // different columns form a single logical line. + let mut all_page_lines: Vec = Vec::new(); + all_page_lines.extend(spanning_lines); + for col_lines in per_column_lines { + all_page_lines.extend(col_lines); + } + + // Sort by Y descending (top-first), then by X for same-Y lines + all_page_lines.sort_by(|a, b| { + b.y.partial_cmp(&a.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + a.items + .first() + .map(|i| i.x) + .unwrap_or(0.0) + .partial_cmp(&b.items.first().map(|i| i.x).unwrap_or(0.0)) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + + // Merge lines at the same Y (within tolerance) into single lines + let y_tol = 3.0; + let mut merged: Vec = Vec::new(); + for line in all_page_lines { + if let Some(last) = merged.last_mut() { + if last.page == line.page && (last.y - line.y).abs() < y_tol { + last.items.extend(line.items); + sort_line_items(&mut last.items); + continue; + } + } + merged.push(line); + } + + all_lines.extend(merged); + } } } @@ -3761,4 +3933,66 @@ mod tests { // Latin is not CJK assert!(!is_cjk_char('A')); } + + #[test] + fn test_newspaper_layout_detection() { + // Two dense columns (>15 lines each) with matching Y positions → newspaper + let make_line = |y: f32, x: f32, page: u32| TextLine { + y, + page, + items: vec![TextItem { + text: "text".into(), + x, + y, + width: 100.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }], + }; + + let col1: Vec = (0..20) + .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) + .collect(); + let col2: Vec = (0..20) + .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) + .collect(); + + assert!(is_newspaper_layout(&[col1, col2])); + } + + #[test] + fn test_tabular_layout_detection() { + // Sparse columns (<15 lines) → tabular, not newspaper + let make_line = |y: f32, x: f32, page: u32| TextLine { + y, + page, + items: vec![TextItem { + text: "text".into(), + x, + y, + width: 100.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }], + }; + + let col1: Vec = (0..5) + .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1)) + .collect(); + let col2: Vec = (0..5) + .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) + .collect(); + + assert!(!is_newspaper_layout(&[col1, col2])); + } } diff --git a/src/markdown.rs b/src/markdown.rs index 6a3861b..9608c5f 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -623,9 +623,10 @@ fn to_markdown_from_lines_with_tables_and_images( } } - // Paragraph break (large Y gap relative to document's typical line spacing) + // Paragraph break: large forward Y gap (normal) or large backward jump + // (newspaper columns emitted sequentially on the same page). let y_gap = prev_y - line.y; - let is_para_break = y_gap > para_threshold; + let is_para_break = y_gap.abs() > para_threshold; if is_para_break && in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -840,9 +841,10 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> } } - // Paragraph break (large Y gap relative to document's typical line spacing) + // Paragraph break: large forward Y gap (normal) or large backward jump + // (newspaper columns emitted sequentially on the same page). let y_gap = prev_y - line.y; - let is_para_break = y_gap > para_threshold; + let is_para_break = y_gap.abs() > para_threshold; if is_para_break && in_paragraph { output.push_str("\n\n"); in_paragraph = false;