From 6b8f4d0295ef6e8c1bd19406defb3c85d8c4d765 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 9 Mar 2026 21:49:47 -0700 Subject: [PATCH] test: add inline unit tests to postprocess, format, grid, and detect_rects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 48 new tests (140 → 188 total) covering previously untested private functions with synthetic data. No PDF fixtures needed. Co-Authored-By: Claude Opus 4.6 --- src/markdown/postprocess.rs | 188 ++++++++++++ src/tables/detect_rects.rs | 578 ++++++++++++++++++++++++++++++++++++ src/tables/format.rs | 232 +++++++++++++++ src/tables/grid.rs | 339 +++++++++++++++++++++ 4 files changed, 1337 insertions(+) diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs index 3f34d16..2bd9970 100644 --- a/src/markdown/postprocess.rs +++ b/src/markdown/postprocess.rs @@ -273,3 +273,191 @@ fn format_urls(text: &str) -> String { } result } + +#[cfg(test)] +mod tests { + use super::*; + + // --- collapse_dot_leaders --- + + #[test] + fn test_collapse_dot_leaders_four_or_more_dots() { + assert_eq!( + collapse_dot_leaders("Introduction............................1"), + "Introduction ... 1" + ); + } + + #[test] + fn test_collapse_dot_leaders_three_dots_unchanged() { + assert_eq!(collapse_dot_leaders("wait...what"), "wait...what"); + } + + #[test] + fn test_collapse_dot_leaders_no_dots() { + assert_eq!(collapse_dot_leaders("Hello World"), "Hello World"); + } + + #[test] + fn test_collapse_dot_leaders_mixed() { + let input = "Chapter 1.......10\nSome text... ok\nChapter 2........20"; + let result = collapse_dot_leaders(input); + assert!(result.contains("Chapter 1 ... 10")); + assert!(result.contains("Some text... ok")); + assert!(result.contains("Chapter 2 ... 20")); + } + + // --- fix_hyphenation --- + + #[test] + fn test_fix_hyphenation_spaced_hyphen() { + assert_eq!(fix_hyphenation("Limoeiro - Norte"), "Limoeiro-Norte"); + } + + #[test] + fn test_fix_hyphenation_list_item_unchanged() { + assert_eq!( + fix_hyphenation("- item one\n- item two"), + "- item one\n- item two" + ); + } + + #[test] + fn test_fix_hyphenation_accented_chars() { + assert_eq!(fix_hyphenation("São - Paulo"), "São-Paulo"); + } + + #[test] + fn test_fix_hyphenation_multiple_instances() { + assert_eq!( + fix_hyphenation("one - two and three - four"), + "one-two and three-four" + ); + } + + // --- is_page_number_line --- + + #[test] + fn test_is_page_number_digits_1_to_4() { + assert!(is_page_number_line("1")); + assert!(is_page_number_line("42")); + assert!(is_page_number_line("123")); + assert!(is_page_number_line("9999")); + assert!(!is_page_number_line("12345")); + } + + #[test] + fn test_is_page_number_page_x() { + assert!(is_page_number_line("Page 5")); + assert!(is_page_number_line("page 12")); + } + + #[test] + fn test_is_page_number_page_x_of_y() { + assert!(is_page_number_line("Page 3 of 10")); + assert!(is_page_number_line("page 1 of 5")); + } + + #[test] + fn test_is_page_number_x_of_y() { + assert!(is_page_number_line("3 of 10")); + } + + #[test] + fn test_is_page_number_centered_dash() { + assert!(is_page_number_line("- 5 -")); + assert!(is_page_number_line("-12-")); + } + + #[test] + fn test_is_page_number_page_of() { + assert!(is_page_number_line("Page of")); + assert!(is_page_number_line("page of 10")); + } + + #[test] + fn test_is_page_number_empty() { + assert!(!is_page_number_line("")); + } + + #[test] + fn test_is_page_number_non_match() { + assert!(!is_page_number_line("Hello World")); + assert!(!is_page_number_line("Chapter 1")); + assert!(!is_page_number_line("Total: 500")); + } + + // --- remove_page_numbers --- + + #[test] + fn test_remove_page_numbers_isolated_number() { + let input = "Some text\n\n42\n\nMore text"; + let result = remove_page_numbers(input); + assert!(!result.contains("\n42\n")); + assert!(result.contains("Some text")); + assert!(result.contains("More text")); + } + + #[test] + fn test_remove_page_numbers_before_break() { + let input = "Content\n\n5\n---\nNext page"; + let result = remove_page_numbers(input); + assert!(!result.contains("\n5\n")); + } + + #[test] + fn test_remove_page_numbers_in_context_kept() { + let input = "Line A\nLine B\n42\nLine C\nLine D"; + let result = remove_page_numbers(input); + assert!(result.contains("42")); + } + + #[test] + fn test_remove_page_numbers_multiple_patterns() { + let input = "\n1\n\nContent\n\n2\n\n---\nMore\n\n3\n"; + let result = remove_page_numbers(input); + assert!(!result.contains("\n1\n")); + assert!(!result.contains("\n2\n")); + assert!(!result.contains("\n3\n")); + } + + #[test] + fn test_remove_page_numbers_empty() { + assert_eq!(remove_page_numbers(""), ""); + } + + // --- format_urls --- + + #[test] + fn test_format_urls_bare_url() { + let result = format_urls("Visit https://example.com for info"); + assert!(result.contains("[https://example.com](https://example.com)")); + } + + #[test] + fn test_format_urls_already_linked() { + let input = "[click](https://example.com)"; + assert_eq!(format_urls(input), input); + } + + #[test] + fn test_format_urls_inside_brackets() { + let input = "[https://example.com](https://example.com)"; + let result = format_urls(input); + assert!(!result.contains("[[")); + } + + #[test] + fn test_format_urls_multiple() { + let input = "See https://a.com and https://b.com"; + let result = format_urls(input); + assert!(result.contains("[https://a.com](https://a.com)")); + assert!(result.contains("[https://b.com](https://b.com)")); + } + + #[test] + fn test_format_urls_no_urls() { + let input = "No links here"; + assert_eq!(format_urls(input), input); + } +} diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 6e31175..f690faa 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -1217,3 +1217,581 @@ fn cluster_x_positions(items: &[(usize, &TextItem)], min_threshold: f32) -> Vec< }) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::ItemType; + + fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + // --- rects_overlap --- + + #[test] + fn test_rects_overlap_overlapping() { + let a = (0.0, 0.0, 10.0, 10.0); + let b = (5.0, 5.0, 10.0, 10.0); + assert!(rects_overlap(&a, &b, 0.0)); + } + + #[test] + fn test_rects_overlap_touching() { + let a = (0.0, 0.0, 10.0, 10.0); + let b = (10.0, 0.0, 10.0, 10.0); + // Touching at edge — with 0 tolerance, the right edge of a == left edge of b + assert!(rects_overlap(&a, &b, 0.0)); + } + + #[test] + fn test_rects_overlap_separated() { + let a = (0.0, 0.0, 10.0, 10.0); + let b = (20.0, 20.0, 10.0, 10.0); + assert!(!rects_overlap(&a, &b, 0.0)); + } + + #[test] + fn test_rects_overlap_contained() { + let a = (0.0, 0.0, 20.0, 20.0); + let b = (5.0, 5.0, 5.0, 5.0); + assert!(rects_overlap(&a, &b, 0.0)); + } + + #[test] + fn test_rects_overlap_identical() { + let a = (10.0, 10.0, 50.0, 50.0); + assert!(rects_overlap(&a, &a, 0.0)); + } + + #[test] + fn test_rects_overlap_tolerance_expansion() { + let a = (0.0, 0.0, 10.0, 10.0); + let b = (15.0, 0.0, 10.0, 10.0); + // Gap of 5 — with tol=0 they don't overlap + assert!(!rects_overlap(&a, &b, 0.0)); + // With tol=3, each expands by 3 → they overlap + assert!(rects_overlap(&a, &b, 3.0)); + } + + // --- cluster_rects --- + + #[test] + fn test_cluster_rects_empty() { + let rects: Vec<(f32, f32, f32, f32)> = vec![]; + assert!(cluster_rects(&rects, 3.0, 1).is_empty()); + } + + #[test] + fn test_cluster_rects_single_rect() { + let rects = vec![(0.0, 0.0, 10.0, 10.0)]; + // min_size=1 → should return the single rect + let groups = cluster_rects(&rects, 3.0, 1); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], vec![0]); + } + + #[test] + fn test_cluster_rects_all_disconnected() { + let rects = vec![ + (0.0, 0.0, 10.0, 10.0), + (100.0, 100.0, 10.0, 10.0), + (200.0, 200.0, 10.0, 10.0), + ]; + // All separated, min_size=2 → no groups + let groups = cluster_rects(&rects, 0.0, 2); + assert!(groups.is_empty()); + } + + #[test] + fn test_cluster_rects_chain_overlap() { + // A overlaps B, B overlaps C → all in one group + let rects = vec![ + (0.0, 0.0, 10.0, 10.0), + (8.0, 0.0, 10.0, 10.0), + (16.0, 0.0, 10.0, 10.0), + ]; + let groups = cluster_rects(&rects, 0.0, 1); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 3); + } + + #[test] + fn test_cluster_rects_all_connected() { + let rects = vec![ + (0.0, 0.0, 20.0, 20.0), + (5.0, 5.0, 20.0, 20.0), + (10.0, 10.0, 20.0, 20.0), + ]; + let groups = cluster_rects(&rects, 0.0, 1); + assert_eq!(groups.len(), 1); + } + + #[test] + fn test_cluster_rects_min_size_filter() { + // Two separate pairs + one lone rect + let rects = vec![ + (0.0, 0.0, 10.0, 10.0), + (5.0, 0.0, 10.0, 10.0), + (100.0, 100.0, 10.0, 10.0), + ]; + // min_size=2 → only the overlapping pair returned + let groups = cluster_rects(&rects, 0.0, 2); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 2); + } + + // --- snap_edges --- + + #[test] + fn test_snap_edges_empty() { + assert!(snap_edges(&[], 6.0).is_empty()); + } + + #[test] + fn test_snap_edges_single_value() { + assert_eq!(snap_edges(&[42.0], 6.0), vec![42.0]); + } + + #[test] + fn test_snap_edges_within_tolerance_deduped() { + let edges = snap_edges(&[10.0, 12.0, 14.0, 30.0], 6.0); + // 10, 12, 14 are all within 6 of the first → deduplicated + assert_eq!(edges.len(), 2); + assert!((edges[0] - 10.0).abs() < 0.01); + assert!((edges[1] - 30.0).abs() < 0.01); + } + + #[test] + fn test_snap_edges_outside_tolerance_kept() { + let edges = snap_edges(&[10.0, 20.0, 30.0], 5.0); + assert_eq!(edges.len(), 3); + } + + #[test] + fn test_snap_edges_unsorted_input() { + let edges = snap_edges(&[30.0, 10.0, 20.0], 5.0); + // Should be sorted + assert_eq!(edges, vec![10.0, 20.0, 30.0]); + } + + // --- assign_items_to_grid --- + + #[test] + fn test_assign_items_basic() { + let items = vec![ + make_item("A", 15.0, 85.0, 10.0), + make_item("B", 55.0, 85.0, 10.0), + make_item("C", 15.0, 55.0, 10.0), + make_item("D", 55.0, 55.0, 10.0), + ]; + // 2x2 grid: cols at [10, 50, 90], rows at [90, 70, 50] (top-to-bottom) + let col_edges = vec![10.0, 50.0, 90.0]; + let row_edges = vec![90.0, 70.0, 40.0]; + let (cells, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert_eq!(cells.len(), 2); + assert_eq!(cells[0][0], "A"); + assert_eq!(cells[0][1], "B"); + assert_eq!(cells[1][0], "C"); + assert_eq!(cells[1][1], "D"); + assert_eq!(indices.len(), 4); + } + + #[test] + fn test_assign_items_outside_grid() { + let items = vec![make_item("Outside", 500.0, 500.0, 10.0)]; + let col_edges = vec![10.0, 50.0, 90.0]; + let row_edges = vec![90.0, 70.0, 50.0]; + let (_, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert!(indices.is_empty()); + } + + #[test] + fn test_assign_items_wrong_page_filtered() { + let mut item = make_item("A", 15.0, 85.0, 10.0); + item.page = 2; + let items = vec![item]; + let col_edges = vec![10.0, 50.0, 90.0]; + let row_edges = vec![90.0, 70.0, 50.0]; + let (_, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert!(indices.is_empty()); + } + + #[test] + fn test_assign_items_multiple_same_cell() { + let items = vec![ + make_item("Hello", 15.0, 85.0, 10.0), + make_item("World", 20.0, 80.0, 10.0), + ]; + let col_edges = vec![10.0, 50.0]; + let row_edges = vec![90.0, 70.0]; + let (cells, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert_eq!(indices.len(), 2); + assert!(cells[0][0].contains("Hello")); + assert!(cells[0][0].contains("World")); + } + + #[test] + fn test_assign_items_boundary_tolerance() { + // Item right at edge with ±2pt tolerance + let items = vec![make_item("Edge", 9.0, 89.0, 10.0)]; + let col_edges = vec![10.0, 50.0]; + let row_edges = vec![90.0, 70.0]; + let (_, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert_eq!(indices.len(), 1); + } + + #[test] + fn test_assign_items_empty_grid() { + let items = vec![make_item("A", 15.0, 85.0, 10.0)]; + let col_edges = vec![10.0]; // Only 1 edge → 0 columns + let row_edges = vec![90.0]; // Only 1 edge → 0 rows + let (cells, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert!(cells.is_empty()); + assert!(indices.is_empty()); + } + + #[test] + fn test_assign_items_all_assigned() { + let items = vec![ + make_item("A", 15.0, 85.0, 10.0), + make_item("B", 55.0, 85.0, 10.0), + ]; + let col_edges = vec![10.0, 50.0, 90.0]; + let row_edges = vec![90.0, 70.0]; + let (_, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert_eq!(indices.len(), 2); + } + + #[test] + fn test_assign_items_sorted_y_desc_x_asc() { + // Two items in same cell — should sort by Y desc, X asc + let items = vec![ + make_item("Bottom", 15.0, 75.0, 10.0), + make_item("Top", 15.0, 85.0, 10.0), + ]; + let col_edges = vec![10.0, 50.0]; + let row_edges = vec![90.0, 70.0]; + let (cells, _) = assign_items_to_grid(&items, &col_edges, &row_edges, 1); + assert_eq!(cells[0][0], "Top Bottom"); + } + + // --- is_row_stripe_pattern --- + + #[test] + fn test_is_row_stripe_pattern_too_few_rects() { + let rects = vec![(0.0, 0.0, 300.0, 20.0), (0.0, 25.0, 300.0, 20.0)]; + assert!(!is_row_stripe_pattern(&rects)); + } + + #[test] + fn test_is_row_stripe_pattern_narrow_rects() { + let rects = vec![ + (0.0, 0.0, 50.0, 20.0), + (0.0, 25.0, 50.0, 20.0), + (0.0, 50.0, 50.0, 20.0), + ]; + assert!(!is_row_stripe_pattern(&rects)); + } + + #[test] + fn test_is_row_stripe_pattern_uniform_wide() { + let rects = vec![ + (10.0, 0.0, 500.0, 20.0), + (10.0, 25.0, 500.0, 20.0), + (10.0, 50.0, 500.0, 20.0), + (10.0, 75.0, 500.0, 20.0), + ]; + assert!(is_row_stripe_pattern(&rects)); + } + + #[test] + fn test_is_row_stripe_pattern_mixed_widths() { + let rects = vec![ + (10.0, 0.0, 500.0, 20.0), + (10.0, 25.0, 100.0, 20.0), // Very different width + (10.0, 50.0, 500.0, 20.0), + (10.0, 75.0, 50.0, 20.0), // Very different width + ]; + assert!(!is_row_stripe_pattern(&rects)); + } + + #[test] + fn test_is_row_stripe_pattern_75_percent_boundary() { + // 3 of 4 (75%) within tolerance → should pass (> 0.75) + let rects = vec![ + (10.0, 0.0, 500.0, 20.0), + (10.0, 25.0, 505.0, 20.0), + (10.0, 50.0, 495.0, 20.0), + (10.0, 75.0, 100.0, 20.0), // outlier + ]; + // 3/4 = 0.75 — NOT > 0.75, so false + assert!(!is_row_stripe_pattern(&rects)); + } + + // --- propagate_merged_cells --- + + #[test] + fn test_propagate_merged_cells_spanning_rect() { + // A rect spanning 2 rows in column 0 + let col_edges = vec![0.0, 50.0, 100.0]; + let row_edges = vec![100.0, 80.0, 60.0]; // 2 rows + let mut cells = vec![ + vec!["Top".to_string(), "A".to_string()], + vec!["Bottom".to_string(), "B".to_string()], + ]; + // Rect spanning both rows in col 0 + let group_rects = vec![(0.0, 60.0, 50.0, 40.0)]; + let skip = vec![false]; + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + assert_eq!(cells[0][0], "Top Bottom"); + assert!(cells[1][0].is_empty()); + } + + #[test] + fn test_propagate_merged_cells_single_row_rect_noop() { + // Use well-separated rows so the rect doesn't bleed into adjacent row + // via the 6pt tolerance in propagate_merged_cells. + let col_edges = vec![0.0, 50.0, 100.0]; + let row_edges = vec![200.0, 100.0, 0.0]; + let mut cells = vec![ + vec!["A".to_string(), "B".to_string()], + vec!["C".to_string(), "D".to_string()], + ]; + // Rect clearly inside row 0 only (y=110..190, row 0 is 100..200) + // ry=110 > row_edges[1]+tol = 106, so it doesn't span into row 1 + let group_rects = vec![(0.0, 110.0, 50.0, 80.0)]; + let skip = vec![false]; + let cells_before = cells.clone(); + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + assert_eq!(cells, cells_before); + } + + #[test] + fn test_propagate_merged_cells_skip_rects_respected() { + let col_edges = vec![0.0, 50.0, 100.0]; + let row_edges = vec![100.0, 80.0, 60.0]; + let mut cells = vec![ + vec!["A".to_string(), "B".to_string()], + vec!["C".to_string(), "D".to_string()], + ]; + let group_rects = vec![(0.0, 60.0, 50.0, 40.0)]; + let skip = vec![true]; // Skip this rect + let cells_before = cells.clone(); + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + assert_eq!(cells, cells_before); + } + + #[test] + fn test_propagate_merged_cells_text_in_multiple_sub_rows() { + let col_edges = vec![0.0, 50.0]; + let row_edges = vec![100.0, 80.0, 60.0, 40.0]; // 3 rows + let mut cells = vec![ + vec!["Line1".to_string()], + vec!["Line2".to_string()], + vec!["Line3".to_string()], + ]; + // Rect spanning all 3 rows + let group_rects = vec![(0.0, 40.0, 50.0, 60.0)]; + let skip = vec![false]; + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + assert_eq!(cells[0][0], "Line1 Line2 Line3"); + assert!(cells[1][0].is_empty()); + assert!(cells[2][0].is_empty()); + } + + #[test] + fn test_propagate_merged_cells_full_width_spanning() { + let col_edges = vec![0.0, 50.0, 100.0]; + let row_edges = vec![100.0, 80.0, 60.0]; + let mut cells = vec![ + vec!["A".to_string(), "X".to_string()], + vec!["B".to_string(), "Y".to_string()], + ]; + // Rect spanning both rows but only column 1 + let group_rects = vec![(50.0, 60.0, 50.0, 40.0)]; + let skip = vec![false]; + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + assert_eq!(cells[0][1], "X Y"); + assert!(cells[1][1].is_empty()); + // Column 0 should be unchanged + assert_eq!(cells[0][0], "A"); + assert_eq!(cells[1][0], "B"); + } + + #[test] + fn test_propagate_merged_cells_empty_cells_preserved() { + let col_edges = vec![0.0, 50.0]; + let row_edges = vec![100.0, 80.0, 60.0]; + let mut cells = vec![vec!["Text".to_string()], vec!["".to_string()]]; + // Rect spanning both rows + let group_rects = vec![(0.0, 60.0, 50.0, 40.0)]; + let skip = vec![false]; + propagate_merged_cells(&mut cells, &col_edges, &row_edges, &group_rects, &skip); + // Only "Text" in first row (empty cell contributes nothing) + assert_eq!(cells[0][0], "Text"); + assert!(cells[1][0].is_empty()); + } + + // --- detect_table_from_rect_group / try_build_grid --- + + // Helper: create a 3-row × 2-col grid of rects with 10pt gaps between rows. + // Gaps prevent propagate_merged_cells from collapsing adjacent rows + // (shared-edge rects bleed via the 6pt tolerance). + // Y layout: row0 y=60..80, row1 y=30..50, row2 y=0..20 + fn make_grid_rects() -> Vec<(f32, f32, f32, f32)> { + vec![ + (10.0, 60.0, 40.0, 20.0), // row0, col0 + (50.0, 60.0, 40.0, 20.0), // row0, col1 + (10.0, 30.0, 40.0, 20.0), // row1, col0 + (50.0, 30.0, 40.0, 20.0), // row1, col1 + (10.0, 0.0, 40.0, 20.0), // row2, col0 + (50.0, 0.0, 40.0, 20.0), // row2, col1 + ] + } + + #[test] + fn test_try_build_grid_basic_valid() { + let items = vec![ + make_item("H1", 15.0, 70.0, 10.0), + make_item("H2", 55.0, 70.0, 10.0), + make_item("D1", 15.0, 40.0, 10.0), + make_item("D2", 55.0, 40.0, 10.0), + make_item("E1", 15.0, 10.0, 10.0), + make_item("E2", 55.0, 10.0, 10.0), + ]; + let group_rects = make_grid_rects(); + let skip = vec![false; 6]; + match try_build_grid(&items, &group_rects, 1, &skip, false) { + GridResult::Ok(table) => { + assert!(table.columns.len() >= 2); + assert!(table.rows.len() >= 2); + } + other => panic!( + "Expected Ok, got {:?}", + match other { + GridResult::FewNonEmptyRows => "FewNonEmptyRows", + GridResult::Failed => "Failed", + GridResult::Ok(_) => unreachable!(), + } + ), + } + } + + #[test] + fn test_try_build_grid_too_few_edges() { + // Only 2 rects → not enough edges for a grid + let items = vec![make_item("A", 15.0, 85.0, 10.0)]; + let group_rects = vec![(10.0, 70.0, 40.0, 20.0), (10.0, 50.0, 40.0, 20.0)]; + let skip = vec![false; 2]; + match try_build_grid(&items, &group_rects, 1, &skip, false) { + GridResult::Failed => {} + _ => panic!("Expected Failed"), + } + } + + #[test] + fn test_try_build_grid_strict_rejects_long_text() { + let long_text = "a".repeat(250); + let mut long_item = make_item(&long_text, 15.0, 70.0, 10.0); + // Override width so the item center stays inside the grid cell + long_item.width = 20.0; + let items = vec![ + long_item, + make_item("H2", 55.0, 70.0, 10.0), + make_item("D1", 15.0, 40.0, 10.0), + make_item("D2", 55.0, 40.0, 10.0), + make_item("E1", 15.0, 10.0, 10.0), + make_item("E2", 55.0, 10.0, 10.0), + ]; + let group_rects = make_grid_rects(); + let skip = vec![false; 6]; + match try_build_grid(&items, &group_rects, 1, &skip, true) { + GridResult::Failed => {} + _ => panic!("Expected Failed due to long text in strict mode"), + } + } + + #[test] + fn test_try_build_grid_empty_column_rejected() { + // All items in column 0 only — column 1 is empty + let items = vec![ + make_item("A", 15.0, 70.0, 10.0), + make_item("B", 15.0, 40.0, 10.0), + make_item("C", 15.0, 10.0, 10.0), + ]; + let group_rects = make_grid_rects(); + let skip = vec![false; 6]; + match try_build_grid(&items, &group_rects, 1, &skip, false) { + GridResult::Failed => {} + _ => panic!("Expected Failed due to empty column"), + } + } + + #[test] + fn test_try_build_grid_no_items() { + let items: Vec = vec![]; + let group_rects = make_grid_rects(); + let skip = vec![false; 6]; + match try_build_grid(&items, &group_rects, 1, &skip, false) { + GridResult::Failed => {} + _ => panic!("Expected Failed with no items"), + } + } + + #[test] + fn test_detect_table_from_rect_group_valid() { + let items = vec![ + make_item("H1", 15.0, 70.0, 10.0), + make_item("H2", 55.0, 70.0, 10.0), + make_item("D1", 15.0, 40.0, 10.0), + make_item("D2", 55.0, 40.0, 10.0), + make_item("E1", 15.0, 10.0, 10.0), + make_item("E2", 55.0, 10.0, 10.0), + ]; + let group_rects = make_grid_rects(); + let result = detect_table_from_rect_group(&items, &group_rects, 1); + assert!(result.is_some()); + } + + // --- extract_hint_region --- + + #[test] + fn test_extract_hint_region_valid_small_cluster() { + let rects = vec![ + (10.0, 100.0, 200.0, 30.0), + (10.0, 140.0, 200.0, 30.0), + (10.0, 180.0, 200.0, 30.0), + ]; + let hint = extract_hint_region(&rects); + assert!(hint.is_some()); + let hint = hint.unwrap(); + assert!(hint.y_top > hint.y_bottom); + } + + #[test] + fn test_extract_hint_region_too_few_rects() { + let rects = vec![(10.0, 100.0, 200.0, 30.0)]; + assert!(extract_hint_region(&rects).is_none()); + } + + #[test] + fn test_extract_hint_region_too_many_rects() { + let rects: Vec<(f32, f32, f32, f32)> = (0..10) + .map(|i| (10.0, 100.0 + i as f32 * 30.0, 200.0, 25.0)) + .collect(); + assert!(extract_hint_region(&rects).is_none()); + } +} diff --git a/src/tables/format.rs b/src/tables/format.rs index b6a9e52..2793b1e 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -180,3 +180,235 @@ fn is_footnote_row(text: &str) -> bool { false } + +#[cfg(test)] +mod tests { + use super::*; + + // --- is_footnote_row --- + + #[test] + fn test_is_footnote_row_parenthesized_number() { + assert!(is_footnote_row("(1)")); + assert!(is_footnote_row("(23)")); + } + + #[test] + fn test_is_footnote_row_number_paren() { + assert!(is_footnote_row("1)")); + assert!(is_footnote_row("12)")); + } + + #[test] + fn test_is_footnote_row_note_colon() { + assert!(is_footnote_row("Note: some text")); + assert!(is_footnote_row("note: lowercase")); + } + + #[test] + fn test_is_footnote_row_notes_colon() { + assert!(is_footnote_row("Notes: multiple")); + assert!(is_footnote_row("NOTES: uppercase")); + } + + #[test] + fn test_is_footnote_row_plain_text_false() { + assert!(!is_footnote_row("Regular cell text")); + assert!(!is_footnote_row("Amount")); + } + + #[test] + fn test_is_footnote_row_empty_false() { + assert!(!is_footnote_row("")); + } + + // --- clean_table_cells --- + + #[test] + fn test_clean_table_cells_empty_rows_removed() { + let cells = vec![ + vec!["A".into(), "B".into()], + vec!["".into(), "".into()], + vec!["C".into(), "D".into()], + ]; + let (cleaned, _) = clean_table_cells(&cells); + assert_eq!(cleaned.len(), 2); + assert_eq!(cleaned[0], vec!["A", "B"]); + assert_eq!(cleaned[1], vec!["C", "D"]); + } + + #[test] + fn test_clean_table_cells_footnote_extracted() { + let cells = vec![ + vec!["Header".into(), "Value".into()], + vec!["Data".into(), "100".into()], + vec!["(1)".into(), "See appendix".into()], + ]; + let (cleaned, footnotes) = clean_table_cells(&cells); + assert_eq!(cleaned.len(), 2); + assert_eq!(footnotes.len(), 1); + assert!(footnotes[0].contains("(1)")); + assert!(footnotes[0].contains("See appendix")); + } + + #[test] + fn test_clean_table_cells_continuation_row_merged() { + let cells = vec![ + vec!["Header".into(), "Col2".into()], + vec!["Row1".into(), "Short".into()], + vec!["".into(), "continued text here".into()], + ]; + let (cleaned, _) = clean_table_cells(&cells); + // The continuation row should merge into the previous row + assert_eq!(cleaned.len(), 2); + assert!(cleaned[1][1].contains("Short")); + assert!(cleaned[1][1].contains("continued text here")); + } + + #[test] + fn test_clean_table_cells_short_subheader_not_merged() { + let cells = vec![ + vec!["Header".into(), "Col2".into()], + vec!["Row1".into(), "Data".into()], + vec!["".into(), "JAN".into()], + ]; + let (cleaned, _) = clean_table_cells(&cells); + // Short subheader (<=5 chars, single non-empty cell) should not merge + assert_eq!(cleaned.len(), 3); + } + + #[test] + fn test_clean_table_cells_numeric_data_row_not_merged() { + let cells = vec![ + vec!["Header".into(), "A".into(), "B".into(), "C".into()], + vec!["Row1".into(), "10".into(), "20".into(), "30".into()], + vec!["".into(), "40".into(), "50".into(), "60".into()], + ]; + let (cleaned, _) = clean_table_cells(&cells); + // Numeric data row with empty first col should not merge + assert_eq!(cleaned.len(), 3); + } + + #[test] + fn test_clean_table_cells_header_row_not_merged() { + // Continuation requires cleaned.len() > 1 (don't merge into header) + let cells = vec![ + vec!["Header".into(), "Col2".into()], + vec!["".into(), "continuation text goes here".into()], + ]; + let (cleaned, _) = clean_table_cells(&cells); + // Should not merge into first row (header) + assert_eq!(cleaned.len(), 2); + } + + #[test] + fn test_clean_table_cells_all_empty() { + let cells = vec![vec!["".into(), "".into()], vec![" ".into(), "".into()]]; + let (cleaned, footnotes) = clean_table_cells(&cells); + assert!(cleaned.is_empty()); + assert!(footnotes.is_empty()); + } + + #[test] + fn test_clean_table_cells_mixed_scenario() { + let cells = vec![ + vec!["Name".into(), "Score".into()], + vec!["Alice".into(), "95".into()], + vec!["".into(), "".into()], + vec!["Bob".into(), "87".into()], + vec!["Note: graded on curve".into(), "".into()], + ]; + let (cleaned, footnotes) = clean_table_cells(&cells); + assert_eq!(cleaned.len(), 3); // header + Alice + Bob (empty row removed) + assert_eq!(footnotes.len(), 1); + assert!(footnotes[0].contains("Note:")); + } + + // --- table_to_markdown --- + + #[test] + fn test_table_to_markdown_basic() { + let table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0, 460.0], + cells: vec![ + vec!["Name".into(), "Age".into()], + vec!["Alice".into(), "30".into()], + vec!["Bob".into(), "25".into()], + ], + item_indices: vec![], + }; + let md = table_to_markdown(&table); + assert!(md.contains("| Name")); + assert!(md.contains("| ---")); + assert!(md.contains("| Alice")); + assert!(md.contains("| Bob")); + } + + #[test] + fn test_table_to_markdown_single_row() { + let table = Table { + columns: vec![100.0], + rows: vec![500.0], + cells: vec![vec!["Only".into(), "Row".into()]], + item_indices: vec![], + }; + let md = table_to_markdown(&table); + assert!(md.contains("| Only")); + assert!(md.contains("| ---")); + } + + #[test] + fn test_table_to_markdown_empty_table() { + let table = Table { + columns: vec![], + rows: vec![], + cells: vec![], + item_indices: vec![], + }; + assert_eq!(table_to_markdown(&table), ""); + } + + #[test] + fn test_table_to_markdown_footnotes_appended() { + let table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0, 460.0], + cells: vec![ + vec!["Header".into(), "Value".into()], + vec!["Data".into(), "100".into()], + vec!["(1)".into(), "Footnote text".into()], + ], + item_indices: vec![], + }; + let md = table_to_markdown(&table); + assert!(md.contains("(1) Footnote text")); + } + + #[test] + fn test_table_to_markdown_unicode_content() { + let table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0], + cells: vec![ + vec!["名前".into(), "年齢".into()], + vec!["太郎".into(), "25".into()], + ], + item_indices: vec![], + }; + let md = table_to_markdown(&table); + assert!(md.contains("名前")); + assert!(md.contains("太郎")); + } + + #[test] + fn test_table_to_markdown_empty_first_row() { + let table = Table { + columns: vec![100.0], + rows: vec![500.0], + cells: vec![vec![]], + item_indices: vec![], + }; + assert_eq!(table_to_markdown(&table), ""); + } +} diff --git a/src/tables/grid.rs b/src/tables/grid.rs index 739bc45..7ebd99c 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -306,3 +306,342 @@ pub(crate) fn recover_header_row( table.cells.insert(0, header_cells); table.item_indices.extend(header_indices); } + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::ItemType; + + fn make_item(text: &str, x: f32, y: f32, font_size: f32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + // --- find_column_index --- + + #[test] + fn test_find_column_index_exact_match() { + let columns = vec![100.0, 200.0, 300.0]; + assert_eq!(find_column_index(&columns, 100.0), Some(0)); + assert_eq!(find_column_index(&columns, 200.0), Some(1)); + assert_eq!(find_column_index(&columns, 300.0), Some(2)); + } + + #[test] + fn test_find_column_index_closest_within_threshold() { + let columns = vec![100.0, 200.0, 300.0]; + assert_eq!(find_column_index(&columns, 105.0), Some(0)); + assert_eq!(find_column_index(&columns, 195.0), Some(1)); + } + + #[test] + fn test_find_column_index_outside_threshold() { + let columns = vec![100.0, 200.0, 300.0]; + // Threshold is clamped to min 25, max 50 based on min gap / 2 + // Min gap = 100, threshold = clamp(50, 25, 50) = 50 + assert_eq!(find_column_index(&columns, 500.0), None); + } + + #[test] + fn test_find_column_index_single_column() { + let columns = vec![150.0]; + // Single column → threshold = 50.0 + assert_eq!(find_column_index(&columns, 150.0), Some(0)); + assert_eq!(find_column_index(&columns, 170.0), Some(0)); + } + + #[test] + fn test_find_column_index_empty_columns() { + let columns: Vec = vec![]; + assert_eq!(find_column_index(&columns, 100.0), None); + } + + // --- find_row_index --- + + #[test] + fn test_find_row_index_exact_match() { + let rows = vec![500.0, 480.0, 460.0]; + assert_eq!(find_row_index(&rows, 500.0), Some(0)); + assert_eq!(find_row_index(&rows, 480.0), Some(1)); + } + + #[test] + fn test_find_row_index_within_threshold() { + let rows = vec![500.0, 480.0, 460.0]; + assert_eq!(find_row_index(&rows, 505.0), Some(0)); + assert_eq!(find_row_index(&rows, 475.0), Some(1)); + } + + #[test] + fn test_find_row_index_outside_threshold() { + let rows = vec![500.0, 480.0, 460.0]; + // threshold is 15.0 + assert_eq!(find_row_index(&rows, 400.0), None); + } + + #[test] + fn test_find_row_index_single_row() { + let rows = vec![500.0]; + assert_eq!(find_row_index(&rows, 500.0), Some(0)); + assert_eq!(find_row_index(&rows, 510.0), Some(0)); + } + + // --- find_column_boundaries --- + + #[test] + fn test_find_column_boundaries_empty() { + let items: Vec<(usize, &TextItem)> = vec![]; + assert_eq!( + find_column_boundaries(&items, TableDetectionMode::SmallFont), + vec![] + ); + } + + #[test] + fn test_find_column_boundaries_two_clusters() { + // Items at x=100 and x=200 with enough repetition + let items_data: Vec = (0..10) + .map(|i| { + let x = if i % 2 == 0 { 100.0 } else { 200.0 }; + make_item("Cell", x, 500.0 - (i as f32 * 20.0), 10.0) + }) + .collect(); + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let cols = find_column_boundaries(&items, TableDetectionMode::SmallFont); + assert_eq!(cols.len(), 2); + } + + #[test] + fn test_find_column_boundaries_single_item() { + let item = make_item("Solo", 100.0, 500.0, 10.0); + let items: Vec<(usize, &TextItem)> = vec![(0, &item)]; + // Single item won't pass the min_items_per_col filter (needs >=2) + let cols = find_column_boundaries(&items, TableDetectionMode::SmallFont); + assert!(cols.is_empty()); + } + + #[test] + fn test_find_column_boundaries_body_font_paragraph_rejection() { + // All items at same X → >60% in one column → rejected in BodyFont mode + let items_data: Vec = (0..10) + .map(|i| make_item("Text", 100.0, 500.0 - (i as f32 * 20.0), 10.0)) + .collect(); + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let cols = find_column_boundaries(&items, TableDetectionMode::BodyFont); + assert!(cols.is_empty()); + } + + #[test] + fn test_find_column_boundaries_min_items_filter() { + // Create 10 items at x=100 and 1 item at x=300 + // The single outlier should be filtered out + let mut items_data: Vec = (0..10) + .map(|i| make_item("Cell", 100.0, 500.0 - (i as f32 * 20.0), 10.0)) + .collect(); + items_data.push(make_item("Lone", 300.0, 500.0, 10.0)); + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let cols = find_column_boundaries(&items, TableDetectionMode::SmallFont); + // Only the cluster at x=100 should survive + assert!(cols.len() <= 1); + } + + // --- find_row_boundaries --- + + #[test] + fn test_find_row_boundaries_empty() { + let items: Vec<(usize, &TextItem)> = vec![]; + assert_eq!(find_row_boundaries(&items), vec![]); + } + + #[test] + fn test_find_row_boundaries_descending_order() { + let items_data = vec![ + make_item("A", 100.0, 500.0, 10.0), + make_item("B", 100.0, 480.0, 10.0), + make_item("C", 100.0, 460.0, 10.0), + ]; + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let rows = find_row_boundaries(&items); + assert_eq!(rows.len(), 3); + // Should be in descending order + assert!(rows[0] > rows[1]); + assert!(rows[1] > rows[2]); + } + + #[test] + fn test_find_row_boundaries_clustering() { + // Items close together should cluster into one row + let items_data = vec![ + make_item("A", 100.0, 500.0, 10.0), + make_item("B", 200.0, 501.0, 10.0), + make_item("C", 100.0, 480.0, 10.0), + ]; + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let rows = find_row_boundaries(&items); + assert_eq!(rows.len(), 2); // 500 and 501 cluster together + } + + #[test] + fn test_find_row_boundaries_single_row() { + let items_data = vec![make_item("A", 100.0, 500.0, 10.0)]; + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let rows = find_row_boundaries(&items); + assert_eq!(rows.len(), 1); + assert!((rows[0] - 500.0).abs() < 0.01); + } + + #[test] + fn test_find_row_boundaries_items_at_same_y() { + let items_data = vec![ + make_item("A", 100.0, 500.0, 10.0), + make_item("B", 200.0, 500.0, 10.0), + make_item("C", 300.0, 500.0, 10.0), + ]; + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let rows = find_row_boundaries(&items); + assert_eq!(rows.len(), 1); + } + + // --- join_cell_items --- + + #[test] + fn test_join_cell_items_single_item() { + let item = make_item("Hello", 100.0, 500.0, 10.0); + assert_eq!(join_cell_items(&[&item]), "Hello"); + } + + #[test] + fn test_join_cell_items_multiple_spaced() { + let a = make_item("Hello", 100.0, 500.0, 10.0); + let b = make_item("World", 150.0, 500.0, 10.0); + assert_eq!(join_cell_items(&[&a, &b]), "Hello World"); + } + + #[test] + fn test_join_cell_items_hyphen_no_space() { + let a = make_item("pre", 100.0, 500.0, 10.0); + let b = make_item("-", 120.0, 500.0, 10.0); + let c = make_item("fix", 130.0, 500.0, 10.0); + assert_eq!(join_cell_items(&[&a, &b, &c]), "pre-fix"); + } + + #[test] + fn test_join_cell_items_subscript_no_space() { + let a = make_item("H", 100.0, 500.0, 12.0); + let b = make_item("2", 110.0, 497.0, 8.0); // smaller font, Y offset + assert_eq!(join_cell_items(&[&a, &b]), "H2"); + } + + #[test] + fn test_join_cell_items_empty_items_skipped() { + let a = make_item("Hello", 100.0, 500.0, 10.0); + let b = make_item(" ", 120.0, 500.0, 10.0); + let c = make_item("World", 150.0, 500.0, 10.0); + assert_eq!(join_cell_items(&[&a, &b, &c]), "Hello World"); + } + + // --- recover_header_row --- + + #[test] + fn test_recover_header_row_prepends_header() { + let all_items = vec![ + make_item("Col1", 100.0, 520.0, 12.0), // body font, above table + make_item("Col2", 200.0, 520.0, 12.0), // body font, above table + make_item("A", 100.0, 500.0, 8.0), // small font, in table + make_item("B", 200.0, 500.0, 8.0), + ]; + let mut table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0], + cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]], + item_indices: vec![2, 3], + }; + + recover_header_row(&mut table, &all_items, 9.0); + assert_eq!(table.cells.len(), 3); + assert_eq!(table.cells[0], vec!["Col1", "Col2"]); + } + + #[test] + fn test_recover_header_row_no_candidates() { + let all_items = vec![ + make_item("A", 100.0, 500.0, 8.0), + make_item("B", 200.0, 500.0, 8.0), + ]; + let mut table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0], + cells: vec![vec!["A".into(), "B".into()]], + item_indices: vec![0, 1], + }; + + let rows_before = table.rows.len(); + recover_header_row(&mut table, &all_items, 9.0); + assert_eq!(table.rows.len(), rows_before); + } + + #[test] + fn test_recover_header_row_too_far_above() { + let all_items = vec![ + make_item("Col1", 100.0, 600.0, 12.0), // way above + make_item("Col2", 200.0, 600.0, 12.0), + make_item("A", 100.0, 500.0, 8.0), + make_item("B", 200.0, 500.0, 8.0), + ]; + let mut table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0, 480.0], + cells: vec![vec!["A".into(), "B".into()], vec!["C".into(), "D".into()]], + item_indices: vec![2, 3], + }; + + let rows_before = table.rows.len(); + recover_header_row(&mut table, &all_items, 9.0); + assert_eq!(table.rows.len(), rows_before); + } + + #[test] + fn test_recover_header_row_single_column_populated() { + // Only 1 column populated → not a real header + let all_items = vec![ + make_item("OnlyCol1", 100.0, 520.0, 12.0), + make_item("A", 100.0, 500.0, 8.0), + make_item("B", 200.0, 500.0, 8.0), + ]; + let mut table = Table { + columns: vec![100.0, 200.0], + rows: vec![500.0], + cells: vec![vec!["A".into(), "B".into()]], + item_indices: vec![1, 2], + }; + + let rows_before = table.rows.len(); + recover_header_row(&mut table, &all_items, 9.0); + assert_eq!(table.rows.len(), rows_before); + } + + #[test] + fn test_recover_header_row_empty_table() { + let all_items = vec![make_item("Col1", 100.0, 520.0, 12.0)]; + let mut table = Table { + columns: vec![], + rows: vec![], + cells: vec![], + item_indices: vec![], + }; + + recover_header_row(&mut table, &all_items, 9.0); + assert!(table.cells.is_empty()); + } +}