chore(refactor): Better organization of the codebase, split modules, update docs, add AGENTS.md
This commit is contained in:
Abimael Martell
@@ -0,0 +1,893 @@
|
||||
//! Text extraction from PDF using lopdf
|
||||
//!
|
||||
//! This module extracts text with position information for structure detection.
|
||||
|
||||
mod content_stream;
|
||||
mod fonts;
|
||||
mod layout;
|
||||
mod links;
|
||||
mod xobjects;
|
||||
|
||||
use crate::text_utils::is_rtl_text;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{PdfRect, TextItem};
|
||||
use crate::PdfError;
|
||||
use lopdf::{Document, Object, ObjectId};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
|
||||
use content_stream::extract_page_text_items;
|
||||
use links::{extract_form_fields, extract_page_links};
|
||||
|
||||
// Re-export public types so existing `crate::extractor::X` paths keep working.
|
||||
pub use crate::text_utils::{is_bold_font, is_italic_font};
|
||||
pub use crate::types::{ItemType, TextLine};
|
||||
pub use layout::group_into_lines;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract text from PDF file as plain string
|
||||
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_file(&path)?;
|
||||
let doc = Document::load(path)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
/// Extract text from PDF memory buffer
|
||||
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
let doc = Document::load_mem(buffer)?;
|
||||
extract_text_from_doc(&doc)
|
||||
}
|
||||
|
||||
/// Extract text from loaded document
|
||||
fn extract_text_from_doc(doc: &Document) -> Result<String, PdfError> {
|
||||
let pages = doc.get_pages();
|
||||
let page_nums: Vec<u32> = pages.keys().cloned().collect();
|
||||
|
||||
doc.extract_text(&page_nums)
|
||||
.map_err(|e| PdfError::Parse(e.to_string()))
|
||||
}
|
||||
|
||||
/// Extract text with position information from PDF file
|
||||
pub fn extract_text_with_positions<P: AsRef<Path>>(path: P) -> Result<Vec<TextItem>, PdfError> {
|
||||
extract_text_with_positions_pages(path, None)
|
||||
}
|
||||
|
||||
/// Extract text with positions from a file, limited to specific pages.
|
||||
///
|
||||
/// `page_filter` is an optional set of 1-indexed page numbers to process.
|
||||
/// When `None`, all pages are processed.
|
||||
pub fn extract_text_with_positions_pages<P: AsRef<Path>>(
|
||||
path: P,
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<Vec<TextItem>, PdfError> {
|
||||
let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?;
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Extract text with positions and rectangles from a file.
|
||||
pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
|
||||
path: P,
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
|
||||
// Read the raw PDF bytes for ToUnicode extraction
|
||||
let pdf_bytes = std::fs::read(path.as_ref())?;
|
||||
crate::validate_pdf_bytes(&pdf_bytes)?;
|
||||
let font_cmaps = FontCMaps::from_pdf_bytes(&pdf_bytes);
|
||||
|
||||
let doc = Document::load_mem(&pdf_bytes)?;
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
|
||||
}
|
||||
|
||||
/// Extract text with positions from memory buffer
|
||||
pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result<Vec<TextItem>, PdfError> {
|
||||
extract_text_with_positions_mem_pages(buffer, None)
|
||||
}
|
||||
|
||||
/// Extract text with positions from memory buffer, limited to specific pages.
|
||||
pub fn extract_text_with_positions_mem_pages(
|
||||
buffer: &[u8],
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<Vec<TextItem>, PdfError> {
|
||||
let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?;
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Extract text with positions and rectangles from memory buffer.
|
||||
pub(crate) fn extract_text_with_positions_mem_and_rects(
|
||||
buffer: &[u8],
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
|
||||
crate::validate_pdf_bytes(buffer)?;
|
||||
// Extract ToUnicode CMaps from raw PDF bytes
|
||||
let font_cmaps = FontCMaps::from_pdf_bytes(buffer);
|
||||
|
||||
let doc = Document::load_mem(buffer)?;
|
||||
extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Orchestration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract positioned text and rectangles from loaded document
|
||||
fn extract_positioned_text_from_doc(
|
||||
doc: &Document,
|
||||
font_cmaps: &FontCMaps,
|
||||
page_filter: Option<&HashSet<u32>>,
|
||||
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
|
||||
// If raw byte scanning found no CMaps, populate from the document model.
|
||||
// This handles PDFs with compressed object streams where raw scanning fails.
|
||||
let mut font_cmaps_owned;
|
||||
let font_cmaps = if font_cmaps.by_obj_num.is_empty() {
|
||||
font_cmaps_owned = font_cmaps.clone();
|
||||
populate_cmaps_from_doc(doc, &mut font_cmaps_owned);
|
||||
&font_cmaps_owned
|
||||
} else {
|
||||
font_cmaps
|
||||
};
|
||||
|
||||
let pages = doc.get_pages();
|
||||
let mut all_items = Vec::new();
|
||||
let mut all_rects = Vec::new();
|
||||
|
||||
// Build page ObjectId → page number map for form field extraction
|
||||
let page_id_to_num: HashMap<ObjectId, u32> =
|
||||
pages.iter().map(|(num, &id)| (id, *num)).collect();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if let Some(filter) = page_filter {
|
||||
if !filter.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
|
||||
all_items.extend(items);
|
||||
all_rects.extend(rects);
|
||||
|
||||
// Extract hyperlinks from page annotations
|
||||
let links = extract_page_links(doc, page_id, *page_num);
|
||||
all_items.extend(links);
|
||||
}
|
||||
|
||||
// Extract AcroForm field values
|
||||
let form_items = extract_form_fields(doc, &page_id_to_num);
|
||||
all_items.extend(form_items);
|
||||
|
||||
Ok((all_items, all_rects))
|
||||
}
|
||||
|
||||
/// Populate FontCMaps from the lopdf document model for ToUnicode streams
|
||||
/// that weren't found by raw byte scanning (e.g. in compressed object streams).
|
||||
fn populate_cmaps_from_doc(doc: &Document, font_cmaps: &mut FontCMaps) {
|
||||
use crate::tounicode::ToUnicodeCMap;
|
||||
|
||||
for (_page_num, &page_id) in doc.get_pages().iter() {
|
||||
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
|
||||
for (font_name, font_dict) in &fonts {
|
||||
if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") {
|
||||
if let Ok(obj_ref) = tounicode_ref.as_reference() {
|
||||
let obj_num = obj_ref.0;
|
||||
if font_cmaps.by_obj_num.contains_key(&obj_num) {
|
||||
continue;
|
||||
}
|
||||
// Try to get the stream content via lopdf
|
||||
if let Ok(stream) = doc.get_object(obj_ref) {
|
||||
if let Ok(stream) = stream.as_stream() {
|
||||
if let Ok(data) = stream.decompressed_content() {
|
||||
if let Some(cmap) = ToUnicodeCMap::parse(&data) {
|
||||
let resource_name =
|
||||
String::from_utf8_lossy(font_name).to_string();
|
||||
let base_name = font_dict
|
||||
.get(b"BaseFont")
|
||||
.ok()
|
||||
.and_then(|o| o.as_name().ok())
|
||||
.map(|n| String::from_utf8_lossy(n).to_string());
|
||||
|
||||
// Store by object number
|
||||
font_cmaps.by_obj_num.insert(obj_num, cmap.clone());
|
||||
// Store by resource name
|
||||
font_cmaps
|
||||
.by_name
|
||||
.insert(resource_name.clone(), cmap.clone());
|
||||
if let Some(base) = base_name {
|
||||
let unique_key = format!("{}_{}", base, obj_num);
|
||||
font_cmaps.by_name.insert(unique_key, cmap.clone());
|
||||
font_cmaps.by_name.insert(base, cmap);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (used by submodules via `super::`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Multiply two 2D transformation matrices
|
||||
/// Matrix format: [a, b, c, d, e, f] representing:
|
||||
/// | a b 0 |
|
||||
/// | c d 0 |
|
||||
/// | e f 1 |
|
||||
pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] {
|
||||
[
|
||||
m1[0] * m2[0] + m1[1] * m2[2],
|
||||
m1[0] * m2[1] + m1[1] * m2[3],
|
||||
m1[2] * m2[0] + m1[3] * m2[2],
|
||||
m1[2] * m2[1] + m1[3] * m2[3],
|
||||
m1[4] * m2[0] + m1[5] * m2[2] + m2[4],
|
||||
m1[4] * m2[1] + m1[5] * m2[3] + m2[5],
|
||||
]
|
||||
}
|
||||
|
||||
/// Merge adjacent text items on the same line into single items.
|
||||
///
|
||||
/// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each
|
||||
/// group by X, then merges consecutive items that share a similar font size
|
||||
/// and are close horizontally.
|
||||
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if items.is_empty() {
|
||||
return items;
|
||||
}
|
||||
|
||||
// Group items by (page, Y position) with 5pt tolerance
|
||||
let y_tolerance = 5.0;
|
||||
let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new();
|
||||
|
||||
for item in &items {
|
||||
let found = line_groups
|
||||
.iter_mut()
|
||||
.find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance);
|
||||
if let Some((_, _, group)) = found {
|
||||
group.push(item);
|
||||
} else {
|
||||
line_groups.push((item.page, item.y, vec![item]));
|
||||
}
|
||||
}
|
||||
|
||||
// Sort each group by X position (direction-aware)
|
||||
for (_, _, group) in &mut line_groups {
|
||||
let rtl = is_rtl_text(group.iter().map(|i| &i.text));
|
||||
if rtl {
|
||||
group.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal));
|
||||
} else {
|
||||
group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal));
|
||||
}
|
||||
}
|
||||
|
||||
// Sort groups by page then Y descending (top of page first)
|
||||
line_groups.sort_by(|a, b| {
|
||||
a.0.cmp(&b.0)
|
||||
.then_with(|| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal))
|
||||
});
|
||||
|
||||
let mut merged = Vec::new();
|
||||
|
||||
for (_, _, group) in &line_groups {
|
||||
let mut i = 0;
|
||||
while i < group.len() {
|
||||
let first = group[i];
|
||||
let mut text = first.text.clone();
|
||||
let mut end_x = first.x + first.width;
|
||||
let x_gap_max = first.font_size * 0.5;
|
||||
|
||||
let mut j = i + 1;
|
||||
while j < group.len() {
|
||||
let next = group[j];
|
||||
// Must be similar font size (within 20%)
|
||||
if (next.font_size - first.font_size).abs() > first.font_size * 0.20 {
|
||||
break;
|
||||
}
|
||||
let gap = next.x - end_x;
|
||||
if gap > x_gap_max {
|
||||
break;
|
||||
}
|
||||
if gap < -first.font_size * 0.5 {
|
||||
break;
|
||||
}
|
||||
// Insert space at word boundaries
|
||||
if gap > first.font_size * 0.08 {
|
||||
text.push(' ');
|
||||
}
|
||||
text.push_str(&next.text);
|
||||
end_x = next.x + next.width;
|
||||
j += 1;
|
||||
}
|
||||
|
||||
merged.push(TextItem {
|
||||
text,
|
||||
x: first.x,
|
||||
y: first.y,
|
||||
width: end_x - first.x,
|
||||
height: first.height,
|
||||
font: first.font.clone(),
|
||||
font_size: first.font_size,
|
||||
page: first.page,
|
||||
is_bold: first.is_bold,
|
||||
is_italic: first.is_italic,
|
||||
item_type: first.item_type.clone(),
|
||||
});
|
||||
|
||||
i = j;
|
||||
}
|
||||
}
|
||||
|
||||
merged
|
||||
}
|
||||
|
||||
/// Helper to get f32 from Object
|
||||
pub(crate) fn get_number(obj: &Object) -> Option<f32> {
|
||||
match obj {
|
||||
Object::Integer(i) => Some(*i as f32),
|
||||
Object::Real(r) => Some(*r),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items};
|
||||
use crate::types::{ItemType, TextLine};
|
||||
use layout::{detect_columns, is_newspaper_layout};
|
||||
|
||||
#[test]
|
||||
fn test_group_into_lines() {
|
||||
let items = vec![
|
||||
TextItem {
|
||||
text: "Hello".into(),
|
||||
x: 100.0,
|
||||
y: 700.0,
|
||||
width: 50.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "World".into(),
|
||||
x: 160.0,
|
||||
y: 700.0,
|
||||
width: 50.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "Next line".into(),
|
||||
x: 100.0,
|
||||
y: 680.0,
|
||||
width: 80.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert_eq!(lines[0].text(), "Hello World");
|
||||
assert_eq!(lines[1].text(), "Next line");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bold_italic_detection() {
|
||||
// Test bold detection
|
||||
assert!(is_bold_font("Arial-Bold"));
|
||||
assert!(is_bold_font("TimesNewRoman-Bold"));
|
||||
assert!(is_bold_font("Helvetica-BoldOblique"));
|
||||
assert!(is_bold_font("ABCDEF+ArialMT-Bold"));
|
||||
assert!(is_bold_font("NotoSans-Black"));
|
||||
assert!(is_bold_font("Roboto-SemiBold"));
|
||||
assert!(!is_bold_font("Arial"));
|
||||
assert!(!is_bold_font("TimesNewRoman-Italic"));
|
||||
|
||||
// Test italic detection
|
||||
assert!(is_italic_font("Arial-Italic"));
|
||||
assert!(is_italic_font("TimesNewRoman-Italic"));
|
||||
assert!(is_italic_font("Helvetica-Oblique"));
|
||||
assert!(is_italic_font("ABCDEF+ArialMT-Italic"));
|
||||
assert!(is_italic_font("Helvetica-BoldOblique"));
|
||||
assert!(!is_italic_font("Arial"));
|
||||
assert!(!is_italic_font("TimesNewRoman-Bold"));
|
||||
|
||||
// Test bold-italic detection
|
||||
assert!(is_bold_font("Arial-BoldItalic"));
|
||||
assert!(is_italic_font("Arial-BoldItalic"));
|
||||
assert!(is_bold_font("Helvetica-BoldOblique"));
|
||||
assert!(is_italic_font("Helvetica-BoldOblique"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_word_level_items_get_spaces() {
|
||||
// Simulate CID font per-word items touching with gap=0
|
||||
let items = vec![
|
||||
TextItem {
|
||||
text: "the".into(),
|
||||
x: 100.0,
|
||||
y: 500.0,
|
||||
width: 19.5,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "Prague".into(),
|
||||
x: 119.5,
|
||||
y: 500.0,
|
||||
width: 42.0,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "Rules".into(),
|
||||
x: 161.5,
|
||||
y: 500.0,
|
||||
width: 35.0,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "the Prague Rules");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_char_items_still_join() {
|
||||
// Per-glyph positioning: single chars should join into words
|
||||
let items = vec![
|
||||
TextItem {
|
||||
text: "N".into(),
|
||||
x: 100.0,
|
||||
y: 500.0,
|
||||
width: 8.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "A".into(),
|
||||
x: 108.0,
|
||||
y: 500.0,
|
||||
width: 8.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "V".into(),
|
||||
x: 116.0,
|
||||
y: 500.0,
|
||||
width: 8.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "NAV");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_per_glyph_word_boundaries() {
|
||||
// Per-character PDF rendering (e.g. SEC filings): each glyph is a
|
||||
// separate TextItem. Intra-word gaps are ≈ 0, word gaps ≈ 2.0 at
|
||||
// font_size 13.3 (ratio 0.15). Must detect word boundaries correctly.
|
||||
fn char_item(ch: &str, x: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: ch.into(),
|
||||
x,
|
||||
y: 719.3,
|
||||
width,
|
||||
height: 13.3,
|
||||
font: "F4".into(),
|
||||
font_size: 13.3,
|
||||
page: 1,
|
||||
is_bold: true,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}
|
||||
}
|
||||
|
||||
// "Item 2" — gap of 2.0 between 'm' and '2' at font_size 13.3
|
||||
let items = vec![
|
||||
char_item("I", 24.3, 3.1),
|
||||
char_item("t", 27.5, 2.7),
|
||||
char_item("e", 30.1, 3.5),
|
||||
char_item("m", 33.7, 6.7),
|
||||
char_item("2", 42.3, 4.0), // gap = 42.3 - 40.4 = 1.9
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "Item 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_per_glyph_words_not_merged() {
|
||||
// Verify multiple words from per-character rendering get spaces between them
|
||||
fn char_item(ch: &str, x: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: ch.into(),
|
||||
x,
|
||||
y: 705.5,
|
||||
width,
|
||||
height: 13.3,
|
||||
font: "F5".into(),
|
||||
font_size: 13.3,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}
|
||||
}
|
||||
|
||||
// "of the" — three words, each with ~2px word gaps
|
||||
let items = vec![
|
||||
char_item("o", 100.0, 4.0),
|
||||
char_item("f", 104.0, 2.7),
|
||||
// word gap: 108.7 → 110.7 (gap = 4.0)
|
||||
char_item("t", 110.7, 2.7),
|
||||
char_item("h", 113.4, 4.4),
|
||||
char_item("e", 117.8, 3.5),
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "of the");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cjk_items_join_without_spaces() {
|
||||
// Japanese text items touching at gap=0 should join without spaces
|
||||
let items = vec![
|
||||
TextItem {
|
||||
text: "である".into(),
|
||||
x: 100.0,
|
||||
y: 500.0,
|
||||
width: 24.0,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "履行義務".into(),
|
||||
x: 124.0,
|
||||
y: 500.0,
|
||||
width: 32.0,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "を識別す".into(),
|
||||
x: 156.0,
|
||||
y: 500.0,
|
||||
width: 32.0,
|
||||
height: 12.0,
|
||||
font: "C2_0".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
|
||||
let lines = group_into_lines(items);
|
||||
assert_eq!(lines.len(), 1);
|
||||
assert_eq!(lines[0].text(), "である履行義務を識別す");
|
||||
}
|
||||
|
||||
fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem {
|
||||
TextItem {
|
||||
text: text.into(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_two_columns() {
|
||||
let mut items = Vec::new();
|
||||
// Left column at x=72, right column at x=350, gutter ~278-350
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Left text here", 72.0, y, 200.0));
|
||||
items.push(make_item("Right text here", 350.0, y, 200.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols);
|
||||
assert!(cols[0].x_min < cols[1].x_min);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_three_columns() {
|
||||
let mut items = Vec::new();
|
||||
// Three columns at x=50, x=220, x=390
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Col one", 50.0, y, 140.0));
|
||||
items.push(make_item("Col two", 220.0, y, 140.0));
|
||||
items.push(make_item("Col three", 390.0, y, 140.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_width_bleed_tolerance() {
|
||||
let mut items = Vec::new();
|
||||
// Two columns with a clear gutter
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("Left text", 72.0, y, 200.0));
|
||||
items.push(make_item("Right text", 350.0, y, 200.0));
|
||||
}
|
||||
// Add a few items that bleed across the gutter
|
||||
for i in 0..3 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item("wide", 72.0, y, 320.0));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert!(
|
||||
cols.len() >= 2,
|
||||
"Width bleed should not prevent column detection, got {:?}",
|
||||
cols
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_column_no_false_split() {
|
||||
let mut items = Vec::new();
|
||||
// Single column: items spanning full width
|
||||
for i in 0..30 {
|
||||
let y = 700.0 - (i as f32) * 14.0;
|
||||
items.push(make_item(
|
||||
"This is a full-width paragraph of text",
|
||||
72.0,
|
||||
y,
|
||||
468.0,
|
||||
));
|
||||
}
|
||||
let cols = detect_columns(&items, 1);
|
||||
assert!(
|
||||
cols.len() <= 1,
|
||||
"Full-width text should not be split into columns, got {:?}",
|
||||
cols
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_rtl_char() {
|
||||
// Hebrew alef
|
||||
assert!(is_rtl_char('\u{05D0}'));
|
||||
// Arabic alif
|
||||
assert!(is_rtl_char('\u{0627}'));
|
||||
// Latin 'A' is not RTL
|
||||
assert!(!is_rtl_char('A'));
|
||||
// CJK is not RTL
|
||||
assert!(!is_rtl_char('\u{4E00}'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_rtl_text() {
|
||||
// Majority Hebrew with digits → RTL
|
||||
assert!(is_rtl_text(["\u{05E9}\u{05DC}\u{05D5}\u{05DD} 123"].iter()));
|
||||
// Majority Latin → not RTL
|
||||
assert!(!is_rtl_text(["Hello world"].iter()));
|
||||
// Empty → not RTL
|
||||
assert!(!is_rtl_text(std::iter::empty::<&str>()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rtl_line_sorting() {
|
||||
let mut items = vec![
|
||||
TextItem {
|
||||
text: "\u{05D0}".into(), // alef at x=100
|
||||
x: 100.0,
|
||||
y: 700.0,
|
||||
width: 10.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "\u{05D1}".into(), // bet at x=200 (rightmost)
|
||||
x: 200.0,
|
||||
y: 700.0,
|
||||
width: 10.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
sort_line_items(&mut items);
|
||||
// RTL: rightmost (higher X) comes first
|
||||
assert_eq!(items[0].x, 200.0);
|
||||
assert_eq!(items[1].x, 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ltr_unaffected() {
|
||||
let mut items = vec![
|
||||
TextItem {
|
||||
text: "Hello".into(),
|
||||
x: 100.0,
|
||||
y: 700.0,
|
||||
width: 50.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
TextItem {
|
||||
text: "World".into(),
|
||||
x: 200.0,
|
||||
y: 700.0,
|
||||
width: 50.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page: 1,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
},
|
||||
];
|
||||
sort_line_items(&mut items);
|
||||
// LTR: leftmost comes first
|
||||
assert_eq!(items[0].x, 100.0);
|
||||
assert_eq!(items[1].x, 200.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hangul_is_cjk() {
|
||||
// Hangul Jamo
|
||||
assert!(is_cjk_char('\u{1100}'));
|
||||
// Hangul Compatibility Jamo
|
||||
assert!(is_cjk_char('\u{3131}'));
|
||||
// Hangul Syllable '가'
|
||||
assert!(is_cjk_char('\u{AC00}'));
|
||||
// Latin is not CJK
|
||||
assert!(!is_cjk_char('A'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_newspaper_layout_detection() {
|
||||
// Two dense columns (>15 lines each) with matching Y positions → newspaper
|
||||
let make_line = |y: f32, x: f32, page: u32| TextLine {
|
||||
y,
|
||||
page,
|
||||
items: vec![TextItem {
|
||||
text: "text".into(),
|
||||
x,
|
||||
y,
|
||||
width: 100.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}],
|
||||
};
|
||||
|
||||
let col1: Vec<TextLine> = (0..20)
|
||||
.map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
|
||||
.collect();
|
||||
let col2: Vec<TextLine> = (0..20)
|
||||
.map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1))
|
||||
.collect();
|
||||
|
||||
assert!(is_newspaper_layout(&[col1, col2]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tabular_layout_detection() {
|
||||
// Sparse columns (<15 lines) → tabular, not newspaper
|
||||
let make_line = |y: f32, x: f32, page: u32| TextLine {
|
||||
y,
|
||||
page,
|
||||
items: vec![TextItem {
|
||||
text: "text".into(),
|
||||
x,
|
||||
y,
|
||||
width: 100.0,
|
||||
height: 12.0,
|
||||
font: "F1".into(),
|
||||
font_size: 12.0,
|
||||
page,
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
}],
|
||||
};
|
||||
|
||||
let col1: Vec<TextLine> = (0..5)
|
||||
.map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
|
||||
.collect();
|
||||
let col2: Vec<TextLine> = (0..5)
|
||||
.map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1))
|
||||
.collect();
|
||||
|
||||
assert!(!is_newspaper_layout(&[col1, col2]));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user