From dca267ac4418b73149346c76b902850f6cda0a55 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Feb 2026 16:42:04 -0800 Subject: [PATCH] feat(extractor): Add RTL text support, vertical WMode, and Korean Hangul Detect RTL scripts (Hebrew, Arabic, Syriac, etc.) by Unicode ranges and sort line items by X descending when a line is predominantly RTL. Fix gap calculation in should_join_items() to be direction-neutral. Extract WMode from Type0 font dictionaries for vertical text foundation. Add missing Korean Hangul ranges to is_cjk_char(). Co-Authored-By: Claude Opus 4.6 --- src/extractor.rs | 197 ++++++++++++++++++++++++++++++++++++++++++++--- src/tables.rs | 13 +++- 2 files changed, 195 insertions(+), 15 deletions(-) diff --git a/src/extractor.rs b/src/extractor.rs index 2953c84..728546d 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -31,6 +31,8 @@ struct FontWidthInfo { /// For Type1/TrueType: 0.001 (widths in 1000ths of em) /// For Type3: FontMatrix[0] (e.g., 0.00048828125 for 2048-unit grid) units_scale: f32, + /// Writing mode: 0 = horizontal (default), 1 = vertical + wmode: u8, } /// All font width info for a page, keyed by font resource name @@ -193,6 +195,7 @@ fn parse_simple_font_widths( space_width, is_cid: false, units_scale, + wmode: 0, }) } @@ -240,12 +243,22 @@ fn parse_type0_widths(doc: &Document, font_dict: &lopdf::Dictionary) -> Option Some(*n as u8), + _ => None, + }) + .unwrap_or(0); + Some(FontWidthInfo { widths, default_width, space_width, is_cid: true, units_scale: 0.001, // CID fonts use standard 1000-unit system + wmode, }) } @@ -709,8 +722,13 @@ fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { // When we have accurate width from font metrics, use a tight threshold if prev_item.width > 0.0 { - let prev_end_x = prev_item.x + prev_item.width; - let gap = curr_item.x - prev_end_x; + let gap = if prev_item.x <= curr_item.x { + // LTR: prev is left of curr + curr_item.x - (prev_item.x + prev_item.width) + } else { + // RTL: prev is right of curr + prev_item.x - (curr_item.x + curr_item.width) + }; let font_size = prev_item.font_size; // Never join across column-scale gaps @@ -1669,9 +1687,14 @@ fn merge_text_items(items: Vec) -> Vec { } } - // Sort each group by X position + // Sort each group by X position (direction-aware) for (_, _, group) in &mut line_groups { - group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + let rtl = is_rtl_text(group.iter().map(|i| &i.text)); + if rtl { + group.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } } // Sort groups by page then Y descending (top of page first) @@ -2533,17 +2556,64 @@ fn effective_font_size(base_size: f32, text_matrix: &[f32; 6]) -> f32 { /// Check if a character is CJK (Chinese, Japanese, Korean). /// CJK languages don't use spaces between words, so word-boundary /// heuristics should not apply when CJK characters are involved. -fn is_cjk_char(c: char) -> bool { +pub(crate) fn is_cjk_char(c: char) -> bool { matches!(c, - '\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation + '\u{1100}'..='\u{11FF}' // Hangul Jamo + | '\u{3000}'..='\u{303F}' // CJK Symbols and Punctuation | '\u{3040}'..='\u{309F}' // Hiragana | '\u{30A0}'..='\u{30FF}' // Katakana + | '\u{3130}'..='\u{318F}' // Hangul Compatibility Jamo | '\u{4E00}'..='\u{9FFF}' // CJK Unified Ideographs + | '\u{AC00}'..='\u{D7AF}' // Hangul Syllables | '\u{F900}'..='\u{FAFF}' // CJK Compatibility Ideographs | '\u{FF00}'..='\u{FFEF}' // Halfwidth and Fullwidth Forms ) } +pub(crate) fn is_rtl_char(c: char) -> bool { + matches!(c, + '\u{0590}'..='\u{05FF}' // Hebrew + | '\u{0600}'..='\u{06FF}' // Arabic + | '\u{0700}'..='\u{074F}' // Syriac + | '\u{0750}'..='\u{077F}' // Arabic Supplement + | '\u{0780}'..='\u{07BF}' // Thaana + | '\u{07C0}'..='\u{07FF}' // NKo + | '\u{0800}'..='\u{083F}' // Samaritan + | '\u{0840}'..='\u{085F}' // Mandaic + | '\u{08A0}'..='\u{08FF}' // Arabic Extended-A + | '\u{FB1D}'..='\u{FB4F}' // Hebrew Presentation Forms + | '\u{FB50}'..='\u{FDFF}' // Arabic Presentation Forms-A + | '\u{FE70}'..='\u{FEFF}' // Arabic Presentation Forms-B + ) +} + +pub(crate) fn is_rtl_text(texts: I) -> bool +where + I: Iterator, + S: AsRef, +{ + let (mut rtl, mut ltr) = (0u32, 0u32); + for t in texts { + for c in t.as_ref().chars() { + if is_rtl_char(c) { + rtl += 1; + } else if c.is_alphabetic() && !is_cjk_char(c) { + ltr += 1; + } + } + } + rtl > 0 && rtl > ltr +} + +fn sort_line_items(items: &mut [TextItem]) { + let rtl = is_rtl_text(items.iter().map(|i| &i.text)); + if rtl { + items.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + items.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } +} + /// Detect if a font name indicates bold style /// Common patterns: "Bold", "Bd", "Black", "Heavy", "Demi", "Semi" (semi-bold) pub fn is_bold_font(font_name: &str) -> bool { @@ -3064,9 +3134,7 @@ pub fn group_into_lines(items: Vec) -> Vec { if let Some(last) = merged.last_mut() { if last.page == line.page && (last.y - line.y).abs() < y_tol { last.items.extend(line.items); - last.items.sort_by(|a, b| { - a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal) - }); + sort_line_items(&mut last.items); continue; } } @@ -3192,10 +3260,9 @@ fn group_single_column(items: Vec) -> Vec { } } - // Sort items within each line by X position (left to right) + // Sort items within each line by X position (direction-aware) for line in &mut lines { - line.items - .sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + sort_line_items(&mut line.items); } lines @@ -3588,4 +3655,110 @@ mod tests { cols ); } + + #[test] + fn test_is_rtl_char() { + // Hebrew alef + assert!(is_rtl_char('\u{05D0}')); + // Arabic alif + assert!(is_rtl_char('\u{0627}')); + // Latin 'A' is not RTL + assert!(!is_rtl_char('A')); + // CJK is not RTL + assert!(!is_rtl_char('\u{4E00}')); + } + + #[test] + fn test_is_rtl_text() { + // Majority Hebrew with digits → RTL + assert!(is_rtl_text(["\u{05E9}\u{05DC}\u{05D5}\u{05DD} 123"].iter())); + // Majority Latin → not RTL + assert!(!is_rtl_text(["Hello world"].iter())); + // Empty → not RTL + assert!(!is_rtl_text(std::iter::empty::<&str>())); + } + + #[test] + fn test_rtl_line_sorting() { + let mut items = vec![ + TextItem { + text: "\u{05D0}".into(), // alef at x=100 + x: 100.0, + y: 700.0, + width: 10.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "\u{05D1}".into(), // bet at x=200 (rightmost) + x: 200.0, + y: 700.0, + width: 10.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + sort_line_items(&mut items); + // RTL: rightmost (higher X) comes first + assert_eq!(items[0].x, 200.0); + assert_eq!(items[1].x, 100.0); + } + + #[test] + fn test_ltr_unaffected() { + let mut items = vec![ + TextItem { + text: "Hello".into(), + x: 100.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + TextItem { + text: "World".into(), + x: 200.0, + y: 700.0, + width: 50.0, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }, + ]; + sort_line_items(&mut items); + // LTR: leftmost comes first + assert_eq!(items[0].x, 100.0); + assert_eq!(items[1].x, 200.0); + } + + #[test] + fn test_hangul_is_cjk() { + // Hangul Jamo + assert!(is_cjk_char('\u{1100}')); + // Hangul Compatibility Jamo + assert!(is_cjk_char('\u{3131}')); + // Hangul Syllable '가' + assert!(is_cjk_char('\u{AC00}')); + // Latin is not CJK + assert!(!is_cjk_char('A')); + } } diff --git a/src/tables.rs b/src/tables.rs index 0037856..4102f7b 100644 --- a/src/tables.rs +++ b/src/tables.rs @@ -2,7 +2,7 @@ //! //! Detects tabular data in PDF text items and converts to markdown tables. -use crate::extractor::{PdfRect, TextItem}; +use crate::extractor::{is_rtl_text, PdfRect, TextItem}; /// Detection mode controls thresholds for table validation #[derive(Debug, Clone, Copy, PartialEq)] @@ -968,8 +968,15 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode for row_items in &mut cell_items { let mut row_cells = Vec::with_capacity(columns.len()); for col_items in row_items.iter_mut() { - // Sort by X position - col_items.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + // Sort by X position (direction-aware) + let rtl = is_rtl_text(col_items.iter().map(|i| &i.text)); + if rtl { + col_items + .sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal)); + } else { + col_items + .sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + } // Join items with subscript-aware spacing let text = join_cell_items(col_items);