chore(refactor): Better organization of the codebase, split modules, update docs, add AGENTS.md

src/types.rs

@@ -0,0 +1,237 @@
+//! Shared types used across the extraction and markdown pipelines.
+//!
+//! Centralises `TextItem`, `TextLine`, `PdfRect`, font-width / encoding
+//! type aliases, and the `ItemType` enum so that every module can import
+//! them from one place.
+
+use std::collections::HashMap;
+
+use crate::text_utils::should_join_items;
+
+// ── Font types (crate-internal) ──────────────────────────────────────
+
+/// Font encoding map: maps byte codes to Unicode characters
+pub(crate) type FontEncodingMap = HashMap<u8, char>;
+
+/// All font encodings for a page
+pub(crate) type PageFontEncodings = HashMap<String, FontEncodingMap>;
+
+/// Font width information extracted from PDF font dictionaries
+#[derive(Debug, Clone)]
+#[allow(dead_code)]
+pub(crate) struct FontWidthInfo {
+    /// Glyph widths: maps character code to width in font units
+    pub(crate) widths: HashMap<u16, u16>,
+    /// Default width for glyphs not in the widths table
+    pub(crate) default_width: u16,
+    /// Width of the space character (code 32) if known
+    pub(crate) space_width: u16,
+    /// Whether this is a CID font (2-byte character codes)
+    pub(crate) is_cid: bool,
+    /// Scale factor to convert font units to text space units.
+    /// For Type1/TrueType: 0.001 (widths in 1000ths of em)
+    /// For Type3: FontMatrix[0] (e.g., 0.00048828125 for 2048-unit grid)
+    pub(crate) units_scale: f32,
+    /// Writing mode: 0 = horizontal (default), 1 = vertical
+    pub(crate) wmode: u8,
+}
+
+/// All font width info for a page, keyed by font resource name
+pub(crate) type PageFontWidths = HashMap<String, FontWidthInfo>;
+
+// ── Public types ─────────────────────────────────────────────────────
+
+/// Type of extracted item
+#[derive(Debug, Clone, Default)]
+pub enum ItemType {
+    /// Regular text content
+    #[default]
+    Text,
+    /// Image placeholder
+    Image,
+    /// Hyperlink (with URL)
+    Link(String),
+    /// Form field (name: value)
+    FormField,
+}
+
+/// A rectangle from a PDF `re` operator (cell boundary, border, etc.)
+#[derive(Debug, Clone)]
+pub struct PdfRect {
+    pub x: f32,
+    pub y: f32,
+    pub width: f32,
+    pub height: f32,
+    pub page: u32,
+}
+
+/// A text item with position information
+#[derive(Debug, Clone)]
+pub struct TextItem {
+    /// The text content
+    pub text: String,
+    /// X position on page
+    pub x: f32,
+    /// Y position on page (PDF coordinates, origin at bottom-left)
+    pub y: f32,
+    /// Width of text
+    pub width: f32,
+    /// Height (approximated from font size)
+    pub height: f32,
+    /// Font name
+    pub font: String,
+    /// Font size
+    pub font_size: f32,
+    /// Page number (1-indexed)
+    pub page: u32,
+    /// Whether the font is bold
+    pub is_bold: bool,
+    /// Whether the font is italic
+    pub is_italic: bool,
+    /// Type of item (text, image, link)
+    pub item_type: ItemType,
+}
+
+/// A line of text (grouped text items)
+#[derive(Debug, Clone)]
+pub struct TextLine {
+    pub items: Vec<TextItem>,
+    pub y: f32,
+    pub page: u32,
+}
+
+impl TextLine {
+    pub fn text(&self) -> String {
+        self.text_with_formatting(false, false)
+    }
+
+    /// Get text with optional bold/italic markdown formatting
+    pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String {
+        if !format_bold && !format_italic {
+            return self.text_plain();
+        }
+
+        let mut result = String::new();
+        let mut current_bold = false;
+        let mut current_italic = false;
+
+        for (i, item) in self.items.iter().enumerate() {
+            let text = item.text.as_str();
+            let text_trimmed = text.trim();
+
+            // Skip empty items
+            if text_trimmed.is_empty() {
+                continue;
+            }
+
+            // Determine spacing
+            let needs_space = if i == 0 || result.is_empty() {
+                false
+            } else {
+                let prev_item = &self.items[i - 1];
+                self.needs_space_between(prev_item, item, &result)
+            };
+
+            // Preserve leading whitespace from the item text.
+            // Items like " means any person" have a leading space that indicates
+            // a word boundary. needs_space_between returns false for these (because
+            // space_already_exists), but we still need to emit the space since
+            // we push text_trimmed below (which strips it).
+            let has_leading_space = text.starts_with(' ');
+
+            // Check for style changes
+            let item_bold = format_bold && item.is_bold;
+            let item_italic = format_italic && item.is_italic;
+
+            // Close previous styles if they change
+            if current_italic && !item_italic {
+                result.push('*');
+                current_italic = false;
+            }
+            if current_bold && !item_bold {
+                result.push_str("**");
+                current_bold = false;
+            }
+
+            // Add space: either from spacing logic or preserved from item text
+            if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) {
+                result.push(' ');
+            }
+
+            // Open new styles
+            if item_bold && !current_bold {
+                result.push_str("**");
+                current_bold = true;
+            }
+            if item_italic && !current_italic {
+                result.push('*');
+                current_italic = true;
+            }
+
+            result.push_str(text_trimmed);
+        }
+
+        // Close any remaining open styles
+        if current_italic {
+            result.push('*');
+        }
+        if current_bold {
+            result.push_str("**");
+        }
+
+        result
+    }
+
+    /// Get plain text without formatting
+    fn text_plain(&self) -> String {
+        let mut result = String::new();
+        for (i, item) in self.items.iter().enumerate() {
+            let text = item.text.as_str();
+            if i == 0 {
+                result.push_str(text);
+            } else {
+                let prev_item = &self.items[i - 1];
+                if self.needs_space_between(prev_item, item, &result) {
+                    result.push(' ');
+                }
+                result.push_str(text);
+            }
+        }
+        result
+    }
+
+    /// Determine if a space is needed between two items
+    fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
+        let text = item.text.as_str();
+
+        // Don't add space before/after hyphens for hyphenated words
+        let prev_ends_with_hyphen = result.ends_with('-');
+        let curr_is_hyphen = text.trim() == "-";
+        let curr_starts_with_hyphen = text.starts_with('-');
+
+        // Detect subscript/superscript: smaller font size and/or Y offset
+        let font_ratio = item.font_size / prev_item.font_size;
+        let reverse_font_ratio = prev_item.font_size / item.font_size;
+        let y_diff = (item.y - prev_item.y).abs();
+
+        let is_sub_super = font_ratio < 0.85 && y_diff > 1.0;
+        let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
+
+        // Use position-based spacing detection
+        let should_join = should_join_items(prev_item, item);
+
+        // Check if space already exists
+        let prev_ends_with_space = result.ends_with(' ');
+        let curr_starts_with_space = text.starts_with(' ');
+        let space_already_exists = prev_ends_with_space || curr_starts_with_space;
+
+        // Add space unless one of these conditions applies
+        !(prev_ends_with_hyphen
+            || curr_is_hyphen
+            || curr_starts_with_hyphen
+            || is_sub_super
+            || was_sub_super
+            || should_join
+            || space_already_exists)
+    }
+}