From ba1fbcbff605aac0b342037541a4ec7ae0340793 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 12 Mar 2026 09:20:34 -0700 Subject: [PATCH 01/46] fix(arabic): NFKC normalize presentation forms and reverse visual-order RTL text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arabic PDFs using ToUnicode maps to presentation forms (U+FB50-FDFF, U+FE70-FEFE) produced unreadable output. Apply NFKC normalization to convert back to base Arabic (U+0600-06FF), then reverse character order within items that had presentation forms (visual→logical order). Improves Arabic PDF word overlap with Mistral from ~4.5% to ~78.6%. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 1 + src/text_utils.rs | 156 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index d625bde..22ae989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ env_logger = "0.11" # Text processing regex = "1.10" once_cell = "1.19" +unicode-normalization = "0.1" # TrueType font parsing (for Identity-H CID font cmap extraction) ttf-parser = "0.25" diff --git a/src/text_utils.rs b/src/text_utils.rs index a6929bb..b27beef 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -5,6 +5,7 @@ //! and markdown pipelines. use crate::types::TextItem; +use unicode_normalization::UnicodeNormalization; /// Check if a character is CJK (Chinese, Japanese, Korean). /// CJK languages don't use spaces between words, so word-boundary @@ -40,6 +41,12 @@ pub(crate) fn is_rtl_char(c: char) -> bool { ) } +fn is_arabic_presentation_form(c: char) -> bool { + // U+FEFF is BOM/ZWNJ, not an Arabic presentation form despite falling + // in the Presentation Forms-B codepoint range. + matches!(c, '\u{FB50}'..='\u{FDFF}' | '\u{FE70}'..='\u{FEFE}') +} + pub(crate) fn is_rtl_text(texts: I) -> bool where I: Iterator, @@ -105,6 +112,9 @@ pub fn is_italic_font(font_name: &str) -> bool { /// Expand Unicode ligature characters to their component characters. /// This makes extracted text more searchable and semantically correct. +/// Also applies NFKC normalization (converts Arabic presentation forms to base +/// characters, decomposes Latin ligatures, etc.) and reverses visual-order +/// Arabic text back to logical order when presentation forms are detected. pub(crate) fn expand_ligatures(text: &str) -> String { // Strip null bytes and other control characters (except newline/tab) let text = if text @@ -118,9 +128,26 @@ pub(crate) fn expand_ligatures(text: &str) -> String { text.to_string() }; + // Detect Arabic presentation forms before normalization — their presence + // signals visual-order storage that needs reversal after NFKC. + let had_presentation_forms = text.chars().any(is_arabic_presentation_form); + + // Apply NFKC normalization only when Arabic presentation forms are present. + // This converts forms (U+FB50-FDFF, U+FE70-FEFF) back to base Arabic + // (U+0600-06FF). We avoid broad NFKC on all non-ASCII text because it + // would convert NBSP (U+00A0) to regular space, breaking downstream logic. + // Latin ligatures are already handled by the explicit match arms below. + let text = if had_presentation_forms { + text.nfkc().collect::() + } else { + text + }; + let mut result = String::with_capacity(text.len()); for ch in text.chars() { match ch { + // Keep explicit ligature expansion as fallback for fonts that bypass + // NFKC (e.g. custom ToUnicode mappings to PUA codepoints) '\u{FB00}' => result.push_str("ff"), '\u{FB01}' => result.push_str("fi"), '\u{FB02}' => result.push_str("fl"), @@ -141,9 +168,74 @@ pub(crate) fn expand_ligatures(text: &str) -> String { _ => result.push(ch), } } + + // If the original text had Arabic presentation forms, the characters are in + // visual (LTR screen) order. After NFKC normalization, reverse to restore + // logical reading order. + if had_presentation_forms { + result = reverse_visual_arabic(&result); + } + result } +/// Reverse visual-order Arabic text to logical order. +/// +/// Pure RTL text (no ASCII alphanumerics) gets a simple character reversal. +/// Mixed content (embedded numbers or Latin words) splits into LTR and non-LTR +/// runs: run order is reversed, and only non-LTR runs are reversed internally. +fn reverse_visual_arabic(text: &str) -> String { + // Check if there are any LTR runs (ASCII letters or digits) + let has_ltr = text.chars().any(|c| c.is_ascii_alphanumeric()); + + if !has_ltr { + // Pure RTL: simple reversal + return text.chars().rev().collect(); + } + + // Mixed content: split into runs of LTR (ASCII alphanumeric + adjacent + // punctuation like '.', ',', '/', '-') vs non-LTR (Arabic + spaces + other). + let chars: Vec = text.chars().collect(); + let mut runs: Vec<(bool, String)> = Vec::new(); // (is_ltr, content) + + let mut i = 0; + while i < chars.len() { + let is_ltr = chars[i].is_ascii_alphanumeric() + || (chars[i].is_ascii_punctuation() && is_adjacent_to_ascii_alnum(&chars, i)); + + let mut run = String::new(); + while i < chars.len() { + let c = chars[i]; + let c_is_ltr = c.is_ascii_alphanumeric() + || (c.is_ascii_punctuation() && is_adjacent_to_ascii_alnum(&chars, i)); + if c_is_ltr != is_ltr { + break; + } + run.push(c); + i += 1; + } + runs.push((is_ltr, run)); + } + + // Reverse run order and reverse non-LTR runs internally + runs.reverse(); + let mut result = String::with_capacity(text.len()); + for (is_ltr, content) in &runs { + if *is_ltr { + result.push_str(content); + } else { + result.extend(content.chars().rev()); + } + } + result +} + +/// Check if the character at `idx` is adjacent to an ASCII alphanumeric character. +fn is_adjacent_to_ascii_alnum(chars: &[char], idx: usize) -> bool { + (idx > 0 && chars[idx - 1].is_ascii_alphanumeric()) + || (idx + 1 < chars.len() && chars[idx + 1].is_ascii_alphanumeric()) +} + /// Decode a PDF text string (ActualText, etc.) that may be UTF-16BE (BOM \xFE\xFF) /// or PDFDocEncoding (Latin-1 superset). pub(crate) fn decode_text_string(bytes: &[u8]) -> String { @@ -425,4 +517,68 @@ mod tests { // NBSP (U+00A0) should NOT be normalized assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b"); } + + #[test] + fn nfkc_arabic_presentation_forms() { + // Arabic Presentation Form-B: FEE1 = MEEM medial, FEF3 = YEH initial + // NFKC maps these to base Arabic + reversal restores logical order + let input = "\u{FEE1}\u{FEF3}"; // visual order: medial meem, initial yeh + let result = expand_ligatures(input); + // After NFKC: base Arabic chars; after reversal: logical order + assert!( + !result.chars().any(is_arabic_presentation_form), + "presentation forms should be normalized: {result:?}" + ); + assert!( + result.chars().any(|c| matches!(c, '\u{0600}'..='\u{06FF}')), + "should contain base Arabic characters: {result:?}" + ); + } + + #[test] + fn no_reversal_for_base_arabic() { + // Base Arabic already in logical order — no presentation forms means no reversal + let input = "\u{0645}\u{0631}\u{062D}\u{0628}\u{0627}"; // مرحبا + let result = expand_ligatures(input); + assert_eq!(result, input, "base Arabic should pass through unchanged"); + } + + #[test] + fn latin_text_unaffected() { + assert_eq!(expand_ligatures("Hello World"), "Hello World"); + } + + #[test] + fn reverse_visual_arabic_pure_rtl() { + // Pure RTL: simple reversal + let input = "\u{0628}\u{0627}"; // ba (visual order) + let result = reverse_visual_arabic(input); + assert_eq!(result, "\u{0627}\u{0628}"); // ab (logical order) + } + + #[test] + fn reverse_visual_arabic_with_ltr_run() { + // Mixed: Arabic + embedded number "123" + Arabic + // Visual order: أ 123 ب → runs: [أ], [123], [ب] + // Reversed runs: [ب], [123], [أ] + // Non-LTR reversed internally: ب, 123, أ + let input = "\u{0623}123\u{0628}"; + let result = reverse_visual_arabic(input); + assert_eq!(result, "\u{0628}123\u{0623}"); + } + + #[test] + fn arabic_presentation_form_detection() { + // Presentation Forms-A range + assert!(is_arabic_presentation_form('\u{FB50}')); + assert!(is_arabic_presentation_form('\u{FDFF}')); + // Presentation Forms-B range (excludes U+FEFF which is BOM) + assert!(is_arabic_presentation_form('\u{FE70}')); + assert!(is_arabic_presentation_form('\u{FEFE}')); + assert!(!is_arabic_presentation_form('\u{FEFF}')); + // Base Arabic — NOT presentation form + assert!(!is_arabic_presentation_form('\u{0645}')); + // Latin + assert!(!is_arabic_presentation_form('A')); + } } From 80a3b81ff90fd36df7d07cf4a95eddcfa01e9a3f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 12 Mar 2026 11:19:30 -0700 Subject: [PATCH 02/46] perf(tables): remove cosmetic padding from markdown tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop column-width padding from table output. Saves ~37% bytes on table-heavy PDFs. Markdown renders identically — padding was purely cosmetic. Optimizes token efficiency for AI agent consumers. Co-Authored-By: Claude Opus 4.6 --- src/tables/format.rs | 38 +- src/tables/mod.rs | 10 +- tests/snapshots/2013-app2.md | 592 ++++++++++++------------- tests/snapshots/nexo-price-en.md | 10 +- tests/snapshots/real-estate-pricing.md | 16 +- tests/snapshots/td9264.md | 50 +-- tests/snapshots/thermo-freon12.md | 114 ++--- 7 files changed, 409 insertions(+), 421 deletions(-) diff --git a/src/tables/format.rs b/src/tables/format.rs index 8b69169..9d442a4 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -17,33 +17,21 @@ pub fn table_to_markdown(table: &Table) -> String { let num_cols = cleaned_cells[0].len(); let mut output = String::new(); - // Calculate column widths for alignment (capped to avoid massive whitespace padding) - const MAX_COL_WIDTH: usize = 40; - let col_widths: Vec = (0..num_cols) - .map(|col| { - cleaned_cells - .iter() - .map(|row| row.get(col).map(|c| c.len()).unwrap_or(0)) - .max() - .unwrap_or(3) - .clamp(3, MAX_COL_WIDTH) - }) - .collect(); - - // Output each row + // Compact format: no padding, minimal separators. Optimized for token + // efficiency — AI agents are the primary consumer, not human eyes. for (row_idx, row) in cleaned_cells.iter().enumerate() { output.push('|'); - for (col_idx, cell) in row.iter().enumerate() { - let width = col_widths[col_idx]; - output.push_str(&format!(" {:width$} |", cell, width = width)); + for cell in row.iter() { + output.push_str(cell); + output.push('|'); } output.push('\n'); // Add separator after header row if row_idx == 0 { output.push('|'); - for width in &col_widths { - output.push_str(&format!(" {} |", "-".repeat(*width))); + for _ in 0..num_cols { + output.push_str("---|"); } output.push('\n'); } @@ -340,10 +328,10 @@ mod tests { item_indices: vec![], }; let md = table_to_markdown(&table); - assert!(md.contains("| Name")); - assert!(md.contains("| ---")); - assert!(md.contains("| Alice")); - assert!(md.contains("| Bob")); + assert!(md.contains("|Name|")); + assert!(md.contains("|---|")); + assert!(md.contains("|Alice|")); + assert!(md.contains("|Bob|")); } #[test] @@ -355,8 +343,8 @@ mod tests { item_indices: vec![], }; let md = table_to_markdown(&table); - assert!(md.contains("| Only")); - assert!(md.contains("| ---")); + assert!(md.contains("|Only|")); + assert!(md.contains("|---|")); } #[test] diff --git a/src/tables/mod.rs b/src/tables/mod.rs index a85418c..043dacb 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -359,9 +359,9 @@ mod tests { }; let md = table_to_markdown(&table); - assert!(md.contains("| Header 1")); - assert!(md.contains("| ---")); - assert!(md.contains("| Cell 1")); + assert!(md.contains("|Header 1|")); + assert!(md.contains("|---|")); + assert!(md.contains("|Cell 1|")); } #[test] @@ -721,12 +721,12 @@ mod tests { let md = table_to_markdown(&table); // JAN and FEB should be on their own rows, not merged into adjacent rows assert!( - md.contains("| JAN"), + md.contains("|JAN|"), "JAN should be on its own row, got:\n{}", md ); assert!( - md.contains("| FEB"), + md.contains("|FEB|"), "FEB should be on its own row, got:\n{}", md ); diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index e595beb..0411127 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -1,298 +1,298 @@ -| | Date | Procurement Title | PE | Bidder | Amount | -| --- | ---- | ---------------------------------------- | --------- | ----------------------------------- | ---------------------------------------- | -| | JAN | | | | | -| 1 | 8/1 | Procurement of Criticals Spare Parts for Engine Maintenance on Mahe | PUC | Wartsila Eastern Africa Ltd | Euro97,922.30 | -| 2 | 8/1 | Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin | PUC | Wartsila Eastern Africa Ltd | Euro270,982.90 | -| 3 | 15/1 | Manufacturing and Deliveries of 900 Students Desks | MOE | | SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00 | -| 4 | 15/1 | Renovation Works at Mont Fleuri Secondary School | MOE | Sai-Fu Enterprise | SR2,815,771.00 | -| 5 | 15/1 | La Gogue to Mont Simpson Raw Water Transfer | PUC | Vijay Construction | SR7,816,058.00 | -| 6 | 15/1 | Storm Water Channel Project at Au Cap-Additional Works | DOE | United Concrete Products (Sey)Ltd | SR184,300.00 | -| 7 | 22/1 | Procurement of Security at ex-Maritime Training Centre | SFA | Elite Surveillance Security Agency | SR30,000.00 | -| 8 | 22/1 | Installation of Sewerage Treatment Plant at Anse Gaulette | MLUH | Green Island Construction Compnay | SR4,524,884.85 | -| 9 | 29/1 | Procurement of Cylinder Liner | PUC | Wartsila Eastern Africa Ltd | Euro28,186.75 | -| 10 | 29/1 | Procurement of Service Pack for Coupling | PUC | Wartsila Eastern Africa Ltd | Euro11,151.00 | -| 11 | 29/1 | Construction of Drainage for Roads A & B Eve Island Praslin | MLUH | Ascent Projects Sey | SR2,931,174.00 | -| 12 | 29/1 | Procurement of Electric Cables for Perseverance Infrastructure-Variations | PUC | Indian Ocean Export Company Pty Ltd | USD768.12 | -| | FEB | | | | | -| 13 | 5/2 | Procurement of DI Pipes and Fittings for Le Rocher Refurbishment | PUC | Legend General Supply | USD43,807.50 | -| 14 | 5/2 | Procurement of Bearings for ABB Turbo Charger | PUC | ABB France | EURO 28,966.15 | -| 15 | 5/2 | Procurement of Alfa Laval Separator Spares | PUC | ALFA LAVAL (Pty) Ltd | Euro 41,191,30 | -| 16 | 5/2 | constraction of 6*2 bedroom Houses- Mont Buxton | MLUH | O-NIVO Construction | SR3,688,844.00 | -| 17 | 5/2 | Procurement of Critical Spares frr Major Overhaul-Set A41 PUC | PUC | Wartsila Eastern Africa Ltd | EURO 104,880.10 | -| 18 | 5/2 | Procurement of Vehcile x1 | MOE | Abhaye Valabhji Pty Ltd | SR 650,000.00 | -| 19 | 5/2 | Procurement of Vehicle x1 | SBFA | Abhaye Valabhji Pty Ltd | SR 585,000.00 | -| 20 | 12/2 | Procurement of Turbo charger Rotor for Engine on Praslin | PUC | Marine Power International FZC | Euro 835,669.00 | -| 21 | 12/2 | Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin | PUC | Overseae Tractor S.A | USD 20,546.12 | -| 22 | 12/2 | Procurement of Air Cooler Cartridge for Wartsila Engine | PUC | Marine Power International FZC | Euro 36,508.93 | -| 23 | 12/2 | Procurement of Spares for Genset 8P on Praslin | PUC | Wartsila Eastern Africa Ltd | Euro 288,202.50 | -| 24 | 19/2 | Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka | SLTA | Esparon's Enterprise | SR 1,105,069.00 | -| 25 | 19/2 | Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41 | PUC | Marine Power International | Euro 144,726.26 | -| 26 | 19/2 | Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31 | PUC | Marine Power International FZC | Euro 143,746.75 | -| 27 | 19/2 | Procurement of Critical Spares for Wartsila Engine-Engine Set B11 | PUC | Marine Power International FZC | Euro 44,089.42 | -| 28 | 26/2 | Procurement for Vehcile x 2 | Judiciary | PMC Auto | SR1,349,551.00 | -| 29 | 26/2 | Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares) | PUC | Wartsila Eastern Africa Ltd | Euro 202,898.40 | -| 30 | 26/2 | Procurement of Safety Spare Patrs for 8MW Engines - (Turbo) | PUC | ABB France | Euro 132,833.33 | -| | MAR | | | | | -| 31 | 5/3 | Procurement of Spare Parts for the Asphalt Plant- Petite Paris | SLTA | Astec Factory (USA) | USD 101,337.81 | -| 32 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines | PUC | Marine Power International FZC | Euro 44,050.00 | -| 33 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines B11 | PUC | Marine Power International FZC | Euro 98,743.00 | -| 34 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines 8B | PUC | Marine Power International FZC | Euro 103,215.00 | -| 35 | 5/3 | Procurement of Critical Spare Parts for Wartsila Engines 8B | PUC | Wartsila Global Logistic | Euro 97,979.10 | -| 36 | 5/3 | Procurement Spare Parts for the 8MW Engines | PUC | Wartsila Global Logistic | Euro 41,444.00 | -| 37 | 5/3 | Procurement of Voltage Districbution Boxes and Fuses-Variations | PUC | Indian Ocean Export Company Pty Ltd | GBP1,300.00 | -| 38 | 5/3 | 33Kv South Mahe Project-Variations | PUC | United Concrete Products (Sey)Ltd | SR459,684.00 | -| 39 | 5/3 | construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations | SFRS | Furui Construction Pty Ltd | SR184,836.03 | -| 40 | 5/3 | Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services | MLHU | GIBB ( Mauritius) | USD138,050.00 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 1 Roy & Sons Import | SR 396,000.00 | -| | | | | | 1 | -| 41 | 12/3 | Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools | MOE | Lot 3 Roy & Sons Import | SR650,000.00 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 4 HIS & PJ Enterprise | SR 1,753,614.70 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 5 Roy & Sons Import | SR 1,080.000.00 | -| 42 | 19/3 | Procurement of Medium Voltage Cables | PUC | Nexans France Lens | Euro 52,354.45 | -| 43 | 19/3 | Procurement of Brass/Bronze Fitting | PUC | Jainsons Industries | Euro 97,718.48 | -| 44 | 19/3 | Procurement of Training Vessel for Maritime Training Centre | SFA | Neil Marine (Sri Lanka) | USD 284,521.15 | -| 44 | 19/3 | Procurement of Training Vessel for Maritime Training Centre-Equipment | SFA | Neil Marine (Sri Lanka) | USD 97,861.50 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 1 | MOE | Print V Care | SR 349,450.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 2 | MOE | Print V Care | SR 392,500.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 3 | MOE | JD's Stationery & Educational Centre | SR 19,150.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 5 | MOE | Vitoria Computer Services Pty LTd | SR 94,500.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 6 | MOE | Roy & Sons Imports | SR 114,750.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 7 | MOE | Trade Supplies Pty Ltd | SR 35,224.00 | -| 46 | 26/3 | Procurement of Microsoft Windows License | STB | Victoria Computer Service | SR 977,150.40 | -| 47 | 26/3 | | Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations PUC | Ras Tek Pvt Ltd | Euro 58,000.00 | -| | APR | | | | | -| 48 | 2/4 | Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin | PUC | Vijay Construction | SR 623,025.00 | -| 49 | 2/4 | Supply of Fish to the Prison Department | | Prison Service Mr. Danny Loizeau | SR 27.50 per KG | -| 50 | 2/4 | Supply of Polo T- Shirt and Jeans -Lot 1 | PUC | Magilyn Ltee | USD 28,885.00 | -| 50 | 2/4 | Supply of Polo T- Shirt and Jeans -Lot 2 | PUC | Magilyn Ltee | USD 35,910.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps for Rocher Caiman Pump Stations | PUC | Bluezone Mauritius | Euro 30,761.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3 | PUC | Bluezone Mauritius | Euro 24,254.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps Sewerage Pump Station | PUC | Bluezone Mauritius | Euro 65,203.00 | -| 52 | 2/4 | Procurement of Virtual Studio Equipment | SBC | New Tek Europe | Euro 27,240.00 | -| 53 | 2/4 | Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works | PUC | Tracetebel Engineering | Euro 244,810.00 | -| 54 | 2/4 | Security in MOH's Institution-Lot 1 | MOH | Alliance Security | SR 58,968.00 | -| 55 | 2/4 | Procurement of Bearings for TurboCharger Wartsila Engines | PUC | ABB France | Euro 57,191.50 | -| 56 | 2/4 | Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine | PUC | Marine Power International | Euro 33,769.00 | -| 57 | 2/4 | Procurement of Lighting Equipments | PUC | Thorn Europhane | Euro 38,982.30 | -| 58 | 9/4 | Construction of Fooothpath at Olivier Maradan Street | SLTA | Benoiton Construction | SR 1,548.520.00 | -| 59 | 9/4 | Procurement of Utrasonic Cleaning Machine | PUC | IOP Marine (Denmark) | Euro 31,740.00 | -| 60 | 9/4 | Implementation of Nwe Financial System | SCAA | Blanche Birger | Euro 65,779.50 | -| 61 | 9/4 | Construction of Fuel station and Admin Block | SPTC | Onivo Construction | SR2,196.748.00 | -| 62 | 9/4 | Procurement of Plastic Chairs | MOE | J Galt International | ZAR 498,560.00 | -| 63 | 9/4 | Security Service in All Education Institution -Lot 2 | MOE | Xtreme Security Service | SR 28,000.00 | -| 64 | 16/4 | Procurement of Critical Spare Parts for Wartsila Engines B41 | PUC | Wartsila Global Logistic Service | Euro 37,460.92 | -| 65 | 16/4 | Security Service STC Premises-Supermarket | STC | Isles Security Agency Ltd | SR 908,107.20 | -| 65 | 16/4 | Security Service STC Premises-Meat & Veg | STC | Allaince Security | SR 456,183.36 | -| 65 | 16/4 | Security Service STC Premises-Warehouse, Duty free, BDR Complex | STC | Isles Security Agency Ltd | SR 490,752.00 | -| 65 | 16/4 | Security Service STC Premises-Head Office | STC | Isles Security Agency Ltd | SR 503,712.00 | -| 65 | 16/4 | Security Service STC Premises-Praslin ( Amitie Store /Duty-Free | STC | Alliance Security | SR 327,598.56 | -| 66 | 16/4 | Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract | MLUH | Allied Builders | SR 1,732,989.80 | -| 67 | 16/4 | Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials | MLUH | Ascent Projects (Sey) Pty Ltd | SR 3,888,632.42 | -| 68 | 23/4 | Procurement of Vehicle | SFA | Exel Motors | SR 519,869.79 | -| 69 | 23/4 | Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson | SCAA | Smiths Detection | Euro 368,000.00 | -| | | | | | 2 | -| 71 | 23/4 | Procurement of Ultracsonic Cleaning Machine | PUC | IOP Marine (Denmark) | Euro 1,250.00 | -| 72 | 30/4 | Procurement of Vehicle | PA | Abhaye Valabhji Pty Ltd | SR575,000.00 | -| 73 | 30/4 | Procurement of ID Cards | DICT | Blanche Birger Bureautique | Euro 43,500.00 | -| 74 | 30/4 | Procurement of Electrical Meters | PUC | ISKRAEMECO | Euro 55,733.60 | -| 75 | 30/4 | Procurement of Grunfos Pump for Le Rocher Pump Station | PUC | Bluezone Mauritius Ltd | Euro 37,910.60 | -| 76 | 30/4 | Procurement of A3 and A4 Photocopr Paper | MOE | Islad Motors Co Ltd | SR552,000.00 | -| | MAY | | | | | -| 77 | 7/5 | Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power | PUC | Technoglass | US$ 156, 920.00 | -| 77 | 7/5 | Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite | PUC | Technoglass | US$ 83,980.00 | -| 78 | 7/5 | Procurement of Connecting Rod (Variations) | PUC | Wartsila Eastern Africa Ltd | Euro 1,200.00 | -| 79 | 7/5 | Project Management Consultancy-Extension of contract | SIBA | Philippe Adrienne Consultancy Engineers | SR 225.000.00 | -| 80 | 7/5 | Procurement of Liquid Chlorine | PUC | Al Afaq LLC | USD 77,088.00 | -| 81 | 21/5 | Procurement of Agriculture Inputs | SAA | Rodley Mathieu | SR1,241,225.00 | -| 82 | 21/5 | Procurement of Metelogical Equipment | MEE | Vaisala | Euro 33,815.00 | -| 83 | 21/5 | Procurement of Turbocharger for Rotor Shaft for Wartsila Engine | PUC | Marine Power International | Euro 39,945.00 | -| 84 | 21/5 | Provision of Security Srevices for MOH's Institutions | MOH | Alliance Security | SR 19,656.00 | -| 85 | 21/5 | Procurement of Low Voltage ABC Cables | PUC | Nextans | Euro 50,118.69 | -| 86 | 28/5 | Procurement of HDPE pipes-Pipe for water applications | PUC | STR Marketing- | USD 68,030.02 | -| 86 | 28/5 | Procurement of HDPE pipes-Pipe for sewerage applications | PUC | STR Marketing | USD 12,509.55 | -| 87 | 28/5 | Procurement of Incenerator for Baie Ste Anne Praslin Hospital | MOH | Incinco Limited | GBP 101,992.00 | -| 88 | 28/5 | Geotechnical Survey on Ile Soleil | | 2020 Development Ltd Geoconsul Ltee | SR 741,520.00 | -| 89 | 28/5 | Extra Works at Palais De Justice | | The Judiciary Quingjian Group Co | SR 1,614,226.00 | -| 90 | 28/5 | Fire Fighting and rescue training Course | SFRS | Emergency Training Solution Pty Ltd | ZAR 824,453.80 | -| 91 | 28/5 | Procurement of Charger Air Cooler for Wartsila Engine at Power Station C | PUC | Marine Power International FZC | Euro 38,217.00 | -| | JUN | | | | | -| 92 | 4/6 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brits Pale | ZAR 438,081.83 | -| 93 | 4/6 | | Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC | Wartsila Global Service | Euro 74,768.40 | -| 94 | 4/6 | Procurement of Cylinder Liner and Pistons | PUC | Wartsila Eastern Africa | Euro 70,082.73 | -| 95 | 4/6 | Procurement of Digital Microwave Equipment | SBC | MOCHINO | Euro 125,845.00 | -| 96 | 4/6 | Procurement of Vehicle 1 | LWMA | Abhaye Valabhji Pty Ltd-Jeep | SR 475,000.00 | -| 96 | 4/6 | Procurement of Vehicle 1 | LWMA | EHW Seychelles Ltd- Car | SR 267,850.00 | -| 97 | 4/6 | Procurement of Forged filter Ball-Valves | PUC | Jainsons Malleables | USD 11,000.00 | -| 98 | 11/6 | Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka | SLTA | Bazil Construction | Sr 1,081,530.00 | -| 99 | 11/6 | Procurement of Seychelles Paswsports | DIA | Groupe Impimerie Nationale | Euro 102,400.00 | -| 100 | 19/6 | Anse Boileau Footpath and Drainage Construction Phase II | SLTA | TCH Building Contractor | SR 960,107.50 | -| 101 | 21/6 | Procurement of Vehicle x 1 | PSD | Abhaye Valabhji-Bus x1 | SR 475,000.00 | -| 101 | 21/6 | Procurement of Vehicle x 1 | PSD | PMC Auto- Car x1 | SR 267,850.00 | -| 102 | 21/6 | Renovation Works on Block A- Beau Vollon Secondary School | MOE | Sai-Fu Enterprise Company Ltd | SR 2,253,300.00 | -| 103 | 21/6 | Procurement of 35 VMS Terminal Accessories | SFA | Communication Specialist Ltd | Euro 63,770.00 | -| 104 | 21/6 | Renewal of SFA's Themis FMC Services | SFA | CLS- France | Euro 36,000.00 | -| 105 | 21/6 | Procurement of Electrical Spares for Wartsila Engine | PUC | Wartsila Eastern Africa | Euro 1,757.00 | -| 106 | 21/6 | Procurement of Class D water Meters | PUC | Elster Metering Limited (Pty) Ltd | USD 109,7051.00 | -| 107 | 21/6 | Procurement of Sludge Incinerator | PUC | Atlas Incinerator | € 94,470.00 | -| 108 | 21/6 | Procurement of Technical Services for Crankshaft Grinding | PUC | Goltens | USD 76,075.00 | -| 109 | 21/6 | Procurement of WAS Pumps | PUC | Netzsch Southern Africa Pty Ltd | Euro 56,657.20 | -| | | | | | 3 | -| 111 | 21/6 | Procurement of Fitting and Pipes ( Stock Replenishment) | PUC | STR Marketing Ltee | USD 60,688.90 | -| 112 | 21/6 | Refurbishment of Pharmaceutical Production Unit-Contract Extension | MOH | Mahe Design | SR808,375.00 | -| 113 | 21/6 | Technical Service for Repair on Generator-1B | PUC | Goltens | USD 75,600.00 | -| 114 | 21/6 | Procurement of Gate Valves | PUC | AVK Valves Southern africa (Pty) Ltd | ZAR 461,608.44 | -| 115 | 25/6 | Procurement of Hot-Dipped Galvanised Materials | PUC | HDSA Shipping (Pty) Ltd | Zar 435,614.00 | -| 116 | 25/6 | Procurement of Critical Spares for Wartsila Engine B51- Lot 1 | PUC | Wartsila Global Services | Euro 92,472.70 | -| 117 | 25/6 | Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin | PUC | RUYSCH | Euro 83,262.64 | -| 118 | 25/6 | | Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013 PUC | Tornado Group | USD 266,820.82 | -| 119 | 25/6 | Procurement of X-ray Screening Machine for VVIP Lounge | SCAA | Smiths Detection | Euro 101,800.00 | -| | JUL | | | | | -| 120 | 2/7 | Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road) | SLTA | Esparon's Enterprise | SR 183,410.00 | -| 121 | 2/7 | Bridge Renovation at Cascade | SLTA | Benioton Construction | SR 1,361,100.00 | -| 122 | 2/7 | Procurement of Bitumen | SLTA | Termcotank S.A | USD 519,418.20 | -| 123 | 2/7 | Procurement of Vehicle x 2 | STB | PMC Auto Pty Ltd | SR 960,126.00 | -| 124 | 2/7 | | Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning MFA | John Nevill | SR 180,000.00 | -| 125 | 9/7 | Procurement of Stationery Items Lot 4 | MOE | JD's Stationey EDU Centre | SR 629,950.00 | -| 126 | 9/7 | Procurement of Atomic Spectrometer | SBS | SMM Instrument (Pty) Ltd | USD 181,147.00 | -| 127 | 9/7 | Operation and Maintenance of Containerised Desalination Units by Tornado-2012 | PUC | Tornado Group | USD 45,174.62 | -| 128 | 9/7 | Procurement of Sensors and Transmitters for Wartsila Engines | PUC | Wartsila Global Logistic Services | Euro 50,360.00 | -| 129 | 9/7 | Procurement of Bulk Water Meters Strainers | PUC | Elster Metering Limited (Pty) Ltd | ZAR 931,966.00 | -| 130 | 9/7 | Procurement of Gudgeon Pins for engine 8P | PUC | Wartsila Global Logistic Services | Euro 23,460.00 | -| 131 | 9/7 | Procurement of Piston for Wartsila Engine A21 | PUC | Wartsila Global Logistic Services | Euro 219,622.00 | -| 132 | 9/7 | Procurement of Piston for Wartsila Engine B11 | PUC | Marine Power International FZC | Euro 243,799.98 | -| 133 | 9/7 | Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance | MLUH | Sai-Fu Enterprise Company Ltd | SR 17,376,392.65 | -| 134 | 9/7 | La Louise Non- Performance Pipieline Replacement | PUC | Ascent Projects Sey Pty Ltd | SR 1,289,375.00 | -| 135 | 9/7 | Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital | MOH | Des Iles Environment Solutions | SR 1,462,875.00 | -| 136 | 9/7 | Consultancy Services for Quality management System (QMS) | DE | Mr. John Horack | USD 22,325.00 | -| 137 | 9/7 | Construction of New Road at Cascade Primary School | MLUH | Esparon's Enterprise | SR 2,443,814.00 | -| 138 | 9/7 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brits Pale Pty Ltd | ZAR 518,694.02 | -| 139 | 9/7 | Procurement of Non-Critical Spares- Specialized Tools | PUC | Marine Power International FZC | Euro 10,866.55 | -| 140 | 9/7 | Upgarding of Roche Caiman Road and Roundabout | SLTA | Bazil Construction | SR 1,592,002.00 | -| 141 | 9/7 | Procurement of Crankshaft for Engine 5B at New Port Station | PUC | A&D Sales | GBP 65,375.00 | -| 142 | 16/7 | Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital | MOH | Des Iles Environment Solutions | SR1,462,875.00 | -| 143 | 16/7 | Consultancy Service for Quality Management System (QMS) | DE | Mr. John Horack | CND$22,325.00 | -| 144 | 16/7 | Construction of New Road at Cascade Primary School | SLTA | Esparon's Enterprise | SR 2,443,814.00 | -| 145 | 16/7 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brit Pale Pty Ltd | ZAR518,694.02 | -| 146 | 16/7 | Procurement od Non- critcal spares-specialized tools | PUC | Marine International FZC | Euro10,866.55 | -| 147 | 16/7 | Upgrading of Roche Caiman Road and roundabout | SLTA | Bazil Construction | SR1,592,002.00 | -| 148 | 16/7 | Procurement of Crankshaft for Engine 5B at New Port Station | PUC | A&D Sales | GBP 65,375.00 | -| 149 | 23/7 | Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area. | MLUH | LC International | SR2,814,108.00 | -| 150 | 23/7 | Procurement of CT Scan Tube | MOH | Ireland Blyth Ltd from Mauritius | Euro 122,000.00 | -| 151 | 23/7 | Procurement of Ultraviolet disinfection System for Sewerage Treatment | PUC | Orica Wtercare | SR851,104.72 | -| 152 | 23/7 | | Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps PUC | Hidrostal Sewage Sa Pty Ltd | Euro97,498.32 | -| 153 | 23/7 | (B)Procurement of Grundfos Pumps for water pumping | PUC | Bluezone Mauritius | Euro 13,410.00 | -| 154 | 23/7 | Procurement of spare for Mirrlees Radiator | PUC | Covard Heat Transfer Ltd | GBP32,042.43 | -| | | | | | 4 | -| 156 | 23/7 | General renovation works to Block B at Belonie Secondary School | MOE | Belvedere Builders | SR869,505.75 | -| 157 | 30/7 | Procurement of Engine Block and Crankshaft for Engine A11 | PUC | Ras Tek Pvt Ltd | Euro798,650.00 | -| 158 | 30/7 | procurement of Wartsila Engine spares | PUC | Wartsila Eastern Africa ltd | Euro158,424.00 | -| 159 | 30/7 | Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) | SLTA | G&S Enterpise | SR1,113,010.00 | -| 160 | 30/7 | Procurement of transfer pump control panel | PUC | CA Engineering Consultancy Pte Ltd | SGD14,600.00 | -| 161 | 30/7 | Consultancy service for North to South Victoria Bye- Pass road and utilities organisation | MLUH | Sonnel Seychelles LTD | SR1,332,000.00 | -| 162 | 30/7 | Procurement of the supply of sodium cardonate | PUC | HPL Chemical LTD | USD42,600.00 | -| | AUG | | | | | -| 163 | 6/8 | Procurement of Vehichels X 4 | MOH | Kim-Koom & Co Pty Ltd | SR1,100,000.00 | -| 164 | 6/8 | | Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe WMF | Mr. Donal Ernesta | | -| 164 | 6/8 | | Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe WMF | Mr. Kali Deenudayali | | -| 165 | 13/8 | Procurement of exercise books | MOE | JD's Stationey EDU Centre | SR,1,700,000.00 | -| 166 | 13/8 | Procurement of High Pressure pump spares -BZM00003022 | PUC | Bluezone Mauritius Ltd | Euro27,712.73 | -| 166 | 13/8 | Procurement of High Pressure pump spares -BZM00003023 | PUC | Bluezone Mauritius Ltd | Euro12,639.55 | -| 167 | 13/8 | Procurement of CR64 pump spares | PUC | Bluezone Mauritius Ltd | Euro31,822.00 | -| 168 | 13/8 | Construction of access road at Ex-Deltel- Anse Royale | MLUH | Benoiton Construction Pty Ltd | SR4,585,441.12 | -| 169 | 13/8 | Renovation work to one classroom block at Pionte Larue Secondary School | MOE | Belverdere Builders | SR1,114,575.00 | -| 170 | 13/8 | Completion of Amitie Housing Project 12 x 3 Bedrooms | MLUH | Allied Builders Sey Ltd | 6,006,410.37 | -| 171 | 13/8 | Copolia Road widening-Phase 2 | SLTA | Belverdere Builders | SR1,037,235.00 | -| 172 | 20/8 | Procurement of ABB Turbocharger Cartridge | PUC | ABB France | Euro106,266.66 | -| 173 | 20/8 | Procurement of services to carry out the full refit and overhaul of tug Alouette | SPA | SECREN (Madagascar) | Euro189,618.42 | -| 174 | 20/8 | Awarding of cranshatf and Block replacement solution for A11 engine | PUC | Wartsila | Euro800,000.00 | -| 175 | 29/8 | Servicing of geartrain for Wartsila Engine 18V32LN | PUC | Wartsila Eastern Africa | Euro 21,141.90 | -| 176 | 29/8 | Spare parts for Wartsila Engine 18V32LN | PUC | Wartsila Eastern Africa | Euro38,440.00 | -| 177 | 29/8 | Procurement for sience equipment and chemical for 2013 | MOE | Findel Education | GBP37,831.26 | -| 178 | 29/8 | Installation of fencing at Mont Fleuri Secodary School | MOE | Donald Builbing & Contractor Pty Ltd | SR1,761,034.00 | -| 179 | 29/8 | | Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA | Furui Construction Pty Ltd | SR277,521.45 | -| 180 | 29/8 | Propsed road widening and drainage improvement at Quincy Vilage | SLTA | TCH Contractor | SR1,015,806.24 | -| 181 | 3/9 | Tender for procurement of windows licenses | STB | Victoria Computer Service | SR788,808.00 | -| 182 | 3/9 | Procurement of Bitumen in drums for the asphalt production Praslin | SLTA | Benzene International Pte Ltd | Euro87,220.00 | -| 183 | 3/9 | Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S | PUC | Wartsila Eastern Africa | Euro9,861.00 | -| 184 | 3/9 | Procurement of gear train parts for Stork Wartsila Engine SW280 | PUC | Wartsila Eastern Africa | Euro7,931.00 | -| | SEP | | | | | -| 185 | 10/9 | Cleaning and maintenance of wetlands and rivers on Praslin | ED | "W" Cleaning Service | SR769,590.00 | -| 186 | 10/9 | Re-construction and partition of Independence House | MLUH | Green Island Construction Co Pty | SR868,022.94 | -| 187 | 10/9 | Procurement of critical spare fro major overhaul | PUC | Wartsila Eastern Afirca | Euro33,623.00 | -| 188 | 10/9 | Procurement of control panel for six pumps station | PUC | CA Engineering Consultancy Pte Ltd | SGD37,400.00 | -| 189 | 17/9 | Security service for Wellness Centre | MOH | Alliance Security | SR78,624.00 | -| 190 | 17/9 | Procurement of reinforce plastic (FRP) handrails and grating | PUC | Webforge Group | SR839,795.02 | -| 191 | 17/9 | Procurement of helital fitting | PUC | Cu A1 Engineering (Pty) Ltd | ZAR343,000.00 | -| 192 | 17/9 | Procurement of Flygt pumps | PUC | Aqualia DPI LTD | Euro56,080.00 | -| 193 | 17/9 | Procurement of black-up and emergency pumps | PUC | M.A.H.Y Khoory & Co | UAE186,650.00 | -| 194 | 17/9 | Procurement of additional requirement of polo t-shirts and jeans | PUC | Magilyn Ltee | USD42,127.50 | -| 195 | 17/9 | Procurement of Pressure filters | PUC | Barr +Wray | GBP379,921.00 | -| 196 | 17/9 | Procurement of grunfos pump for water pumping | PUC | Blue Zone Mauritius | Euro108,574.00 | -| | | | | | 5 | -| 198 | 24/9 | Procurement of Technical Services for Crankshaft Grinding on Engine 6B | PUC | Golten Co Ltd | USD92,654.00 | -| 199 | 24/9 | Procurement of technical service for repair of generator 1B- varations | PUC | Golten Co Ltd | USD84,337.00 | -| 200 | 24/9 | Procurement of spare parts fro Wartsila Engine A21 | PUC | Wartsila Global Logistic | Euro235,579.30 | -| 201 | 24/9 | Procurement of vehicle x 2 | SLTA | Abhaye Valabhji Pty Ltd | SR1000.000.00 | -| | OCT | | | | | -| 202 | 1/10 | Proposed new traffic lane to 5th June Avenue | SLTA | Divy Constrution | SR2,864,589.00 | -| 203 | 1/10 | | Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA | G & S Enterprise | SR200,448.00 | -| 204 | 1/10 | Proposed Reconstrcution of Burnt House-Au Cap | MLUH | Furui Construction | SR946,130.00 | -| 205 | 1/10 | Variation on the project associated with the procurement of seven 100m3/day containerised plant | PUC | Tornado Group (UAE) | USD172,500.00 | -| 206 | 1/10 | Works on the breaker system at Bel Omber desalination plant | PUC | United Concrete Products (Sey)Ltd | SR1,998,993.11 | -| 207 | 1/10 | Procurement of Viking Johnson fittings | PUC | Viking Johnson (UK) | GBP92,286.50 | -| 208 | 1/10 | Procurement of HDPE Pipes and Fittings | PUC | STR Marketing Ltee | USD196,672.57 | -| 209 | 1/10 | Procurement of Piston and Gudgeon Pins | PUC | Marine Power International FZC | Euro277,098.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 1 | PUC | Ms. Suzanne Edmond | SR500.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 2 | PUC | Ms. Suzanne Edmond | SR200.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 3 | PUC | Ms. Suzanne Edmond | SR475.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 4 | PUC | Sey Sytle | SR450.00 | -| 211 | 8/10 | Procurement of vehicle x 2 | FIU | PMC Auto Pty Ltd | SR526,864.00 | -| 212 | 15/10 | Renovation Works to Le Chantier Mall-Variation Works | SSF | Allied Builders Sey Ltd | SR1,673,752.51 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 1 | MSACDS | Eagle Watch Security Services | SR96,600.00 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 2 | MSACDS | Alliance Security Services | SR103,012.00 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 4 | MSACDS | Security Protection Services | SR71,000.00 | -| 214 | 15/10 | Anse Boileau Footpath and Drainage Construction Phase 2 - Variations | SLTA | TCH Contractor | SR326,780.00 | -| 215 | 15/10 | Construction of Fooothpath at Olivier Maradan Street-Variations | | Benoiton Construction Pty Ltd | SR368,642.50 | -| 216 | 15/10 | Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014 | PUC | Tornado Group (UAE) | USD1,214,400.00 | -| 217 | 15/10 | Procurement of Non Critical Spare Parts for Wartsila Engine 5B | | MAN Diesel & Turbo (UK) | GDP88,360.88 | -| 218 | 15/10 | Procurement of Meter Boxes | PUC | CAHORS (France) | Euro35,221.20 | -| 219 | 15/10 | Construction of Services Road to Commercial Zone West of Inter Island Quay | MLUH | Allied Builders Sey Ltd | SR6,234,343.36 | -| 220 | 15/10 | Proposed New traffic lane from Roche Caiman to Eden Island | SLTA | Franky's Constrcution | SR3,017,155.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | Seytex | SR695,810.58 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | VCS Pty Ltd | SR36,430.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | KIS Distribution Company | SR10,500.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | MNM General Supply | SR16,216.20 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | Roy & Sons Import | SR107,625.00 | -| 222 | 22/10 | Road Widening and Drainage Improvement at Quincy Village-Variations | SLTA | TCH Building Contractor | SR328,213.75 | -| 223 | 22/10 | Procurement of Grunfos Pumps for Mare Aux Cochons | PUC | Bluezone Mauritius | Euro44,914.00 | -| 224 | 22/10 | Procurement of Vehicle x 3 | MLUH | Sun Motors | SR2,310,000.00 | -| 225 | 22/10 | Procurement of Agriculture Inputs | SAA | Launch Export (SA) | ZAR720,100.00 | -| 226 | 22/10 | Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09 | | Allied Builders Sey Ltd | SR14,964,125.91 | -| 227 | 22/10 | Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08 | | Allied Builders Sey Ltd | SR35,927,978.92 | -| 228 | 29/10 | Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri | MLUH | Allied Builders Sey Ltd | SR10,372,824.02 | -| 229 | 29/10 | Construction of One Block of Condominium Flats at Pointe Larue | MLUH | Franky's Constrcution | SR18,063,876.00 | -| 230 | 29/10 | Proposed Construction of Micro-Enterprise Building at Providence | MLUH | O Nivo Construction | SR28,462,412.00 | -| 231 | 29/10 | Renovation of English River Secondary Schools | MLUH | Divy Constrution | SR2,133,560.00 | -| 232 | 29/10 | Renovation and Partitioning of Independence House-Variation | MLUH | Green Island Construction Co Pty | SR1,847,477.43 | -| | | | | | 6 | +||Date|Procurement Title|PE|Bidder|Amount| +|---|---|---|---|---|---| +||JAN||||| +|1|8/1|Procurement of Criticals Spare Parts for Engine Maintenance on Mahe|PUC|Wartsila Eastern Africa Ltd|Euro97,922.30| +|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| +|3|15/1|Manufacturing and Deliveries of 900 Students Desks|MOE||SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00| +|4|15/1|Renovation Works at Mont Fleuri Secondary School|MOE|Sai-Fu Enterprise|SR2,815,771.00| +|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| +|6|15/1|Storm Water Channel Project at Au Cap-Additional Works|DOE|United Concrete Products (Sey)Ltd|SR184,300.00| +|7|22/1|Procurement of Security at ex-Maritime Training Centre|SFA|Elite Surveillance Security Agency|SR30,000.00| +|8|22/1|Installation of Sewerage Treatment Plant at Anse Gaulette|MLUH|Green Island Construction Compnay|SR4,524,884.85| +|9|29/1|Procurement of Cylinder Liner|PUC|Wartsila Eastern Africa Ltd|Euro28,186.75| +|10|29/1|Procurement of Service Pack for Coupling|PUC|Wartsila Eastern Africa Ltd|Euro11,151.00| +|11|29/1|Construction of Drainage for Roads A & B Eve Island Praslin|MLUH|Ascent Projects Sey|SR2,931,174.00| +|12|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| +||FEB||||| +|13|5/2|Procurement of DI Pipes and Fittings for Le Rocher Refurbishment|PUC|Legend General Supply|USD43,807.50| +|14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| +|15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| +|16|5/2|constraction of 6*2 bedroom Houses- Mont Buxton|MLUH|O-NIVO Construction|SR3,688,844.00| +|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| +|18|5/2|Procurement of Vehcile x1|MOE|Abhaye Valabhji Pty Ltd|SR 650,000.00| +|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| +|20|12/2|Procurement of Turbo charger Rotor for Engine on Praslin|PUC|Marine Power International FZC|Euro 835,669.00| +|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| +|22|12/2|Procurement of Air Cooler Cartridge for Wartsila Engine|PUC|Marine Power International FZC|Euro 36,508.93| +|23|12/2|Procurement of Spares for Genset 8P on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro 288,202.50| +|24|19/2|Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka|SLTA|Esparon's Enterprise|SR 1,105,069.00| +|25|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41|PUC|Marine Power International|Euro 144,726.26| +|26|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31|PUC|Marine Power International FZC|Euro 143,746.75| +|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| +|28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| +|29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| +|30|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| +||MAR||||| +|31|5/3|Procurement of Spare Parts for the Asphalt Plant- Petite Paris|SLTA|Astec Factory (USA)|USD 101,337.81| +|32|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines|PUC|Marine Power International FZC|Euro 44,050.00| +|33|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines B11|PUC|Marine Power International FZC|Euro 98,743.00| +|34|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines 8B|PUC|Marine Power International FZC|Euro 103,215.00| +|35|5/3|Procurement of Critical Spare Parts for Wartsila Engines 8B|PUC|Wartsila Global Logistic|Euro 97,979.10| +|36|5/3|Procurement Spare Parts for the 8MW Engines|PUC|Wartsila Global Logistic|Euro 41,444.00| +|37|5/3|Procurement of Voltage Districbution Boxes and Fuses-Variations|PUC|Indian Ocean Export Company Pty Ltd|GBP1,300.00| +|38|5/3|33Kv South Mahe Project-Variations|PUC|United Concrete Products (Sey)Ltd|SR459,684.00| +|39|5/3|construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations|SFRS|Furui Construction Pty Ltd|SR184,836.03| +|40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| +||||||1| +|41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 4 HIS & PJ Enterprise|SR 1,753,614.70| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 5 Roy & Sons Import|SR 1,080.000.00| +|42|19/3|Procurement of Medium Voltage Cables|PUC|Nexans France Lens|Euro 52,354.45| +|43|19/3|Procurement of Brass/Bronze Fitting|PUC|Jainsons Industries|Euro 97,718.48| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre|SFA|Neil Marine (Sri Lanka)|USD 284,521.15| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre-Equipment|SFA|Neil Marine (Sri Lanka)|USD 97,861.50| +|45|19/3|Procurement of Stationery Items-Lot 1|MOE|Print V Care|SR 349,450.00| +|45|19/3|Procurement of Stationery Items-Lot 2|MOE|Print V Care|SR 392,500.00| +|45|19/3|Procurement of Stationery Items-Lot 3|MOE|JD's Stationery & Educational Centre|SR 19,150.00| +|45|19/3|Procurement of Stationery Items-Lot 5|MOE|Vitoria Computer Services Pty LTd|SR 94,500.00| +|45|19/3|Procurement of Stationery Items-Lot 6|MOE|Roy & Sons Imports|SR 114,750.00| +|45|19/3|Procurement of Stationery Items-Lot 7|MOE|Trade Supplies Pty Ltd|SR 35,224.00| +|46|26/3|Procurement of Microsoft Windows License|STB|Victoria Computer Service|SR 977,150.40| +|47|26/3||Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations PUC|Ras Tek Pvt Ltd|Euro 58,000.00| +||APR||||| +|48|2/4|Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin|PUC|Vijay Construction|SR 623,025.00| +|49|2/4|Supply of Fish to the Prison Department||Prison Service Mr. Danny Loizeau|SR 27.50 per KG| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 1|PUC|Magilyn Ltee|USD 28,885.00| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 2|PUC|Magilyn Ltee|USD 35,910.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations|PUC|Bluezone Mauritius|Euro 30,761.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3|PUC|Bluezone Mauritius|Euro 24,254.00| +|51|2/4|Procurement of Grundfos Pumps Sewerage Pump Station|PUC|Bluezone Mauritius|Euro 65,203.00| +|52|2/4|Procurement of Virtual Studio Equipment|SBC|New Tek Europe|Euro 27,240.00| +|53|2/4|Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works|PUC|Tracetebel Engineering|Euro 244,810.00| +|54|2/4|Security in MOH's Institution-Lot 1|MOH|Alliance Security|SR 58,968.00| +|55|2/4|Procurement of Bearings for TurboCharger Wartsila Engines|PUC|ABB France|Euro 57,191.50| +|56|2/4|Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine|PUC|Marine Power International|Euro 33,769.00| +|57|2/4|Procurement of Lighting Equipments|PUC|Thorn Europhane|Euro 38,982.30| +|58|9/4|Construction of Fooothpath at Olivier Maradan Street|SLTA|Benoiton Construction|SR 1,548.520.00| +|59|9/4|Procurement of Utrasonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 31,740.00| +|60|9/4|Implementation of Nwe Financial System|SCAA|Blanche Birger|Euro 65,779.50| +|61|9/4|Construction of Fuel station and Admin Block|SPTC|Onivo Construction|SR2,196.748.00| +|62|9/4|Procurement of Plastic Chairs|MOE|J Galt International|ZAR 498,560.00| +|63|9/4|Security Service in All Education Institution -Lot 2|MOE|Xtreme Security Service|SR 28,000.00| +|64|16/4|Procurement of Critical Spare Parts for Wartsila Engines B41|PUC|Wartsila Global Logistic Service|Euro 37,460.92| +|65|16/4|Security Service STC Premises-Supermarket|STC|Isles Security Agency Ltd|SR 908,107.20| +|65|16/4|Security Service STC Premises-Meat & Veg|STC|Allaince Security|SR 456,183.36| +|65|16/4|Security Service STC Premises-Warehouse, Duty free, BDR Complex|STC|Isles Security Agency Ltd|SR 490,752.00| +|65|16/4|Security Service STC Premises-Head Office|STC|Isles Security Agency Ltd|SR 503,712.00| +|65|16/4|Security Service STC Premises-Praslin ( Amitie Store /Duty-Free|STC|Alliance Security|SR 327,598.56| +|66|16/4|Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract|MLUH|Allied Builders|SR 1,732,989.80| +|67|16/4|Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials|MLUH|Ascent Projects (Sey) Pty Ltd|SR 3,888,632.42| +|68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| +|69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| +||||||2| +|71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| +|72|30/4|Procurement of Vehicle|PA|Abhaye Valabhji Pty Ltd|SR575,000.00| +|73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| +|74|30/4|Procurement of Electrical Meters|PUC|ISKRAEMECO|Euro 55,733.60| +|75|30/4|Procurement of Grunfos Pump for Le Rocher Pump Station|PUC|Bluezone Mauritius Ltd|Euro 37,910.60| +|76|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| +||MAY||||| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power|PUC|Technoglass|US$ 156, 920.00| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite|PUC|Technoglass|US$ 83,980.00| +|78|7/5|Procurement of Connecting Rod (Variations)|PUC|Wartsila Eastern Africa Ltd|Euro 1,200.00| +|79|7/5|Project Management Consultancy-Extension of contract|SIBA|Philippe Adrienne Consultancy Engineers|SR 225.000.00| +|80|7/5|Procurement of Liquid Chlorine|PUC|Al Afaq LLC|USD 77,088.00| +|81|21/5|Procurement of Agriculture Inputs|SAA|Rodley Mathieu|SR1,241,225.00| +|82|21/5|Procurement of Metelogical Equipment|MEE|Vaisala|Euro 33,815.00| +|83|21/5|Procurement of Turbocharger for Rotor Shaft for Wartsila Engine|PUC|Marine Power International|Euro 39,945.00| +|84|21/5|Provision of Security Srevices for MOH's Institutions|MOH|Alliance Security|SR 19,656.00| +|85|21/5|Procurement of Low Voltage ABC Cables|PUC|Nextans|Euro 50,118.69| +|86|28/5|Procurement of HDPE pipes-Pipe for water applications|PUC|STR Marketing-|USD 68,030.02| +|86|28/5|Procurement of HDPE pipes-Pipe for sewerage applications|PUC|STR Marketing|USD 12,509.55| +|87|28/5|Procurement of Incenerator for Baie Ste Anne Praslin Hospital|MOH|Incinco Limited|GBP 101,992.00| +|88|28/5|Geotechnical Survey on Ile Soleil||2020 Development Ltd Geoconsul Ltee|SR 741,520.00| +|89|28/5|Extra Works at Palais De Justice||The Judiciary Quingjian Group Co|SR 1,614,226.00| +|90|28/5|Fire Fighting and rescue training Course|SFRS|Emergency Training Solution Pty Ltd|ZAR 824,453.80| +|91|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| +||JUN||||| +|92|4/6|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale|ZAR 438,081.83| +|93|4/6||Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC|Wartsila Global Service|Euro 74,768.40| +|94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| +|95|4/6|Procurement of Digital Microwave Equipment|SBC|MOCHINO|Euro 125,845.00| +|96|4/6|Procurement of Vehicle 1|LWMA|Abhaye Valabhji Pty Ltd-Jeep|SR 475,000.00| +|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| +|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| +|98|11/6|Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka|SLTA|Bazil Construction|Sr 1,081,530.00| +|99|11/6|Procurement of Seychelles Paswsports|DIA|Groupe Impimerie Nationale|Euro 102,400.00| +|100|19/6|Anse Boileau Footpath and Drainage Construction Phase II|SLTA|TCH Building Contractor|SR 960,107.50| +|101|21/6|Procurement of Vehicle x 1|PSD|Abhaye Valabhji-Bus x1|SR 475,000.00| +|101|21/6|Procurement of Vehicle x 1|PSD|PMC Auto- Car x1|SR 267,850.00| +|102|21/6|Renovation Works on Block A- Beau Vollon Secondary School|MOE|Sai-Fu Enterprise Company Ltd|SR 2,253,300.00| +|103|21/6|Procurement of 35 VMS Terminal Accessories|SFA|Communication Specialist Ltd|Euro 63,770.00| +|104|21/6|Renewal of SFA's Themis FMC Services|SFA|CLS- France|Euro 36,000.00| +|105|21/6|Procurement of Electrical Spares for Wartsila Engine|PUC|Wartsila Eastern Africa|Euro 1,757.00| +|106|21/6|Procurement of Class D water Meters|PUC|Elster Metering Limited (Pty) Ltd|USD 109,7051.00| +|107|21/6|Procurement of Sludge Incinerator|PUC|Atlas Incinerator|€ 94,470.00| +|108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| +|109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| +||||||3| +|111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| +|112|21/6|Refurbishment of Pharmaceutical Production Unit-Contract Extension|MOH|Mahe Design|SR808,375.00| +|113|21/6|Technical Service for Repair on Generator-1B|PUC|Goltens|USD 75,600.00| +|114|21/6|Procurement of Gate Valves|PUC|AVK Valves Southern africa (Pty) Ltd|ZAR 461,608.44| +|115|25/6|Procurement of Hot-Dipped Galvanised Materials|PUC|HDSA Shipping (Pty) Ltd|Zar 435,614.00| +|116|25/6|Procurement of Critical Spares for Wartsila Engine B51- Lot 1|PUC|Wartsila Global Services|Euro 92,472.70| +|117|25/6|Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin|PUC|RUYSCH|Euro 83,262.64| +|118|25/6||Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013 PUC|Tornado Group|USD 266,820.82| +|119|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| +||JUL||||| +|120|2/7|Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road)|SLTA|Esparon's Enterprise|SR 183,410.00| +|121|2/7|Bridge Renovation at Cascade|SLTA|Benioton Construction|SR 1,361,100.00| +|122|2/7|Procurement of Bitumen|SLTA|Termcotank S.A|USD 519,418.20| +|123|2/7|Procurement of Vehicle x 2|STB|PMC Auto Pty Ltd|SR 960,126.00| +|124|2/7||Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning MFA|John Nevill|SR 180,000.00| +|125|9/7|Procurement of Stationery Items Lot 4|MOE|JD's Stationey EDU Centre|SR 629,950.00| +|126|9/7|Procurement of Atomic Spectrometer|SBS|SMM Instrument (Pty) Ltd|USD 181,147.00| +|127|9/7|Operation and Maintenance of Containerised Desalination Units by Tornado-2012|PUC|Tornado Group|USD 45,174.62| +|128|9/7|Procurement of Sensors and Transmitters for Wartsila Engines|PUC|Wartsila Global Logistic Services|Euro 50,360.00| +|129|9/7|Procurement of Bulk Water Meters Strainers|PUC|Elster Metering Limited (Pty) Ltd|ZAR 931,966.00| +|130|9/7|Procurement of Gudgeon Pins for engine 8P|PUC|Wartsila Global Logistic Services|Euro 23,460.00| +|131|9/7|Procurement of Piston for Wartsila Engine A21|PUC|Wartsila Global Logistic Services|Euro 219,622.00| +|132|9/7|Procurement of Piston for Wartsila Engine B11|PUC|Marine Power International FZC|Euro 243,799.98| +|133|9/7|Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance|MLUH|Sai-Fu Enterprise Company Ltd|SR 17,376,392.65| +|134|9/7|La Louise Non- Performance Pipieline Replacement|PUC|Ascent Projects Sey Pty Ltd|SR 1,289,375.00| +|135|9/7|Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR 1,462,875.00| +|136|9/7|Consultancy Services for Quality management System (QMS)|DE|Mr. John Horack|USD 22,325.00| +|137|9/7|Construction of New Road at Cascade Primary School|MLUH|Esparon's Enterprise|SR 2,443,814.00| +|138|9/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale Pty Ltd|ZAR 518,694.02| +|139|9/7|Procurement of Non-Critical Spares- Specialized Tools|PUC|Marine Power International FZC|Euro 10,866.55| +|140|9/7|Upgarding of Roche Caiman Road and Roundabout|SLTA|Bazil Construction|SR 1,592,002.00| +|141|9/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|142|16/7|Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR1,462,875.00| +|143|16/7|Consultancy Service for Quality Management System (QMS)|DE|Mr. John Horack|CND$22,325.00| +|144|16/7|Construction of New Road at Cascade Primary School|SLTA|Esparon's Enterprise|SR 2,443,814.00| +|145|16/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brit Pale Pty Ltd|ZAR518,694.02| +|146|16/7|Procurement od Non- critcal spares-specialized tools|PUC|Marine International FZC|Euro10,866.55| +|147|16/7|Upgrading of Roche Caiman Road and roundabout|SLTA|Bazil Construction|SR1,592,002.00| +|148|16/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|149|23/7|Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area.|MLUH|LC International|SR2,814,108.00| +|150|23/7|Procurement of CT Scan Tube|MOH|Ireland Blyth Ltd from Mauritius|Euro 122,000.00| +|151|23/7|Procurement of Ultraviolet disinfection System for Sewerage Treatment|PUC|Orica Wtercare|SR851,104.72| +|152|23/7||Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps PUC|Hidrostal Sewage Sa Pty Ltd|Euro97,498.32| +|153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| +|154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| +||||||4| +|156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| +|157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00| +|158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00| +|159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00| +|160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00| +|161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00| +|162|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| +||AUG||||| +|163|6/8|Procurement of Vehichels X 4|MOH|Kim-Koom & Co Pty Ltd|SR1,100,000.00| +|164|6/8||Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe WMF|Mr. Donal Ernesta|| +|164|6/8||Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe WMF|Mr. Kali Deenudayali|| +|165|13/8|Procurement of exercise books|MOE|JD's Stationey EDU Centre|SR,1,700,000.00| +|166|13/8|Procurement of High Pressure pump spares -BZM00003022|PUC|Bluezone Mauritius Ltd|Euro27,712.73| +|166|13/8|Procurement of High Pressure pump spares -BZM00003023|PUC|Bluezone Mauritius Ltd|Euro12,639.55| +|167|13/8|Procurement of CR64 pump spares|PUC|Bluezone Mauritius Ltd|Euro31,822.00| +|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| +|169|13/8|Renovation work to one classroom block at Pionte Larue Secondary School|MOE|Belverdere Builders|SR1,114,575.00| +|170|13/8|Completion of Amitie Housing Project 12 x 3 Bedrooms|MLUH|Allied Builders Sey Ltd|6,006,410.37| +|171|13/8|Copolia Road widening-Phase 2|SLTA|Belverdere Builders|SR1,037,235.00| +|172|20/8|Procurement of ABB Turbocharger Cartridge|PUC|ABB France|Euro106,266.66| +|173|20/8|Procurement of services to carry out the full refit and overhaul of tug Alouette|SPA|SECREN (Madagascar)|Euro189,618.42| +|174|20/8|Awarding of cranshatf and Block replacement solution for A11 engine|PUC|Wartsila|Euro800,000.00| +|175|29/8|Servicing of geartrain for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro 21,141.90| +|176|29/8|Spare parts for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro38,440.00| +|177|29/8|Procurement for sience equipment and chemical for 2013|MOE|Findel Education|GBP37,831.26| +|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| +|179|29/8||Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA|Furui Construction Pty Ltd|SR277,521.45| +|180|29/8|Propsed road widening and drainage improvement at Quincy Vilage|SLTA|TCH Contractor|SR1,015,806.24| +|181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| +|182|3/9|Procurement of Bitumen in drums for the asphalt production Praslin|SLTA|Benzene International Pte Ltd|Euro87,220.00| +|183|3/9|Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S|PUC|Wartsila Eastern Africa|Euro9,861.00| +|184|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| +||SEP||||| +|185|10/9|Cleaning and maintenance of wetlands and rivers on Praslin|ED|"W" Cleaning Service|SR769,590.00| +|186|10/9|Re-construction and partition of Independence House|MLUH|Green Island Construction Co Pty|SR868,022.94| +|187|10/9|Procurement of critical spare fro major overhaul|PUC|Wartsila Eastern Afirca|Euro33,623.00| +|188|10/9|Procurement of control panel for six pumps station|PUC|CA Engineering Consultancy Pte Ltd|SGD37,400.00| +|189|17/9|Security service for Wellness Centre|MOH|Alliance Security|SR78,624.00| +|190|17/9|Procurement of reinforce plastic (FRP) handrails and grating|PUC|Webforge Group|SR839,795.02| +|191|17/9|Procurement of helital fitting|PUC|Cu A1 Engineering (Pty) Ltd|ZAR343,000.00| +|192|17/9|Procurement of Flygt pumps|PUC|Aqualia DPI LTD|Euro56,080.00| +|193|17/9|Procurement of black-up and emergency pumps|PUC|M.A.H.Y Khoory & Co|UAE186,650.00| +|194|17/9|Procurement of additional requirement of polo t-shirts and jeans|PUC|Magilyn Ltee|USD42,127.50| +|195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| +|196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| +||||||5| +|198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| +|199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| +|200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| +|201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00| +||OCT||||| +|202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| +|203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00| +|204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| +|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| +|206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| +|207|1/10|Procurement of Viking Johnson fittings|PUC|Viking Johnson (UK)|GBP92,286.50| +|208|1/10|Procurement of HDPE Pipes and Fittings|PUC|STR Marketing Ltee|USD196,672.57| +|209|1/10|Procurement of Piston and Gudgeon Pins|PUC|Marine Power International FZC|Euro277,098.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 1|PUC|Ms. Suzanne Edmond|SR500.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 2|PUC|Ms. Suzanne Edmond|SR200.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 3|PUC|Ms. Suzanne Edmond|SR475.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 4|PUC|Sey Sytle|SR450.00| +|211|8/10|Procurement of vehicle x 2|FIU|PMC Auto Pty Ltd|SR526,864.00| +|212|15/10|Renovation Works to Le Chantier Mall-Variation Works|SSF|Allied Builders Sey Ltd|SR1,673,752.51| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 4|MSACDS|Security Protection Services|SR71,000.00| +|214|15/10|Anse Boileau Footpath and Drainage Construction Phase 2 - Variations|SLTA|TCH Contractor|SR326,780.00| +|215|15/10|Construction of Fooothpath at Olivier Maradan Street-Variations||Benoiton Construction Pty Ltd|SR368,642.50| +|216|15/10|Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014|PUC|Tornado Group (UAE)|USD1,214,400.00| +|217|15/10|Procurement of Non Critical Spare Parts for Wartsila Engine 5B||MAN Diesel & Turbo (UK)|GDP88,360.88| +|218|15/10|Procurement of Meter Boxes|PUC|CAHORS (France)|Euro35,221.20| +|219|15/10|Construction of Services Road to Commercial Zone West of Inter Island Quay|MLUH|Allied Builders Sey Ltd|SR6,234,343.36| +|220|15/10|Proposed New traffic lane from Roche Caiman to Eden Island|SLTA|Franky's Constrcution|SR3,017,155.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Seytex|SR695,810.58| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|VCS Pty Ltd|SR36,430.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|KIS Distribution Company|SR10,500.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|MNM General Supply|SR16,216.20| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Roy & Sons Import|SR107,625.00| +|222|22/10|Road Widening and Drainage Improvement at Quincy Village-Variations|SLTA|TCH Building Contractor|SR328,213.75| +|223|22/10|Procurement of Grunfos Pumps for Mare Aux Cochons|PUC|Bluezone Mauritius|Euro44,914.00| +|224|22/10|Procurement of Vehicle x 3|MLUH|Sun Motors|SR2,310,000.00| +|225|22/10|Procurement of Agriculture Inputs|SAA|Launch Export (SA)|ZAR720,100.00| +|226|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09||Allied Builders Sey Ltd|SR14,964,125.91| +|227|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08||Allied Builders Sey Ltd|SR35,927,978.92| +|228|29/10|Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri|MLUH|Allied Builders Sey Ltd|SR10,372,824.02| +|229|29/10|Construction of One Block of Condominium Flats at Pointe Larue|MLUH|Franky's Constrcution|SR18,063,876.00| +|230|29/10|Proposed Construction of Micro-Enterprise Building at Providence|MLUH|O Nivo Construction|SR28,462,412.00| +|231|29/10|Renovation of English River Secondary Schools|MLUH|Divy Constrution|SR2,133,560.00| +|232|29/10|Renovation and Partitioning of Independence House-Variation|MLUH|Green Island Construction Co Pty|SR1,847,477.43| +||||||6| -| 298 | 17/12 | Renovation of Glacis Health Centre-Variations | MOH | F & P Construction | SR400,000.00 | -| --- | ----- | ---------------------------------------- | ---- | ---------------------------------------- | -------------- | -| 299 | 17/12 | Supply of New Equipment for Kitchen | | Prison Service K. K. Chua | SR943,000.00 | -| 300 | 23/12 | Procurement of Vehicle x 1 | MLUH | Sun Motors | SR770,000.00 | -| 300 | 23/12 | Procurement of Vehicle x 1 | MLUH | Abhaye Valabhji | SR435,000.00 | -| 301 | 23/12 | Procurement of Spares for LT Water Circulating Pump | PUC | Wartsila Global Logistcs Services | Euro 19,389.00 | -| 302 | 23/12 | Procurement of Filter Cartridge-Desalination Plants | PUC | Trans Crescent Technical Equipment Company | Euro22,030.00 | -| 303 | 23/12 | Procurement of LED Streetlights | SLTA | Lighting Orient (China) | USD104,282.00 | -| 304 | 23/12 | Proposed New Road at Cascade Primary School-Variations | SLTA | Esparon's Enterprise | SR861,400.00 | -| 305 | 23/12 | General renovation at School Section at MOE Headquarters-Variations | MOE | Prime Builders | SR2,761,935.75 | -| 306 | 23/12 | Remedial Works at Mont Fleuri Primary School and Creche-Variations | MOE | Sai-Fu Enterprise Company Ltd | SR1,006,778.50 | -| 307 | 23/12 | General Renovation of Toilet at La Digue School | MOE | Furui Construction | SR338,785.40 | -| 308 | 26/12 | Procurement of 2500 Ream of A4 Paper | MOE | Island Motors | SR1,200,000.00 | -| 309 | 26/12 | Procurement of Additional of 400 Desktop Computers | MOE | Orion Computers | SR3,100,000.00 | -| 310 | 26/12 | Procuremet of Vehcile x 5 | MOE | PMC Auto Pty Ltd | SR1,374,897.00 | -| 311 | 26/12 | Procurement of Canon Ink / Riso Meter | MOE | Paradise Computer Services | SR473,000.00 | -| 312 | 26/12 | Supply of Metal Fencing | MOE | BBT (UK) | GBP105,278.00 | -| 313 | 26/12 | Procurement of Mobile Dental Clinic / Surgeries x 2 | MOH | Quayle Dental (UK) | GBP233,842.00 | -| | | | | | 9 | +|298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| +|---|---|---|---|---|---| +|299|17/12|Supply of New Equipment for Kitchen||Prison Service K. K. Chua|SR943,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Sun Motors|SR770,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Abhaye Valabhji|SR435,000.00| +|301|23/12|Procurement of Spares for LT Water Circulating Pump|PUC|Wartsila Global Logistcs Services|Euro 19,389.00| +|302|23/12|Procurement of Filter Cartridge-Desalination Plants|PUC|Trans Crescent Technical Equipment Company|Euro22,030.00| +|303|23/12|Procurement of LED Streetlights|SLTA|Lighting Orient (China)|USD104,282.00| +|304|23/12|Proposed New Road at Cascade Primary School-Variations|SLTA|Esparon's Enterprise|SR861,400.00| +|305|23/12|General renovation at School Section at MOE Headquarters-Variations|MOE|Prime Builders|SR2,761,935.75| +|306|23/12|Remedial Works at Mont Fleuri Primary School and Creche-Variations|MOE|Sai-Fu Enterprise Company Ltd|SR1,006,778.50| +|307|23/12|General Renovation of Toilet at La Digue School|MOE|Furui Construction|SR338,785.40| +|308|26/12|Procurement of 2500 Ream of A4 Paper|MOE|Island Motors|SR1,200,000.00| +|309|26/12|Procurement of Additional of 400 Desktop Computers|MOE|Orion Computers|SR3,100,000.00| +|310|26/12|Procuremet of Vehcile x 5|MOE|PMC Auto Pty Ltd|SR1,374,897.00| +|311|26/12|Procurement of Canon Ink / Riso Meter|MOE|Paradise Computer Services|SR473,000.00| +|312|26/12|Supply of Metal Fencing|MOE|BBT (UK)|GBP105,278.00| +|313|26/12|Procurement of Mobile Dental Clinic / Surgeries x 2|MOH|Quayle Dental (UK)|GBP233,842.00| +||||||9| diff --git a/tests/snapshots/nexo-price-en.md b/tests/snapshots/nexo-price-en.md index 2279856..9fbc2cd 100644 --- a/tests/snapshots/nexo-price-en.md +++ b/tests/snapshots/nexo-price-en.md @@ -1,10 +1,10 @@ 본 가격표는 국내 거주 중인 외국인을 위한 한국어 가격표의 비공식 번역본입니다. ※ The post-tax benefit sales price is provided for your reference only, reflecting the current tax benefits and eco-friendly vehicle individual consumption tax reductions. 본 가격표와 한국어 가격표의 내용이 상이한 경우 한국어 가격표의 내용이 우선하므로, 반드시 한국어 가격표의 내용을 확인하십시오. The final sales price may vary depending on the addition of optional items and whether the eco-friendly vehicle criteria are met, so please be sure to check the quotation. This price list is an unofficial translation of the Korean price list for the convenience of foreign residents in South Korea. ※ Please check the Korean price list for information on colors, details, and fuel consumption for each model. If the price list differs from the Korean price list, please check the contents of the Korean price list first. ※ All optional item prices are listed based on pre-tax reduction amounts. The actual sales price, which reflects the total individual consumption tax reduction including optional items, may differ depending on applicable tax benefits. ※ The items (specifications, colors, etc.) and prices listed in this pricing table are subject to change without prior notice depending on the holding of new car launch events, improvements made in automobile performance, introduction of related laws and regulations, and changes in company circumstances. The all-new NEXO Release Date: June 10, 2025 / (Unit: KRW) -| Classification Exclusive Exclusive | Selling price before tax benefit Supply value(surtax) 80,509,000 73,190,000(7,319,000) | Selling price after tax benefit 76,435,000 | Standard equipment • Powertrain/Performance: Fuel cell system(150kW drive motor, lithium-ion battery, and reducer), Regenerative braking system, Column-Type Shift By Wire(vibration warning), Drive mode select • Safety: 9 airbag system(1st-row advanced/center side airbags, 1st/2nd-row side airbags, and rollover-resistant curtain airbags), Multi-Collision Brake System, Active hood system(for pedestrian protection), Safety unlock function, Artificial engine sound(for pedestrian protection), Child seat fastening system (2 in 2nd-row), Fire extinguisher for vehicles, Pedal Misapplication Safety Assist • Smart Safety Technology: Forward Collision-avoidance Assist(vehicles/ pedestrians/two-wheeled vehicles/junction turning/front oncoming), Smart Cruise Control with Stop & Go, Lane Keeping Assist, Lane Following Assist 2, Blind-spot Collision Warning(driving), Blind-spot Collision-avoidance Assist(forward exit), Rear Cross-traffic Collision-avoidance Assist, Safety Exit Assist, Driver Attention Warning, High Beam Assist, Advanced Rear Occupant Alert, Intelligent Speed Limit Assist, Hands-On Detection, Highway Driving Assist, Navigation-based Smart Cruise Control(safety speed zone/curve control), Vibration warning steering wheel • Exterior: Full LED headlamps(projection type), LED turn signal lamps (front and rear), LED Daytime Running Lights, LED positioning lights, LED rear combination lamps, LED third brake lights, 18-inch alloy wheels & tires, Solar glass(windshield), Double-glazed soundproof glass(windshield, and 1st/ 2nd-row doors), Outside mirror(heating, power-folding, power adjustment, | Options (before tax benefit) ▶ Hi-pass(e hi-pass) [200,000] | -| ---------------------------------- | ---------------------------------------- | ---------------------------------------- | ---------------------------------------- | ---------------------------------------- | -| | with 3.5% individual consumption tax applied 79,287,000 | with 3.5% individual consumption tax applied 76,435,000 | and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather- | | -| Special | 83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000 | 79,275,000 with 3.5% individual consumption tax applied 79,275,000 | upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's | ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN | -| Prestige | 87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000 | 83,445,000 with 3.5% individual consumption tax applied 83,445,000 | seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(fron t and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design | sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires | +|Classification Exclusive Exclusive|Selling price before tax benefit Supply value(surtax) 80,509,000 73,190,000(7,319,000)|Selling price after tax benefit 76,435,000|Standard equipment • Powertrain/Performance: Fuel cell system(150kW drive motor, lithium-ion battery, and reducer), Regenerative braking system, Column-Type Shift By Wire(vibration warning), Drive mode select • Safety: 9 airbag system(1st-row advanced/center side airbags, 1st/2nd-row side airbags, and rollover-resistant curtain airbags), Multi-Collision Brake System, Active hood system(for pedestrian protection), Safety unlock function, Artificial engine sound(for pedestrian protection), Child seat fastening system (2 in 2nd-row), Fire extinguisher for vehicles, Pedal Misapplication Safety Assist • Smart Safety Technology: Forward Collision-avoidance Assist(vehicles/ pedestrians/two-wheeled vehicles/junction turning/front oncoming), Smart Cruise Control with Stop & Go, Lane Keeping Assist, Lane Following Assist 2, Blind-spot Collision Warning(driving), Blind-spot Collision-avoidance Assist(forward exit), Rear Cross-traffic Collision-avoidance Assist, Safety Exit Assist, Driver Attention Warning, High Beam Assist, Advanced Rear Occupant Alert, Intelligent Speed Limit Assist, Hands-On Detection, Highway Driving Assist, Navigation-based Smart Cruise Control(safety speed zone/curve control), Vibration warning steering wheel • Exterior: Full LED headlamps(projection type), LED turn signal lamps (front and rear), LED Daytime Running Lights, LED positioning lights, LED rear combination lamps, LED third brake lights, 18-inch alloy wheels & tires, Solar glass(windshield), Double-glazed soundproof glass(windshield, and 1st/ 2nd-row doors), Outside mirror(heating, power-folding, power adjustment,|Options (before tax benefit) ▶ Hi-pass(e hi-pass) [200,000]| +|---|---|---|---|---| +||with 3.5% individual consumption tax applied 79,287,000|with 3.5% individual consumption tax applied 76,435,000|and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather-|| +|Special|83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|79,275,000 with 3.5% individual consumption tax applied 79,275,000|upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| +|Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(fron t and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| **Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR diff --git a/tests/snapshots/real-estate-pricing.md b/tests/snapshots/real-estate-pricing.md index 75f2112..b3962da 100644 --- a/tests/snapshots/real-estate-pricing.md +++ b/tests/snapshots/real-estate-pricing.md @@ -28,9 +28,9 @@ R E V I E W 8 5 -800 -| 1982 | 1986 | 1990 | 1998 | 2006 | -| ---- | --------- | ---------- | ---------- | ------ | -| | Apartment | Industrial | Office-CBD | Retail | +|1982|1986|1990|1998|2006| +|---|---|---|---|---| +||Apartment|Industrial|Office-CBD|Retail| 1982 1986 1990 1994 1998 2002 2006 @@ -38,11 +38,11 @@ R E V I E W 8 5 **Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office** -| | Multifamily | Industrial | CBD Office | -| ---------- | ----------- | ---------- | ---------- | -| Industrial | 0.937 | | | -| CBDOffice | 0.924 | | | -| Retail | 0.922 | 0.969 | 0.964 | +||Multifamily|Industrial|CBD Office| +|---|---|---|---| +|Industrial|0.937||| +|CBDOffice|0.924||| +|Retail|0.922|0.969|0.964| more about investing in tax losses than burst, cap rates spreads steadily com- real estate cash streams. When tax laws pressed, recently falling to approximately dramatically changed in 1986, cap rate zero. And if NOI cap rate spreads are spreads rose, though they generally roughly zero, cash flow cap rate spreads remained negative due to the availability (after reserves for tenant improvements, of excess leverage through 1990 and pro-leasing commissions, and capital expendi- jections of strong cash flow growth, in tures) are well below zero. spite of weak fundamentals. This compression of cap rates and cap Throughout the first two-thirds of the rate spreads over the past five years has 1990s, spreads substantially widened as generated enormous wealth for real estate capital abandoned real estate. Spreads fur-owners. In fact, the combination of cheap ther widened in the latter part of the debt and cap rate compression covered a 1990s, as investors scorned cash flow dur-multitude of property underwriting ing the tech bubble and treasury rates errors made during the past five years, as drifted downward. As the tech bubble neither cap rate compression nor narrow- diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index ca2e20c..d1b5abf 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -68,10 +68,10 @@ Federal income tax return (including any amended return filed on or before the d 2009. Par. 53. For each entry in the “Location” column of the following table, remove the language in the “Remove” column and add the language in the “Add” column in its place: Location Remove Add The last sentence of the The following rules shall The rules described in introductory text to be applicable in paragraph (a) of §1.302- §1.302-4 determining whether the 4T and in paragraphs (b) specific requirements of through (g) of this section section 302(c)(2) are apply in determining met: whether the specific requirements of section 302(c)(2) are met. §1.338(h)(10)-1(f) §1.331-1(d), and §1.332-§1.331-1T(d) and §1.332- 6 6T -| The last sentence of | paragraph (a)(2)(ii) of this | paragraph (a) of §1.382- | -| ----------------------- | ---------------------------- | ------------------------- | -| §1.382-2T(h)(4)(vi)(B) | section | 11T | -| The first sentence of | §1.382-2T(a)(2)(ii) | §1.382-11T(a) | +|The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| +|---|---|---| +|§1.382-2T(h)(4)(vi)(B)|section|11T| +|The first sentence of|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| §1.382-6(b)(2)(i) The second sentence of paragraph (c) of this paragraphs (c)(1), (c)(3), §1.382-8(a) section (c)(4) and (c)(5) of this section and paragraph @@ -109,18 +109,18 @@ paragraphs (c)(1), (c)(3), (c)(4), and (c)(5) of this section, and paragraph (c)(2) of §1.382-8T -| §1.382-8(g), Example | paragraph (c)(2) of this | paragraph (c)(2) of | -| --------------------- | ------------------------ | ------------------- | -| The first sentence of | paragraph (c)(2) of this | paragraph (c)(2) of | -| §1.382-8(g), Example | section | §1.382-8T | +|§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| +|---|---|---| +|The first sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| +|§1.382-8(g), Example|section|§1.382-8T| (2)(c) section §1.382-8T (2)(e) -| §1.382-8(g), Example | paragraph (c)(2) of this | paragraph (c)(2) of | -| --------------------- | ------------------------- | ------------------------ | -| §1.382-8(g), Example | paragraphs (c)(1) and (2) | paragraph (c)(1) of this | +|§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| +|---|---|---| +|§1.382-8(g), Example|paragraphs (c)(1) and (2)|paragraph (c)(1) of this| (3)(b) section §1.382-8T (3)(c)(1)(B) of this section section and paragraph @@ -139,19 +139,19 @@ paragraph (c)(2) of this section paragraph (c)(2) of this section (c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T -| The first sentence of | paragraph (b)(4)(iv) of | paragraph (b)(4)(iv) of | -| ----------------------- | ----------------------- | ----------------------- | -| §1.1502-32(b)(4)(v)(A) | this section | §1.1502-32T | -| The first sentence of | paragraph (b)(4)(iv) of | paragraph (b)(4)(iv) of | -| §1.1502-32(b)(4)(v)(B) | this section | §1.1502-32T | +|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| +|---|---|---| +|§1.1502-32(b)(4)(v)(A)|this section|§1.1502-32T| +|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| +|§1.1502-32(b)(4)(v)(B)|this section|§1.1502-32T| §1.1502-35(c)(4)(ii)(B) §1.1502-76(b)(2)(ii)(D) §1.1502-76(b)(2)(ii)(A)(2) paragraph (b)(2)(ii)(D) of this section §1.1502-92(e)(1) §1.382-2T(a)(2)(ii) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-92(e)(2) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) The second sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) -| The last sentence of | paragraph (f) of this | paragraph (f) of §1.1502- | -| --------------------- | ------------------------ | -------------------------- | -| §1.1502-95(b)(3) | section | 95T | -| The last sentence of | subdivision (ii) of this | paragraph (c)(2)(i) of | -| §1.1563-1(c)(2)(iv), | subparagraph | §1.1563-1T | +|The last sentence of|paragraph (f) of this|paragraph (f) of §1.1502-| +|---|---|---| +|§1.1502-95(b)(3)|section|95T| +|The last sentence of|subdivision (ii) of this|paragraph (c)(2)(i) of| +|§1.1563-1(c)(2)(iv),|subparagraph|§1.1563-1T| Example (1) The last sentence of §1.1563-1(c)(2)(iv), Example (1) The third sentence of §1.1563-1(c)(2)(iv), Example (2) The third sentence of §1.1563-1(c)(2)(iv), Example (2) @@ -163,10 +163,10 @@ subdivisions (ii), (iii), and (iv) of this subparagraph §1.1502-76T(b)(2)(ii)(D) paragraph (b)(2)(ii)(D) of §1.1502-76T §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) -| The first sentence of | §1.332-6(b), 1.368-3(a), | §1.332-6T(a), §1.368- | -| --------------------- | ------------------------- | ----------------------- | -| §1.6043-2(a) | or 1.1081-11 | 3T(a), or §1.1081-11T | -| The first sentence of | §1.6012-2 | paragraphs (a), (b) and | +|The first sentence of|§1.332-6(b), 1.368-3(a),|§1.332-6T(a), §1.368-| +|---|---|---| +|§1.6043-2(a)|or 1.1081-11|3T(a), or §1.1081-11T| +|The first sentence of|§1.6012-2|paragraphs (a), (b) and| §301.6011-5T(a) (twice) diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 1c3ba51..ffaf571 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -46,63 +46,63 @@ l **Freon** **®** **12 Saturation Properties-Temperature Table** -| Temp | Pressure | Volume | | Density | | | Enthalpy | | Entropy | | Temp | -| ---- | -------- | ----------- | ------------- | ---------------- | -------- | -------- | ---------------- | -------- | ------------------ | -------- | ---- | -| °C | [kPa] | [m Liquid v | /kg] Vapour v | [kg/m3] Liquid d | Vapour d | Liquid H | [kJ/kg] Latent H | Vapour H | [kJ/K-kg] Liquid S | Vapour S | °C | -| -100 | 1.2 | 0.0006 | 10.0000 | 1679.0 | 0.100 | 113.3 | 192.8 | 306.1 | 0.6077 | 1.7210 | -100 | -| -99 | 1.3 | 0.0006 | 9.1670 | 1677.0 | 0.109 | 114.1 | 192.4 | 306.5 | 0.6124 | 1.7170 | -99 | -| -98 | 1.4 | 0.0006 | 8.4100 | 1674.0 | 0.119 | 115.0 | 192.0 | 307.0 | 0.6171 | 1.7130 | -98 | -| -97 | 1.6 | 0.0006 | 7.7250 | 1671.0 | 0.129 | 115.8 | 191.6 | 307.4 | 0.6218 | 1.7100 | -97 | -| -96 | 1.7 | 0.0006 | 7.1040 | 1669.0 | 0.141 | 116.6 | 191.3 | 307.9 | 0.6264 | 1.7060 | -96 | -| -95 | 1.9 | 0.0006 | 6.5400 | 1666.0 | 0.153 | 117.4 | 190.9 | 308.3 | 0.6310 | 1.7030 | -95 | -| -94 | 2.0 | 0.0006 | 6.0270 | 1663.0 | 0.166 | 118.2 | 190.6 | 308.8 | 0.6356 | 1.6990 | -94 | -| -93 | 2.2 | 0.0006 | 5.5610 | 1661.0 | 0.180 | 119.1 | 190.1 | 309.2 | 0.6402 | 1.6960 | -93 | -| -92 | 2.4 | 0.0006 | 5.1360 | 1658.0 | 0.195 | 119.9 | 189.8 | 309.7 | 0.6448 | 1.6920 | -92 | -| -91 | 2.6 | 0.0006 | 4.7480 | 1655.0 | 0.211 | 120.7 | 189.4 | 310.1 | 0.6493 | 1.6890 | -91 | -| -90 | 2.9 | 0.0006 | 4.3950 | 1653.0 | 0.228 | 121.5 | 189.1 | 310.6 | 0.6538 | 1.6860 | -90 | -| -89 | 3.1 | 0.0006 | 4.0720 | 1650.0 | 0.246 | 122.4 | 188.6 | 311.0 | 0.6583 | 1.6830 | -89 | -| -88 | 3.4 | 0.0006 | 3.7760 | 1648.0 | 0.265 | 123.2 | 188.3 | 311.5 | 0.6628 | 1.6800 | -88 | -| -87 | 3.6 | 0.0006 | 3.5050 | 1645.0 | 0.285 | 124.0 | 188.0 | 312.0 | 0.6672 | 1.6770 | -87 | -| -86 | 3.9 | 0.0006 | 3.2570 | 1642.0 | 0.307 | 124.8 | 187.6 | 312.4 | 0.6716 | 1.6740 | -86 | -| -85 | 4.3 | 0.0006 | 3.0290 | 1640.0 | 0.330 | 125.7 | 187.2 | 312.9 | 0.6761 | 1.6710 | -85 | -| -84 | 4.6 | 0.0006 | 2.8190 | 1637.0 | 0.355 | 126.5 | 186.8 | 313.3 | 0.6804 | 1.6680 | -84 | -| -83 | 5.0 | 0.0006 | 2.6270 | 1634.0 | 0.381 | 127.3 | 186.5 | 313.8 | 0.6848 | 1.6660 | -83 | -| -82 | 5.3 | 0.0006 | 2.4490 | 1632.0 | 0.408 | 128.1 | 186.2 | 314.3 | 0.6892 | 1.6630 | -82 | -| -81 | 5.8 | 0.0006 | 2.2860 | 1629.0 | 0.437 | 129.0 | 185.7 | 314.7 | 0.6935 | 1.6600 | -81 | -| -80 | 6.2 | 0.0006 | 2.1360 | 1626.0 | 0.468 | 129.8 | 185.4 | 315.2 | 0.6978 | 1.6580 | -80 | -| -79 | 6.7 | 0.0006 | 1.9970 | 1624.0 | 0.501 | 130.6 | 185.1 | 315.7 | 0.7021 | 1.6550 | -79 | -| -78 | 7.1 | 0.0006 | 1.8680 | 1621.0 | 0.535 | 131.5 | 184.6 | 316.1 | 0.7064 | 1.6530 | -78 | -| -77 | 7.7 | 0.0006 | 1.7490 | 1618.0 | 0.572 | 132.3 | 184.3 | 316.6 | 0.7106 | 1.6500 | -77 | -| -76 | 8.2 | 0.0006 | 1.6390 | 1616.0 | 0.610 | 133.1 | 184.0 | 317.1 | 0.7149 | 1.6480 | -76 | -| -75 | 8.8 | 0.0006 | 1.5380 | 1613.0 | 0.650 | 134.0 | 183.5 | 317.5 | 0.7191 | 1.6450 | -75 | -| -74 | 9.4 | 0.0006 | 1.4430 | 1610.0 | 0.693 | 134.8 | 183.2 | 318.0 | 0.7233 | 1.6430 | -74 | -| -73 | 10.1 | 0.0006 | 1.3560 | 1608.0 | 0.738 | 135.6 | 182.9 | 318.5 | 0.7275 | 1.6410 | -73 | -| -72 | 10.8 | 0.0006 | 1.2740 | 1605.0 | 0.785 | 136.5 | 182.4 | 318.9 | 0.7317 | 1.6390 | -72 | -| -71 | 11.5 | 0.0006 | 1.1990 | 1602.0 | 0.834 | 137.3 | 182.1 | 319.4 | 0.7358 | 1.6370 | -71 | -| -70 | 12.3 | 0.0006 | 1.1290 | 1600.0 | 0.886 | 138.2 | 181.7 | 319.9 | 0.7400 | 1.6340 | -70 | -| -69 | 13.1 | 0.0006 | 1.0630 | 1597.0 | 0.941 | 139.0 | 181.3 | 320.3 | 0.7441 | 1.6320 | -69 | -| -68 | 14.0 | 0.0006 | 1.0020 | 1594.0 | 0.998 | 139.8 | 181.0 | 320.8 | 0.7482 | 1.6300 | -68 | -| -67 | 14.9 | 0.0006 | 0.9455 | 1591.0 | 1.058 | 140.7 | 180.6 | 321.3 | 0.7523 | 1.6280 | -67 | -| -66 | 15.8 | 0.0006 | 0.8925 | 1589.0 | 1.120 | 141.5 | 180.3 | 321.8 | 0.7564 | 1.6260 | -66 | -| -65 | 16.8 | 0.0006 | 0.8430 | 1586.0 | 1.186 | 142.4 | 179.8 | 322.2 | 0.7604 | 1.6250 | -65 | -| -64 | 17.9 | 0.0006 | 0.7968 | 1583.0 | 1.255 | 143.2 | 179.5 | 322.7 | 0.7645 | 1.6230 | -64 | -| -63 | 19.0 | 0.0006 | 0.7536 | 1581.0 | 1.327 | 144.1 | 179.1 | 323.2 | 0.7685 | 1.6210 | -63 | -| -62 | 20.1 | 0.0006 | 0.7132 | 1578.0 | 1.402 | 144.9 | 178.8 | 323.7 | 0.7726 | 1.6190 | -62 | -| -61 | 21.3 | 0.0006 | 0.6754 | 1575.0 | 1.481 | 145.8 | 178.3 | 324.1 | 0.7766 | 1.6170 | -61 | -| -60 | 22.6 | 0.0006 | 0.6399 | 1572.0 | 1.563 | 146.6 | 178.0 | 324.6 | 0.7806 | 1.6160 | -60 | -| -59 | 24.0 | 0.0006 | 0.6067 | 1570.0 | 1.648 | 147.5 | 177.6 | 325.1 | 0.7845 | 1.6140 | -59 | -| -58 | 25.4 | 0.0006 | 0.5755 | 1567.0 | 1.738 | 148.3 | 177.3 | 325.6 | 0.7885 | 1.6120 | -58 | -| -57 | 26.8 | 0.0006 | 0.5463 | 1564.0 | 1.831 | 149.2 | 176.8 | 326.0 | 0.7924 | 1.6110 | -57 | -| -56 | 28.4 | 0.0006 | 0.5188 | 1561.0 | 1.928 | 150.0 | 176.5 | 326.5 | 0.7964 | 1.6090 | -56 | -| -55 | 30.0 | 0.0006 | 0.4930 | 1559.0 | 2.029 | 150.9 | 176.1 | 327.0 | 0.8003 | 1.6080 | -55 | -| -54 | 31.6 | 0.0006 | 0.4687 | 1556.0 | 2.134 | 151.7 | 175.8 | 327.5 | 0.8042 | 1.6060 | -54 | -| -53 | 33.4 | 0.0006 | 0.4458 | 1553.0 | 2.243 | 152.6 | 175.4 | 328.0 | 0.8081 | 1.6050 | -53 | -| -52 | 35.2 | 0.0007 | 0.4243 | 1550.0 | 2.357 | 153.5 | 174.9 | 328.4 | 0.8120 | 1.6030 | -52 | -| -51 | 37.1 | 0.0007 | 0.4040 | 1548.0 | 2.475 | 154.3 | 174.6 | 328.9 | 0.8159 | 1.6020 | -51 | -| -50 | 39.1 | 0.0007 | 0.3849 | 1545.0 | 2.598 | 155.2 | 174.2 | 329.4 | 0.8197 | 1.6000 | -50 | -| -49 | 41.2 | 0.0007 | 0.3669 | 1542.0 | 2.725 | 156.0 | 173.9 | 329.9 | 0.8236 | 1.5990 | -49 | -| -48 | 43.4 | 0.0007 | 0.3499 | 1539.0 | 2.858 | 156.9 | 173.4 | 330.3 | 0.8274 | 1.5980 | -48 | -| -47 | 45.6 | 0.0007 | 0.3339 | 1536.0 | 2.995 | 157.8 | 173.0 | 330.8 | 0.8313 | 1.5960 | -47 | +|Temp|Pressure|Volume||Density|||Enthalpy||Entropy||Temp| +|---|---|---|---|---|---|---|---|---|---|---|---| +|°C|[kPa]|[m Liquid v|/kg] Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| +|-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| +|-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99| +|-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98| +|-97|1.6|0.0006|7.7250|1671.0|0.129|115.8|191.6|307.4|0.6218|1.7100|-97| +|-96|1.7|0.0006|7.1040|1669.0|0.141|116.6|191.3|307.9|0.6264|1.7060|-96| +|-95|1.9|0.0006|6.5400|1666.0|0.153|117.4|190.9|308.3|0.6310|1.7030|-95| +|-94|2.0|0.0006|6.0270|1663.0|0.166|118.2|190.6|308.8|0.6356|1.6990|-94| +|-93|2.2|0.0006|5.5610|1661.0|0.180|119.1|190.1|309.2|0.6402|1.6960|-93| +|-92|2.4|0.0006|5.1360|1658.0|0.195|119.9|189.8|309.7|0.6448|1.6920|-92| +|-91|2.6|0.0006|4.7480|1655.0|0.211|120.7|189.4|310.1|0.6493|1.6890|-91| +|-90|2.9|0.0006|4.3950|1653.0|0.228|121.5|189.1|310.6|0.6538|1.6860|-90| +|-89|3.1|0.0006|4.0720|1650.0|0.246|122.4|188.6|311.0|0.6583|1.6830|-89| +|-88|3.4|0.0006|3.7760|1648.0|0.265|123.2|188.3|311.5|0.6628|1.6800|-88| +|-87|3.6|0.0006|3.5050|1645.0|0.285|124.0|188.0|312.0|0.6672|1.6770|-87| +|-86|3.9|0.0006|3.2570|1642.0|0.307|124.8|187.6|312.4|0.6716|1.6740|-86| +|-85|4.3|0.0006|3.0290|1640.0|0.330|125.7|187.2|312.9|0.6761|1.6710|-85| +|-84|4.6|0.0006|2.8190|1637.0|0.355|126.5|186.8|313.3|0.6804|1.6680|-84| +|-83|5.0|0.0006|2.6270|1634.0|0.381|127.3|186.5|313.8|0.6848|1.6660|-83| +|-82|5.3|0.0006|2.4490|1632.0|0.408|128.1|186.2|314.3|0.6892|1.6630|-82| +|-81|5.8|0.0006|2.2860|1629.0|0.437|129.0|185.7|314.7|0.6935|1.6600|-81| +|-80|6.2|0.0006|2.1360|1626.0|0.468|129.8|185.4|315.2|0.6978|1.6580|-80| +|-79|6.7|0.0006|1.9970|1624.0|0.501|130.6|185.1|315.7|0.7021|1.6550|-79| +|-78|7.1|0.0006|1.8680|1621.0|0.535|131.5|184.6|316.1|0.7064|1.6530|-78| +|-77|7.7|0.0006|1.7490|1618.0|0.572|132.3|184.3|316.6|0.7106|1.6500|-77| +|-76|8.2|0.0006|1.6390|1616.0|0.610|133.1|184.0|317.1|0.7149|1.6480|-76| +|-75|8.8|0.0006|1.5380|1613.0|0.650|134.0|183.5|317.5|0.7191|1.6450|-75| +|-74|9.4|0.0006|1.4430|1610.0|0.693|134.8|183.2|318.0|0.7233|1.6430|-74| +|-73|10.1|0.0006|1.3560|1608.0|0.738|135.6|182.9|318.5|0.7275|1.6410|-73| +|-72|10.8|0.0006|1.2740|1605.0|0.785|136.5|182.4|318.9|0.7317|1.6390|-72| +|-71|11.5|0.0006|1.1990|1602.0|0.834|137.3|182.1|319.4|0.7358|1.6370|-71| +|-70|12.3|0.0006|1.1290|1600.0|0.886|138.2|181.7|319.9|0.7400|1.6340|-70| +|-69|13.1|0.0006|1.0630|1597.0|0.941|139.0|181.3|320.3|0.7441|1.6320|-69| +|-68|14.0|0.0006|1.0020|1594.0|0.998|139.8|181.0|320.8|0.7482|1.6300|-68| +|-67|14.9|0.0006|0.9455|1591.0|1.058|140.7|180.6|321.3|0.7523|1.6280|-67| +|-66|15.8|0.0006|0.8925|1589.0|1.120|141.5|180.3|321.8|0.7564|1.6260|-66| +|-65|16.8|0.0006|0.8430|1586.0|1.186|142.4|179.8|322.2|0.7604|1.6250|-65| +|-64|17.9|0.0006|0.7968|1583.0|1.255|143.2|179.5|322.7|0.7645|1.6230|-64| +|-63|19.0|0.0006|0.7536|1581.0|1.327|144.1|179.1|323.2|0.7685|1.6210|-63| +|-62|20.1|0.0006|0.7132|1578.0|1.402|144.9|178.8|323.7|0.7726|1.6190|-62| +|-61|21.3|0.0006|0.6754|1575.0|1.481|145.8|178.3|324.1|0.7766|1.6170|-61| +|-60|22.6|0.0006|0.6399|1572.0|1.563|146.6|178.0|324.6|0.7806|1.6160|-60| +|-59|24.0|0.0006|0.6067|1570.0|1.648|147.5|177.6|325.1|0.7845|1.6140|-59| +|-58|25.4|0.0006|0.5755|1567.0|1.738|148.3|177.3|325.6|0.7885|1.6120|-58| +|-57|26.8|0.0006|0.5463|1564.0|1.831|149.2|176.8|326.0|0.7924|1.6110|-57| +|-56|28.4|0.0006|0.5188|1561.0|1.928|150.0|176.5|326.5|0.7964|1.6090|-56| +|-55|30.0|0.0006|0.4930|1559.0|2.029|150.9|176.1|327.0|0.8003|1.6080|-55| +|-54|31.6|0.0006|0.4687|1556.0|2.134|151.7|175.8|327.5|0.8042|1.6060|-54| +|-53|33.4|0.0006|0.4458|1553.0|2.243|152.6|175.4|328.0|0.8081|1.6050|-53| +|-52|35.2|0.0007|0.4243|1550.0|2.357|153.5|174.9|328.4|0.8120|1.6030|-52| +|-51|37.1|0.0007|0.4040|1548.0|2.475|154.3|174.6|328.9|0.8159|1.6020|-51| +|-50|39.1|0.0007|0.3849|1545.0|2.598|155.2|174.2|329.4|0.8197|1.6000|-50| +|-49|41.2|0.0007|0.3669|1542.0|2.725|156.0|173.9|329.9|0.8236|1.5990|-49| +|-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48| +|-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47| **3** From 152de8b56db2757af7039e98d3c2ea53aa48155f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 12 Mar 2026 17:54:49 -0700 Subject: [PATCH 03/46] feat(text): adaptive join threshold for Canva-style letter-spaced PDFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Canva-generated PDFs render text character-by-character with CSS-style letter-spacing (~0.5-0.9× font_size). The hardcoded 0.10 threshold caused every character to get a space inserted ("K a r i b i b"). Detect Canva pages via fix_letterspaced_items (≥50% items match "a b c" pattern), compute an IQR-based threshold (median × 1.55) on the gap distribution BEFORE space removal, then propagate per-page thresholds through PageThresholds → group_into_lines_with_thresholds → TextLine → should_join_items. Co-Authored-By: Claude Opus 4.6 --- src/extractor/layout.rs | 26 ++- src/extractor/mod.rs | 29 ++- src/lib.rs | 3 +- src/markdown/mod.rs | 14 +- src/text_utils.rs | 459 ++++++++++++++++++++++++++++++++++++- src/types.rs | 22 +- tests/integration_tests.rs | 5 + 7 files changed, 537 insertions(+), 21 deletions(-) diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index ee582a8..cdc4de6 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -1,5 +1,7 @@ //! Column detection, line grouping, and reading-order layout. +use std::collections::HashMap; + use crate::text_utils::{effective_width, sort_line_items}; use crate::types::{TextItem, TextLine}; use log::debug; @@ -367,6 +369,16 @@ fn split_column_stragglers(lines: Vec) -> (Vec, Vec) -> Vec { + group_into_lines_with_thresholds(items, &HashMap::new()) +} + +/// Group text items into lines, using pre-computed per-page adaptive thresholds +/// from Canva-style letter-spacing detection. Falls back to computing the +/// threshold from item gaps when no pre-computed value is available. +pub(crate) fn group_into_lines_with_thresholds( + items: Vec, + page_thresholds: &HashMap, +) -> Vec { if items.is_empty() { return Vec::new(); } @@ -387,12 +399,17 @@ pub fn group_into_lines(items: Vec) -> Vec { for page in pages { let page_items: Vec = items.iter().filter(|i| i.page == page).cloned().collect(); + // Use pre-computed threshold from fix_letterspaced_items if available + // (computed before embedded-space removal, with full signal). + // Non-Canva pages use the default 0.10 threshold. + let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10); + // Detect columns for this page let columns = detect_columns(&page_items, page); if columns.len() <= 1 { // Single column - use simple sorting - let lines = group_single_column(page_items); + let lines = group_single_column(page_items, adaptive_threshold); all_lines.extend(lines); } else { // Multi-column - separate spanning items from column items @@ -461,12 +478,12 @@ pub fn group_into_lines(items: Vec) -> Vec { let mut per_column_lines: Vec> = Vec::new(); for col_items in col_buckets { - let lines = group_single_column(col_items); + let lines = group_single_column(col_items, adaptive_threshold); per_column_lines.push(lines); } // Process spanning items as their own group - let spanning_lines = group_single_column(spanning_items); + let spanning_lines = group_single_column(spanning_items, adaptive_threshold); let is_newspaper = is_newspaper_layout(&per_column_lines); debug!( @@ -618,7 +635,7 @@ fn should_use_y_sorting(items: &[TextItem]) -> bool { /// Group items from a single column into lines /// Uses heuristics to decide between PDF stream order and Y-position sorting. -fn group_single_column(items: Vec) -> Vec { +fn group_single_column(items: Vec, adaptive_threshold: f32) -> Vec { if items.is_empty() { return Vec::new(); } @@ -687,6 +704,7 @@ fn group_single_column(items: Vec) -> Vec { items: vec![item], y, page, + adaptive_threshold, }); } } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index e0ea518..2fa49cd 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -25,6 +25,7 @@ pub use crate::text_utils::{is_bold_font, is_italic_font}; pub use crate::types::{ItemType, TextLine}; pub(crate) use layout::detect_columns; pub use layout::group_into_lines; +pub(crate) use layout::group_into_lines_with_thresholds; // --------------------------------------------------------------------------- // Public API @@ -96,7 +97,9 @@ pub(crate) fn extract_text_with_positions_and_rects>( Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) + let (extraction, _thresholds) = + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; + Ok(extraction) } /// Extract text with positions from memory buffer @@ -127,23 +130,31 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) + let (extraction, _thresholds) = + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; + Ok(extraction) } // --------------------------------------------------------------------------- // Orchestration // --------------------------------------------------------------------------- +/// Per-page adaptive join thresholds from Canva-style letter-spacing detection. +pub(crate) type PageThresholds = HashMap; + /// Extract positioned text, rectangles, and line segments from a pre-loaded document. +/// +/// Also returns per-page adaptive join thresholds for Canva-style pages. pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result { +) -> Result<(PageExtraction, PageThresholds), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); let mut all_rects = Vec::new(); let mut all_lines = Vec::new(); + let mut page_thresholds: PageThresholds = HashMap::new(); // Build page ObjectId → page number map for form field extraction let page_id_to_num: HashMap = @@ -155,7 +166,12 @@ pub(crate) fn extract_positioned_text_from_doc( continue; } } - let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + let (mut items, rects, lines) = + extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + let threshold = crate::text_utils::fix_letterspaced_items(&mut items); + if threshold > 0.10 { + page_thresholds.insert(*page_num, threshold); + } debug!( "page {}: {} text items, {} rects, {} lines", page_num, @@ -194,7 +210,7 @@ pub(crate) fn extract_positioned_text_from_doc( let form_items = extract_form_fields(doc, &page_id_to_num); all_items.extend(form_items); - Ok((all_items, all_rects, all_lines)) + Ok(((all_items, all_rects, all_lines), page_thresholds)) } // --------------------------------------------------------------------------- @@ -824,6 +840,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, @@ -856,6 +873,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, @@ -888,6 +906,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, diff --git a/src/lib.rs b/src/lib.rs index aebab7a..7011967 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -351,7 +351,7 @@ fn process_document( }; let (markdown, layout, has_encoding_issues) = match extracted { - Some((items, rects, lines)) => { + Some(((items, rects, lines), page_thresholds)) => { let layout = compute_layout_complexity(&items, &rects, &lines); let md = if options.mode == ProcessMode::Analyze { @@ -362,6 +362,7 @@ fn process_document( options.markdown, &rects, &lines, + &page_thresholds, )) }; diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 5a8184b..8a8e241 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -16,7 +16,7 @@ pub use convert::to_markdown_from_lines; use std::collections::{HashMap, HashSet}; -use crate::extractor::group_into_lines; +use crate::extractor::group_into_lines_with_thresholds; use crate::types::{PdfLine, PdfRect, TextItem}; use analysis::calculate_font_stats_from_items; @@ -452,7 +452,7 @@ pub fn to_markdown_from_items_with_rects( options: MarkdownOptions, rects: &[crate::types::PdfRect], ) -> String { - to_markdown_from_items_with_rects_and_lines(items, options, rects, &[]) + to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new()) } /// Convert positioned text items to markdown, using rectangles and line segments for table detection. @@ -464,6 +464,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( options: MarkdownOptions, rects: &[crate::types::PdfRect], pdf_lines: &[crate::types::PdfLine], + page_thresholds: &HashMap, ) -> String { use crate::tables::{ detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown, @@ -781,7 +782,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // items from different side-by-side zones (e.g. left/right month columns // in a calendar) don't merge into the same line. let lines = if page_band_splits.is_empty() { - group_into_lines(non_table_items) + group_into_lines_with_thresholds(non_table_items, page_thresholds) } else { // Separate items into band-split pages and non-split pages let mut split_page_items: HashMap> = HashMap::new(); @@ -794,7 +795,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } // Process unsplit pages normally - let mut all_lines = group_into_lines(unsplit_items); + let mut all_lines = group_into_lines_with_thresholds(unsplit_items, page_thresholds); // Process each split page's bands independently, then interleave // by Y position so paired zones (e.g. left/right months) appear together. let mut split_pages: Vec = split_page_items.keys().copied().collect(); @@ -811,7 +812,10 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .cloned() .collect(); if !band_items.is_empty() { - page_lines.extend(group_into_lines(band_items)); + page_lines.extend(group_into_lines_with_thresholds( + band_items, + page_thresholds, + )); } } // Sort by Y descending (top to bottom) so left and right diff --git a/src/text_utils.rs b/src/text_utils.rs index b27beef..3f9d8ce 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -278,12 +278,267 @@ pub(crate) fn is_cid_font(font: &str) -> bool { font.starts_with("C2_") || font.starts_with("C0_") } +/// Detect and fix Canva-style letter-spacing within text items. +/// +/// Canva-generated PDFs render text character-by-character with CSS-style +/// letter-spacing. The TJ handler inserts spaces between each character, +/// producing items like `"a r i b"` instead of `"arib"`. This function +/// detects such items by checking if the text follows a strict pattern of +/// alternating single characters and spaces, then removes the spurious spaces. +/// +/// Only activates when ≥50% of items on the page are letter-spaced, to avoid +/// false positives on normal PDFs with short items like `"a b"`. +/// +/// Returns the adaptive join threshold for this page: DEFAULT (0.10) for normal +/// pages, or a higher Otsu-derived threshold for Canva-style pages. +pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + + if items.is_empty() { + return DEFAULT; + } + + // Check if the item text matches "x y z" pattern (single chars separated by spaces) + fn is_letterspaced(text: &str) -> bool { + let trimmed = text.trim(); + let chars: Vec = trimmed.chars().collect(); + // Need at least 3 chars: "a b" = ['a', ' ', 'b'] + if chars.len() < 3 { + return false; + } + // Pattern: non-space, space, non-space, space, ... + chars + .iter() + .enumerate() + .all(|(i, &c)| if i % 2 == 0 { c != ' ' } else { c == ' ' }) + } + + // Count how many items are letter-spaced vs total non-trivial items + let mut letterspaced_count = 0u32; + let mut total_text_items = 0u32; + for item in items.iter() { + let trimmed = item.text.trim(); + if trimmed.is_empty() || trimmed.len() < 3 { + continue; + } + total_text_items += 1; + if is_letterspaced(&item.text) { + letterspaced_count += 1; + } + } + + // Only fix if ≥50% of substantial items are letter-spaced + if total_text_items < 4 || letterspaced_count * 2 < total_text_items { + return DEFAULT; + } + // Compute threshold BEFORE removing spaces. Since we've confirmed this + // is a Canva-style page (≥50% letterspaced), use the ungated variant + // that includes all pairs — the char-count guard in the normal function + // would filter out long letterspaced items like "i s s i o n" (11 chars). + let threshold = compute_canva_join_threshold(items); + + // Remove spaces from letter-spaced items + for item in items.iter_mut() { + if is_letterspaced(&item.text) { + let fixed: String = item.text.chars().filter(|&c| c != ' ').collect(); + item.text = fixed; + } + } + + threshold +} + +/// Compute Otsu join threshold for a confirmed Canva-style page. +/// +/// Like [`compute_single_char_join_threshold`] but without the per-pair +/// char-count guard, since we already know the page has Canva-style +/// letter-spacing. Uses all adjacent pairs for maximum sample size. +fn compute_canva_join_threshold(items: &[TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + const MIN_SAMPLES: usize = 8; + + let mut ratios: Vec = Vec::new(); + for pair in items.windows(2) { + let prev = &pair[0]; + let curr = &pair[1]; + + // Skip CJK pairs + let prev_c = prev.text.trim().chars().last(); + let curr_c = curr.text.trim().chars().next(); + if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) { + continue; + } + + if prev.width <= 0.0 || prev.font_size <= 0.0 { + continue; + } + + let gap = if prev.x <= curr.x { + curr.x - (prev.x + prev.width) + } else { + prev.x - (curr.x + curr.width) + }; + + let ratio = gap / prev.font_size; + + if !(0.0..=3.0).contains(&ratio) { + continue; + } + + ratios.push(ratio); + } + + if ratios.len() < MIN_SAMPLES { + return DEFAULT; + } + + ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + // If all gaps are tight (max < 0.40), use default — normal PDF + let max_ratio = ratios[ratios.len() - 1]; + if max_ratio < 0.40 { + return DEFAULT; + } + + // If the minimum gap is below 0.40, there's a mix of tight and wide gaps, + // meaning this isn't a uniform letter-spacing PDF — use default. + if ratios[0] < 0.40 { + return DEFAULT; + } + + // Median-based threshold: for Canva letter-spacing, the median gap ratio + // is dominated by the letter-spacing value (~0.55× font_size). Word gaps + // are consistently ~1.8× the letter-spacing. Using median × 1.55 places + // the threshold between the widest intra-word gaps and the narrowest + // inter-word gaps. + let median = ratios[ratios.len() / 2]; + (median * 1.55).clamp(0.50, 2.0) +} + +/// Compute an adaptive join threshold for text items on a line. +/// +/// Uses Otsu's method on the gap/font_size ratio distribution to find the +/// natural split between intra-word and inter-word gaps. With per-pair +/// char-count guard (both items ≥ 5 chars → skip). Used only in tests; +/// production code uses `compute_canva_join_threshold` via `fix_letterspaced_items`. +#[cfg(test)] +fn compute_single_char_join_threshold(items: &[TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + const MIN_SAMPLES: usize = 8; + + // Collect gap/font_size ratios for adjacent pairs involving at least one + // short fragment (< 5 chars). This detects per-character rendering + // (Canva-style) without being fooled by uniform word-level spacing. + let mut ratios: Vec = Vec::new(); + for pair in items.windows(2) { + let prev = &pair[0]; + let curr = &pair[1]; + + let prev_chars = prev.text.trim().chars().count(); + let curr_chars = curr.text.trim().chars().count(); + + // Require at least one item to be a short fragment. + // Pairs of long words (both ≥ 5 chars) indicate normal text. + if prev_chars >= 5 && curr_chars >= 5 { + continue; + } + + // Skip CJK pairs + let prev_c = prev.text.trim().chars().last(); + let curr_c = curr.text.trim().chars().next(); + if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) { + continue; + } + + if prev.width <= 0.0 || prev.font_size <= 0.0 { + continue; + } + + let gap = if prev.x <= curr.x { + curr.x - (prev.x + prev.width) + } else { + prev.x - (curr.x + curr.width) + }; + + let ratio = gap / prev.font_size; + + // Skip negative gaps and huge gaps (> 3× font_size) + if !(0.0..=3.0).contains(&ratio) { + continue; + } + + ratios.push(ratio); + } + + if ratios.len() < MIN_SAMPLES { + return DEFAULT; + } + + ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + // If all gaps are tight (max < 0.40), use default — normal PDF + let max_ratio = ratios[ratios.len() - 1]; + if max_ratio < 0.40 { + return DEFAULT; + } + + // If the minimum gap is below 0.40, there's a mix of tight and wide gaps, + // meaning this isn't a uniform letter-spacing PDF — use default. + // Canva-style letter-spacing has min gaps ≈ 0.5× font_size; normal + // justified text gaps are ≈ 0.15–0.30× font_size. + if ratios[0] < 0.40 { + return DEFAULT; + } + + // All gaps are wide (≥0.25× font_size) — Canva-style letter-spacing. + // Use Otsu to find the split between intra-word and inter-word gaps. + let n = ratios.len() as f32; + let total_sum: f32 = ratios.iter().sum(); + + let mut best_threshold = DEFAULT; + let mut best_variance = f32::NEG_INFINITY; + + let mut w0: f32 = 0.0; + let mut sum0: f32 = 0.0; + + for i in 0..ratios.len() - 1 { + w0 += 1.0; + sum0 += ratios[i]; + + let w1 = n - w0; + if w1 == 0.0 { + break; + } + + let mean0 = sum0 / w0; + let mean1 = (total_sum - sum0) / w1; + let variance = w0 * w1 * (mean0 - mean1).powi(2); + + // Only consider thresholds at value boundaries (skip duplicates) + if i + 1 < ratios.len() && (ratios[i + 1] - ratios[i]).abs() < 1e-6 { + continue; + } + + if variance > best_variance { + best_variance = variance; + // Place threshold midway between the two classes + best_threshold = (ratios[i] + ratios[i + 1]) / 2.0; + } + } + + best_threshold.clamp(0.05, 2.0) +} + /// Determine if two adjacent text items should be joined without a space /// based on their physical positions on the page and character case. /// Uses a hybrid approach: position-based with case-aware thresholds. /// CID fonts emit one word per text operator with gaps ≈ 0 between words. /// Non-CID (Type1/TrueType) fonts emit phrases or fragments. -pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { +pub(crate) fn should_join_items( + prev_item: &TextItem, + curr_item: &TextItem, + single_char_threshold: f32, +) -> bool { // If either text explicitly has leading/trailing spaces, respect them if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') { return false; @@ -365,6 +620,12 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b } } + // When the adaptive threshold indicates Canva-style letter-spacing + // (all gaps wide), use it uniformly for all pair types. + if single_char_threshold > 0.20 { + return gap < font_size * single_char_threshold; + } + // Single-character fragment joined to a multi-character item: use a // moderately generous threshold to rejoin split words like "b" + "illion" // or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words. @@ -385,7 +646,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b return gap < font_size * 0.25; } } - return gap < font_size * 0.10; + return gap < font_size * single_char_threshold; } // With accurate widths, a gap < 15% of font size means glyphs are @@ -473,6 +734,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b #[cfg(test)] mod tests { use super::*; + use crate::types::ItemType; #[test] fn strip_soft_hyphen() { @@ -581,4 +843,197 @@ mod tests { // Latin assert!(!is_arabic_presentation_form('A')); } + + /// Helper to create a single-char TextItem at a given x position with width. + fn make_char_item(ch: char, x: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: ch.to_string(), + x, + y: 100.0, + width, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + #[test] + fn otsu_threshold_sec_style_tight_gaps() { + // SEC-style: intra-word gaps ≈ 0, word gap ≈ 0.15× font_size + // All gaps tight → should return default 0.10 + let fs = 12.0; + let char_w = fs * 0.5; + let mut items = Vec::new(); + // 15 chars with gap ≈ 0 (intra-word) + for i in 0..15 { + let x = 100.0 + i as f32 * (char_w + fs * 0.01); + items.push(make_char_item('a', x, char_w, fs)); + } + // Word gap + let word_x = items.last().unwrap().x + char_w + fs * 0.15; + items.push(make_char_item('b', word_x, char_w, fs)); + // 5 more tight chars + for i in 1..5 { + let x = word_x + i as f32 * (char_w + fs * 0.01); + items.push(make_char_item('c', x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + // Max gap is 0.15, but most are 0.01 → max < 0.20 → default + assert!( + (threshold - 0.10).abs() < 0.01, + "SEC-style should return default ~0.10, got {threshold}" + ); + } + + #[test] + fn otsu_threshold_canva_style_wide_gaps() { + // Canva-style: intra-word gaps ≈ 0.6× font_size, word gaps ≈ 1.2× font_size + let fs = 12.0; + let char_w = fs * 0.5; + let intra_gap = fs * 0.6; + let word_gap = fs * 1.2; + let mut items = Vec::new(); + + // Word 1: 8 chars with intra-word spacing + for i in 0..8 { + let x = 100.0 + i as f32 * (char_w + intra_gap); + items.push(make_char_item('K', x, char_w, fs)); + } + // Word gap + let word_x = items.last().unwrap().x + char_w + word_gap; + items.push(make_char_item('T', word_x, char_w, fs)); + // Word 2: 7 more chars + for i in 1..7 { + let x = word_x + i as f32 * (char_w + intra_gap); + items.push(make_char_item('o', x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + // Should find threshold between 0.6 and 1.2 → roughly 0.9 + assert!( + threshold > 0.5 && threshold < 1.1, + "Canva-style should find threshold ~0.9, got {threshold}" + ); + } + + #[test] + fn otsu_threshold_few_samples_returns_default() { + // < 8 single-char pairs → default + let fs = 12.0; + let char_w = fs * 0.5; + let items: Vec = (0..5) + .map(|i| make_char_item('x', 100.0 + i as f32 * (char_w + 1.0), char_w, fs)) + .collect(); + + let threshold = compute_single_char_join_threshold(&items); + assert!( + (threshold - 0.10).abs() < 0.01, + "few samples should return default 0.10, got {threshold}" + ); + } + + #[test] + fn fix_letterspaced_items_returns_adaptive_threshold() { + // Simulate Canva page with many letter-spaced items and word gaps. + // Needs ≥8 inter-item gaps for the threshold to be computed. + let fs = 12.0; + let char_w = fs * 0.5; + let letter_gap = fs * 0.6; // 0.6× font_size between items + let word_gap = fs * 1.2; // 1.2× font_size between words + + let words: Vec<&str> = vec![ + "H e l l o", + "W o r l d", + "F o o", + "B a r", + "B a z", + "Q u x", + "T e s t", + "D a t a", + "M o r e", + "T e x t", + ]; + + let mut items = Vec::new(); + let mut x = 100.0; + for (wi, word) in words.iter().enumerate() { + let char_count = word.chars().filter(|c| !c.is_whitespace()).count(); + let w = char_count as f32 * char_w + (char_count - 1) as f32 * letter_gap; + items.push(TextItem { + text: word.to_string(), + x, + y: 100.0, + width: w, + height: fs, + font: "TestFont".to_string(), + font_size: fs, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + }); + // Alternate between letter-gap and word-gap to create bimodal distribution + x += w + if wi % 3 == 2 { word_gap } else { letter_gap }; + } + + let threshold = fix_letterspaced_items(&mut items); + + // Threshold should be above default (Canva-style detected) + assert!( + threshold > 0.50, + "Canva page should get threshold > 0.50, got {threshold}" + ); + + // Spaces should be removed from letter-spaced items + assert_eq!(items[0].text, "Hello"); + assert_eq!(items[1].text, "World"); + assert_eq!(items[2].text, "Foo"); + assert_eq!(items[9].text, "Text"); + } + + #[test] + fn canva_style_items_join_correctly() { + // Simulate Canva PDF: "Hello" with 0.6× font_size letter-spacing + let fs = 12.0; + let char_w = fs * 0.5; + let intra_gap = fs * 0.6; + let word_gap = fs * 1.2; + + let mut items = Vec::new(); + let chars = ['H', 'e', 'l', 'l', 'o']; + for (i, &ch) in chars.iter().enumerate() { + let x = 100.0 + i as f32 * (char_w + intra_gap); + items.push(make_char_item(ch, x, char_w, fs)); + } + // Space then "W" + let w_x = items.last().unwrap().x + char_w + word_gap; + items.push(make_char_item('W', w_x, char_w, fs)); + let chars2 = ['o', 'r', 'l', 'd']; + for (i, &ch) in chars2.iter().enumerate() { + let x = w_x + (i + 1) as f32 * (char_w + intra_gap); + items.push(make_char_item(ch, x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + + // Intra-word pairs should join + assert!( + should_join_items(&items[0], &items[1], threshold), + "H+e should join with threshold {threshold}" + ); + assert!( + should_join_items(&items[3], &items[4], threshold), + "l+o should join with threshold {threshold}" + ); + // Word boundary should NOT join + assert!( + !should_join_items(&items[4], &items[5], threshold), + "o+W (word boundary) should NOT join with threshold {threshold}" + ); + } } diff --git a/src/types.rs b/src/types.rs index 7798f7b..1288b3c 100644 --- a/src/types.rs +++ b/src/types.rs @@ -125,6 +125,10 @@ pub struct TextLine { pub items: Vec, pub y: f32, pub page: u32, + /// Adaptive join threshold from page-level letter-spacing detection. + /// Default 0.10 for normal PDFs; higher for Canva-style PDFs. + #[doc(hidden)] + pub adaptive_threshold: f32, } impl TextLine { @@ -138,6 +142,8 @@ impl TextLine { return self.text_plain(); } + let single_char_threshold = self.adaptive_threshold; + let mut result = String::new(); let mut current_bold = false; let mut current_italic = false; @@ -156,7 +162,7 @@ impl TextLine { false } else { let prev_item = &self.items[i - 1]; - self.needs_space_between(prev_item, item, &result) + self.needs_space_between(prev_item, item, &result, single_char_threshold) }; // Preserve leading whitespace from the item text. @@ -211,6 +217,8 @@ impl TextLine { /// Get plain text without formatting fn text_plain(&self) -> String { + let single_char_threshold = self.adaptive_threshold; + let mut result = String::new(); for (i, item) in self.items.iter().enumerate() { let text = item.text.as_str(); @@ -218,7 +226,7 @@ impl TextLine { result.push_str(text); } else { let prev_item = &self.items[i - 1]; - if self.needs_space_between(prev_item, item, &result) { + if self.needs_space_between(prev_item, item, &result, single_char_threshold) { result.push(' '); } result.push_str(text); @@ -228,7 +236,13 @@ impl TextLine { } /// Determine if a space is needed between two items - fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool { + fn needs_space_between( + &self, + prev_item: &TextItem, + item: &TextItem, + result: &str, + single_char_threshold: f32, + ) -> bool { let text = item.text.as_str(); // Don't add space before/after hyphens for hyphenated words @@ -245,7 +259,7 @@ impl TextLine { let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; // Use position-based spacing detection - let should_join = should_join_items(prev_item, item); + let should_join = should_join_items(prev_item, item, single_char_threshold); // Check if space already exists let prev_ends_with_space = result.ends_with(' '); diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2864a2d..1d2cd85 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -138,6 +138,7 @@ fn test_text_line_text_method() { items, y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), "Hello World"); } @@ -149,6 +150,7 @@ fn test_text_line_single_item() { items, y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), "Single"); } @@ -159,6 +161,7 @@ fn test_text_line_empty() { items: vec![], y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), ""); } @@ -473,11 +476,13 @@ fn test_markdown_from_lines_basic() { items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)], y: 700.0, page: 1, + adaptive_threshold: 0.10, }, TextLine { items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)], y: 680.0, page: 1, + adaptive_threshold: 0.10, }, ]; let md = to_markdown_from_lines(lines, MarkdownOptions::default()); From 04aa6d4ae5b1488204cfc4dbe0580a14f700c9ff Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 12 Mar 2026 23:23:02 -0700 Subject: [PATCH 04/46] feat(text): width-based joining for Canva single-char items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use character-width ratios instead of font_size for Canva-style PDFs: - Single-char prev: gap/prev.width < 1.25 - Multi→single: gap/avg_prev_char_width < 1.25 - Multi→multi: page-level threshold (gap/font_size) Also adds second Canva detection path for per-character rendering without embedded spaces (>50% single-char items). Improves ebgt7isj04ophcq word accuracy from ~3% to 64%. Co-Authored-By: Claude Opus 4.6 --- src/text_utils.rs | 185 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 149 insertions(+), 36 deletions(-) diff --git a/src/text_utils.rs b/src/text_utils.rs index 3f9d8ce..09f635a 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -329,6 +329,19 @@ pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 { // Only fix if ≥50% of substantial items are letter-spaced if total_text_items < 4 || letterspaced_count * 2 < total_text_items { + // Second detection path: per-character rendering without embedded spaces. + // Canva sometimes emits each character as a separate TextItem (no "a b c" + // pattern within items). Detect by checking if >50% of items are single chars. + let single_char_count = items + .iter() + .filter(|i| i.text.trim().chars().count() == 1) + .count(); + if items.len() >= 10 && single_char_count * 2 >= items.len() { + let threshold = compute_canva_join_threshold(items); + if threshold > 0.40 { + return threshold; + } + } return DEFAULT; } // Compute threshold BEFORE removing spaces. Since we've confirmed this @@ -348,21 +361,39 @@ pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 { threshold } -/// Compute Otsu join threshold for a confirmed Canva-style page. +/// Compute join threshold for a confirmed Canva-style page. /// -/// Like [`compute_single_char_join_threshold`] but without the per-pair -/// char-count guard, since we already know the page has Canva-style -/// letter-spacing. Uses all adjacent pairs for maximum sample size. +/// Uses `median × 1.55` on the gap/font_size ratio distribution. The page-level +/// threshold is used for multi-char item pairs; single-char pairs use +/// character-width–based joining in `should_join_items` instead. fn compute_canva_join_threshold(items: &[TextItem]) -> f32 { const DEFAULT: f32 = 0.10; const MIN_SAMPLES: usize = 8; + let ratios = collect_gap_ratios(items); + if ratios.len() < MIN_SAMPLES { + return DEFAULT; + } + + let mut sorted: Vec = ratios; + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + if sorted[sorted.len() - 1] < 0.40 || sorted[0] < 0.40 { + return DEFAULT; + } + + let median = sorted[sorted.len() / 2]; + (median * 1.55).clamp(0.50, 2.0) +} + +/// Collect positive gap/font_size ratios from adjacent item pairs, +/// filtering out CJK, zero-width, and out-of-range values. +fn collect_gap_ratios(items: &[TextItem]) -> Vec { let mut ratios: Vec = Vec::new(); for pair in items.windows(2) { let prev = &pair[0]; let curr = &pair[1]; - // Skip CJK pairs let prev_c = prev.text.trim().chars().last(); let curr_c = curr.text.trim().chars().next(); if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) { @@ -381,38 +412,11 @@ fn compute_canva_join_threshold(items: &[TextItem]) -> f32 { let ratio = gap / prev.font_size; - if !(0.0..=3.0).contains(&ratio) { - continue; + if (0.0..=3.0).contains(&ratio) { + ratios.push(ratio); } - - ratios.push(ratio); } - - if ratios.len() < MIN_SAMPLES { - return DEFAULT; - } - - ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - // If all gaps are tight (max < 0.40), use default — normal PDF - let max_ratio = ratios[ratios.len() - 1]; - if max_ratio < 0.40 { - return DEFAULT; - } - - // If the minimum gap is below 0.40, there's a mix of tight and wide gaps, - // meaning this isn't a uniform letter-spacing PDF — use default. - if ratios[0] < 0.40 { - return DEFAULT; - } - - // Median-based threshold: for Canva letter-spacing, the median gap ratio - // is dominated by the letter-spacing value (~0.55× font_size). Word gaps - // are consistently ~1.8× the letter-spacing. Using median × 1.55 places - // the threshold between the widest intra-word gaps and the narrowest - // inter-word gaps. - let median = ratios[ratios.len() / 2]; - (median * 1.55).clamp(0.50, 2.0) + ratios } /// Compute an adaptive join threshold for text items on a line. @@ -621,8 +625,25 @@ pub(crate) fn should_join_items( } // When the adaptive threshold indicates Canva-style letter-spacing - // (all gaps wide), use it uniformly for all pair types. + // (all gaps wide), use character-width–based joining. + // + // Canva renders text character-by-character with CSS-style letter-spacing. + // For single-char prev items, gap/char_width gives a clean separation + // (~0.9–1.05 for letter gaps, ~1.5+ for word gaps). + // For multi-char prev, avg_char_width normalizes for character mix. + // Multi→multi pairs use the page-level threshold (gap/font_size). if single_char_threshold > 0.20 { + if prev_chars == 1 { + // Single-char prev: its rendered width is an accurate reference + return gap < prev_item.width * 1.25; + } + if curr_chars == 1 { + // Multi→single: avg char width of prev normalises for + // wide/narrow character mix (e.g. "ilw" includes i,l,w) + let avg_char_width = prev_item.width / prev_chars as f32; + return gap < avg_char_width * 1.25; + } + // Both multi-char: use page-level threshold return gap < font_size * single_char_threshold; } @@ -1036,4 +1057,96 @@ mod tests { "o+W (word boundary) should NOT join with threshold {threshold}" ); } + + /// Helper to create a multi-char TextItem at a given position. + fn make_text_item(text: &str, x: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y: 100.0, + width, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + } + } + + #[test] + fn canva_width_based_single_char_prev_join() { + // Canva-style: single-char prev uses gap/prev.width < 1.25 + let fs = 12.0; + let threshold = 0.90; // Canva page threshold + + // "K" (w=7.9) → "a" (gap=8.12): letter gap, ratio=1.028 → JOIN + let k = make_text_item("K", 100.0, 7.9, fs); + let a = make_text_item("a", 115.9, 6.0, fs); + assert!( + should_join_items(&k, &a, threshold), + "K→a: gap/width={:.3}, should join", + (a.x - (k.x + k.width)) / k.width + ); + + // "f" (w=4.0) → "K" (gap=10.47): word boundary, ratio=2.618 → SPLIT + let f = make_text_item("f", 193.0, 4.0, fs); + let k2 = make_text_item("K", 207.47, 7.9, fs); + assert!( + !should_join_items(&f, &k2, threshold), + "f→K: gap/width={:.3}, should split", + (k2.x - (f.x + f.width)) / f.width + ); + } + + #[test] + fn canva_width_based_multi_to_single_join() { + // Multi→single: uses avg_char_width of prev + let fs = 12.0; + let threshold = 0.90; + + // "ilw" (w=23.6, 3 chars) → "a" (gap=9.42): intra-word, avg=7.87, ratio=1.197 → JOIN + let ilw = make_text_item("ilw", 320.0, 23.6, fs); + let a = make_text_item("a", 353.0, 6.0, fs); + assert!( + should_join_items(&ilw, &a, threshold), + "ilw→a: avg_ratio={:.3}, should join (intra-word 'railway')", + (a.x - (ilw.x + ilw.width)) / (ilw.width / 3.0) + ); + + // "rich" (w=34.8, 4 chars) → "m" (gap=14.01): word boundary, avg=8.7, ratio=1.610 → SPLIT + let rich = make_text_item("rich", 229.0, 34.8, fs); + let m = make_text_item("m", 277.8, 10.7, fs); + assert!( + !should_join_items(&rich, &m, threshold), + "rich→m: avg_ratio={:.3}, should split (word boundary)", + (m.x - (rich.x + rich.width)) / (rich.width / 4.0) + ); + } + + #[test] + fn canva_width_based_multi_to_multi_page_threshold() { + // Multi→multi: uses page-level threshold (gap/font_size < threshold) + let fs = 12.0; + let threshold = 0.90; + + // "rib" (w=25.0) → "ib" (gap=7.01): intra-word, r=0.584 → JOIN + let rib = make_text_item("rib", 236.0, 25.0, fs); + let ib = make_text_item("ib", 268.0, 14.0, fs); + assert!( + should_join_items(&rib, &ib, threshold), + "rib→ib: ratio={:.3}, should join (intra-word)", + (ib.x - (rib.x + rib.width)) / fs + ); + + // "ized" (w=35.9) → "fo" (gap=13.92): word boundary, r=1.160 → SPLIT + let ized = make_text_item("ized", 142.0, 35.9, fs); + let fo = make_text_item("fo", 191.8, 13.8, fs); + assert!( + !should_join_items(&ized, &fo, threshold), + "ized→fo: ratio={:.3}, should split (word boundary)", + (fo.x - (ized.x + ized.width)) / fs + ); + } } From 76ea52680bc13e0cdca5c1729a22ee0b9174ad44 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Mar 2026 17:38:00 -0700 Subject: [PATCH 05/46] feat(layout): detect sidebar annotations as newspaper columns Narrow annotation columns beside wide body columns were being Y-interleaved with body text, producing garbled reading order. Add sidebar detection with width ratio, line balance, and sparse density guards to correctly trigger sequential column reading. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/layout.rs | 136 +++++++++++++++++++++++++++++++++++++++- src/extractor/mod.rs | 38 +++++++++-- 2 files changed, 167 insertions(+), 7 deletions(-) diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index cdc4de6..471ec81 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -257,21 +257,78 @@ fn is_page_number(item: &TextItem) -> bool { /// Group text items into lines, with multi-column support /// Detect newspaper-style columns: independent text flows that should be read /// sequentially (all of col1, then col2) rather than Y-interleaved. -pub(crate) fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { +pub(crate) fn is_newspaper_layout( + per_column_lines: &[Vec], + columns: &[ColumnRegion], +) -> bool { if per_column_lines.len() < 2 { return false; } // Each column must independently have substantial content let min_lines = per_column_lines.iter().map(|c| c.len()).min().unwrap_or(0); + let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0); + + if min_lines < 5 { + return false; + } + if min_lines < 15 { + // Sidebar detection: a narrow annotation column beside a wide body column. + // Guards: + // - Only 2 columns (sidebars are body+sidebar, not 3+ columns) + // - width_ratio < 0.50: sidebar is much narrower than body + // - line_balance < 0.35: sidebar has significantly fewer lines + // - max_lines >= 20: body column has substantial prose content + // - narrower column has fewer lines (not a dense reference column) + if columns.len() == 2 && per_column_lines.len() == 2 { + let w0 = columns[0].x_max - columns[0].x_min; + let w1 = columns[1].x_max - columns[1].x_min; + let width_ratio = w0.min(w1) / w0.max(w1); + let line_balance = if max_lines > 0 { + min_lines as f32 / max_lines as f32 + } else { + 1.0 + }; + let narrow_width = w0.min(w1); + if width_ratio < 0.50 && line_balance < 0.35 && max_lines >= 20 && narrow_width >= 160.0 + { + let narrower_idx = if w0 < w1 { 0 } else { 1 }; + let fewest_idx = if per_column_lines[0].len() <= per_column_lines[1].len() { + 0 + } else { + 1 + }; + if narrower_idx == fewest_idx { + // Sparse density check: sidebar annotations are spread thinly + // across the page height while regular two-column text is dense. + // Compare average Y-gap between successive lines in each column. + let narrow = &per_column_lines[narrower_idx]; + let wide = &per_column_lines[1 - narrower_idx]; + let avg_gap = |lines: &[TextLine]| -> f32 { + if lines.len() < 2 { + return 0.0; + } + let mut ys: Vec = lines.iter().map(|l| l.y).collect(); + ys.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let span = ys.last().unwrap() - ys.first().unwrap(); + span / (lines.len() as f32 - 1.0) + }; + let narrow_gap = avg_gap(narrow); + let wide_gap = avg_gap(wide); + // Sidebar annotations have >2.5x the average gap of body text + if wide_gap > 0.0 && narrow_gap / wide_gap >= 2.5 { + return true; + } + } + } + } return false; } // Dense balanced columns (similar line counts) are newspaper regardless of Y-alignment. // By this point table items are already removed, so two dense balanced columns // of remaining text are independent prose flows. - let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0); let balance_ratio = min_lines as f32 / max_lines as f32; if balance_ratio > 0.7 { return true; @@ -485,7 +542,7 @@ pub(crate) fn group_into_lines_with_thresholds( // Process spanning items as their own group let spanning_lines = group_single_column(spanning_items, adaptive_threshold); - let is_newspaper = is_newspaper_layout(&per_column_lines); + let is_newspaper = is_newspaper_layout(&per_column_lines, &columns); debug!( "page {}: layout={}", page, @@ -834,4 +891,77 @@ mod tests { cols.len() ); } + + /// Helper: build a Vec with `n` lines at given X, starting at Y=700. + fn make_lines(n: usize, x: f32) -> Vec { + (0..n) + .map(|i| { + let y = 700.0 - i as f32 * 14.0; + let item = make_item(1, x, y, "SomeText__"); + TextLine { + y, + page: 1, + adaptive_threshold: 0.10, + items: vec![item], + } + }) + .collect() + } + + #[test] + fn sidebar_layout_detected_as_newspaper() { + // Wide body column (x 0..400) with 40 lines, + // narrow sidebar (x 420..590, width 170) with 12 lines. + // width_ratio = 170/400 = 0.425, line_balance = 12/40 = 0.30 → sidebar → newspaper + // Sidebar lines have ~3x gap of body lines (sparse annotations). + let body = make_lines(40, 50.0); + let sidebar: Vec = (0..12) + .map(|i| { + let y = 693.0 - i as f32 * 45.0; // sparse annotations: ~3x body gap + let item = make_item(1, 440.0, y, "SomeText__"); + TextLine { + y, + page: 1, + adaptive_threshold: 0.10, + items: vec![item], + } + }) + .collect(); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 400.0, + }, + ColumnRegion { + x_min: 420.0, + x_max: 590.0, + }, + ]; + assert!( + is_newspaper_layout(&[body, sidebar], &cols), + "Wide body + narrow sidebar should be detected as newspaper" + ); + } + + #[test] + fn borderless_table_not_misclassified() { + // Two columns of similar width and equal line counts → borderless table, not newspaper. + // width_ratio = 250/300 = 0.83 (> 0.50), so sidebar guard fails → false. + let col1 = make_lines(10, 50.0); + let col2 = make_lines(10, 350.0); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 550.0, + }, + ]; + assert!( + !is_newspaper_layout(&[col1, col2], &cols), + "Equal-width equal-row columns should NOT be newspaper (borderless table)" + ); + } } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 2fa49cd..767506f 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -342,7 +342,7 @@ mod tests { use super::*; use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items}; use crate::types::{ItemType, TextLine}; - use layout::{detect_columns, is_newspaper_layout}; + use layout::{detect_columns, is_newspaper_layout, ColumnRegion}; #[test] fn test_group_into_lines() { @@ -863,7 +863,17 @@ mod tests { .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(is_newspaper_layout(&[col1, col2], &cols)); } #[test] @@ -897,7 +907,17 @@ mod tests { .map(|i| make_line(685.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(is_newspaper_layout(&[col1, col2], &cols)); } #[test] @@ -929,6 +949,16 @@ mod tests { .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(!is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(!is_newspaper_layout(&[col1, col2], &cols)); } } From d9c2143c320c21e48d76337b06ed998dce94fd4a Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Mar 2026 20:50:24 -0700 Subject: [PATCH 06/46] feat: tagged PDF structure tree support (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: tagged PDF structure tree support for semantic markdown generation Parse /StructTreeRoot from tagged PDFs and use semantic roles (H1-H6, P, LI, BlockQuote, Code, Caption) to improve markdown output. Structure tree headings add to font-size heuristics without suppressing them. Coverage threshold (≥50%) ensures only properly tagged PDFs activate this path. Phase 1: Parse structure tree with role maps, MCID collection, flattening Phase 2: Capture MCIDs from BMC/BDC operators, tag TextItems Phase 3: Structure-aware markdown generation in convert loop Co-Authored-By: Claude Opus 4.6 * fix: accumulate consecutive code lines into single fenced block Per-line code fencing produced broken markdown for multi-line code blocks (separate ``` open/close per line). Unify struct-tree Code role and font-based monospace detection into a single is_code_line check with in_code_block state for proper accumulation. Co-Authored-By: Claude Opus 4.6 * test: add tagged PDF fixture with Firecrawl docs content Synthetic 7-page PDF with rich structure tree exercising H1, H2, H3, P, Code, LI, Caption, TH, TD roles. Generated via fpdf2 script. Integration test verifies struct tree parsing and code fence output. Co-Authored-By: Claude Opus 4.6 * chore: remove python PDF generator script from repo Keep the generated fixture PDF but don't track the generator script. Co-Authored-By: Claude Opus 4.6 * fix: handle malformed bare-name struct types in tagged PDFs Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code in structure elements. lopdf silently drops these objects since bare tokens are invalid PDF syntax. Add a pre-processor that scans for known bare struct type names and prepends / before loading. Unifies path and memory loading through the same fix pipeline. Co-Authored-By: Claude Opus 4.6 * chore: update lopdf dependency to main branch The firecrawl/zlib-checksum-encrypted branch was merged and deleted. Point to main which includes all previously merged fixes. Co-Authored-By: Claude Opus 4.6 * chore: switch lopdf to upstream repo pinned at 845cd3d Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- Cargo.toml | 2 +- src/extractor/content_stream.rs | 92 +- src/extractor/layout.rs | 1 + src/extractor/links.rs | 2 + src/extractor/mod.rs | 23 + src/extractor/xobjects.rs | 2 + src/lib.rs | 37 +- src/markdown/convert.rs | 435 ++++++++- src/markdown/mod.rs | 39 +- src/structure_tree.rs | 1035 ++++++++++++++++++++++ src/tables/detect_heuristic.rs | 1 + src/tables/detect_lines.rs | 1 + src/tables/detect_rects.rs | 1 + src/tables/financial.rs | 1 + src/tables/grid.rs | 3 + src/tables/mod.rs | 4 + src/text_utils.rs | 3 + src/types.rs | 3 + tests/fixtures/bare_name_struct.pdf | Bin 0 -> 1865 bytes tests/fixtures/firecrawl_docs_tagged.pdf | Bin 0 -> 64026 bytes tests/integration_tests.rs | 38 + 21 files changed, 1650 insertions(+), 73 deletions(-) create mode 100644 src/structure_tree.rs create mode 100644 tests/fixtures/bare_name_struct.pdf create mode 100644 tests/fixtures/firecrawl_docs_tagged.pdf diff --git a/Cargo.toml b/Cargo.toml index 22ae989..eea0ced 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector" [dependencies] # PDF parsing -lopdf = { git = "https://github.com/firecrawl/lopdf", branch = "firecrawl/zlib-checksum-encrypted", features = ["rayon"] } +lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "845cd3d4648d9cefb7b5def5fb387df61ba3a0e5", features = ["rayon"] } # Error handling thiserror = "2.0" diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 2ce06e6..ff4a3c1 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -124,10 +124,18 @@ pub(crate) fn extract_page_text_items( let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; - // Marked content (ActualText) tracking - let mut marked_content_stack: Vec> = Vec::new(); + // Marked content tracking: (ActualText, MCID) per nesting level + struct MarkedContentEntry { + actual_text: Option, + mcid: Option, + } + let mut marked_content_stack: Vec = Vec::new(); let mut suppress_glyph_extraction = false; let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry + /// Get the innermost MCID from the marked content stack. + fn current_mcid(stack: &[MarkedContentEntry]) -> Option { + stack.iter().rev().find_map(|e| e.mcid) + } for op in &content.operations { trace!("{} {:?}", op.operator, op.operands); @@ -292,6 +300,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -440,6 +449,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -496,6 +506,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -531,11 +542,15 @@ pub(crate) fn extract_page_text_items( } "BMC" => { // Begin Marked Content (no properties) - marked_content_stack.push(None); + marked_content_stack.push(MarkedContentEntry { + actual_text: None, + mcid: None, + }); } "BDC" => { - // Begin Marked Content with properties — extract ActualText + // Begin Marked Content with properties — extract ActualText and MCID let mut actual_text: Option = None; + let mut mcid: Option = None; if op.operands.len() >= 2 { let dict = match &op.operands[1] { Object::Dictionary(d) => Some(d.clone()), @@ -549,47 +564,56 @@ pub(crate) fn extract_page_text_items( _ => None, }; } + if let Ok(Object::Integer(id)) = d.get(b"MCID") { + mcid = Some(*id); + } } } if actual_text.is_some() { suppress_glyph_extraction = true; actual_text_start_tm = Some(text_matrix); } - marked_content_stack.push(actual_text); + marked_content_stack.push(MarkedContentEntry { actual_text, mcid }); } "EMC" => { // End Marked Content — emit ActualText item with correct width - if let Some(Some(at)) = marked_content_stack.pop() { - // Compute width from text matrix advancement during BDC..EMC - if let Some(start_tm) = actual_text_start_tm.take() { - let combined = multiply_matrices(&start_tm, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - // Width in device space from text matrix delta - let delta_ts = text_matrix[4] - start_tm[4]; - let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; - let width = (delta_ts * scale_x).abs(); - if !at.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&at), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); + if let Some(entry) = marked_content_stack.pop() { + if let Some(at) = entry.actual_text { + // Compute width from text matrix advancement during BDC..EMC + if let Some(start_tm) = actual_text_start_tm.take() { + let combined = multiply_matrices(&start_tm, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + // Width in device space from text matrix delta + let delta_ts = text_matrix[4] - start_tm[4]; + let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; + let width = (delta_ts * scale_x).abs(); + if !at.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&at), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + mcid: entry + .mcid + .or_else(|| current_mcid(&marked_content_stack)), + }); + } } + suppress_glyph_extraction = + marked_content_stack.iter().any(|e| e.actual_text.is_some()); } - suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); } } "re" => { diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index 471ec81..312c6ca 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -796,6 +796,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } diff --git a/src/extractor/links.rs b/src/extractor/links.rs index f7e9090..74a1fae 100644 --- a/src/extractor/links.rs +++ b/src/extractor/links.rs @@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V is_bold: false, is_italic: false, item_type: ItemType::Link(url), + mcid: None, }); } } @@ -316,5 +317,6 @@ pub(crate) fn walk_form_fields( is_bold: false, is_italic: false, item_type: ItemType::FormField, + mcid: None, }); } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 767506f..2b85eed 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -319,6 +319,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { is_bold: first.is_bold, is_italic: first.is_italic, item_type: first.item_type.clone(), + mcid: first.mcid, }); i = j; @@ -359,6 +360,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -372,6 +374,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Next line".into(), @@ -385,6 +388,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -438,6 +442,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Prague".into(), @@ -451,6 +456,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Rules".into(), @@ -464,6 +470,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -488,6 +495,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "A".into(), @@ -501,6 +509,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "V".into(), @@ -514,6 +523,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -540,6 +550,7 @@ mod tests { is_bold: true, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -573,6 +584,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -607,6 +619,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "履行義務".into(), @@ -620,6 +633,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "を識別す".into(), @@ -633,6 +647,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -654,6 +669,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -765,6 +781,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "\u{05D1}".into(), // bet at x=200 (rightmost) @@ -778,6 +795,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -801,6 +819,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -814,6 +833,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -853,6 +873,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -896,6 +917,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -939,6 +961,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index c9e5da1..e011b88 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -408,6 +408,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } @@ -549,6 +550,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } diff --git a/src/lib.rs b/src/lib.rs index 7011967..4acabe5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ pub mod extractor; pub mod glyph_names; pub mod markdown; pub mod process_mode; +pub mod structure_tree; pub mod tables; pub mod text_utils; pub mod tounicode; @@ -273,20 +274,21 @@ pub fn process_pdf_mem_with_config( /// are combined here, but lopdf loads the full doc in `load()` so we extract /// page count from it directly to avoid the metadata-only round-trip. fn load_document_from_path>(path: P) -> Result<(Document, u32), PdfError> { - let doc = match Document::load(&path) { - Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_with_password(&path, "")?, - Err(e) => return Err(e.into()), - }; - let page_count = doc.get_pages().len() as u32; - Ok((doc, page_count)) + let buffer = std::fs::read(&path)?; + load_document_from_mem(&buffer) } /// Load a PDF from a memory buffer. fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> { - let doc = match Document::load_mem(buffer) { + // Fix malformed struct element names before parsing. Some PDF generators + // write bare names (/S Code) instead of proper PDF names (/S /Code), which + // causes lopdf to silently drop the entire object. + let fixed = structure_tree::fix_bare_struct_names(buffer); + let buf = fixed.as_ref(); + + let doc = match Document::load_mem(buf) { Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buffer, "")?, + Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buf, "")?, Err(e) => return Err(e.into()), }; let page_count = doc.get_pages().len() as u32; @@ -350,6 +352,22 @@ fn process_document( Some(extracted?) }; + // Parse structure tree for tagged PDFs (reuses the loaded document) + let struct_roles = structure_tree::StructTree::from_doc(&doc).and_then(|tree| { + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + if roles.is_empty() { + None + } else { + log::debug!( + "structure tree: {} pages with MCID roles, {} total MCIDs", + roles.len(), + tree.mcid_count() + ); + Some(roles) + } + }); + let (markdown, layout, has_encoding_issues) = match extracted { Some(((items, rects, lines), page_thresholds)) => { let layout = compute_layout_complexity(&items, &rects, &lines); @@ -363,6 +381,7 @@ fn process_document( &rects, &lines, &page_thresholds, + struct_roles.as_ref(), )) }; diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 42fb107..1fef66f 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; +use crate::structure_tree::StructRole; use crate::types::TextLine; use super::analysis::{ @@ -13,6 +14,50 @@ use super::postprocess::clean_markdown; use super::preprocess::{merge_drop_caps, merge_heading_lines}; use super::MarkdownOptions; +/// Resolve the dominant structure role for a text line by looking up its items' MCIDs. +/// +/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span). +/// These wrapper roles don't carry useful semantic info for markdown generation. +fn resolve_line_struct_role( + line: &TextLine, + struct_roles: &std::collections::HashMap>, +) -> Option { + let page_roles = struct_roles.get(&line.page)?; + for item in &line.items { + if let Some(mcid) = item.mcid { + if let Some(role) = page_roles.get(&mcid) { + match role { + // Skip container/wrapper roles — not useful for line classification + StructRole::Document + | StructRole::Part + | StructRole::Art + | StructRole::Sect + | StructRole::Div + | StructRole::NonStruct + | StructRole::Span + | StructRole::Private => continue, + _ => return Some(role.clone()), + } + } + } + } + None +} + +/// Map a StructRole heading variant to a markdown heading level (1–6). +fn struct_role_heading_level(role: &StructRole) -> Option { + match role { + StructRole::H => Some(1), // Generic heading → H1 + StructRole::H1 => Some(1), + StructRole::H2 => Some(2), + StructRole::H3 => Some(3), + StructRole::H4 => Some(4), + StructRole::H5 => Some(5), + StructRole::H6 => Some(6), + _ => None, + } +} + /// Merge continuation tables that span across page breaks. /// /// When consecutive pages each have exactly one table with the same number of columns @@ -182,6 +227,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( page_tables: std::collections::HashMap>, page_images: std::collections::HashMap>, band_split_pages: &HashSet, + struct_roles: Option< + &std::collections::HashMap>, + >, ) -> String { if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { return String::new(); @@ -215,6 +263,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let mut in_list = false; let mut in_paragraph = false; let mut last_list_x: Option = None; + let mut in_code_block = false; let mut prev_had_dot_leaders = false; let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); @@ -233,6 +282,10 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( if line.page != current_page { // Flush current page's remaining tables and images if current_page > 0 { + if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } flush_page_tables_and_images( current_page, &page_tables, @@ -352,7 +405,25 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Detect figure/table captions and source citations // These should be on their own line followed by a paragraph break - if is_caption_line(plain_trimmed) { + let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(&line, roles)); + + // Determine if this line is code (struct-tree or font-based) for block accumulation + let is_code_line = struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Code)) + || (options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))); + + // Close code block when transitioning to non-code + if in_code_block && !is_code_line { + output.push_str("```\n"); + in_code_block = false; + } + + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Caption)) + || is_caption_line(plain_trimmed) + { if in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -362,27 +433,48 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( continue; } - // Detect headers by font size - // Note: Headers typically shouldn't have bold markers since they're already emphasized - // Skip very short text (drop caps/labels) and very long text (body paragraphs) - if options.detect_headers + // Detect headers: structure-tree headings win, then font-size heuristics. + // Structure roles ADD headings (e.g. same-size text tagged H2) but do NOT + // suppress headings that the font heuristic would detect (some tagged PDFs + // mark obvious headings as P or Span). + let struct_heading = struct_role.as_ref().and_then(struct_role_heading_level); + let heuristic_heading = if options.detect_headers && plain_trimmed.len() > 3 && plain_trimmed.split_whitespace().count() <= 15 { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers) - { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let prefix = "#".repeat(header_level); - // Use plain text for headers to avoid redundant formatting - output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); - in_list = false; - continue; + detect_header_level(line_font_size, base_size, &heading_tiers) + } else { + None + }; + + if let Some(level) = struct_heading.or(heuristic_heading) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + let prefix = "#".repeat(level); + // Use plain text for headers to avoid redundant formatting + output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); + in_list = false; + continue; + } + + // Structure-tree list item (LI only — LBody is a continuation, not a new item) + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::LI)) + && !is_list_item(plain_trimmed) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(&format!("- {}", trimmed)); + output.push('\n'); + in_list = true; + last_list_x = line.items.first().map(|i| i.x); + continue; } // Detect list items @@ -428,18 +520,32 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( } } - // Detect code blocks by font - if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; + // Structure-tree block quote + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::BlockQuote)) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + output.push_str(&format!("> {}\n", trimmed)); + continue; + } + + // Code block accumulation (struct-tree Code role or monospace font) + if is_code_line { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + if !in_code_block { + output.push_str("```\n"); + in_code_block = true; + } + output.push_str(plain_trimmed); + output.push('\n'); + continue; } // Regular text - join lines within same paragraph with space @@ -456,6 +562,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( prev_had_dot_leaders = cur_dot_leaders; } + // Close any trailing code block + if in_code_block { + output.push_str("```\n"); + } + // Flush current page and any remaining pages with tables/images // (handles table-only pages after the last text line, and trailing image-only pages) flush_page_tables_and_images( @@ -680,3 +791,271 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Clean up and post-process clean_markdown(output, &options) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::structure_tree::StructRole; + use crate::types::TextItem; + use std::collections::HashMap; + + fn make_item(text: &str, page: u32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x: 72.0, + y: 700.0, + width: 100.0, + height: 12.0, + font: "Helvetica".to_string(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid, + } + } + + fn make_line(items: Vec) -> TextLine { + let y = items.first().map(|i| i.y).unwrap_or(0.0); + let page = items.first().map(|i| i.page).unwrap_or(1); + TextLine { + items, + y, + page, + adaptive_threshold: 0.10, + } + } + + #[test] + fn test_struct_role_heading() { + let lines = vec![ + make_line(vec![make_item("Introduction", 1, Some(0))]), + make_line(vec![{ + let mut item = make_item("Body text here.", 1, Some(1)); + item.y = 680.0; + item + }]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::H1); + page_roles.insert(1i64, StructRole::P); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("# Introduction"), + "Should have H1 heading: {md}" + ); + assert!( + md.contains("Body text here."), + "Should have body text: {md}" + ); + } + + #[test] + fn test_struct_role_list_item() { + let lines = vec![make_line(vec![make_item("First item", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::LI); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("- First item"), + "Should format as list item: {md}" + ); + } + + #[test] + fn test_struct_role_blockquote() { + let lines = vec![make_line(vec![make_item("Quoted text", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::BlockQuote); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("> Quoted text"), + "Should format as blockquote: {md}" + ); + } + + #[test] + fn test_struct_role_heading_levels() { + let mcids = vec![ + (StructRole::H1, "Title"), + (StructRole::H2, "Section"), + (StructRole::H3, "Subsection"), + ]; + + let mut lines = Vec::new(); + let mut page_roles = HashMap::new(); + for (i, (role, text)) in mcids.iter().enumerate() { + let mut item = make_item(text, 1, Some(i as i64)); + item.y = 700.0 - (i as f32 * 30.0); + lines.push(make_line(vec![item])); + page_roles.insert(i as i64, role.clone()); + } + + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!(md.contains("# Title"), "H1 → #: {md}"); + assert!(md.contains("## Section"), "H2 → ##: {md}"); + assert!(md.contains("### Subsection"), "H3 → ###: {md}"); + } + + #[test] + fn test_no_struct_roles_falls_back_to_heuristics() { + let mut item = make_item("Big Title", 1, None); + item.font_size = 24.0; + item.height = 24.0; + + let lines = vec![ + make_line(vec![item]), + make_line(vec![{ + let mut body = make_item("Normal body text.", 1, None); + body.y = 660.0; + body + }]), + ]; + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + None, + ); + + assert!( + md.contains("# Big Title"), + "Font heuristic should detect heading: {md}" + ); + } + + #[test] + fn test_resolve_line_struct_role_skips_containers() { + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Div); + page_roles.insert(1i64, StructRole::H2); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let line = make_line(vec![ + make_item("Part ", 1, Some(0)), + make_item("Title", 1, Some(1)), + ]); + + let role = resolve_line_struct_role(&line, &roles); + assert_eq!(role, Some(StructRole::H2)); + } + + #[test] + fn test_struct_role_code() { + let lines = vec![make_line(vec![make_item("fn main() {}", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("```\nfn main() {}\n```"), + "Should format as code block: {md}" + ); + } + + #[test] + fn test_struct_role_code_multiline_accumulation() { + let mut line1 = make_item("fn main() {", 1, Some(0)); + line1.y = 700.0; + let mut line2 = make_item(" println!(\"hello\");", 1, Some(1)); + line2.y = 688.0; + let mut line3 = make_item("}", 1, Some(2)); + line3.y = 676.0; + + let lines = vec![ + make_line(vec![line1]), + make_line(vec![line2]), + make_line(vec![line3]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + page_roles.insert(1i64, StructRole::Code); + page_roles.insert(2i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + // Should produce a single fenced block, not three separate ones + assert!( + md.contains("```\nfn main() {\nprintln!(\"hello\");\n}\n```"), + "Should accumulate consecutive code lines into one block: {md}" + ); + // Should NOT have adjacent fences + assert!( + !md.contains("```\n```"), + "Should not have adjacent close/open fences: {md}" + ); + } +} diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 8a8e241..ed0cbd5 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -452,7 +452,7 @@ pub fn to_markdown_from_items_with_rects( options: MarkdownOptions, rects: &[crate::types::PdfRect], ) -> String { - to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new()) + to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new(), None) } /// Convert positioned text items to markdown, using rectangles and line segments for table detection. @@ -465,6 +465,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( rects: &[crate::types::PdfRect], pdf_lines: &[crate::types::PdfLine], page_thresholds: &HashMap, + struct_roles: Option<&HashMap>>, ) -> String { use crate::tables::{ detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown, @@ -757,6 +758,40 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } + // Check structure tree coverage on ALL text items (before table filtering) + // to decide whether to use structure-aware markdown generation. + let struct_roles_coverage_ok = struct_roles.is_some_and(|roles| { + let total = text_items.len(); + if total == 0 { + return false; + } + let tagged = text_items + .iter() + .filter(|item| { + item.mcid + .and_then(|mcid| { + roles + .get(&item.page) + .and_then(|page_roles| page_roles.get(&mcid)) + }) + .is_some() + }) + .count(); + let coverage = tagged as f32 / total as f32; + log::debug!( + "structure tree coverage: {}/{} items ({:.0}%)", + tagged, + total, + coverage * 100.0 + ); + coverage >= 0.5 + }); + let effective_struct_roles = if struct_roles_coverage_ok { + struct_roles + } else { + None + }; + // Filter out table items and process the rest let non_table_items: Vec = text_items .into_iter() @@ -841,6 +876,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( page_tables, page_images, &band_split_page_set, + effective_struct_roles, ) } @@ -926,6 +962,7 @@ mod tests { is_bold: false, is_italic: false, item_type: crate::types::ItemType::Text, + mcid: None, } } diff --git a/src/structure_tree.rs b/src/structure_tree.rs new file mode 100644 index 0000000..d7e440a --- /dev/null +++ b/src/structure_tree.rs @@ -0,0 +1,1035 @@ +//! Tagged PDF structure tree parser. +//! +//! Reads the `/StructTreeRoot` from the document catalog and builds an +//! in-memory tree of [`StructElement`] nodes. Each leaf maps back to +//! content-stream marked content via MCID (Marked Content ID), which lets +//! downstream code attach semantic roles (heading, paragraph, table cell, +//! list item, …) to extracted [`TextItem`]s. + +use log::debug; +use lopdf::{Document, Object, ObjectId}; +use std::borrow::Cow; +use std::collections::HashMap; + +// ─── Standard structure types ──────────────────────────────────────── + +/// Standard PDF structure element types (ISO 32000-1, Table 333–340). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum StructRole { + Document, + Part, + Art, + Sect, + Div, + BlockQuote, + Caption, + TOC, + TOCI, + Index, + NonStruct, + Private, + // Heading & paragraph + H, + H1, + H2, + H3, + H4, + H5, + H6, + P, + // List + L, + LI, + Lbl, + LBody, + // Table + Table, + TR, + TH, + TD, + THead, + TBody, + TFoot, + // Inline + Span, + Quote, + Note, + Reference, + BibEntry, + Code, + Link, + Annot, + // Illustration + Figure, + Formula, + Form, + // Ruby / Warichu (CJK) + Ruby, + RB, + RT, + RP, + Warichu, + WT, + WP, + // Fallback + Other(String), +} + +impl StructRole { + fn from_name(name: &str) -> Self { + match name { + "Document" => Self::Document, + "Part" => Self::Part, + "Art" => Self::Art, + "Sect" => Self::Sect, + "Div" => Self::Div, + "BlockQuote" => Self::BlockQuote, + "Caption" => Self::Caption, + "TOC" => Self::TOC, + "TOCI" => Self::TOCI, + "Index" => Self::Index, + "NonStruct" => Self::NonStruct, + "Private" => Self::Private, + "H" => Self::H, + "H1" => Self::H1, + "H2" => Self::H2, + "H3" => Self::H3, + "H4" => Self::H4, + "H5" => Self::H5, + "H6" => Self::H6, + "P" => Self::P, + "L" => Self::L, + "LI" => Self::LI, + "Lbl" => Self::Lbl, + "LBody" => Self::LBody, + "Table" => Self::Table, + "TR" => Self::TR, + "TH" => Self::TH, + "TD" => Self::TD, + "THead" => Self::THead, + "TBody" => Self::TBody, + "TFoot" => Self::TFoot, + "Span" => Self::Span, + "Quote" => Self::Quote, + "Note" => Self::Note, + "Reference" => Self::Reference, + "BibEntry" => Self::BibEntry, + "Code" => Self::Code, + "Link" => Self::Link, + "Annot" => Self::Annot, + "Figure" => Self::Figure, + "Formula" => Self::Formula, + "Form" => Self::Form, + "Ruby" => Self::Ruby, + "RB" => Self::RB, + "RT" => Self::RT, + "RP" => Self::RP, + "Warichu" => Self::Warichu, + "WT" => Self::WT, + "WP" => Self::WP, + other => Self::Other(other.to_string()), + } + } + + /// Resolve a possibly-custom tag name through a role map. + fn from_name_with_role_map(name: &str, role_map: &HashMap) -> Self { + // Follow role map chain (max 8 hops to avoid cycles) + let mut current = name.to_string(); + for _ in 0..8 { + let role = Self::from_name(¤t); + if !matches!(role, Self::Other(_)) { + return role; + } + if let Some(mapped) = role_map.get(current.as_str()) { + current = mapped.clone(); + } else { + return role; + } + } + Self::Other(name.to_string()) + } +} + +// ─── Marked content reference ──────────────────────────────────────── + +/// A leaf reference linking a structure element to content-stream content. +#[derive(Debug, Clone)] +pub struct MarkedContentRef { + /// The Marked Content ID used in the content stream's `BDC`/`BMC`. + pub mcid: i64, + /// Page ObjectId this content belongs to (from `/Pg` key). + pub page_id: Option, +} + +// ─── Structure element ─────────────────────────────────────────────── + +/// A node in the PDF structure tree. +#[derive(Debug, Clone)] +pub struct StructElement { + /// Semantic role (H1, P, Table, TD, …). + pub role: StructRole, + /// Alternative text for figures / illustrations. + pub alt_text: Option, + /// Actual text override (e.g. for ligatures). + pub actual_text: Option, + /// Language override (e.g. "en-US"). + pub lang: Option, + /// Direct marked-content references (leaf content). + pub content_refs: Vec, + /// Child structure elements. + pub children: Vec, +} + +// ─── Structure tree (top level) ────────────────────────────────────── + +/// Parsed PDF structure tree. +/// +/// Built from `/StructTreeRoot` in the document catalog. Use +/// [`StructTree::from_doc`] to parse, then [`StructTree::mcid_to_roles`] +/// to get per-page MCID → role lookup tables. +#[derive(Debug, Clone)] +pub struct StructTree { + /// Root children (the top-level structure elements). + pub children: Vec, +} + +impl StructTree { + /// Attempt to parse the structure tree from a PDF document. + /// + /// Returns `None` if the PDF is not tagged (no `/StructTreeRoot`). + pub fn from_doc(doc: &Document) -> Option { + let catalog = doc.catalog().ok()?; + let struct_root_obj = catalog.get(b"StructTreeRoot").ok()?; + let struct_root = resolve_dict(doc, struct_root_obj)?; + + // Parse role map: custom tag → standard tag + let role_map = parse_role_map(doc, struct_root); + debug!("structure tree: {} role map entries", role_map.len()); + + // Parse child elements from /K + let children = parse_kids(doc, struct_root, &role_map, None, 0); + debug!("structure tree: {} top-level elements", children.len()); + + if children.is_empty() { + return None; + } + + Some(StructTree { children }) + } + + /// Build per-page MCID → StructRole lookup. + /// + /// Returns a map: page_number (1-indexed) → (MCID → StructRole). + /// The `page_ids` map should come from `doc.get_pages()`. + pub fn mcid_to_roles( + &self, + page_ids: &std::collections::BTreeMap, + ) -> HashMap> { + // Invert: ObjectId → page number + let obj_to_page: HashMap = + page_ids.iter().map(|(&num, &id)| (id, num)).collect(); + + let mut result: HashMap> = HashMap::new(); + self.collect_mcid_roles(&self.children, &obj_to_page, &mut result); + result + } + + fn collect_mcid_roles( + &self, + elements: &[StructElement], + obj_to_page: &HashMap, + result: &mut HashMap>, + ) { + for elem in elements { + for mcref in &elem.content_refs { + if let Some(page_id) = mcref.page_id { + if let Some(&page_num) = obj_to_page.get(&page_id) { + result + .entry(page_num) + .or_default() + .insert(mcref.mcid, elem.role.clone()); + } + } + } + self.collect_mcid_roles(&elem.children, obj_to_page, result); + } + } + + /// Count total marked-content references across the tree. + pub fn mcid_count(&self) -> usize { + fn count(elements: &[StructElement]) -> usize { + elements + .iter() + .map(|e| e.content_refs.len() + count(&e.children)) + .sum() + } + count(&self.children) + } + + /// Build a flat list of structure elements with their roles and MCIDs, + /// preserving document order. Useful for structure-aware markdown generation. + pub fn flatten(&self) -> Vec { + let mut out = Vec::new(); + flatten_recursive(&self.children, &mut out, 0); + out + } +} + +/// A flattened view of a structure element for linear traversal. +#[derive(Debug, Clone)] +pub struct FlatStructElement { + /// Semantic role. + pub role: StructRole, + /// Nesting depth (0 = top-level). + pub depth: usize, + /// Alt text (figures). + pub alt_text: Option, + /// Direct MCIDs with page ObjectIds. + pub content_refs: Vec, + /// Number of child elements (in the original tree). + pub child_count: usize, +} + +fn flatten_recursive(elements: &[StructElement], out: &mut Vec, depth: usize) { + for elem in elements { + out.push(FlatStructElement { + role: elem.role.clone(), + depth, + alt_text: elem.alt_text.clone(), + content_refs: elem.content_refs.clone(), + child_count: elem.children.len(), + }); + flatten_recursive(&elem.children, out, depth + 1); + } +} + +// ─── Parsing helpers ───────────────────────────────────────────────── + +/// Parse the `/RoleMap` dictionary (custom tag → standard tag). +fn parse_role_map(doc: &Document, struct_root: &lopdf::Dictionary) -> HashMap { + let mut map = HashMap::new(); + let Ok(rm_obj) = struct_root.get(b"RoleMap") else { + return map; + }; + let Some(rm_dict) = resolve_dict(doc, rm_obj) else { + return map; + }; + for (key, val) in rm_dict.iter() { + let key_str = String::from_utf8_lossy(key).to_string(); + if let Ok(name) = val.as_name() { + let val_str = String::from_utf8_lossy(name).to_string(); + map.insert(key_str, val_str); + } + } + map +} + +/// Max recursion depth for structure tree parsing (prevents stack overflow on +/// malformed PDFs). +const MAX_DEPTH: usize = 64; + +/// Parse child elements from a `/K` entry. +fn parse_kids( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, +) -> Vec { + if depth >= MAX_DEPTH { + return Vec::new(); + } + + let Ok(k_obj) = dict.get(b"K") else { + return Vec::new(); + }; + + // /Pg on this element (inherited by children) + let page_id = get_page_ref(doc, dict).or(inherited_page); + + match k_obj { + Object::Array(arr) => { + let mut children = Vec::new(); + for item in arr { + let resolved = resolve_obj(doc, item); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + } + children + } + other => { + let resolved = resolve_obj(doc, other); + let mut children = Vec::new(); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + children + } + } +} + +/// Parse a single child (either a struct element dict or an MCID integer). +fn parse_kid( + doc: &Document, + obj: &Object, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + match obj { + // Direct MCID integer — create a leaf wrapper + Object::Integer(mcid) => { + // This is a bare MCID at the struct-element level. + // We attach it to the parent element, so we create a wrapper struct element. + // Actually, bare MCIDs inside /K are content refs for the parent, + // not separate child elements. We handle this at the caller level. + // For now, create a minimal Span wrapper. + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id: inherited_page, + }], + children: Vec::new(), + }); + } + Object::Dictionary(d) => { + parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out); + } + Object::Stream(s) => { + // Some PDFs wrap struct elements in streams (rare) + parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out); + } + _ => {} + } +} + +/// Parse a dictionary that could be either a struct element or a marked-content +/// reference (MCR) dictionary. +fn parse_struct_element_dict( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + if depth >= MAX_DEPTH { + return; + } + // Check if this is a marked-content reference dict (has /Type /MCR) + if is_mcr_dict(dict) { + if let Ok(Object::Integer(mcid)) = dict.get(b"MCID") { + let page_id = get_page_ref(doc, dict).or(inherited_page); + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id, + }], + children: Vec::new(), + }); + } + return; + } + + // Check if this is an object reference dict (has /Type /OBJR) — skip these + if is_objr_dict(dict) { + return; + } + + // It's a struct element — parse its /S (structure type) + let role_name = match dict.get(b"S") { + Ok(s_obj) => { + let resolved = resolve_obj(doc, s_obj); + match resolved.as_name() { + Ok(name) => String::from_utf8_lossy(name).to_string(), + Err(_) => return, + } + } + Err(_) => return, + }; + + let role = StructRole::from_name_with_role_map(&role_name, role_map); + let page_id = get_page_ref(doc, dict).or(inherited_page); + + // Extract optional attributes + let alt_text = get_text_string(dict, b"Alt"); + let actual_text = get_text_string(dict, b"ActualText"); + let lang = get_text_string(dict, b"Lang"); + + // Parse children from /K + let mut content_refs = Vec::new(); + let mut children = Vec::new(); + + if let Ok(k_obj) = dict.get(b"K") { + let k_resolved = resolve_obj(doc, k_obj); + match k_resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Array(arr) => { + for item in arr { + let resolved = resolve_obj(doc, item); + match resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else if is_objr_dict(d) { + // Skip object references + } else { + parse_struct_element_dict( + doc, + d, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + } + Object::Stream(s) => { + parse_struct_element_dict( + doc, + &s.dict, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + _ => {} + } + } + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else { + parse_struct_element_dict(doc, d, role_map, page_id, depth + 1, &mut children); + } + } + _ => {} + } + } + + out.push(StructElement { + role, + alt_text, + actual_text, + lang, + content_refs, + children, + }); +} + +/// Check if dict has `/Type /MCR`. +fn is_mcr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"MCR") +} + +/// Check if dict has `/Type /OBJR`. +fn is_objr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"OBJR") +} + +/// Get the `/Pg` page reference from a dictionary. +fn get_page_ref(doc: &Document, dict: &lopdf::Dictionary) -> Option { + let pg = dict.get(b"Pg").ok()?; + match pg { + Object::Reference(id) => Some(*id), + _ => { + let resolved = resolve_obj(doc, pg); + if let Object::Reference(id) = resolved { + Some(*id) + } else { + None + } + } + } +} + +/// Extract a text string from a dictionary key (handles PDF text encoding). +fn get_text_string(dict: &lopdf::Dictionary, key: &[u8]) -> Option { + let obj = dict.get(key).ok()?; + match obj { + Object::String(bytes, _) => Some(crate::text_utils::decode_text_string(bytes)), + _ => None, + } +} + +/// Resolve an Object reference, returning the target object. +fn resolve_obj<'a>(doc: &'a Document, obj: &'a Object) -> &'a Object { + match obj { + Object::Reference(id) => doc.get_object(*id).unwrap_or(obj), + _ => obj, + } +} + +/// Resolve an Object to a dictionary (handling references). +fn resolve_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a lopdf::Dictionary> { + match obj { + Object::Dictionary(d) => Some(d), + Object::Reference(id) => doc.get_dictionary(*id).ok(), + _ => None, + } +} + +// ─── PDF byte pre-processing ──────────────────────────────────────── + +/// Fix malformed structure element `/S` entries in raw PDF bytes. +/// +/// Some PDF generators (notably fpdf2) write bare names like `/S Code` +/// instead of the correct `/S /Code`. lopdf cannot parse dictionaries +/// containing bare tokens, so the entire object is silently dropped. +/// +/// This function scans for the pattern `/S ` inside struct +/// element dictionaries and prepends `/` to make them valid PDF names. +/// Returns `Cow::Borrowed` if no fixes were needed. +pub fn fix_bare_struct_names(buf: &[u8]) -> Cow<'_, [u8]> { + // Quick check: if no StructTreeRoot, nothing to fix + if !contains_bytes(buf, b"/StructTreeRoot") { + return Cow::Borrowed(buf); + } + + // Known struct type names that may appear as bare tokens. + // We only fix names that are valid PDF structure types to avoid + // false positives on arbitrary dictionary values. + const KNOWN_NAMES: &[&[u8]] = &[ + b"Document", + b"Part", + b"Art", + b"Sect", + b"Div", + b"BlockQuote", + b"Caption", + b"TOC", + b"TOCI", + b"Index", + b"NonStruct", + b"Private", + b"H", + b"H1", + b"H2", + b"H3", + b"H4", + b"H5", + b"H6", + b"P", + b"L", + b"LI", + b"Lbl", + b"LBody", + b"Table", + b"TR", + b"TH", + b"TD", + b"THead", + b"TBody", + b"TFoot", + b"Span", + b"Quote", + b"Note", + b"Reference", + b"BibEntry", + b"Code", + b"Link", + b"Annot", + b"Figure", + b"Formula", + b"Form", + b"Ruby", + b"RB", + b"RT", + b"RP", + b"Warichu", + b"WT", + b"WP", + ]; + + let pattern = b"/S "; + let mut result: Option> = None; + let mut pos = 0; + + while pos + pattern.len() < buf.len() { + let Some(idx) = find_bytes(&buf[pos..], pattern).map(|i| i + pos) else { + break; + }; + + let after = idx + pattern.len(); + // Check if the next char is already '/' (correct name) or not + if after < buf.len() && buf[after] == b'/' { + pos = after; + continue; + } + + // Try to match a known bare struct name at this position + let mut matched = false; + for name in KNOWN_NAMES { + let end = after + name.len(); + if end <= buf.len() + && &buf[after..end] == *name + // Must be followed by a delimiter (whitespace, newline, /, >) + && (end >= buf.len() || matches!(buf[end], b'\n' | b'\r' | b' ' | b'/' | b'>')) + { + // Found a bare name — lazily allocate output buffer + let out = result.get_or_insert_with(|| buf[..after].to_vec()); + // Append everything from last position up to the bare name + if out.len() < after { + out.extend_from_slice(&buf[out.len()..after]); + } + out.push(b'/'); + out.extend_from_slice(name); + pos = end; + matched = true; + debug!( + "fix_bare_struct_names: patched /S {} → /S /{}", + String::from_utf8_lossy(name), + String::from_utf8_lossy(name) + ); + break; + } + } + + if !matched { + pos = after; + } + } + + match result { + Some(mut out) => { + // Append remaining bytes + if out.len() < buf.len() { + out.extend_from_slice(&buf[out.len()..]); + } + Cow::Owned(out) + } + None => Cow::Borrowed(buf), + } +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + find_bytes(haystack, needle).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_struct_role_from_name() { + assert_eq!(StructRole::from_name("H1"), StructRole::H1); + assert_eq!(StructRole::from_name("P"), StructRole::P); + assert_eq!(StructRole::from_name("Table"), StructRole::Table); + assert_eq!(StructRole::from_name("TD"), StructRole::TD); + assert_eq!( + StructRole::from_name("CustomTag"), + StructRole::Other("CustomTag".to_string()) + ); + } + + #[test] + fn test_struct_role_with_role_map() { + let mut role_map = HashMap::new(); + role_map.insert("Heading1".to_string(), "H1".to_string()); + role_map.insert("Body".to_string(), "P".to_string()); + // Chain: MyTag → Heading1 → H1 + role_map.insert("MyTag".to_string(), "Heading1".to_string()); + + assert_eq!( + StructRole::from_name_with_role_map("Heading1", &role_map), + StructRole::H1 + ); + assert_eq!( + StructRole::from_name_with_role_map("Body", &role_map), + StructRole::P + ); + assert_eq!( + StructRole::from_name_with_role_map("MyTag", &role_map), + StructRole::H1 + ); + // Standard names bypass the map + assert_eq!( + StructRole::from_name_with_role_map("H2", &role_map), + StructRole::H2 + ); + } + + #[test] + fn test_struct_role_role_map_cycle() { + // A→B→A cycle should not infinite-loop + let mut role_map = HashMap::new(); + role_map.insert("A".to_string(), "B".to_string()); + role_map.insert("B".to_string(), "A".to_string()); + + let role = StructRole::from_name_with_role_map("A", &role_map); + // Should terminate (as Other) rather than loop forever + assert!(matches!(role, StructRole::Other(_))); + } + + #[test] + fn test_flat_struct_element() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + let flat = tree.flatten(); + assert_eq!(flat.len(), 3); + assert_eq!(flat[0].role, StructRole::Document); + assert_eq!(flat[0].depth, 0); + assert_eq!(flat[1].role, StructRole::H1); + assert_eq!(flat[1].depth, 1); + assert_eq!(flat[2].role, StructRole::P); + assert_eq!(flat[2].depth, 1); + } + + #[test] + fn test_mcid_count() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![ + MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }, + MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }, + ], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 2, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + assert_eq!(tree.mcid_count(), 3); + } + + #[test] + fn test_mcid_to_roles() { + use std::collections::BTreeMap; + + let page_id: ObjectId = (5, 0); + let mut page_ids = BTreeMap::new(); + page_ids.insert(1u32, page_id); + + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + ], + }], + }; + + let roles = tree.mcid_to_roles(&page_ids); + let page1 = roles.get(&1).unwrap(); + assert_eq!(page1.get(&0), Some(&StructRole::H1)); + assert_eq!(page1.get(&1), Some(&StructRole::P)); + } + + #[test] + fn test_fix_bare_struct_names() { + // Verify the byte-level pre-processor fixes bare names. + // All inputs include /StructTreeRoot to pass the early-return guard. + let input = b"/StructTreeRoot /S Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!( + fixed.windows(b"/S /Code".len()).any(|w| w == b"/S /Code"), + "Should fix bare Code: {:?}", + String::from_utf8_lossy(&fixed) + ); + + // Already correct — should return borrowed + let input = b"/StructTreeRoot /S /Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + + // Multiple bare names + let input = b"/StructTreeRoot /S H1\n/foo\n/S P\n/bar"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S /H1"), "Should fix H1: {s}"); + assert!(s.contains("/S /P"), "Should fix P: {s}"); + + // Unknown name should not be touched + let input = b"/StructTreeRoot /S FooBar\n"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S FooBar"), "Should not fix unknown: {s}"); + + // No StructTreeRoot — skip entirely + let input = b"/S Code\nno struct tree"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + } + + #[test] + fn test_bare_name_struct_types() { + // Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code. + // lopdf silently drops objects with invalid tokens. Our pre-processor + // fixes these before loading. + let raw = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap(); + let fixed = fix_bare_struct_names(&raw); + let doc = Document::load_mem(fixed.as_ref()).unwrap(); + + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "Should parse bare-name struct tree"); + let tree = tree.unwrap(); + + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + + assert!( + roles.iter().any(|r| matches!(r, StructRole::H1)), + "Should find H1 from bare name: {:?}", + roles + ); + assert!( + roles.iter().any(|r| matches!(r, StructRole::Code)), + "Should find Code from bare name: {:?}", + roles + ); + } + + #[test] + fn test_parse_real_tagged_pdf() { + let doc = Document::load("tests/fixtures/2013-app2.pdf").unwrap(); + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "2013-app2.pdf should have a structure tree"); + let tree = tree.unwrap(); + + // Should have a non-trivial structure + assert!(!tree.children.is_empty()); + assert!( + tree.mcid_count() > 0, + "Should have marked content references" + ); + + // Flatten and verify we get heading/paragraph/table elements + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + assert!( + roles.iter().any(|r| matches!(r, StructRole::P)), + "Should contain paragraph elements" + ); + + // Verify mcid_to_roles produces a populated map + let page_ids = doc.get_pages(); + let role_map = tree.mcid_to_roles(&page_ids); + assert!(!role_map.is_empty(), "Should have MCID→role mappings"); + } +} diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 873caa0..4576d9a 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -105,6 +105,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec Option> is_bold: item.is_bold, is_italic: item.is_italic, item_type: item.item_type.clone(), + mcid: item.mcid, }); } Some(sub_items) diff --git a/src/tables/grid.rs b/src/tables/grid.rs index ceb9e69..e3361ef 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -381,6 +381,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -727,6 +728,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); @@ -762,6 +764,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 043dacb..1e9e0b8 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -231,6 +231,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec Vec, } /// A line of text (grouped text items) diff --git a/tests/fixtures/bare_name_struct.pdf b/tests/fixtures/bare_name_struct.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bb7af199f6f613cea3bc2ca6bc70dbbfa944353f GIT binary patch literal 1865 zcmbtVO^6&-5LOg4Pl&kSMIjcFRio^__j;yhd%|SHOwV?9*{rk7EGilH(CK+Qw)XTJ zx?eB5LA)e*5kW*ENDxgfIjGgm;{Gs9=gWcs%oyI&Cw4CLuoNUcVwEt60ICv@5G#n3I`r}jFbE28JS4Y>tf zh8DAG&Kee2WjM|y0d;(}2w@Tgh^mz0wZQwa5+Qp&NKLH1?!ffv_&kF$P+QlWri%l>`)9WU$IYvPjwJ7<&D+SN-_*Ped=n}=^1n}7a%?&}}7kKey?>xuL9&Zn2D zIONFV?>^ps<5*oQV3gfR3uca5!;OU_v08Ld1e-$no;%Z!L66Xd5PAAxALxsISPx@= zY`_*Lr&X#l-58W-)*lU%D`4fWQBcy6FQWesubVmPk0yXRaX4shC_={o9lxE*A(CE4 zDCkn0G!&26Lbgi-lyYNqw@3G-^<4g1xZ?oJ(5-5?bOf}xw7h}q@xx#l{k^)+0cp=1 z5YKeU^+dJ_9}o5Fcv7^QrI!pa%3kP&Kn!xQY4kDf^T_}g1cUBf(et#@17kPj#WW{K z+2KPU(tNPRusCUK#;h1Vzfll}{}Ya#PD{Q=$|Z|c>#SnW Y|NDtl%dv7Jm3EDnIVaQ83oVEI1%RjtG5`Po literal 0 HcmV?d00001 diff --git a/tests/fixtures/firecrawl_docs_tagged.pdf b/tests/fixtures/firecrawl_docs_tagged.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e4b3a78b1193e31027208e40e741dd12fa71fe67 GIT binary patch literal 64026 zcmeFa2|QI@`~ROI6^#;!ijpYP*?XTUq%zAa${2Exc`T6zG%7@?D9uSEB_R|_6Vjk4 zL#PlHr3|I|?{m)X?zGo=>B~zu)iw`~RQ&^?L5^XSL5~uf6v5Uh7)dwbt76%ysmX z2`Y&2ytmySZwM1))MWf!y@WM1gjKcuL;XTz$ik}2*SZDE*u&^wSqc*w|0Xm3O=bL> zK>v-UFoF5^1jYr9!m1`Tx3wXtrun(i?+T$;{<^z~OOUsrpS!<|hK4HrziDnVAwi)u^kP-B0Gi*@H6d&L{jgfe zsOqf^F_7^K_3=Tq>8i#q8~j5ud0=d>WUy2 zzu*9uAe!Hr4Kk|QXo(JO-P$!YOZ{c^r!c4gO~|hbS<-_2Lxa}Pf@PpTs>r-5SVrwv z<>*hxQZsrbe$`BGtxpIo2yF}>mk^o`ZH>PhO<2{K=I0UODFdlN6#CDCLxN~7zQSQg zj@mx=dv#kha$u#{@SI6%UV*ERj3%#%Xt6yKczcH2(v5+S&e}m93W;kIokU;1v{JT| zO3fC}o~@)U|Mf_=m38F#x250a#tzA9%ra5&9sKd_-qYtr!8x0Tr!*fqHnZe#`RfZ4 zOiPtS&IT78n)Q9$;2CdNJxXoes?M?yA@N*gk8kRiH4;Q98&Zw-8T>d@W>(kJQf!cN z=B~F>y6z9*v#SHCgKs+9y54yPm2ADb{>03j3of2o&KC8{iUyy2TN02ubam2`JC5Rc zuQw1?Utf;W4_tXB@0?6^H-DmW?=W%Wa2PQ;swU0lw)ey@B}yq3ngQn?Z``-*sfuVC zZG7|7fSNwr?IOh;lZNH@t+kC78!|T7*#Bi+MA7LI0fSTGyF(OX(>)S&r7qN!_AQ$E zavD)dH|p-%ysX-Ta6++o!kNk|dJgA=!&GL-9lj}j({bs(n~q8D@E5b;ctNLR+mtLD zT5m>1@MH3yV5_mZOU_-JfFmC`&@;3{1<=Uvs%t_XYn;v zuhWs#lCSlNtHmvIB&^EbTDlzEJCi^@uH7kAwjk!ow<}S$2SXCXCM~l$mr<&9kS4F@ z^KHnM81!H&-}`kdN^5T_D63X@Q><$8vQ*R5_9f0Re}B1j*uuz7_xkj>V>H)|pAJ16 z2nhbcm$F+m#!?{rSa-y{Z#5OhITw%C-aA+```e>yA*9xz2wimt>e8KGJ}4|E1@-z= zk-aP04$7NcFEgp8eLp#;d1-}Y*!jV`$~P)AXNY9%Rr}!Ze#3#ZGh)HA)@^PYtC0n> zyt>!M4OQoUR^`j|ypU#nGIDB7`uBtasp~&B8Tj}gy13VH&6LP2>ov1)yI%6hOkbz< zy6x`L!pPoN3D0E*{ca6UzfID+KX6H_bl5X!dy3z5p`!7QJI9Rsx=(+qpu!W;=vtxB z>!#g-LyfW3v(}xfJ2-n$m570ySB0l@MXBr7<=K>j6+z2qJf_*+4Yxlc=BzQQQ&(xv z&mQIaXQhet0&5m6sJCg|q1@3h)#>cdo3Dpk!_xL_{`vV$-}Kzj;nuc+R}CxWiZw^| z3|6f(2va;SpSh3klk?N_w5Bxamvy1GM@{A1c4fxh$@zG=WlVUse^J#50lT&nD_^Ev zQL4YJJ_b2=^!}BmXrm{wIr>9_>*{0%w!WD0IciH`KOfOEVu#p?j-skljT4tRsgQLC zXKRGI535W~@}>4~==U59f2ZklhR<>6bz)g*qG?rmg2Uz2TYb01ZO=!(#<{6Ibkk6G zBQ%NLoj2y%9*@xlV+-9Zk3)_}+uH*3ZGPUq_+(O8V(Gb+7v>Ij^iL}(+baJ0_N?H} zO^+@=Pkukwth=Ob<^-{xyQLM9osNDnGmlQZ+GZ)U%tlGbt02C>>Q-E&`II83tOdO^ z;rsRDt#i7+p8r}mO#v%+{0mq zG$=_n=~Eh;6L%K~>~@|joRKCp2w#tRWH#Z&#KS-Gpzicuy#PfyPf0K5mbc@NP+TuX zj*8BmbMcWze0t3pO4H1l@~boFke5wDkom%gbz`?Ygr|Rx++!zwth`0GzowHW^-gfi zkn@-wm#*gaZw&ArY&{7NsG7}hUFu$6=ALi*!v)^+e6o7etvSxA_Nhb5rYTHR4b3~M zbIEy~eBILmuUSSe_kxkyY3Ev@3bT!20q%iHQ#+Dk<_}~NSHkB?ZB;LdnH^BMo8{l$ zFwRQKX`P*6i21@xHxmk1eBCI%nZ&Q&O+2`97o3)bH276adAIqv?=fT5e22D3m5BGJ z-y5&nl6F^H;?j!!#j-lGr9blJJUu`5vq<{eA@LVY zs;^}WBmH+)R>w6u6`c6A?4GmMT!XNbTETWZFWA#M^YQ$ENE1J=i$@#X9d?E8)NkLj zWFk^A?sZ<9yW3M8>xf(J_5Bu6<DMC_f?WPuZU#Kp<@NDz& z!6k9Tyo{Oay;Ar6q&R)6Si|3xi)wsEH9jBoBM>(hsgfUj5h@H^1s#{z(7&DR=tr#9B zzG-#ve8kJ|FeQZCY<2zkW95gR<#Wj4Qf1Y&58F~&YP0L2r1lwB86>5aHs1BUrBPJ4 z)yQp5hDd1S>X5p=6>Sxf_uk)?EA&kb+nFvPHxPPL&-vm!a&+pvfzSrM_t%md_ZYi& zrR^K(7Rr*}!JZ9B; z-`ef!dDW%jd>IWy{vhS6lAl(Fe$2~iF>M)AmT5f`?r+fhP($iNT=JrId*|C;6Vuf` z(sbSL`Ph2XwQ#nrXH$+vgYP!dtF~zI6QReYi2Qj^UhE04ICgx6|L4n5HA^a_x6NPm z`MsTwR7{-4)NQoDgYG)(MyKe{U7~)t`kdYlanFf+UwCI;mR$EBclV1-_lIL2n!3TA z2ZgO2btPw6emv(S_qk{6keAHMPxTSYyGsHhq;93g&Q`q=)qKaR*Xp8t>EicJ&;2jf zxUHYR;(EaRD-n~XWi>S37@%#OLiet)VIT9e=swynr{N!WhV;M4ouNB*sB6XaxPG}a zg#X~q&@1t)W}Ms^0tr$3)16u6Yt~>e`NxBc)8Bm1wkx|`)V)$?VV?B4&o35^$vEE? zbY9#;+d=wbP4|*h=YM{8e_za3xk!1ptJ=rKHB}E5_Zd$wZt63bTsG55RCQPDk2BYr zBQ#^m6i0VWt%+_GG~FKYldt*p{)f|D`#i5ss2N{7=G@rF!S~RW8?AGy}O&$R{8oGhy6>^*dKRb3J%^ zyZG3e`NirPV=YsqQ*unN@3K8yDWCfx;pCk#m-3y*-bIm<7hSk~`N4|Zq>D@Pb7=E@ ze75e@6kX>b>Jn_>P%}Z$^?vz;{j0W=AG+luA#LVUn#z|}XPJh^#1++z z4{LLs$H*G4yMG~DSF_H2sh~$%ht6eB!_F}=LPrg>j&$37p8NVpZ}z*wm3d?J?|QDR z*0i%iw&mMCduikQbKKL3n%7KYXK!eYj@`PbI=iD@tW$Qp1TA^Dbk2QU#Ysj1&sU$x0~ctx8GYQ;jA2V(8=-Y!}YeB+bXBMn{Imj=>Y?i;YkWd{HMn>jGvx- z>gd*Z{;TiCNN=9ykRZ@6G6`Nc=6n3~qAmFcXa6YQZ~Vr1%lVd#1aaM?XVh+*JUC$^ zV;_`?>@72xZM=XeV4*Z4Qbz5F`;1;L16G2d(n zYpeLqdd$|@_S$^Ejc~EQft0xY=f_uHT^Zl=+WMw>(zili6P;52xmF9OmPwMoxDlox zFDi5e_wT#wI5;6A=)fBvoAc+l*Db3YkZ*mErQF{C7GxIX1_ zTWq#t-pR`xt-56v*GLoqO@_Uq%pp= z614s|dMZwz<=>WdOq#7a(dp>1nZp}C6_Uf-WAcwg%z{NN=WF!mZ*ZC`@OtTsG;)`E zpi=r#{brQ|?zvXcXWmUxc;)9^p`210J-wskgO%I@!V}fTq`kU=1+%AgJ$U(|D6i>` zvFFdv$<;Xn7u98Al>%l>en!ktp0ZNGac-*a4C66AYkl$80_RY}bQ^8{S&n&Y(xBO_0}wa}VX`9$g} zpVeU_!o}tr1)QFR^3bNd@d|OD`l^fUdlfIuhcYk5wz>zz%z$aIfI*7=tocdFf~EL<9WOI>96h(UUr@P%_}t0kig zkz4i0_UP+2#3spa&7EVb|fQ-K*>QYAMkj6I069*g!! z##wEw-1o{paG^?S+J-IS&Pw^dmkbteb3cE3@S~ZJS@n$9Z{OZYS2xi@daKuOGplQq z3LR6uDfQh(%cpxcZ5jID6Mb)8L%@TbHAfGthb07E@dzxaj6T?xX}kSx)YWCz&u6X+ zD7sN$@`8{z74{%$3!l54Q4p0omVD-(9kf{`@xz$9S#Ftr#97X`Pm5pUd5h@EvG&F;luA z-`XA5Yjq+-YyG26b@yw~YMqvPJDH~q#!E6%4sV#f_+IAn&rdJ=$vYfzO*d*8r; z!fB3WtG?PB?>acVOs%P=Wa`sAhanT?gD$P@ZrjeSY;av+Z9QfsrEu@cpAo~`?p*E4 zdhzyW!5qm1a44lmLVgWc{|u#AG~s_dGQbS6;D%D@KMbYxO8lxB$Dx#p;35Nd-~1a$ z65mTUSLl*tCO%XXH7prc9{oa2cR(Te%~qE~S|Pi2=i9j%9&n@RCj5+Cd2vwo>*I5q z&+!+vQ;R!3^(KtlvATIx)ko)D@k?GvlV(?ZPn|>#`1DA~9;%E=m2_w<9ugirn{)Vm zw11$yT%*q5god`v&5_I7@?An~Vk!kyZq7N9X*BK4l*ZxMHKnutRTdw6dfcYvX1l;r zGrz@UW?Dx#F8($oowdHF<>rvBncS+5vPgO9-i^lsMSNrrcb5kCX$g;>ZG&jO^t*mF zseNIY$);|NcX4-C<~Tlx+7grJxMQe%Peh;c$|BjJg>sWL+Ks}`CRd#H*sifMH(=k2 zB8!TJ=f#c$hM%;gjF+NKstvNcwPBjaThgbYg_j);rg)#}=bw3?o0y^bE{kNjJs2y632@xFS_AbOjI5xztP^e zj+F7J7sk~(dBa(@-qeRFDSqjBlJ!@6bxRis6ppKR@`HVp*IzsI zGK2rJ;0NTzx!|tf)LQ_+I+GIhF@qzASNx5GWhh>l6Todv|R9d2C7lnB7)S zK8TU$CLJnU;>VwRuwl=JZPnjnpq0}T9$$89a{RL1s`c5`PRm0&64|?UhQH7cS-i(f zFiiEDWRudO+43Z>7e_nY-9&>A)s3xk3W3|rKWBdxoK~p3$Wm?R1I?-Xa+(dYCDvPz zCcWDKIPy@_*kPyj@YW)wAQmzzV!$SY<-|Kyv%=_p0 zuUmKiLv+jSnMv=i`XhtikcG9O4jCH`O>Mb3sCRteWNPc)ld&!vPn~Vri8}y>q82f09evKFw=!d{TmemFx6ZV}(oF z-6y(N?;ymbM5Gr=7Az3o|N5qw{_+0AR15WcsjlAIjkfjD^(%E6zX)7w)Hr@)_j{pF zUrY_clZ(E1W^0xg>F~|gj?g^)SfNHu`zEs7}W2Ndh-nsDT-uz1^Uth3#vBB(( z>$?Mdfm$E04=0vIU8+Odng`qQOq^C zVDPQ`bVc@}lh@=twuGg5=#IA5cc4#kJYUS#6SFNTsL^m7~?i;K;^>tmiTI#HyU%tJ6`te)kvG~m; zgWb+SI_1F=ihBsD*|OBr568cy8rQ#0&{`5)__+CO>5bci;cbnuEW`xCU-?pYs$jkZTytst1KT6x)FV8X& zTp2CH-~J>|ZpUXt+cjKRBU&Rb6j|@IZGnU#G2rQh$*TpTKKMO3V7vI1=W%j@@1A>) zh$1g9Jxt5UwDWYLO|yx;>>c$sI%V-ylioAmoguG^6QS!7l8k^;&u;l)=^DGIZisq; zU+?YSM*iIXkDtU*j49vqFg^{UY zO2kWBu9;*x)+)8lXyN6I1(lKFDW?cy?-8aIe4ceE@6p)a=~teux$H#$%SFlhtg`;oU`^?Jxav zvu__Ph<;!qakhI@@(Ee#XK?|8;&$GLq|4r_hshI&ez&^68NOK8@ZshY%JR$K?sXpX z_9=AGT{9S%uKZ)of^+gBfg25y7WzZf%Ev}$p6)j6h%1P=ZAFxH=?JJ?cF}FmVTBDT zVYl9w)@I0yd_=^{l^gjwQQgCJFq%8Nr*!1&b)2BIv)Y;oFIhd)nu`{wo%gOe64CsE)Q!u{< z`G4FG0zKn`PPzZb{SPeT@#joLeQxmwkyse{uLxE^y&Bf`1i?gic zt{Ck4I`*o_6~o=`{zcVI1(PDWb(bs;$O!@1l1IOiY>cbFsUvmWoT{h;6X=zGYI@E`_+0dve8Yo!7YwP1H@yx;L@A|J9f0 zoh1`){8;ogEONcTGyfZ94XNFB%Y|#-K7AZay(DE<>squqvNFS<=~=nr>v-GRiP`+a z0r{g(`<9+ktqJxiSGq89zqG<#-?pOk#-Yg%-Yj!4)s;Q9uwY*<<$z4fzz(y`ODw*d zmK1&qmFs=8EvIyg=%Ln?l@8kjWo7LTR9bJ)(0g{Pe&Ss{{_JFNbO2N6{_XoVu@ z#`c9RXr-wbt@2nHR<=aORCd~-D|5E$oHkqdQ7&HJc|d20X1TBOq;Cd-?dmTtrf=BO zcrmx)$|kAm*sxud)F1w~x~ocJ(ygUi%aa=RsHZhPFueC(Xpi7l`ND=F{(Sv_!|_iA z9ctz;zEcocE>Ljgu!+JLrBS)LkBIG$l4tl^<;r$?Y_C-5B~~3*(X2E%@0xf2lW6lC ztGw|^0&DxH7c{MW7o`7)*jAl$+M4DmFzDB}ZQ{djzL4pcGQ`aXbZ&I)DzK9-adVlo zdl~J8=<b}dD-8n9$o{0qtH)I~Zw3BO&ejRoz z?`O|}Wf3;&`P7eJpj~f7oqB8r+!T^OzRGpDF{`S)^2XxGAAO&yzc;fBGKtl8~-Gnwn}QEs$8QV(#o^AnG3bD6{OP zgk5P`VySZX{^6VbCQ7^Ov_j%NG^dq)jBG2oeJ8W#c(d-9mG6V}&u?%4=)Lbu{Fk2j zH}4gajGkE$rE7?^*U5S%k#qMh&5d+TxBFNIMMZo)*=0M%PDu01t-zUWZ%H3!d{6H$ z)ckU@wSIEv=x6Eb-HOxVq9x8oo_oxn6zT5~{^h)rpw5>8q1v>|tvOrNN^~lvmk+s7 zq=wB0b$nN7sV)(pxAkl4x~89NS9NBkKcLkApnlDEt!&+7uefc*a&VKh3t zw<5x-D-J1bm?4tMUoCG}D12@0qV&L%N~?W!$2RT`nE2eXJ+DqA_sS~e{kj+4E{mM{ z^S7|^X~mP$5}7mEqT6a`Z`I0*E?JCD6;uP|=RN0>3!ksH)7rB-$i`De>5Uy=sI%!+}xus#-ogUw6 zYj1K7$=K>X^C_`@Z`b@SPxi^W70oSCX}9}o6{UBn8x58`*xI7xEMFGsSAPA&;i@Z3 zhfdTks@+@bzUox%hZD6gJf$+K3kFP_w8bP``b}I9KT7DGIBmc;sjZ%x;W0v!Q#T}E?(aJi8dJBeX{`V45|ibJPsJ`s*`#G--*CcWYUO~MD%~O(EZ+h9s zctX&scv{5N$jtmt>n)E4gnWJbY~QGkZ|3?B1kdfzT}=)xSt)xxW5&T({Ru_$pKEs= zH>`W1ZKRZHJ^z4zSFq-Oa*dO4R>-CBzR)q-Zks)TjML61S#JtFLwbI{&-cg48H-xK z-W|75<+YFXY@d10t&Wr{mFYV<^V4cBWjV}KIVEz}Zp|Z-cf+ml=@+anEwSGkZ@Ey^ zMJFcaT669|SiXvgm}**)rp1Y_El8)P#q+-Jcj8tkyt!33NvHp8T%O{?Ni#RpJltn@ zXjQwcV^0-9v|?k*39_W;j-Xhfe<;>ap!oQ``xD>dPX0jW9C2Uoxy!S4I+pC~xvst1#B%%RvzcRN zFRS_)Km1*j!8Jdgg z9SW?iY#AeW`sM1Rgq!geBAVB(kN)a+HcCLO@vg&|ZwiOnb@!3IRvoYv7%-f^ec)Tf zxq(SzPiU==Gb>U*>UTi#bC-3;m2W?kg;(Fu)Aui$w0Eaf+~^Ivri5l6xOCED+l;RB zB=Y99`?GdT4Ytp;_x0QR$!>!81}j0ohMg5xW7-rpZS~EyBkm=%88sd5xpT%QPyd+R z=U0>G|9r(~(ZffGo#HB~`SbZs_f-q6Nl8Xy22@P^9=#^WH&A~(&byMMRDFxIwC4)t zveI%%aX-~B18G?s%1H}vXHJtCGMe1_EN1WH$@=@WU*~>>GR`mjtbeA@V)6b(7w@<4 z>bkSn^OO1hWnW+Do)Q$fCPoX;+yGxbR%`yIPWY*jxbIQ3X$xl0*y0xL9+cIhec|C8 z3G3~T?$-;wd6RP}I>OBKWk^HEeE)Mh&6DDnXLKa!d#-reY422%R}{zN}@3Mnsld*2xN!E#mh6F_n&p&rcu<@pHZ}G{ycDl9;J-XOJa#$IB;p>Itxi9$l$7VcETHrlV zH7eetqon2WtGo4cRF+jW#Z}dgEx%SJOkAZnVE?^DAl=f{B1$Lzki;vgJ*B%Y%#kwN zU$vsC<^U1_>_Ilt-n-SZB2rq@?NPU-?wg) z;}_mGKcM=IpT|R4#Zz8k+m?qu@u%h~nNN|ud8;QdSkqKI>%Od#Uw?<+7Nu3KiXV0R zzS?bgrtDHSKHjRzQTI#HE#W%_hYikIn97RK!o>WiUjDAR^I?P1j_`~Uf{Tl=1pl44 zv4@u_8@C>5t?fCv(XHzTbZh(55TU1=a+>eRPZ=OT>bw{3)hgm4Huc(5TfVrC&|Ogj zZ}OBry;C1IDnvuhlqTX&nLzkH?xNjJmvFHUeoz&yzIT+wPsIA|yB6g`&TC`l-e{=J zTR=3Btcaya^}NeIyiHW8m9j2>R;Iaa#?pC?r4p$V2?2V7xuc>g#L_p-R~co|F;AyW zfikA-@coGo^D9i(S9gDSQ=w9;RrsTEN$BkJHx_@t9>n)p6ne(P!C`iE*2 z#pHJ0krKb9mgd?u@MErj@dcwV(F;S)(D>|E_rG5}W|hLNRqsUnIugZ&2@3*VZ??=a z9nPJU@D|NkYK2^Se^w>MQo3mOyM47c$JyyvJH;KFnP0!IPkh^n&5rVEhx3#9_MGio z)-id(*z#Iq3j_W)_XAHn%UQL@^kH+|Q=-^r&8=zv8i%XDZ1s&gY2n%REgb#B?fywx zl&k$-b1}WJu58J^Yo4RX5)XC=h##?--dd(K z%@IgF`N3yi%xJ-JgrBy(NBIQnPm+nt!kV;NP5NGoW z#>tBGss(QYwT5?-7S4-#l#}WfwAHq0UqXU_N!x__EMNJlxT zLXMvAI`2gp>WkL(Jy={?eQ;0Gcl%F+tzJ5dBgJo}C+q43KbDFdJXrct@Kn{@v^VCm z_dJDGJBz>k(QPYw-PqJ1(BxQ@+&l@l6DL0!o_zde&Z$b74QUmsj|N}qLtn?FE&tJX zVtvE+-VCW4v7zq+X8BD&h?jm0MmspJUr%scZJ8=bSZO$&_9|&Zhwo9P=9+u9(4D?z`fvj;nafhwJo>A zXXQl(wMT1(^&*h}SjBa9>+c?Qh_$$*cP8>?vF_E9ijNz+0;BUo9)Afr|M*I-6b6FYpCAc16f1ls_KzNK)`H{~i17k``L&M%A@h(aRRiFt~GKJbBuZ2P?#5VmCS- z?fICZFJQ(|$lLGv4c(3d+3M4e zr}OPUV$@jGQ%w<19rN{D;P_3M!AtLa%9uNOk6*FF?J0^k7kM1{puMd9N~uJ@e}cZ& z-s?VAyYp&jsx$5vT#ihAA?~uQeM+FsM*i|Hugco~mT2+&A!lc_*S{$x3As;9zaGB~ zx^p&7ZB4tOP}yrK;}1o(`vxIpgVShlWOwF?oCD28TK1uDAd{q;2nO#%h|c zEnd}~a54XGkXByzt$yj52U_Fmvv0Tr$$a$YA1_PGuhpm2cBL8`e3*OObonNq)&s-t z4=tpVXp>9ISN?R&M3zqx_N$yQSvXHh+GXQ`c^6;Lt3uawsYS+h1jpPY>tk^lzO{Y@N~t*=%CDbnpZpR!u!sjnq4;M$UcuR zySL;fca9y@3z-`>U}$peo|3__S+3`DCi53(6udetw*HXq$JYFsW8a!K_4pxgCx+L3 ze>AyO#_sZ5Ij`r{N~bHPm_JXPX}@7!k>qC;`6oKns6e)bSF5;%k&#tpO)B^19&lU4}Q!OobXa9iR z{Waa$`^nMkq7`<-gRW|kbDT%VY<-zH%f)=Tzk6@KQTfoqIOVM5&Sb+7#Xa8~eqJ5; z(lB-J z`zF^=TgJOt`Q9`3y17~M$+vUlf}-NXpM|0?3J8`s-k0so>vZ4PaC!2>b&5MK6pA*D zU$o-hqyzi4$Dw~)A={bv+Wqt8-6HLh=3(Q?&%SOt?g?E*ikt2`&6@62@6%vmR2MWO ze@;&Hj%{;36;E&6FL2m(nRb+Ca=iQbS*Hps-y!*J1s`mdUFvq78M1b3>LB4rVsnji zq1`!~h|HN&ljBq!(;9B135~0)?3j}hn9$h0Wm0vtoSe^o71`FHY4dHToa%J*OqI+x zJE-zd|JY))wt0tl>`IX@t2`-$wg{i@>lr$&=G)DT`=_j3oTYd~m!rJIg0X= z)9oQmld5;H=nOpv%MT~_Jk~r@rwOmzYWb#QqAn|n|ve>)VWvU=w>NZ3Qxp9BcTX5zzQ~RW~ zf&(Ut3nSjxPr5Z`{*CW83wO`)g7!MAgvRH;YtM00PqE2;5@j%5GckX-BEm?KD0upU zA))+oeBjHyvzN-Jzg?E`bWm;wag>+nZ4UBQE}| z@ae4Q^*1+?a);koo>AO?qenj2J8|0j6FF)#!+b6Vm>RhS1uk9mjp#9k93^iZSX4h4 zKia>mdVJ+mHwBdwCkaQR->rT^;lJo~>Pysxdp6R-20z0p<(D2hIXzCE7^rfNw#e{& z+Jw<|*3x^-7Uh?jS-u&1e?oL!)?4ZI+4o8>%sfl2lXm*4E98|eI?W`tvQg1^=B5=c z_hsfq9D3Dw!`;k0yyINHu+xJd@oOaX-vvK1Zj)5~Bp)27jt4c1qE@y0k57?Qg^0 zg;ediG-0p6`vy~s03(~FqDoI}1m8UlvR@{n}}OlX`25&8Np_KTYJ5wWxG(o8;z5H`td3K@juc6LQ;Db;h z?8w|0gNrMW{*w##x2qkgFfKuo?Camg3@hlBkR8YEIeKfuxvQDpvQL8=oFWpMGaefq zP$eukeeE(cz3F>Dd7PudWH&Es_%P;VM3=UG_cDmevD& zvrdRbQ3sdys83fRPz$Q$8u{D9Vp7%4{X8L8HA>r9?~sT}-mbMvC%0~r-?UEr%LLOm zlP>6-yOEacW&2rH!bq;7ZqjU-!AW^77t=PwT@#`S8;2>yk|wv8XhlXWZ8<8^KXwZUU$}36&`59cIOU1KIm8`r7u`DfeqGr9>CdDnXD=%p{F-y^QlQHL z5wDr|w!Na>xiWQM-G)oY&3kf#?OWT4dFzvnR44ybnb!*!e9Mq1_~|n| zT6CAz+?Zp_=RGn1aC{@FPORTsSg6YDc8*3{%EIw_d#lNIHrmFcUqsHFf2o-`9Dey( zkdwFM7DL}!NZK{xg7_rTj8GZ!F7Nf@iY+ap;?6A64!zP5p(x@wK8d27_f9}RZSrz* zb{Z|iNqUpS`p7U+i{~uYVu9hX7Gse!3w@fm`1Y?|A8?WH$PB?JLlb*nZ5^AYw@lZ= z894zPz2vXyTwL-#Jf@_hIArXRZLW$RvugK5ph3A~}!MRx~Wbk<9EcL~;WICaK$@U2zW zWn^E*_8Z%FX9`_?HM4oPlFjH6r(V*f6q~EN4tzK%rP4ZU{PcCA4>D~u?~qb;gS36F z%abM{whg6U*C>~cx=|*UJ9|ya`l7GNN1i6o+$YzpZF2Z=7e3SK1%1(f61zEjw(1A@ z_|?NZ`jR$K!U{@`P3$cjHT%ew-F-rVyJMd&Xufu)SyJ`m=VXs|f%+p~xF3#)fEk$PrOqVDb8 z95MSyYW+gbOlNJ8c57OG(Y5)MFVSyrzUu7%VJ6lzrMRK-vuAonrAe{pl?O6`JX9N)e0esTET#WtB^B@zzI5YQ=U zdU|oQc4Vwoy-lv1;*(WBV~pR2luLNMn*JrjSf_Y+-nVb0M_1SW6c}GEcYsYd#JWyw znM*KD58Xv3qiR6&Sw{<5yT(O%nZJ*lu&S;fx@l{zpNEX<%C&w={esv2DnQpyhq{I^ zHqFtunf;G;j zGgqIrfuZQewqH&2Z`+Eo&+gZq7_S4}!>31(Az*#D9wftPPBf9O2g?wcn;lj45OgaY zJ-yD@!WZPf#)=jqW3P(tqC_|R(ZbMoYUqpZ&|8N6sg3<<=*aSxpg-ctbVK;xc4yXr z#Q)xB{v+J*zxA0}J%GP_=8?SIKL+*xtq;oD$Nc4kqPG1%VbHAI&T<)hhye>@D=vMH zD}9@-42&&GVeXP;#;AVnzhz~EwSPgSVCHUEn3)KInJZvnW&{mpW^rJqGY2!dg_(Au&}*OcfGSg~U`LF;z%R6%td0#8e?MRY*(~GE;@jR3S4}$V?S7Q-#b_ zAv0CTOcgRyh0Ii;FjXi_6$(>@!c?I!RVYjq3R8u`RG~0cC`=V9Q-#V@p)ysdOcg3q zh00W+GF7Nd6)IDO%2Xk!F=Ys9OdWz6Q;49(R3fM`r3h+FErJ?TjG)F;Bd9Uu2x^!f zw*HCmYiBwE)BCkW9Sgt`n0wWQ3D|6afc5+YtlKAGeLjKN;R_S6o!JCN|Bdc=hp_rV zSp6WZeh^ka2&*5qyPUwp0^KQ(Juhb41kAVzm~|5{^Cn>SO~4GCfLS;JGjRfD;{?pe z37C}=Ff%7$c22+yoq$<70W)<1X6ppZ*a?`m6EJf}vzZJe2$;bWFpDQ(CQra@o`4xW z0ke7nX7&Wk?g^OT6EMpsV5U#NY#-gr#k>z@{RGVX37Gv8FaQV`0R#*I0>%IVgMffh zK)^5{U>p!I5C|9v1PlcN#sUF@fq>CKz;Ga7JP6E zA27xTjPU_u;|GlK0b_i?7#}dk2aNFnV|>6EA27xTjPU_ue83nVFvbUr@d0Cez!)De z#s`e?0b_i?7#}dk2aNFnV|>6EA27xT{L44`$2nq-NANHA=$C)=%R&0(A^mcZFg{?6 z4;bSE#`u6SK46Rw7~=!R_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6S zK46Rw7~=!R_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6SK46Rw7~=!R z_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6SK46Rw7~=!R_<%7!V2lqK z;{(R{fH6K0j1L6k1Ht$}Fg_5B4+P@_!T3NhJ`ju#1mgq2_&_i|5R4B5;{(C?KrlWK zj1L6k1Ht$}Fg_5B4+P@_!T3NhJ`ju#1mgq2_&_i|5R4B5;{(C?KrlWKj1L4GKM;%$ z1mgq2_&_i|5R4Dx*Bs*4B;waB;@33dpLqn+e?flDBz{dLe$6F*O(rlt5R4Cm$p?L! z^v?lA^h1LD*M$0_v!-7`W9C#z*N5iIGEHJU;;-O^X{axHK9kyiCrjx3?f-L=B}ep3 zF8lvZnCNqJ!vC2GlOuXAnf-q!P1q#+zmq2BBhMmwt>9=w_7~iS;w0<*Z^Hte>luF+%RF zVk5DCM%!E*aj<_bBO4=sz$(ST`Z*76j0^;;5(m@gViSzHDytL+!(U;* z(V1Z6L0F_<4rb3y3fJkw9E_fu6t2sMIT$@RDO`sSb1?Z8I-J`I=jFg044#`5uDgdh zm^(KqTxSn+Fm`TII6nvGVC>waaBdFF!P>b6}P z-mAe!lz0-u`89xL@Fa%wYyh+1NsN9mOPc|GAx~mBUmoZUc@o2U^FV*dlNipQ2V;6X zso`9D!iX4PEc{}51z=QdL#9$D)47kwImv6_ulkOK-_IyC}g zMI5UZ2NF0(B$%5PSfxh90b^+%tJ;V_U@Y)sRT~irjAeo>YG8Da<&APn2QW8tu&RxS z1jdp@R<#ig2aJV}tZE}7fw6p&RgDt~T-%MY2$NNf0|s12WC9~w&#J}&guhUu8vv`? zh*)4OD`iz15etkpI;?6VVu7*5l|>B<`SHR6BiqEPHX;`OSc^*EvNs|Y@X1>+_Q%VU zGM4_bKIw>9U@RbJRT~itjAh8IY9nHSv6z`vZA2_EmPE6vjfe%t!fIAE4lHnq?K3yO zv1ozez~A=4WhiB=?`D$67BA!vR+<{N#WG09>_j za|48f`E%96O$`tZ#?MuY-n6Xk#liHszDL}|0GNN$eSP#u0CfJzC5ge@Udd7|z`A&l z!c7W*9+n3w+>8LQt-qJT=W+qlVv*t;0%vYDWoawGvUrffSv+7^JV@aT9lemNtm^?m< z3$7mFpE;x<+d%MSHb-|@0JZRzidvFTLgBH&2!3pHwYvB{S;OY?` zwD2+i5!gLGwF|Bv`CYHSN#HP#x?nL_a23gKL2fQIKEn$xBl)exJ;sAi^1>X<|1Z>V z4i7kc@M&LgBFR(TaLx}n zmE=hc=l&oZhJn9$Qk>h5a2N>wLJjBnBO_t~p9Kb&n()+a^ovMKJdTI;Nk_s0^H3jFH4ekVU(^l90+EA5^cQM47KkHZfq6_3>ywUz z1?FK#tZE!s;4)%Ci~}!Nw7_L3JdnUVc8OJOBqT5oZ(>y&2?@+2pjg#LL;^k^Od1gh z_SYR9x#QIJ- zu)rnL#5|^sMT^3L1kRZz=79|?Qh*ZOL1AVUxQ$i#OfUe{Zz(Q$6Xtn)EVTk?`g
    5dFAe~3+4{&l&X-jSm{+jWi*w?Qc_c8a z76;?!S}&aGgZo!`s29%g!2|>kTDTzq%sud+g&P3C+yf6J<~E%Mb^&wncDVQy_#82qSzu{1ZhI7%2hX!M7zbyEyNd|y z1?J%I{Dm5h05G?})05&DAdSfF$8QQIaqxP$KPf&(3< z#icXC?-K@iXOZHH0rP7VtcE)x4wxUcV3ivY2+VI|u*!{y1m-6;Smj290`p59ta8A} z$J?vGu>c0Xc$34y07kxelf%&fhQ4@{!wn5!?29)!`mn_MDn>>HK2;1xzj&%0<6@TD zf#EOS5a8-t-sEspfFUs6zks2G{2DRy!OO zV04T(IUE&WfQ&ae92H=Uj75%%>%{z?6iep`2FX~&IQve_Z&I;pjck~}=aIoU8B5i; zr|D#kg>%>1w>Yny>V z!{2X{OPZPabxu|>P7rVvqu-ZRj1vP~#c%|$iIDK{m^c6koCx4rGu$x1COX1H&2X*} zo7e~sVmJc8l|j7N8IAxpago2T89v?2CMv>%7;YF~6BFS<3^xj}iHPtZM!%GW05j12u!)B7AciA=O)P{5F&qJGA|X78;Rs+82jM{sM*y2B2oGX70>I75Jm7>+JhO>_ z{C&;v2T!w!fAAoNBY;ixg9kDCr7Q%niGA=Oh9iJY?BloC-(>5VUv*}!9;bnTn=6FR zJ+p~_uvCv*Vx0MfX_mUNiGT1UheLr){DUVs91LvYAH2xn^UrMJA3Vw7h6Fb851!<3 zNU({2@FYjSnDxzaa*Vi7s_^Gjvx$H3R6E?zz$X5|lN@evU=#n~Ne(wWu!(>0B!{Dd zP5gr=IUE&i;vYQ7F+cv!Vkm6lA3Vw7s9+QS;7JZg1-M_CWjy3+#>_9;vp(#Is6f~dW3N)3kl$cWnRQ^9Dw_kc@e`= zK;lFJmx(jNeYY|%HNz1AZdT?+3`YRCSD6F*Ryolil z0QV{LB1RvFSO@^ODf1$RBLLi`%!?R~0P;u(z@J(T?o#HZW;g;UoH7X9*u?Ktrf^Cj za23N5K;bwBaOoWJS!N2S90IqR;d9Jj3Wxr>u_I%-RSX9J*!(~E zhbyDf4;B8i3|6CeIk++c{cAC}GRAp%_%ixOOL1i|efb2g451%ZgDWG_BOmxO`o}79 zWhDBc-MBI`0z5BFp)ak*T}PpRIuTb!1-OLJz#4F0gz;^lKYxobPDjL-(GM}kmBC<} zA_)B~DBN{Kw5I~fKpR3xZ1l)%^eAlfC~WkopdL{T)I(npP(LC8)FVP{^k6o6jOg#5 zZ9qiW=n>iIk=W>w+0>8BMvuZqj|%FM(5m6z2T2XoBN0G75=u4jTgK+AKcAPxrjH;Y zZ1m7Z1@zDgf@N&#M`oi(VWUT7qelhx$ZYxuG6B>hGfu($^Lfb-FfKr3Hhlycp`VV0 z`z?_kYQ>iUV+uqjkpRxgWMB-1$Q1e);c$8sz!!kXRB#NSr~&=~x-ps_?f6rV!ls|3 zpdB=D9ZX*`h5Ie)NC0Iha9|nz>$14tlE8kALS|EE3Y$7pfpHL`P{BG=)xbJa8DBv9 zvo2JK9tXvjf$dC%=?CWHu4BUj>IwjTJC#H~B^CEu65y9WR5A&?2k2ixR4O=rP(hyy zB^vNS&^?iWuY!69fKP+&P6T`(v<#>-y6upNw|jJ$rhifEkLQKeNc6?M_%dJ&MZF<_ z540xqIXv#SRKS0R)zF5)Uq?`*f3V`u-=g1?1g`__046YYQsaILv(W=tfYHzLfcFJ` zYM4M`qX+s7s0$3Rg_Z$615BU-ei)2m4!jSD0Qg}rL;(CS7rYMvpPV_Yp8eB7*lI13oai`G%eb{PTT46#D8LTp5)J+ygaU;2tmm=(Az8QGt7) z6$I`9!}NI9pZA3ssUv(DFzx>WqUlc}tN8mXSGBDPn>+1o394(`N@%_(s zL8u^G=rjtr2ay2GabP0jtnxqgi0G({zX#gY0c}M@8wR)zy$M(b)|tqrKOmwZ3E;OB zHgyJlB$!AA>r7Gu>r5hmbta*m1pdAxh`v_n&vqff^pEf0${3+FTp1CV+oQdy8h8&9 z;D^B^G7-E7h5oHY+&!rD6Qc2DfDZzb)c_v^#SMsWprbz6PoiBDa9=X|Yy5rD*9Klk zWTQu9qelY!U39ttv^|*u%zx1?0`QUO-3CxUv^xdHRSE&@*U%|D&|gs?aO|P5X#)xZ zjujLXBz#*@P@@FP=*Q;#`7SABurH>t@ee3euc@r!)KA3Q5IWNZ`bk8M0QBvM z8U*;AXc^!uqr(iq78%Nf#*1)qZo%%cz#bil6YfZ+8|V*~5}eQ`kBqYn(& z20A?go&y~g@Z&y$`h>u9u!$$4SvFv9fLayc$D(Dx{1Uw_*w>*xAy7XeH;rRKs@LLivk3!JL1)hzJ z1`mPXQb5}vQ^EcU4Z#C8iDCpi2Zc==P#|zzpuiyKsA~qiUkU=+8ih^&M?ohb_-CV# zKn5vb934d;h;tznD%fwJQzqbjP|<)B{=QTK*q>6-0S>qh9W1~y&~~Zl#1^;?%q5`R z6`MMPc}WCyX~4QrKwG1t&M5A_Xl4iS8PLcLzz13e*an)40sL{I8X9Q;?tywCU>Wc( z|DQt6prv6Th{E^$ie7r|#yAln2x1fHNwCM_p@#|uX&~n0ulL)4VD~F{%#e`H%+Abv zZ?=tOeJD~ZzRGx2IT)mG>3eLdrzrKRr;Lx3GqeEJQ&_y}DK_=^xPkwu_Vx=@ql_Df zP?#DuQN58TrI`mIQhaaVKU$gQp-3L~Gx)FEr=WK66l~gmL5Wa2z<*^7A|$IF}!WD9?XF>oMWD;|_JJ&UNS_aacr_s#MP@vGhl^HbdbGS~-*H}W4s zh;pkj4~WM45Vrhxjm26Zlcfz3rtby9m3<)~2Z|-2TI&PI>b>{o0l~2y`91>^t*`xC zhvw;c{ya@p*HxSS`TMjSyxF{sBp&lPPC+U jNH4(%`f)q7PygTXo{#)|&Btk84*Wz$pw)WazYf(8fEH_C literal 0 HcmV?d00001 diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1d2cd85..bd599c9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -23,6 +23,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -47,6 +48,7 @@ fn make_text_item_with_font( is_bold: is_bold_font(font), is_italic: is_italic_font(font), item_type: ItemType::Text, + mcid: None, } } @@ -999,3 +1001,39 @@ startxref ); } } + +#[test] +fn test_firecrawl_tagged_pdf_struct_tree() { + use lopdf::Document; + use pdf_inspector::structure_tree::{StructRole, StructTree}; + + let doc = Document::load("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let tree = StructTree::from_doc(&doc).expect("Should have a structure tree"); + + // Verify structure tree contains expected roles + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + assert!(!roles.is_empty(), "Should have MCID roles across pages"); + + let flat = tree.flatten(); + let has_code = flat.iter().any(|e| matches!(e.role, StructRole::Code)); + let has_h1 = flat.iter().any(|e| matches!(e.role, StructRole::H1)); + let has_li = flat.iter().any(|e| matches!(e.role, StructRole::LI)); + let has_caption = flat.iter().any(|e| matches!(e.role, StructRole::Caption)); + assert!(has_code, "Should have Code elements"); + assert!(has_h1, "Should have H1 elements"); + assert!(has_li, "Should have LI elements"); + assert!(has_caption, "Should have Caption elements"); + + // Full conversion: code fences should be generated from Code struct elements + let buf = std::fs::read("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let result = pdf_inspector::process_pdf_mem(&buf).unwrap(); + let md = result.markdown.unwrap(); + let fence_count = md.matches("```").count(); + assert!( + fence_count > 0, + "Should produce code fences from tagged Code elements" + ); + // Fences come in open/close pairs + assert_eq!(fence_count % 2, 0, "Code fences should be balanced"); +} From 7c1ddd9ba922f4dbc507337d5aad7e091f07cd54 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Mar 2026 21:26:02 -0700 Subject: [PATCH 07/46] fix: merge consecutive struct-tree heading lines into single heading When a heading wraps across multiple visual lines, the structure tree creates separate heading nodes per line, producing broken output like `## Snow` / `## Lake`. Extend merge_heading_lines to consider struct-tree roles (H1-H6) in addition to font-size heuristics. A 20-word combined limit prevents merging body text mis-tagged as headings. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/markdown/convert.rs | 4 +- src/markdown/preprocess.rs | 167 ++++++++++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 6 deletions(-) diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 1fef66f..6d256e3 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -248,7 +248,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let heading_tiers = compute_heading_tiers(&lines, base_size); // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); + let lines = merge_heading_lines(lines, base_size, &heading_tiers, struct_roles); // Compute the typical line spacing for paragraph break detection. // For double-spaced documents (like legal/government PDFs), the normal @@ -621,7 +621,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> let heading_tiers = compute_heading_tiers(&lines, base_size); // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); + let lines = merge_heading_lines(lines, base_size, &heading_tiers, None); // Compute the typical line spacing for paragraph break detection let para_threshold = compute_paragraph_threshold(&lines, base_size); diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index fa89efa..d443f75 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -2,20 +2,62 @@ use std::collections::{HashMap, HashSet}; +use crate::structure_tree::StructRole; use crate::types::TextLine; use super::analysis::detect_header_level; +/// Resolve a heading level for a line, considering both struct-tree roles and font heuristics. +/// Struct-tree headings take priority. +fn effective_heading_level( + line: &TextLine, + base_size: f32, + heading_tiers: &[f32], + struct_roles: Option<&HashMap>>, +) -> Option { + // Check struct-tree role first + if let Some(roles) = struct_roles { + if let Some(page_roles) = roles.get(&line.page) { + for item in &line.items { + if let Some(mcid) = item.mcid { + if let Some(role) = page_roles.get(&mcid) { + let level = match role { + StructRole::H => Some(1), + StructRole::H1 => Some(1), + StructRole::H2 => Some(2), + StructRole::H3 => Some(3), + StructRole::H4 => Some(4), + StructRole::H5 => Some(5), + StructRole::H6 => Some(6), + _ => None, + }; + if level.is_some() { + return level; + } + } + } + } + } + } + + // Fall back to font-size heuristic + let font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + detect_header_level(font, base_size, heading_tiers) +} + /// Merge consecutive heading lines at the same level into a single line. /// /// When a heading wraps across multiple text lines (e.g., "About Glenair, the Mission-Critical" /// and "Interconnect Company"), each fragment becomes a separate `# Header` in the output. /// This function detects consecutive lines at the same heading tier on the same page /// with a small Y gap and merges them into one line. +/// +/// Both font-size heuristic headings and struct-tree tagged headings are considered. pub(crate) fn merge_heading_lines( lines: Vec, base_size: f32, heading_tiers: &[f32], + struct_roles: Option<&HashMap>>, ) -> Vec { if lines.is_empty() { return lines; @@ -24,19 +66,23 @@ pub(crate) fn merge_heading_lines( let mut result: Vec = Vec::with_capacity(lines.len()); for line in lines { + let line_level = effective_heading_level(&line, base_size, heading_tiers, struct_roles); let line_font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - let line_level = detect_header_level(line_font, base_size, heading_tiers); // Check if the previous line is a heading at the same level on the same page let should_merge = if let (Some(prev), Some(curr_level)) = (result.last(), line_level) { - let prev_font = prev.items.first().map(|i| i.font_size).unwrap_or(base_size); - let prev_level = detect_header_level(prev_font, base_size, heading_tiers); + let prev_level = effective_heading_level(prev, base_size, heading_tiers, struct_roles); let same_page = prev.page == line.page; let same_level = prev_level == Some(curr_level); let y_gap = prev.y - line.y; // Merge if gap is within ~2x the font size (normal line wrap spacing) let close_enough = y_gap > 0.0 && y_gap < line_font * 2.0; - same_page && same_level && close_enough + // Don't merge if combined text would be too long — real headings are short. + // This prevents merging body-text lines that are mis-tagged as headings. + let prev_words = prev.text().split_whitespace().count(); + let curr_words = line.text().split_whitespace().count(); + let not_too_long = prev_words + curr_words <= 20; + same_page && same_level && close_enough && not_too_long } else { false }; @@ -440,3 +486,116 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec .map(|(_, line)| line) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ItemType, TextItem}; + + fn make_item(text: &str, font_size: f32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x: 0.0, + y: 0.0, + width: 100.0, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid, + } + } + + fn make_line(text: &str, font_size: f32, page: u32, y: f32, mcid: Option) -> TextLine { + TextLine { + items: vec![make_item(text, font_size, mcid)], + y, + page, + adaptive_threshold: 0.10, + } + } + + #[test] + fn test_merge_struct_tree_headings() { + // Two consecutive lines tagged as H2 via struct tree, same font size as body + let lines = vec![ + make_line( + "Historical Context for Operations in Snow", + 12.0, + 1, + 700.0, + Some(10), + ), + make_line("Lake District", 12.0, 1, 686.0, Some(11)), + make_line("Body text paragraph.", 12.0, 1, 660.0, Some(12)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H2); + roles.insert(11i64, StructRole::H2); + roles.insert(12i64, StructRole::P); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should merge two H2 lines into one"); + let merged_text = result[0].text(); + assert!( + merged_text.contains("Snow") && merged_text.contains("Lake"), + "merged heading should contain both fragments: {merged_text}" + ); + } + + #[test] + fn test_no_merge_different_struct_levels() { + // Two consecutive lines tagged as different heading levels + let lines = vec![ + make_line("Chapter 1", 12.0, 1, 700.0, Some(10)), + make_line("Introduction", 12.0, 1, 686.0, Some(11)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H1); + roles.insert(11i64, StructRole::H2); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should not merge different heading levels"); + } + + #[test] + fn test_no_merge_heading_with_body() { + // A heading line followed by a body paragraph line + let lines = vec![ + make_line("Introduction", 12.0, 1, 700.0, Some(10)), + make_line("This is body text.", 12.0, 1, 686.0, Some(11)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H1); + roles.insert(11i64, StructRole::P); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should not merge heading with body text"); + } + + #[test] + fn test_merge_font_headings_still_works() { + // Original font-size based merging should still work without struct roles + let lines = vec![ + make_line("A Very Long Heading That", 18.0, 1, 700.0, None), + make_line("Wraps to Next Line", 18.0, 1, 682.0, None), + make_line("Body text.", 12.0, 1, 660.0, None), + ]; + + let heading_tiers = vec![18.0]; + let result = merge_heading_lines(lines, 12.0, &heading_tiers, None); + assert_eq!(result.len(), 2, "should merge font-based heading lines"); + } +} From 442a169ddfbdb166c9237a99cd6e25e571e8ed70 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Mar 2026 22:22:19 -0700 Subject: [PATCH 08/46] feat(layout): pre-mask spanning lines for multi-column pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-item lines (titles, section headers) that span across detected columns were being split into individual column buckets, corrupting newspaper detection and reading order. Add identify_spanning_lines() that groups items by Y-proximity and marks lines wider than 1.3x the widest column that have no gap at a detected gutter boundary. Uses column-aware thresholds and gutter-interval gap detection for precision — only fires on pages with detected columns, preserves original item order via boolean mask. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/layout.rs | 233 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 5 deletions(-) diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index 312c6ca..f91900f 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -220,6 +220,105 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec columns } +/// Identify items that belong to lines spanning across detected columns. +/// +/// Groups items into rough lines by Y-proximity and marks items whose line's +/// combined X-span exceeds 1.3× the widest column AND has no gap located at +/// a detected gutter boundary. Returns a boolean mask parallel to `items`. +fn identify_spanning_lines(items: &[TextItem], columns: &[ColumnRegion]) -> Vec { + let n = items.len(); + let mut mask = vec![false; n]; + + if n < 3 || columns.len() < 2 { + return mask; + } + + let max_col_width = columns + .iter() + .map(|c| c.x_max - c.x_min) + .fold(0.0_f32, f32::max); + let span_threshold = max_col_width * 1.3; + + // Gutter centers: boundaries between adjacent columns + let gutters: Vec = columns.windows(2).map(|c| c[0].x_max).collect(); + let gutter_tol = 15.0; + let y_tol = 5.0; + + // Build (original_index, y) pairs sorted by Y descending for grouping + let mut indexed: Vec<(usize, f32)> = + items.iter().enumerate().map(|(i, it)| (i, it.y)).collect(); + indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + // Group by Y-proximity into rough lines (as index sets) + let mut groups: Vec> = Vec::new(); + let mut current_group: Vec = Vec::new(); + let mut current_y = f32::NAN; + + for (idx, y) in indexed { + if current_group.is_empty() || (current_y - y).abs() < y_tol { + if current_group.is_empty() { + current_y = y; + } + current_group.push(idx); + } else { + groups.push(std::mem::take(&mut current_group)); + current_y = y; + current_group.push(idx); + } + } + if !current_group.is_empty() { + groups.push(current_group); + } + + for group in groups { + if group.len() < 2 { + continue; + } + + // Sort group indices by X to compute span + let mut sorted_by_x: Vec = group; + sorted_by_x.sort_by(|&a, &b| { + items[a] + .x + .partial_cmp(&items[b].x) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let line_left = items[sorted_by_x[0]].x; + let last = *sorted_by_x.last().unwrap(); + let line_right = items[last].x + effective_width(&items[last]); + let span = line_right - line_left; + + if span <= span_threshold { + continue; + } + + // Check if any inter-item gap falls at a detected gutter boundary. + // If so, this is items from different columns at the same Y, not a + // true spanning line (like a title or section header). + let has_gutter_gap = sorted_by_x.windows(2).any(|pair| { + let left_end = items[pair[0]].x + effective_width(&items[pair[0]]); + let right_start = items[pair[1]].x; + let gap = right_start - left_end; + if gap < 5.0 { + return false; + } + // Check if any gutter falls within the gap interval (with tolerance) + gutters + .iter() + .any(|&g| g > left_end - gutter_tol && g < right_start + gutter_tol) + }); + + if !has_gutter_gap { + for &idx in &sorted_by_x { + mask[idx] = true; + } + } + } + + mask +} + /// Determines if a text item spans across multiple column regions (e.g. full-width headers/titles). fn spans_multiple_columns(item: &TextItem, columns: &[ColumnRegion]) -> bool { let w = effective_width(item); @@ -469,15 +568,28 @@ pub(crate) fn group_into_lines_with_thresholds( let lines = group_single_column(page_items, adaptive_threshold); all_lines.extend(lines); } else { - // Multi-column - separate spanning items from column items + // Multi-column detected. Pre-mask lines that span the full page + // width (titles, section headers, footers). These multi-item lines + // would otherwise be split across column buckets, corrupting + // newspaper detection and reading order. + let spanning_mask = identify_spanning_lines(&page_items, &columns); + let premasked_count = spanning_mask.iter().filter(|&&m| m).count(); + if premasked_count > 0 { + debug!( + "page {}: pre-masked {} spanning-line items", + page, premasked_count + ); + } + + // Partition items preserving original order let mut spanning_items: Vec = Vec::new(); let mut column_items: Vec = Vec::new(); - for item in &page_items { - if spans_multiple_columns(item, &columns) { - spanning_items.push(item.clone()); + for (i, item) in page_items.into_iter().enumerate() { + if spanning_mask[i] || spans_multiple_columns(&item, &columns) { + spanning_items.push(item); } else { - column_items.push(item.clone()); + column_items.push(item); } } @@ -965,4 +1077,115 @@ mod tests { "Equal-width equal-row columns should NOT be newspaper (borderless table)" ); } + + #[test] + fn premask_spanning_title_removed_from_columns() { + // Title spans x=30..550 as 5 adjacent items (no gap near gutter at x=300) + // Two columns: left (x=0..300), right (x=300..600) + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Spanning title: 5 items at Y=750, each ~100pt wide, gaps ~4pt + // No item gap falls near the gutter at x=300 + for i in 0..5 { + items.push(make_item( + 1, + 30.0 + i as f32 * 104.0, + 750.0, + "TitleWord_________", + )); + } + + // Left column body: 20 lines + for i in 0..20 { + items.push(make_item(1, 30.0, 700.0 - i as f32 * 14.0, "LeftText__")); + } + + // Right column body: 20 lines + for i in 0..20 { + items.push(make_item(1, 320.0, 700.0 - i as f32 * 14.0, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + let non_spanning_count = mask.iter().filter(|&&m| !m).count(); + assert_eq!(spanning_count, 5, "Title items should be pre-masked"); + assert_eq!(non_spanning_count, 40, "Column items should remain"); + } + + #[test] + fn premask_does_not_mask_column_items_at_same_y() { + // Two items at same Y with gap at gutter → NOT masked + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Items in two columns at same Y — gap center ~305 is near gutter at 300 + for i in 0..15 { + let y = 700.0 - i as f32 * 14.0; + items.push(make_item(1, 30.0, y, "LeftText__")); + items.push(make_item(1, 320.0, y, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + assert_eq!( + spanning_count, 0, + "Column items with gap at gutter should NOT be pre-masked" + ); + } + + #[test] + fn premask_narrow_line_not_masked() { + // Items that form a line spanning only ~40% of column width → not masked + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Narrow header at top (spans ~240pt, max col width = 300, threshold = 390) + for i in 0..3 { + items.push(make_item( + 1, + 180.0 + i as f32 * 84.0, + 750.0, + "SmallHeader___", + )); + } + + // Two columns below + for i in 0..15 { + let y = 700.0 - i as f32 * 14.0; + items.push(make_item(1, 30.0, y, "LeftText__")); + items.push(make_item(1, 400.0, y, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + assert_eq!(spanning_count, 0, "Narrow header should NOT be pre-masked"); + } } From 3ff40987d6528e592e2cb2702f7281703057dbdd Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 10:40:30 -0700 Subject: [PATCH 09/46] fix(detect): classify tiled scans with garbage OCR as Scanned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanned PDFs with JBIG2/tiled image strips were misclassified as TextBased because no individual image tile exceeded the template threshold. Now checks aggregate image area per page (≥2M pixels). Also adds is_garbage_text() check: if a Mixed/template PDF's extracted text is predominantly non-alphanumeric (<50%), upgrade to Scanned so callers use proper OCR instead of the garbage text layer. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/detector.rs | 23 +++++++++++++++++++---- src/lib.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/detector.rs b/src/detector.rs index ab5f9d8..cb23796 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -197,10 +197,11 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); pages_actually_sampled += 1; log::debug!( - "page {}: text_ops={} images={} image_count={} template={} unique_chars={} path_ops={} vector_text={} image_area={}", + "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={}", page_num, analysis.text_operator_count, analysis.has_images, analysis.image_count, analysis.has_template_image, - analysis.unique_text_chars, analysis.path_op_count, analysis.has_vector_text, + analysis.unique_text_chars, analysis.unique_alphanum_chars, + analysis.path_op_count, analysis.has_vector_text, analysis.total_image_area ); let is_image_dominated = analysis.image_count > 10 @@ -265,9 +266,8 @@ pub(crate) fn detect_from_document( // Classification logic let (pdf_type, confidence) = if has_template_images && pages_with_text > 0 { - // Template-based PDF: has text but images provide essential context - // Classify as Mixed with lower confidence ocr_recommended = true; + // Template-based PDF: has text but images provide essential context (PdfType::Mixed, 0.5 + (0.3 * (1.0 - template_ratio))) } else if text_ratio >= config.text_page_ratio_threshold { ocr_recommended = false; @@ -383,6 +383,8 @@ struct PageAnalysis { image_count: u32, /// Number of unique non-whitespace text characters found in string operands unique_text_chars: u32, + /// Number of unique ASCII alphanumeric bytes (letters + digits) in string operands + unique_alphanum_chars: u32, /// Number of path construction/painting ops (m, l, c, h, f, re, etc.) #[allow(dead_code)] path_op_count: u32, @@ -454,6 +456,11 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { // outlined text produces thousands of path ops. let has_vector_text = path_ops >= 1000 && path_ops > text_ops.saturating_mul(200); + let unique_alphanum_chars = all_unique_chars + .iter() + .filter(|b| b.is_ascii_alphanumeric()) + .count() as u32; + PageAnalysis { text_operator_count: text_ops, has_images, @@ -461,6 +468,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { total_image_area, image_count, unique_text_chars: all_unique_chars.len() as u32, + unique_alphanum_chars, path_op_count: path_ops, has_vector_text, } @@ -801,6 +809,13 @@ fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { } } + // Tiled scans: many small image tiles (e.g., JBIG2 strips) that together + // cover the full page. No individual tile triggers the template threshold, + // but the aggregate area clearly indicates a scanned/image-backed page. + if !has_template_image && total_area >= TEMPLATE_IMAGE_THRESHOLD * 4 { + has_template_image = true; + } + (has_images, total_area, has_template_image) } diff --git a/src/lib.rs b/src/lib.rs index 4acabe5..d5acd94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -391,6 +391,16 @@ fn process_document( None => (None, LayoutComplexity::default(), false), }; + // If the extracted text is predominantly garbage (non-alphanumeric) and + // the PDF is image-backed (Mixed/template), upgrade to Scanned — the text + // layer comes from a bad OCR pass, and callers should use proper OCR. + let (pdf_type, markdown, confidence) = + if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { + (PdfType::Scanned, None, 0.95) + } else { + (pdf_type, markdown, confidence) + }; + Ok(PdfProcessResult { pdf_type, markdown, @@ -444,6 +454,32 @@ fn detect_encoding_issues(markdown: &str) -> bool { false } +/// Check if extracted text is predominantly garbage (non-alphanumeric). +/// +/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---." +/// where most characters are punctuation/symbols. Real text in any language +/// has >50% alphanumeric characters. +fn is_garbage_text(markdown: &str) -> bool { + let mut alphanum = 0usize; + let mut non_alphanum = 0usize; + for ch in markdown.chars() { + if ch.is_whitespace() { + continue; + } + // Skip markdown syntax chars that we add (not from the PDF) + if matches!(ch, '#' | '*' | '|' | '-' | '\n') { + continue; + } + if ch.is_alphanumeric() { + alphanum += 1; + } else { + non_alphanum += 1; + } + } + let total = alphanum + non_alphanum; + total >= 50 && alphanum * 2 < total +} + /// Analyse extracted items and rects for layout complexity. fn compute_layout_complexity( items: &[types::TextItem], From 8e674eeef5eb77fd3f8c34cb4fdfa826229cb23c Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 11:06:49 -0700 Subject: [PATCH 10/46] docs: add CLAUDE.md with project instructions Architecture overview, build/test commands, key design decisions, debugging tips, and coding conventions. Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..05ccaa0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,79 @@ +# pdf-inspector + +Fast PDF text extraction to structured Markdown. CLI binary: `pdf2md`. Detection binary: `detect-pdf`. + +## Build & Test + +```bash +cargo fmt # format +cargo clippy -- -D warnings # lint (enforced, zero warnings) +cargo test # unit + integration tests (267+ unit, 73+ integration) +cargo build --release # release binary for benchmarks +``` + +All three must pass before committing. + +## Binaries + +- `pdf2md` — extract PDF → Markdown. Supports `--json` for structured output. +- `detect-pdf` — classify PDF type (TextBased/Scanned/Mixed/ImageBased). Supports `--analyze --json`. + +## Architecture + +``` +src/ + lib.rs – public API, process_pdf_with_options, encoding issue detection + detector.rs – PDF type classification, tiled-scan detection, page sampling + types.rs – TextItem, TextLine, PdfRect, PdfLine + tounicode.rs – CMap/ToUnicode parsing, CID decoding + text_utils.rs – CJK/RTL handling, Otsu threshold, ligature expansion, NFKC + extractor/ + mod.rs – top-level extraction orchestrator + content_stream.rs – PDF operator state machine (Tj/TJ/Td/Tm/q/Q) + fonts.rs – font width/encoding, CMapDecisionCache, TrueType cmap fallback + layout.rs – column detection (histogram), newspaper/tabular classification, + spanning-line pre-masking, sidebar detection + tables/ + detect_rects.rs – rect-based table detection (union-find clustering) + detect_heuristic.rs – heuristic table detection (gap-histogram, body-font tables) + detect_lines.rs – line-based table detection (H/V line grids) + grid.rs – column/row boundaries, cell assignment + format.rs – table→Markdown formatting, continuation row merging + markdown/ + convert.rs – core line→Markdown loop, struct-tree role support + analysis.rs – font stats, heading tiers, paragraph thresholds + classify.rs – line classification (header, list, code, caption) + preprocess.rs – drop cap merging, heading line merging + postprocess.rs – dot leaders, hyphenation, page numbers, URL formatting +``` + +## Key design decisions + +- **Primary audience is AI agents.** Output optimized for token efficiency and semantic quality, not visual formatting. No cosmetic padding. +- **Three table detection strategies** run in priority order: rect-based → line-based → heuristic. First valid result wins. +- **Column detection** uses horizontal projection histograms with valley detection. Multi-item spanning lines (titles, headers) are pre-masked using column-aware thresholds before column assignment. +- **Newspaper vs tabular** classification determines reading order: newspaper reads columns sequentially, tabular Y-interleaves them. +- **Tiled-scan detection** catches scanned PDFs with JBIG2/strip images where no single tile exceeds the template threshold but aggregate area does (≥2M pixels). +- **Garbage text upgrade** reclassifies Mixed PDFs as Scanned when extracted text is <50% alphanumeric. +- **Tagged PDF support** uses structure tree roles (H1-H6, P, L, Code, BlockQuote) when available, falling back to font-size heuristics. + +## Testing + +- **Unit tests**: inline `#[cfg(test)] mod tests` in each module with synthetic data. +- **Integration tests**: `tests/integration_tests.rs` with fixture PDFs in `tests/fixtures/`. +- **Regression suite**: sibling repo `pdf-evals` with 179+ snapshot PDFs. Run `cargo build --release` then `bench.py test` in that repo before committing. + +## Debugging + +```bash +RUST_LOG=pdf_inspector::extractor::layout=debug cargo run --bin pdf2md -- file.pdf +RUST_LOG=pdf_inspector::tables=debug cargo run --bin pdf2md -- file.pdf +RUST_LOG=pdf_inspector::detector=debug cargo run --release --bin detect-pdf -- file.pdf +``` + +## Conventions + +- Clippy: use `is_some_and(...)` not `map_or(false, ...)` +- lopdf quirk: `ParseError` is private — match by string for `InvalidFileHeader` +- Column limit for tables: 25 (wide statistical tables) +- `propagate_merged_cells` skipped for >10 columns (spanning rects = background fills) From b8211db4b0ddfd1657ccfb727c03798368642bd3 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 11:39:38 -0700 Subject: [PATCH 11/46] feat: extract invisible OCR text layer from Mixed/template PDFs Scanned PDFs with OCR text layers use rendering mode 3 (invisible text) positioned behind page images. Previously we skipped all Tr=3 text. Now for Mixed/template PDFs, if normal extraction produces garbage or empty output, we retry with invisible text included. This unlocks text from OCR-generated PDFs without requiring external OCR. Also adds Windows Unicode BMP (3,1) subtable support to the TrueType cmap fallback, and allows TrueType fonts with explicit encoding to extract their embedded cmap (OCR fonts often lie about encoding). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/content_stream.rs | 12 ++++++---- src/extractor/mod.rs | 21 ++++++++++++++++- src/lib.rs | 33 +++++++++++++++++++++++++- src/tounicode.rs | 41 +++++++++++++++++++++++++++------ 4 files changed, 94 insertions(+), 13 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ff4a3c1..ca1e450 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -25,6 +25,7 @@ pub(crate) fn extract_page_text_items( page_id: ObjectId, page_num: u32, font_cmaps: &FontCMaps, + include_invisible: bool, ) -> Result { use lopdf::content::Content; @@ -252,8 +253,10 @@ pub(crate) fn extract_page_text_items( } continue; } - // Skip invisible (Tr=3) text but still advance text matrix - if text_rendering_mode == 3 { + // Skip invisible (Tr=3) text but still advance text matrix. + // For Mixed/template PDFs, include_invisible=true extracts + // the OCR text layer that sits behind scanned images. + if text_rendering_mode == 3 && !include_invisible { if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -311,7 +314,8 @@ pub(crate) fn extract_page_text_items( if in_text_block && !op.operands.is_empty() { if let Ok(array) = op.operands[0].as_array() { let font_info = font_widths.get(¤t_font); - let is_invisible = text_rendering_mode == 3 || suppress_glyph_extraction; + let is_invisible = (text_rendering_mode == 3 && !include_invisible) + || suppress_glyph_extraction; // Compute space threshold based on font metrics when available let space_threshold = if let Some(font_info) = font_info { @@ -471,7 +475,7 @@ pub(crate) fn extract_page_text_items( line_matrix[4] += (-tl) * line_matrix[2]; line_matrix[5] += (-tl) * line_matrix[3]; text_matrix = line_matrix; - if !(text_rendering_mode == 3 + if !((text_rendering_mode == 3 && !include_invisible) || suppress_glyph_extraction || op.operands.is_empty()) { diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 2b85eed..9be4977 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -149,6 +149,25 @@ pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, +) -> Result<(PageExtraction, PageThresholds), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, false) +} + +/// Extract with option to include invisible (Tr=3) text. +/// Used for Mixed/template PDFs where the OCR text layer is invisible. +pub(crate) fn extract_positioned_text_include_invisible( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, +) -> Result<(PageExtraction, PageThresholds), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, true) +} + +fn extract_positioned_text_impl( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, + include_invisible: bool, ) -> Result<(PageExtraction, PageThresholds), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); @@ -167,7 +186,7 @@ pub(crate) fn extract_positioned_text_from_doc( } } let (mut items, rects, lines) = - extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?; let threshold = crate::text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { page_thresholds.insert(*page_num, threshold); diff --git a/src/lib.rs b/src/lib.rs index d5acd94..db15348 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -342,7 +342,38 @@ fn process_document( // Step 2 — Extraction (reuses the already-loaded document) let extracted = { let font_cmaps = FontCMaps::from_doc(&doc); - extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, options.page_filter.as_ref()) + let result = extractor::extract_positioned_text_from_doc( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ); + + // For Mixed/template PDFs: if normal extraction produces garbage text + // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. + // This unlocks OCR text layers behind scanned images. + if pdf_type == PdfType::Mixed { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _)| e) { + let sample: String = items.iter().take(200).map(|i| i.text.as_str()).collect(); + if is_garbage_text(&sample) || sample.trim().is_empty() { + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } else { + result + } + } else { + // Normal extraction failed — try invisible as fallback + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } + } else { + result + } }; // For Mixed PDFs, extraction failure is non-fatal diff --git a/src/tounicode.rs b/src/tounicode.rs index eeb3e10..727e89f 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -815,6 +815,28 @@ fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option { } } } + // Fallback: Windows Unicode BMP (3,1) — maps Unicode codepoints to GIDs. + // For single-byte fonts, try each byte value as a Unicode codepoint. + // Common in OCR-generated PDFs where byte values correspond to Unicode + // codepoints but the declared encoding (WinAnsiEncoding) is wrong. + if !used_encoding_cmap { + for subtable in cmap_table.subtables { + if subtable.platform_id == ttf_parser::PlatformId::Windows + && subtable.encoding_id == 1 + { + for code in 0x20..=0xFF_u32 { + if let Some(gid) = subtable.glyph_index(code) { + if let Some(&ch) = gid_to_unicode.get(&gid.0) { + let ch = strip_pua_char(ch); + cmap.char_map.entry(code as u16).or_insert(ch.to_string()); + } + } + } + used_encoding_cmap = true; + break; + } + } + } } if !used_encoding_cmap { @@ -1898,13 +1920,6 @@ impl FontCMaps { if font_dict.get(b"ToUnicode").is_ok() { continue; } - // Skip fonts with explicit encoding — they can be decoded by the - // standard encoding path (lopdf) and don't need a fallback CMap. - if let Ok(enc) = font_dict.get(b"Encoding") { - if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() { - continue; - } - } let subtype = match font_dict .get(b"Subtype") .ok() @@ -1916,6 +1931,18 @@ impl FontCMaps { if subtype == b"Type0" { continue; } + // Skip non-TrueType fonts with explicit encoding — they can be + // decoded by the standard encoding path and don't need a fallback. + // TrueType fonts are NOT skipped: OCR-generated PDFs often declare + // WinAnsiEncoding but the embedded font's cmap has the real mapping. + if subtype != b"TrueType" { + if let Ok(enc) = font_dict.get(b"Encoding") { + if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() + { + continue; + } + } + } let font_descriptor = font_dict.get(b"FontDescriptor").ok().and_then(|o| match o { Object::Reference(r) => doc.get_dictionary(*r).ok(), From 258210a82195cc6c4d7c959e02d6292b0f77410f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 11:49:43 -0700 Subject: [PATCH 12/46] fix(postprocess): collapse consecutive spaces in extracted text OCR text layers and some PDF producers emit trailing spaces on each text item, which combine with gap-based joining to produce double spaces ("Vice President"). Now collapses runs of 2+ spaces to single space within lines, preserving leading indentation. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/markdown/postprocess.rs | 35 +++++++++ tests/snapshots/2013-app2.md | 22 +++--- tests/snapshots/nexo-price-en.md | 2 +- tests/snapshots/p1244-1996.md | 18 ++--- tests/snapshots/td9264.md | 116 +++++++++++++++--------------- tests/snapshots/thermo-freon12.md | 8 +-- 6 files changed, 118 insertions(+), 83 deletions(-) diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs index 2bd9970..6da6090 100644 --- a/src/markdown/postprocess.rs +++ b/src/markdown/postprocess.rs @@ -24,6 +24,12 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str text = format_urls(&text); } + // Collapse consecutive spaces within text lines. + // OCR text layers and some PDF producers emit trailing spaces on each + // text item, which combine with gap-based space insertion to produce + // double spaces ("Vice President" instead of "Vice President"). + collapse_consecutive_spaces(&mut text); + // Remove excessive newlines (more than 2 in a row) while text.contains("\n\n\n") { text = text.replace("\n\n\n", "\n\n"); @@ -36,6 +42,35 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str text } +/// Collapse runs of 2+ spaces to a single space within each line. +/// Preserves leading indentation and markdown table pipe alignment. +fn collapse_consecutive_spaces(text: &mut String) { + let mut result = String::with_capacity(text.len()); + for line in text.split('\n') { + if !result.is_empty() { + result.push('\n'); + } + // Preserve leading whitespace + let trimmed = line.trim_start(); + let leading = &line[..line.len() - trimmed.len()]; + result.push_str(leading); + // Collapse inner runs of spaces to single space + let mut prev_space = false; + for ch in trimmed.chars() { + if ch == ' ' { + if !prev_space { + result.push(' '); + } + prev_space = true; + } else { + prev_space = false; + result.push(ch); + } + } + } + *text = result; +} + /// Collapse dot leaders (runs of 4+ dots) into " ... " /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" fn collapse_dot_leaders(text: &str) -> String { diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index 0411127..ace09ee 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -2,10 +2,10 @@ |---|---|---|---|---|---| ||JAN||||| |1|8/1|Procurement of Criticals Spare Parts for Engine Maintenance on Mahe|PUC|Wartsila Eastern Africa Ltd|Euro97,922.30| -|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| +|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| |3|15/1|Manufacturing and Deliveries of 900 Students Desks|MOE||SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00| |4|15/1|Renovation Works at Mont Fleuri Secondary School|MOE|Sai-Fu Enterprise|SR2,815,771.00| -|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| +|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| |6|15/1|Storm Water Channel Project at Au Cap-Additional Works|DOE|United Concrete Products (Sey)Ltd|SR184,300.00| |7|22/1|Procurement of Security at ex-Maritime Training Centre|SFA|Elite Surveillance Security Agency|SR30,000.00| |8|22/1|Installation of Sewerage Treatment Plant at Anse Gaulette|MLUH|Green Island Construction Compnay|SR4,524,884.85| @@ -18,17 +18,17 @@ |14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| |15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| |16|5/2|constraction of 6*2 bedroom Houses- Mont Buxton|MLUH|O-NIVO Construction|SR3,688,844.00| -|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| +|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| |18|5/2|Procurement of Vehcile x1|MOE|Abhaye Valabhji Pty Ltd|SR 650,000.00| -|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| +|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| |20|12/2|Procurement of Turbo charger Rotor for Engine on Praslin|PUC|Marine Power International FZC|Euro 835,669.00| -|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| +|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| |22|12/2|Procurement of Air Cooler Cartridge for Wartsila Engine|PUC|Marine Power International FZC|Euro 36,508.93| |23|12/2|Procurement of Spares for Genset 8P on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro 288,202.50| |24|19/2|Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka|SLTA|Esparon's Enterprise|SR 1,105,069.00| |25|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41|PUC|Marine Power International|Euro 144,726.26| |26|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31|PUC|Marine Power International FZC|Euro 143,746.75| -|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| +|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| |28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| |29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| |30|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| @@ -121,8 +121,8 @@ |94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| |95|4/6|Procurement of Digital Microwave Equipment|SBC|MOCHINO|Euro 125,845.00| |96|4/6|Procurement of Vehicle 1|LWMA|Abhaye Valabhji Pty Ltd-Jeep|SR 475,000.00| -|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| -|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| +|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| +|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| |98|11/6|Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka|SLTA|Bazil Construction|Sr 1,081,530.00| |99|11/6|Procurement of Seychelles Paswsports|DIA|Groupe Impimerie Nationale|Euro 102,400.00| |100|19/6|Anse Boileau Footpath and Drainage Construction Phase II|SLTA|TCH Building Contractor|SR 960,107.50| @@ -198,7 +198,7 @@ |166|13/8|Procurement of High Pressure pump spares -BZM00003022|PUC|Bluezone Mauritius Ltd|Euro27,712.73| |166|13/8|Procurement of High Pressure pump spares -BZM00003023|PUC|Bluezone Mauritius Ltd|Euro12,639.55| |167|13/8|Procurement of CR64 pump spares|PUC|Bluezone Mauritius Ltd|Euro31,822.00| -|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| +|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| |169|13/8|Renovation work to one classroom block at Pionte Larue Secondary School|MOE|Belverdere Builders|SR1,114,575.00| |170|13/8|Completion of Amitie Housing Project 12 x 3 Bedrooms|MLUH|Allied Builders Sey Ltd|6,006,410.37| |171|13/8|Copolia Road widening-Phase 2|SLTA|Belverdere Builders|SR1,037,235.00| @@ -208,7 +208,7 @@ |175|29/8|Servicing of geartrain for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro 21,141.90| |176|29/8|Spare parts for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro38,440.00| |177|29/8|Procurement for sience equipment and chemical for 2013|MOE|Findel Education|GBP37,831.26| -|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| +|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| |179|29/8||Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA|Furui Construction Pty Ltd|SR277,521.45| |180|29/8|Propsed road widening and drainage improvement at Quincy Vilage|SLTA|TCH Contractor|SR1,015,806.24| |181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| @@ -237,7 +237,7 @@ |202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| |203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00| |204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| -|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| +|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| |206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| |207|1/10|Procurement of Viking Johnson fittings|PUC|Viking Johnson (UK)|GBP92,286.50| |208|1/10|Procurement of HDPE Pipes and Fittings|PUC|STR Marketing Ltee|USD196,672.57| diff --git a/tests/snapshots/nexo-price-en.md b/tests/snapshots/nexo-price-en.md index 9fbc2cd..57c887c 100644 --- a/tests/snapshots/nexo-price-en.md +++ b/tests/snapshots/nexo-price-en.md @@ -6,5 +6,5 @@ |Special|83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|79,275,000 with 3.5% individual consumption tax applied 79,275,000|upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| |Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(fron t and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| -**Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR +**Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index a80af44..4548660 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -8,7 +8,7 @@ Department of the Treasury **Internal Revenue Service** # and Report to Employer -**This publication contains:** **Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer +**This publication contains:** **Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer For the period @@ -20,7 +20,7 @@ Name and address of employee **Publication 1244 (Rev. 7-96)** Cat. No. 44472W -**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A , Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.— If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. +**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A , Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.— If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. *(continued on inside of back cover)* @@ -43,9 +43,9 @@ tips of directly from customers received other employees paid tips rec’d. entr 27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals** -**1.** Report total cash tips (col. a) on Form 4070, line 1. -**2.** Report total credit card tips (col. b) on Form 4070, line 2. -**3.** Report total tips paid out (col. c) on Form 4070, line 3. **Page 4** +**1.** Report total cash tips (col. a) on Form 4070, line 1. +**2.** Report total credit card tips (col. b) on Form 4070, line 2. +**3.** Report total tips paid out (col. c) on Form 4070, line 3. **Page 4** Form Employee’s Report (Rev. July 1996) @@ -59,15 +59,15 @@ Employer’s name and address (include establishment name, if different) **1** C **3** Tips paid out -Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3 ) from, 19, to, 19 Signature Date +Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3 ) from, 19, to, 19 Signature Date -**Paperwork Reduction Act Notice.— We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law —each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these +**Paperwork Reduction Act Notice.— We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law —each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these -forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676). +forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676). **Instructions (continued)** -**Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law. +**Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law. **Instructions (continued)** diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index d1b5abf..f53d803 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -1,71 +1,71 @@ -(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows: +(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows: §1.1563-3T Rules for determining stock ownership (temporary). -(a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a) -through (d)(2)(iii). (iv) Statement. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include-- +(a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a) +through (d)(2)(iii). (iv) Statement. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include-- (A) A description of each of the controlled groups in which the corporation -could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and +could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and (B) The following representation: [INSERT NAME AND EMPLOYER IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER OF THE [INSERT DESIGNATION OF GROUP]. -(v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of +(v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in termination of membership in the controlled group in which such corporation has been included. (B) Election not filed. In the event no election is filed in accordance with the -provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service will determine the group in which such corporation is to be included. Such determination will be binding for all subsequent years unless the corporation files a valid election with respect to any such subsequent year or until a change in the stock ownership of the corporation results in termination of membership in the controlled group in which such corporation has been included. +provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service will determine the group in which such corporation is to be included. Such determination will be binding for all subsequent years unless the corporation files a valid election with respect to any such subsequent year or until a change in the stock ownership of the corporation results in termination of membership in the controlled group in which such corporation has been included. -(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3). +(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3). (e) Effective date-- (1) Applicability date. This section applies to any original Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30, 2006. -(2) Expiration date. The applicability of this section will expire on May 26, -2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: §1.6012-2 Corporations required to make returns of income. +(2) Expiration date. The applicability of this section will expire on May 26, +2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: §1.6012-2 Corporations required to make returns of income. * * * * * (c) [Reserved]. For further guidance, see §1.6012-2T(c). * * * * * -(k) [Reserved]. For further guidance, see §1.6012-2T(k)(1). +(k) [Reserved]. For further guidance, see §1.6012-2T(k)(1). -Par. 52. Section 1.6012-2T is added to read as follows: §1.6012-2T Corporations required to make returns of income (temporary). +Par. 52. Section 1.6012-2T is added to read as follows: §1.6012-2T Corporations required to make returns of income (temporary). -(a) through (b) [Reserved]. For further guidance, see §1.6012-2(a) through +(a) through (b) [Reserved]. For further guidance, see §1.6012-2(a) through (b). -(c) Insurance companies-- (1) Domestic life insurance companies-- (i) In -general. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return-- +(c) Insurance companies-- (1) Domestic life insurance companies-- (i) In +general. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return-- (A) A copy of its annual statement which shows the reserves used by the company in computing the taxable income reported on its return; and (B) A copy of Schedule A (real estate) and of Schedule D (bonds and stocks), -or any successor thereto, of such annual statement. (ii) Mutual savings banks. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L. +or any successor thereto, of such annual statement. (ii) Mutual savings banks. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L. -(2) Domestic nonlife insurance companies. Every domestic insurance -company other than a life insurance company shall make a return on Form 1120PC. This includes organizations described in section 501(m)(1) that provide commercial- type insurance and organizations described in section 833. Except as provided in paragraph (c)(4) of this section, such company shall file with its return a copy of its +(2) Domestic nonlife insurance companies. Every domestic insurance +company other than a life insurance company shall make a return on Form 1120PC. This includes organizations described in section 501(m)(1) that provide commercial- type insurance and organizations described in section 833. Except as provided in paragraph (c)(4) of this section, such company shall file with its return a copy of its annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return. -(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and +(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and (c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns -electronically. If an insurance company described in paragraph (c)(1), (c)(2), or +electronically. If an insurance company described in paragraph (c)(1), (c)(2), or -(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). -(5) Definition. For purposes of this section, the term annual statement means +(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). +(5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of -Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. +Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. -(d) through (j) [Reserved]. For further guidance, see §1.6012-2(d) through (j). +(d) through (j) [Reserved]. For further guidance, see §1.6012-2(d) through (j). (k) Effective date-- (1) Applicability date. This section applies to any original Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30, 2006. -(2) Expiration date. The applicability of this section will expire on May 26, -2009. Par. 53. For each entry in the “Location” column of the following table, remove the language in the “Remove” column and add the language in the “Add” column in its place: Location Remove Add The last sentence of the The following rules shall The rules described in introductory text to be applicable in paragraph (a) of §1.302- §1.302-4 determining whether the 4T and in paragraphs (b) +(2) Expiration date. The applicability of this section will expire on May 26, +2009. Par. 53. For each entry in the “Location” column of the following table, remove the language in the “Remove” column and add the language in the “Add” column in its place: Location Remove Add The last sentence of the The following rules shall The rules described in introductory text to be applicable in paragraph (a) of §1.302- §1.302-4 determining whether the 4T and in paragraphs (b) specific requirements of through (g) of this section section 302(c)(2) are apply in determining met: whether the specific requirements of section 302(c)(2) are met. §1.338(h)(10)-1(f) §1.331-1(d), and §1.332-§1.331-1T(d) and §1.332- 6 6T |The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| @@ -178,9 +178,9 @@ paragraphs (d)(2)(ii) and (iii) of this section, and paragraph (d)(2)(iv) of §1 (d) through (j) of §1.6012- 2, and paragraph (c) of §1.6012-2T -PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The authority citation for part 602 continues to read as follows: Authority: 26 U.S.C. 7805. Par. 55. In §602.101, paragraph (b) is amended to read as follows: +PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The authority citation for part 602 continues to read as follows: Authority: 26 U.S.C. 7805. Par. 55. In §602.101, paragraph (b) is amended to read as follows: -1. The following entries to the table are removed: +1. The following entries to the table are removed: §602.101 OMB Control numbers. * * * * * @@ -188,14 +188,14 @@ PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The au CFR part or section where Current OMB identified or described control No. * * * * * -1.332-6…………………………………………………………………. 1545-2019 -1.382-11……………………………………………………………….. 1545-2019 -1.351-3…………………………………………………………………. 1545-2019 -1.355-5…………………………………………………………………. 1545-2019 -1.368-3…………………………………………………………………. 1545-2019 -1.1081-11………………………………………………………………. 1545-2019 +1.332-6…………………………………………………………………. 1545-2019 +1.382-11……………………………………………………………….. 1545-2019 +1.351-3…………………………………………………………………. 1545-2019 +1.355-5…………………………………………………………………. 1545-2019 +1.368-3…………………………………………………………………. 1545-2019 +1.1081-11………………………………………………………………. 1545-2019 * * * * * **______________________________________________________________** -2. The following entries are added in numerical order to the table: +2. The following entries are added in numerical order to the table: §602.101 OMB Control numbers. * * * * * @@ -203,33 +203,33 @@ CFR part or section where Current OMB identified or described control No. CFR part or section where Current OMB identified or described control No. * * * * * -1.302-2T………………………………………………………………… 1545-2019 -1.302-4T………………………………………………………………… 1545-2019 +1.302-2T………………………………………………………………… 1545-2019 +1.302-4T………………………………………………………………… 1545-2019 -1.331-1T………………………………………………………………… 1545-2019 -1.332-6T………………………………………………………………... 1545-2019 -1.338-10T………………………………………………………………. 1545-2019 -1.351-3T………………………………………………………………… 1545-2019 -1.355-5T………………………………………………………………… 1545-2019 -1.368-3T………………………………………………………………… 1545-2019 -1.381(b)-1T…………………………………………………………….. 1545-2019 -1.382-8T………………………………………………………………… 1545-2019 -1.382-11T………………………………………………………………. 1545-2019 -1.1081-11T……………………………………………………………… 1545-2019 -1.1221-2T……………………………………………………………….. 1545-2019 -1.1502-13T……………………………………………………………… 1545-2019 -1.1502-31T……………………………………………………………… 1545-2019 -1.1502-32T……………………………………………………………… 1545-2019 -1.1502-33T……………………………………………………………… 1545-2019 -1.1502-35T……………………………………………………………… 1545-2019 -1.1502-76T……………………………………………………………… 1545-2019 -1.1502-95T……………………………………………………………… 1545-2019 -1.1563-1T……………………………………………………………….. 1545-2019 +1.331-1T………………………………………………………………… 1545-2019 +1.332-6T………………………………………………………………... 1545-2019 +1.338-10T………………………………………………………………. 1545-2019 +1.351-3T………………………………………………………………… 1545-2019 +1.355-5T………………………………………………………………… 1545-2019 +1.368-3T………………………………………………………………… 1545-2019 +1.381(b)-1T…………………………………………………………….. 1545-2019 +1.382-8T………………………………………………………………… 1545-2019 +1.382-11T………………………………………………………………. 1545-2019 +1.1081-11T……………………………………………………………… 1545-2019 +1.1221-2T……………………………………………………………….. 1545-2019 +1.1502-13T……………………………………………………………… 1545-2019 +1.1502-31T……………………………………………………………… 1545-2019 +1.1502-32T……………………………………………………………… 1545-2019 +1.1502-33T……………………………………………………………… 1545-2019 +1.1502-35T……………………………………………………………… 1545-2019 +1.1502-76T……………………………………………………………… 1545-2019 +1.1502-95T……………………………………………………………… 1545-2019 +1.1563-1T……………………………………………………………….. 1545-2019 -1.1563-3T……………………………………………………………….. 1545-2019 -1.6012-2T……………………………………………………………….. 1545-2019 +1.1563-3T……………………………………………………………….. 1545-2019 +1.6012-2T……………………………………………………………….. 1545-2019 * * * * * Mark E. Matthews Deputy Commissioner for Services and Enforcement. -Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy). +Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy). diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index ffaf571..08a217a 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -1,6 +1,6 @@ **Technical Information** -## l T-12 SI +## l T-12 SI DuPont Fluorochemicals @@ -16,10 +16,10 @@ DuPont Fluorochemicals **®** **Thermodynamic Properties of Freon 12 Refrigerant** **(R-12)** **SI Units** -Tables of the thermodynamic **Units** properties of R-12 have been developed and are presented here. P = Pressure in kPa. Absolute This information is based on values calculated using the NIST REFPROP T = Temperature in Celcius Database (McLinden, M.O., Klein, +Tables of the thermodynamic **Units** properties of R-12 have been developed and are presented here. P = Pressure in kPa. Absolute This information is based on values calculated using the NIST REFPROP T = Temperature in Celcius Database (McLinden, M.O., Klein, -S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume -A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter +S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume +A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter H = Enthalpy (kJ/kg) S = Entropy (kJ/kg.K) From 80f9bb7ab2f602dc3192f6c97d279af89c954fb7 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 12:07:21 -0700 Subject: [PATCH 13/46] fix(fonts): revert TrueType cmap override for fonts with explicit encoding The third-pass change to extract embedded cmaps for TrueType fonts with WinAnsiEncoding caused apostrophes and other characters to be dropped in subsetted fonts where the cmap doesn't cover all glyphs. The declared encoding is authoritative for these fonts. The OCR text extraction fix (invisible Tr=3 text) doesn't need this cmap override. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/tounicode.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/tounicode.rs b/src/tounicode.rs index 727e89f..2c25668 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -1920,6 +1920,13 @@ impl FontCMaps { if font_dict.get(b"ToUnicode").is_ok() { continue; } + // Skip fonts with explicit encoding — they can be decoded by the + // standard encoding path (lopdf) and don't need a fallback CMap. + if let Ok(enc) = font_dict.get(b"Encoding") { + if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() { + continue; + } + } let subtype = match font_dict .get(b"Subtype") .ok() @@ -1931,18 +1938,6 @@ impl FontCMaps { if subtype == b"Type0" { continue; } - // Skip non-TrueType fonts with explicit encoding — they can be - // decoded by the standard encoding path and don't need a fallback. - // TrueType fonts are NOT skipped: OCR-generated PDFs often declare - // WinAnsiEncoding but the embedded font's cmap has the real mapping. - if subtype != b"TrueType" { - if let Ok(enc) = font_dict.get(b"Encoding") { - if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() - { - continue; - } - } - } let font_descriptor = font_dict.get(b"FontDescriptor").ok().and_then(|o| match o { Object::Reference(r) => doc.get_dictionary(*r).ok(), From dcdb0a39ac9b626a93c0264cdcf13b8ba0d6db0f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 13:01:12 -0700 Subject: [PATCH 14/46] fix(detect): detect images embedded in Pattern resources Screenshot PDFs (e.g., Chrome "Save as PDF") embed images inside tiling Pattern resources rather than as direct XObject images. Now traverses Pattern resources during image detection, correctly classifying these as Mixed with OCR recommended. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/detector.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/detector.rs b/src/detector.rs index cb23796..e170895 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -806,6 +806,48 @@ fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { TEMPLATE_IMAGE_THRESHOLD, &mut visited, ); + + // Also check Pattern resources: tiling patterns can contain + // XObject images (e.g., screenshots pasted into PDFs via + // Chrome "Save as PDF"). + if let Ok(pattern_obj) = resources.get(b"Pattern") { + let pattern_dict = match pattern_obj { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(dict) => Some(dict), + _ => None, + }; + if let Some(pattern_dict) = pattern_dict { + for (_, value) in pattern_dict.iter() { + let pat_ref = match value.as_reference() { + Ok(r) => r, + _ => continue, + }; + if !visited.insert(pat_ref) { + continue; + } + if let Ok(Object::Stream(stream)) = doc.get_object(pat_ref) { + if let Ok(pat_resources) = stream.dict.get(b"Resources") { + let pat_res_dict = match pat_resources { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(dict) => Some(dict), + _ => None, + }; + if let Some(pat_res) = pat_res_dict { + collect_images_from_resources( + doc, + pat_res, + &mut has_images, + &mut total_area, + &mut has_template_image, + TEMPLATE_IMAGE_THRESHOLD, + &mut visited, + ); + } + } + } + } + } + } } } From e6adb29eb81789b85b8fe82dd4d423fdff8a203d Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 16:24:43 -0700 Subject: [PATCH 15/46] feat(tables): generate hint regions from failed rect clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When rect clusters have valid bounding boxes but insufficient grid structure (e.g. 2x2 edges from outer borders), emit their bounding box as a RectHintRegion so the heuristic detector can be scoped to the table area. Requires reasonable dimensions (100-600pt height, ≤500pt width) and ≥6 text items inside the region. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/tables/detect_rects.rs | 196 ++++++++++++++++++++++++++++++++++++- 1 file changed, 194 insertions(+), 2 deletions(-) diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 249c8ca..29c1f63 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -285,12 +285,12 @@ pub fn detect_tables_from_rects( let mut tables = Vec::new(); let mut hint_regions = Vec::new(); + let mut failed_clusters: Vec> = Vec::new(); // Full grid detection requires ≥ 6 rects if page_rects.len() >= 6 { let clusters = cluster_rects(&page_rects, 3.0, 6); debug!("page {}: {} clusters with >= 6 rects", page, clusters.len()); - for cluster_indices in &clusters { let group_rects: Vec<(f32, f32, f32, f32)> = cluster_indices.iter().map(|&i| page_rects[i]).collect(); @@ -307,13 +307,21 @@ pub fn detect_tables_from_rects( left.len(), right.len() ); + let mut split_found = false; for sub in [&left, &right] { if let Some(table) = detect_table_from_rect_group(items, sub, page) { tables.push(table); + split_found = true; } else if let Some(table) = detect_row_stripe_table(items, sub, page) { tables.push(table); + split_found = true; } } + if !split_found { + failed_clusters.push(group_rects); + } + } else { + failed_clusters.push(group_rects); } } @@ -379,8 +387,11 @@ pub fn detect_tables_from_rects( // from cluster bounding boxes to scope heuristic table detection. // This handles both large decorative-rect clusters (calendars, forms) // and small cell-border clusters on rect-sparse pages. + let mut has_failed_cluster_hints = false; if page_rects.len() >= 6 { let clusters = cluster_rects(&page_rects, 3.0, 6); + + // Generate hints from large clusters (≥30 rects, decorative/calendar style) for cluster_indices in &clusters { let group_rects: Vec<(f32, f32, f32, f32)> = cluster_indices.iter().map(|&i| page_rects[i]).collect(); @@ -415,12 +426,59 @@ pub fn detect_tables_from_rects( }); } } + + // Generate hints from failed clusters (≥6 rects that had valid bounding + // boxes but insufficient grid structure — e.g. outer border or header + // divider with 2x2 edges). These tell us WHERE a table is even though + // the rects don't define column structure. + for fc_rects in &failed_clusters { + if fc_rects.len() < 6 { + continue; + } + let x_left = fc_rects.iter().map(|r| r.0).reduce(f32::min).unwrap(); + let x_right = fc_rects.iter().map(|r| r.0 + r.2).reduce(f32::max).unwrap(); + let y_bottom = fc_rects.iter().map(|r| r.1).reduce(f32::min).unwrap(); + let y_top = fc_rects.iter().map(|r| r.1 + r.3).reduce(f32::max).unwrap(); + let h = y_top - y_bottom; + // Require reasonable height and text items inside the region + let padding = 15.0; + let items_inside = items + .iter() + .filter(|item| { + item.y >= y_bottom - padding + && item.y <= y_top + padding + && item.x >= x_left - padding + && item.x <= x_right + padding + }) + .count(); + let w = x_right - x_left; + // Require reasonable dimensions: height ≥100pt (≈5+ rows), + // width ≤500pt (not page-spanning), height ≤600pt (not full page) + if (100.0..=600.0).contains(&h) && w <= 500.0 && items_inside >= 6 { + debug!( + "page {}: failed-cluster hint from {} rects ({} items): x={:.1}..{:.1} y={:.1}..{:.1} ({:.0}×{:.0})", + page, fc_rects.len(), items_inside, x_left, x_right, y_bottom, y_top, + x_right - x_left, h + ); + hint_regions.push(RectHintRegion { + y_top, + y_bottom, + x_left, + x_right, + cluster_rects: fc_rects.clone(), + }); + has_failed_cluster_hints = true; + } + } + // Deduplicate overlapping hints hint_regions = merge_overlapping_hints(hint_regions); // Require multiple hint regions to confirm a multi-zone layout // (calendars, forms). A single hint is likely a decorative cluster // that would interfere with full-page heuristic detection. - if hint_regions.len() < 2 { + // Exception: failed-cluster hints represent real table boundaries + // confirmed by rect presence, so a single one is meaningful. + if hint_regions.len() < 2 && !has_failed_cluster_hints { hint_regions.clear(); } if !hint_regions.is_empty() { @@ -2294,4 +2352,138 @@ mod tests { assert!((merged[0].x_left - 20.0).abs() < 0.01); assert!((merged[0].x_right - 340.0).abs() < 0.01); } + + #[test] + fn failed_cluster_generates_hint_with_items() { + // A cluster of rects forming an outer border (2 x-edges after snapping) + // that fails grid detection should produce a hint when items are inside. + // Use overlapping rects with the same left/right edges but varied heights + // so row-stripe detection also fails. + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 100.0, 400.0, 200.0), // outer border + (52.0, 102.0, 396.0, 196.0), // inner border (within snap tolerance) + (51.0, 101.0, 398.0, 198.0), // another border variant + (50.0, 100.0, 400.0, 10.0), // top divider (thin) + (50.0, 290.0, 400.0, 10.0), // bottom divider (thin) + (50.0, 195.0, 400.0, 10.0), // middle divider + ]; + // Create text items inside the bounding box (≥6 items) + let mut items: Vec = Vec::new(); + for row in 0..4 { + for col in 0..3 { + items.push(TextItem { + text: format!("cell{}_{}", row, col), + x: 60.0 + col as f32 * 120.0, + y: 120.0 + row as f32 * 40.0, + width: 50.0, + height: 10.0, + font: String::new(), + font_size: 10.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid: None, + }); + } + } + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&items, &rects, 1); + // Grid detection should fail (2 x-edges after snapping: ~50 and ~450) + // If detection fails, we should get a failed-cluster hint + if tables.is_empty() { + assert_eq!(hints.len(), 1, "failed cluster should produce one hint"); + assert!(!hints[0].cluster_rects.is_empty()); + } + // If tables were detected, that's also acceptable + } + + #[test] + fn failed_cluster_no_hint_without_items() { + // Rects with no text items inside → no failed-cluster hint generated. + // Use >6 rects to avoid the rect-sparse path (4-6 rects). + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 100.0, 400.0, 200.0), + (52.0, 102.0, 396.0, 196.0), + (51.0, 101.0, 398.0, 198.0), + (50.0, 100.0, 400.0, 10.0), + (50.0, 290.0, 400.0, 10.0), + (50.0, 195.0, 400.0, 10.0), + (50.0, 150.0, 400.0, 10.0), + (50.0, 250.0, 400.0, 10.0), + ]; + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&[], &rects, 1); + // No items → no table, no hint (items_inside check fails) + if tables.is_empty() { + assert!(hints.is_empty(), "no items inside → no hint"); + } + } + + #[test] + fn failed_cluster_no_hint_narrow_height() { + // Cluster with only 20pt height (header band) should not produce hint + // even with items inside (height < 100pt threshold) + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 650.0, 50.0, 20.0), + (100.0, 650.0, 50.0, 20.0), + (150.0, 650.0, 50.0, 20.0), + (200.0, 650.0, 50.0, 20.0), + (250.0, 650.0, 50.0, 20.0), + (300.0, 650.0, 50.0, 20.0), + (350.0, 650.0, 50.0, 20.0), + (400.0, 650.0, 50.0, 20.0), + ]; + let mut items: Vec = Vec::new(); + for col in 0..8 { + items.push(TextItem { + text: format!("hdr{}", col), + x: 55.0 + col as f32 * 50.0, + y: 655.0, + width: 40.0, + height: 10.0, + font: String::new(), + font_size: 10.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid: None, + }); + } + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&items, &rects, 1); + assert!(tables.is_empty()); + assert!( + hints.is_empty(), + "narrow header band (20pt) should not produce hint" + ); + } } From e5a048f674c08ecbf01dbc2af905e8bb5c5497f6 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 18 Mar 2026 17:11:51 -0700 Subject: [PATCH 16/46] feat(tables): recover label columns for numeric-only tables Two changes to improve balance sheet / financial table detection: 1. split_side_by_side: Don't split when one side is text labels and the other is numeric data at matching Y positions. This prevents splitting a single label+number table into two independent regions. 2. try_add_label_column: After detecting a numeric-only table, look for unclaimed text items to the left at matching Y positions and prepend them as column 0 (row labels). Tested on IN_Annual_Report_2017 balance sheet which now produces proper 3-column tables (Label|2016|2017) instead of separated number tables and paragraph text. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/markdown/mod.rs | 74 +++++++++++++++++++++ src/tables/detect_heuristic.rs | 116 +++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index ed0cbd5..bfc3a66 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -124,6 +124,51 @@ pub(crate) fn split_side_by_side(items: &[TextItem]) -> Vec<(f32, f32)> { return vec![]; } + // Don't split when the left side is text labels and the right side is numeric + // data at matching Y positions — this is a single table (labels + numbers), + // not two independent side-by-side regions. + // Requires ALL THREE: left side is mostly non-numeric, right side is mostly + // numeric, AND high Y-correlation between the two sides. + let is_numeric_item = |item: &&&TextItem| -> bool { + let text = item.text.trim(); + if text.is_empty() { + return false; + } + let data_chars = text + .chars() + .filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c)) + .count(); + data_chars as f32 / text.chars().count() as f32 >= 0.6 + }; + + let left_items: Vec<&TextItem> = items + .iter() + .filter(|i| i.x + i.width / 2.0 < best_split) + .collect(); + let right_items: Vec<&TextItem> = items + .iter() + .filter(|i| i.x + i.width / 2.0 >= best_split) + .collect(); + + if !left_items.is_empty() && !right_items.is_empty() { + let left_numeric_ratio = + left_items.iter().filter(is_numeric_item).count() as f32 / left_items.len() as f32; + let right_numeric_ratio = + right_items.iter().filter(is_numeric_item).count() as f32 / right_items.len() as f32; + + // Left side is mostly text (< 30% numeric) AND right side is mostly numbers (≥ 70%) + if left_numeric_ratio < 0.30 && right_numeric_ratio >= 0.70 { + let y_tol = 5.0; + let y_matches = right_items + .iter() + .filter(|ri| left_items.iter().any(|li| (li.y - ri.y).abs() < y_tol)) + .count(); + if y_matches as f32 / right_items.len() as f32 >= 0.5 { + return vec![]; + } + } + } + vec![(x_min, best_split), (best_split, x_max)] } @@ -998,4 +1043,33 @@ mod tests { .collect(); assert!(split_from_hint_regions(&items, &rects, 1).is_empty()); } + + #[test] + fn no_split_label_plus_number_table() { + // Balance sheet layout: text labels on left, numbers on right. + // Should NOT split because it's one table, not side-by-side regions. + let mut items = Vec::new(); + for row in 0..30 { + // Label at x=50 + let mut label = make_item(50.0, 700.0 - row as f32 * 15.0, 1); + label.text = format!("Row label {}", row); + label.width = 100.0; + items.push(label); + // Number at x=400 + let mut num1 = make_item(400.0, 700.0 - row as f32 * 15.0, 1); + num1.text = format!("{},000.0", 100 + row); + num1.width = 50.0; + items.push(num1); + // Number at x=470 + let mut num2 = make_item(470.0, 700.0 - row as f32 * 15.0, 1); + num2.text = format!("{},500.0", 200 + row); + num2.width = 50.0; + items.push(num2); + } + let split = split_side_by_side(&items); + assert!( + split.is_empty(), + "label+number table should not be split side-by-side" + ); + } } diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 4576d9a..aa1f7ba 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -179,6 +179,14 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo { // Try to recover body-font header row above the small-font table recover_header_row(&mut table, items, table_font_threshold); + // Try to recover a label column from unclaimed items to the left + try_add_label_column( + &mut table, + &table_candidates, + &claimed_indices, + y_min, + y_max, + ); for &idx in &table.item_indices { claimed_indices.insert(idx); } @@ -1073,3 +1081,111 @@ pub(crate) fn find_first_table_row( (first_table_row, excluded_items) } + +/// Try to recover a label column for numeric-only tables. +/// +/// Financial balance sheets often have text labels (row descriptions) to the +/// left of numeric columns. The label X-positions vary due to indentation, +/// so they don't form a consistent column cluster and are excluded from the +/// initial table detection. This function finds unclaimed items at matching +/// Y-positions to the left of the table and prepends them as column 0. +fn try_add_label_column( + table: &mut Table, + all_candidates: &[(usize, &TextItem)], + claimed_indices: &std::collections::HashSet, + y_min: f32, + y_max: f32, +) { + // Only apply to tables with 2-3 numeric columns and ≥5 rows + if table.columns.len() < 2 || table.columns.len() > 3 || table.rows.len() < 5 { + return; + } + + // Check if the table is predominantly numeric (no text labels in any column) + let numeric_cells = table + .cells + .iter() + .flat_map(|row| row.iter()) + .filter(|cell| { + let text = cell.trim(); + if text.is_empty() { + return false; + } + let data_chars = text + .chars() + .filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c)) + .count(); + let total_chars = text.chars().count(); + total_chars > 0 && data_chars as f32 / total_chars as f32 >= 0.6 + }) + .count(); + let total_non_empty = table + .cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + if total_non_empty == 0 || (numeric_cells as f32 / total_non_empty as f32) < 0.7 { + return; + } + + let table_x_min = table.columns.first().copied().unwrap_or(f32::MAX); + let y_tol = 5.0; + + // For each table row, find unclaimed items to the left at the same Y + let mut label_items_per_row: Vec> = Vec::new(); + let mut found_count = 0; + for &row_y in &table.rows { + let mut row_labels: Vec<(usize, &TextItem)> = all_candidates + .iter() + .filter(|(idx, item)| { + !claimed_indices.contains(idx) + && !table.item_indices.contains(idx) + && (item.y - row_y).abs() < y_tol + && item.x < table_x_min - 10.0 + && item.y >= y_min + && item.y <= y_max + }) + .map(|(idx, item)| (*idx, *item)) + .collect(); + row_labels.sort_by(|a, b| { + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal) + }); + if !row_labels.is_empty() { + found_count += 1; + } + label_items_per_row.push(row_labels); + } + + // Require labels for at least 40% of rows + if found_count < table.rows.len() * 2 / 5 { + return; + } + + debug!( + "recovering label column: {}/{} rows have labels to the left", + found_count, + table.rows.len() + ); + + // Prepend label column + let label_col_x = label_items_per_row + .iter() + .flat_map(|items| items.iter().map(|(_, i)| i.x)) + .fold(f32::INFINITY, f32::min); + + table.columns.insert(0, label_col_x); + for (row_idx, row_labels) in label_items_per_row.iter().enumerate() { + let label_text = row_labels + .iter() + .map(|(_, item)| item.text.as_str()) + .collect::>() + .join(" "); + table.cells[row_idx].insert(0, label_text); + for (idx, _) in row_labels { + table.item_indices.push(*idx); + } + } +} From da8c27f3c4f20d0d7936cd58e8c1f0ef58427793 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Thu, 19 Mar 2026 01:27:49 +0100 Subject: [PATCH 17/46] =?UTF-8?q?perf:=20cap=20cluster=5Frects=20component?= =?UTF-8?q?=20size=20to=20avoid=20O(n=C2=B2)=20on=20vector-heavy=20pages?= =?UTF-8?q?=20(#6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pages with tens of thousands of vector-drawing rects (e.g. architectural plans with 36k+ rects) caused cluster_rects and sub-rect deduplication to spend 10-50s in O(n²) loops. No real table has thousands of cell rects, so once a union-find component exceeds 2000 elements we skip further comparisons, and skip the O(n²) sub-rect dedup entirely for such pages. Benchmarks on 8 slow production PDFs show 10-40x speedups on vector-heavy files (12s→0.3s) with no regression on other file types. All 342 tests pass. --- src/tables/detect_rects.rs | 80 ++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 29c1f63..d308c29 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -8,10 +8,11 @@ use crate::types::{PdfRect, TextItem}; use super::Table; -/// Disjoint-set (union-find) for clustering indices. +/// Disjoint-set (union-find) with component sizes for clustering indices. struct UnionFind { parent: Vec, rank: Vec, + size: Vec, } impl UnionFind { @@ -19,6 +20,7 @@ impl UnionFind { Self { parent: (0..n).collect(), rank: vec![0; n], + size: vec![1; n], } } @@ -35,15 +37,24 @@ impl UnionFind { if ra == rb { return; } + let new_size = self.size[ra] + self.size[rb]; if self.rank[ra] < self.rank[rb] { self.parent[ra] = rb; + self.size[rb] = new_size; } else if self.rank[ra] > self.rank[rb] { self.parent[rb] = ra; + self.size[ra] = new_size; } else { self.parent[rb] = ra; + self.size[ra] = new_size; self.rank[ra] += 1; } } + + fn component_size(&mut self, x: usize) -> usize { + let root = self.find(x); + self.size[root] + } } /// Check if two rects overlap after expanding each by `tol` on all sides. @@ -64,8 +75,19 @@ pub(crate) fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32), !(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom) } +/// Maximum component size for rect clustering. No real table has thousands +/// of cell rects — once a component exceeds this, it is a vector drawing or +/// page-spanning clipping path. We skip overlap checks for rects already in +/// an oversized component, keeping the original O(n²) loop but making it +/// effectively O(n) for pathological pages. +const MAX_CLUSTER_RECTS: usize = 2000; + /// Cluster rects by spatial overlap using union-find. /// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned. +/// +/// Skips overlap checks for rects whose component has already exceeded +/// [`MAX_CLUSTER_RECTS`], so pages with tens of thousands of vector-drawing +/// rects complete in milliseconds instead of minutes. pub(crate) fn cluster_rects( rects: &[(f32, f32, f32, f32)], tolerance: f32, @@ -75,9 +97,20 @@ pub(crate) fn cluster_rects( let mut uf = UnionFind::new(n); for i in 0..n { + // If rect i is already in an oversized component, no point comparing + // it against further rects — the component won't be used for table + // detection anyway. + if uf.component_size(i) >= MAX_CLUSTER_RECTS { + continue; + } for j in (i + 1)..n { if rects_overlap(&rects[i], &rects[j], tolerance) { uf.union(i, j); + // Check if the merged component just exceeded the cap — + // if so, no need to test more pairs for rect i. + if uf.component_size(i) >= MAX_CLUSTER_RECTS { + break; + } } } } @@ -253,26 +286,31 @@ pub fn detect_tables_from_rects( // Only remove when the container is a similarly-sized cell (height // ratio < 4×), NOT when the container is a table-wide background // that dwarfs the sub-rect. - let before = page_rects.len(); - let snapshot = page_rects.clone(); - page_rects.retain(|&(ax, ay, aw, ah)| { - let tol = 2.0; - !snapshot.iter().any(|&(bx, by, bw, bh)| { - // b must strictly contain a (b is larger in area) - bw * bh > aw * ah * 1.2 - && bh < ah * 4.0 // container must be similarly sized, not a table background - && bx <= ax + tol - && (bx + bw) >= (ax + aw) - tol - && by <= ay + tol - && (by + bh) >= (ay + ah) - tol - }) - }); - if page_rects.len() < before { - debug!( - "page {}: removed {} contained sub-rects", - page, - before - page_rects.len(), - ); + // + // Skip this O(n²) dedup when there are too many rects — pages with + // thousands of vector-drawing rects won't benefit from cell dedup. + if page_rects.len() < MAX_CLUSTER_RECTS { + let before = page_rects.len(); + let snapshot = page_rects.clone(); + page_rects.retain(|&(ax, ay, aw, ah)| { + let tol = 2.0; + !snapshot.iter().any(|&(bx, by, bw, bh)| { + // b must strictly contain a (b is larger in area) + bw * bh > aw * ah * 1.2 + && bh < ah * 4.0 // container must be similarly sized, not a table background + && bx <= ax + tol + && (bx + bw) >= (ax + aw) - tol + && by <= ay + tol + && (by + bh) >= (ay + ah) - tol + }) + }); + if page_rects.len() < before { + debug!( + "page {}: removed {} contained sub-rects", + page, + before - page_rects.len(), + ); + } } } From 74d416e8ce6ab3ba47390746f7a3dfe464561e96 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 19 Mar 2026 08:36:16 -0700 Subject: [PATCH 18/46] feat(detect): flag pages with gid-encoded fonts for OCR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fonts using raw glyph ID names (gidNNNNN) in their Differences encoding cannot be decoded to Unicode without the original font's cmap table. Detect this pattern during font parsing and add affected pages to pages_needing_ocr so downstream consumers know to use OCR instead. Fixes text extraction on PDFs like Tezukuri_Food-Menu.pdf where the main body font (AcuminVariableConcept) uses gid-encoded glyphs — even PyMuPDF and ODL fail on these. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/content_stream.rs | 8 +++-- src/extractor/fonts.rs | 55 ++++++++++++++++++++++++++------- src/extractor/mod.rs | 31 +++++++++++++------ src/extractor/xobjects.rs | 2 +- src/lib.rs | 27 +++++++++++++--- src/types.rs | 3 +- 6 files changed, 96 insertions(+), 30 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ca1e450..ef7fe61 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -20,13 +20,15 @@ use super::fonts::{ use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; use super::{get_number, multiply_matrices}; +/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates +/// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( doc: &Document, page_id: ObjectId, page_num: u32, font_cmaps: &FontCMaps, include_invisible: bool, -) -> Result { +) -> Result<(PageExtraction, bool), PdfError> { use lopdf::content::Content; let mut items = Vec::new(); @@ -46,7 +48,7 @@ pub(crate) fn extract_page_text_items( let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); // Build font encoding maps from Differences arrays - let font_encodings = build_font_encodings(doc, &fonts); + let (font_encodings, has_gid_fonts) = build_font_encodings(doc, &fonts); // Build font width info for accurate text positioning let font_widths = build_font_widths(doc, &fonts); @@ -877,7 +879,7 @@ pub(crate) fn extract_page_text_items( } let items = super::merge_text_items(items); - Ok((items, rects, lines)) + Ok(((items, rects, lines), has_gid_fonts)) } /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance). diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index e6f3b78..5f9a3f3 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -473,29 +473,37 @@ pub(crate) fn get_operand_bytes(obj: &Object) -> Option<&[u8]> { } } -/// Build encoding maps for all fonts on a page +/// Build encoding maps for all fonts on a page. +/// Returns `(encodings, has_gid_fonts)` where `has_gid_fonts` is true when +/// any font uses raw glyph ID names (gidNNNNN) that can't be decoded. pub(crate) fn build_font_encodings( doc: &Document, fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, -) -> PageFontEncodings { +) -> (PageFontEncodings, bool) { let mut encodings = PageFontEncodings::new(); + let mut has_gid_fonts = false; for (font_name, font_dict) in fonts { let resource_name = String::from_utf8_lossy(font_name).to_string(); - if let Some(encoding_map) = parse_font_encoding(doc, font_dict) { - encodings.insert(resource_name, encoding_map); + if let Some(result) = parse_font_encoding(doc, font_dict) { + if result.gid_glyph_count > 0 { + has_gid_fonts = true; + } + if !result.map.is_empty() { + encodings.insert(resource_name, result.map); + } } } - encodings + (encodings, has_gid_fonts) } /// Parse font encoding from a font dictionary pub(crate) fn parse_font_encoding( doc: &Document, font_dict: &lopdf::Dictionary, -) -> Option { +) -> Option { let encoding_obj = font_dict.get(b"Encoding").ok()?; // Encoding can be a name or a dictionary @@ -519,11 +527,21 @@ pub(crate) fn parse_font_encoding( } } +/// Result of parsing an encoding dictionary's Differences array. +pub(crate) struct EncodingResult { + pub map: FontEncodingMap, + /// Number of glyph names matching the `gidNNNNN` pattern (raw glyph IDs). + /// These indicate a font with unresolvable encoding — the glyph IDs + /// reference the original font's glyph table, but without the original + /// font's cmap there is no way to map them to Unicode. + pub gid_glyph_count: u32, +} + /// Parse an encoding dictionary with Differences array pub(crate) fn parse_encoding_dictionary( doc: &Document, enc_dict: &lopdf::Dictionary, -) -> Option { +) -> Option { let differences = enc_dict.get(b"Differences").ok()?; let diff_array = match differences { @@ -541,6 +559,7 @@ pub(crate) fn parse_encoding_dictionary( let mut encoding_map = FontEncodingMap::new(); let mut current_code: u8 = 0; let mut ligature_count = 0u32; + let mut gid_glyph_count = 0u32; for item in diff_array { match item { @@ -562,6 +581,14 @@ pub(crate) fn parse_encoding_dictionary( ); ligature_count += 1; } + // Detect raw glyph ID names (e.g. "gid00053") that can't be + // mapped to Unicode without the original font's cmap table. + if glyph_name.starts_with("gid") + && glyph_name.len() >= 4 + && glyph_name[3..].chars().all(|c| c.is_ascii_digit()) + { + gid_glyph_count += 1; + } if let Some(ch) = glyph_to_char(&glyph_name) { encoding_map.insert(current_code, ch); } else { @@ -584,11 +611,17 @@ pub(crate) fn parse_encoding_dictionary( ); } - if encoding_map.is_empty() { - None - } else { - Some(encoding_map) + if gid_glyph_count > 0 { + debug!( + " Differences: {} gid-encoded glyphs (unresolvable without original font)", + gid_glyph_count + ); } + + Some(EncodingResult { + map: encoding_map, + gid_glyph_count, + }) } /// Get the CMap lookup key for an Identity-H/V CID font without ToUnicode. diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 9be4977..0105e6b 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -97,7 +97,7 @@ pub(crate) fn extract_text_with_positions_and_rects>( Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds) = + let (extraction, _thresholds, _gid_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -130,7 +130,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds) = + let (extraction, _thresholds, _gid_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -149,7 +149,7 @@ pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, page_filter, false) } @@ -159,7 +159,7 @@ pub(crate) fn extract_positioned_text_include_invisible( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, page_filter, true) } @@ -168,12 +168,13 @@ fn extract_positioned_text_impl( font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, include_invisible: bool, -) -> Result<(PageExtraction, PageThresholds), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); let mut all_rects = Vec::new(); let mut all_lines = Vec::new(); let mut page_thresholds: PageThresholds = HashMap::new(); + let mut gid_encoded_pages: HashSet = HashSet::new(); // Build page ObjectId → page number map for form field extraction let page_id_to_num: HashMap = @@ -185,18 +186,26 @@ fn extract_positioned_text_impl( continue; } } - let (mut items, rects, lines) = + let ((mut items, rects, lines), has_gid_fonts) = extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?; + if has_gid_fonts { + gid_encoded_pages.insert(*page_num); + } let threshold = crate::text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { page_thresholds.insert(*page_num, threshold); } debug!( - "page {}: {} text items, {} rects, {} lines", + "page {}: {} text items, {} rects, {} lines{}", page_num, items.len(), rects.len(), - lines.len() + lines.len(), + if has_gid_fonts { + " [gid-encoded fonts]" + } else { + "" + } ); if log::log_enabled!(log::Level::Trace) { for item in &items { @@ -229,7 +238,11 @@ fn extract_positioned_text_impl( let form_items = extract_form_fields(doc, &page_id_to_num); all_items.extend(form_items); - Ok(((all_items, all_rects, all_lines), page_thresholds)) + Ok(( + (all_items, all_rects, all_lines), + page_thresholds, + gid_encoded_pages, + )) } // --------------------------------------------------------------------------- diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index e011b88..5491bb8 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -156,7 +156,7 @@ fn extract_form_xobject_text_inner( // Get fonts from the Form's Resources let form_fonts = get_form_fonts(doc, &stream.dict); - let font_encodings = build_font_encodings(doc, &form_fonts); + let (font_encodings, _has_gid_fonts) = build_font_encodings(doc, &form_fonts); // Build font width info for the form let font_widths = build_font_widths(doc, &form_fonts); diff --git a/src/lib.rs b/src/lib.rs index db15348..b6de2b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -352,7 +352,7 @@ fn process_document( // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. // This unlocks OCR text layers behind scanned images. if pdf_type == PdfType::Mixed { - if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _)| e) { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _)| e) { let sample: String = items.iter().take(200).map(|i| i.text.as_str()).collect(); if is_garbage_text(&sample) || sample.trim().is_empty() { extractor::extract_positioned_text_include_invisible( @@ -399,8 +399,8 @@ fn process_document( } }); - let (markdown, layout, has_encoding_issues) = match extracted { - Some(((items, rects, lines), page_thresholds)) => { + let (markdown, layout, has_encoding_issues, gid_pages) = match extracted { + Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { let layout = compute_layout_complexity(&items, &rects, &lines); let md = if options.mode == ProcessMode::Analyze { @@ -417,9 +417,14 @@ fn process_document( }; let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m)); - (md, layout, enc) + (md, layout, enc, gid_encoded_pages) } - None => (None, LayoutComplexity::default(), false), + None => ( + None, + LayoutComplexity::default(), + false, + std::collections::HashSet::new(), + ), }; // If the extracted text is predominantly garbage (non-alphanumeric) and @@ -432,6 +437,18 @@ fn process_document( (pdf_type, markdown, confidence) }; + // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list + let mut pages_needing_ocr = pages_needing_ocr; + if !gid_pages.is_empty() { + log::debug!("pages with gid-encoded fonts (need OCR): {:?}", gid_pages); + for page in gid_pages { + if !pages_needing_ocr.contains(&page) { + pages_needing_ocr.push(page); + } + } + pages_needing_ocr.sort_unstable(); + } + Ok(PdfProcessResult { pdf_type, markdown, diff --git a/src/types.rs b/src/types.rs index b7d77f6..78fe2b0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,7 +8,8 @@ use std::collections::HashMap; use crate::text_utils::should_join_items; -/// Result tuple returned by page-level text extraction: text items, rectangles, and line segments. +/// Result tuple returned by page-level text extraction: text items, rectangles, line segments, +/// and whether fonts with unresolvable gid-encoded glyphs were encountered. pub(crate) type PageExtraction = (Vec, Vec, Vec); // ── Font types (crate-internal) ────────────────────────────────────── From 57560f169228bff5a46cce0dbfc7eb25fcc18613 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 19 Mar 2026 14:41:30 -0700 Subject: [PATCH 19/46] feat(tables): column-based table detection for borderless layouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When rect-based, line-based, and heuristic detection all fail to find tables on a page, try building a table directly from the layout engine's column boundaries. Handles borderless tabular layouts like exam/reference grids where columns are defined purely by text alignment. Includes header-row column refinement: when a detected column contains multiple header items, it gets split at the gap between them to recover the correct number of columns. Guards against false positives: requires ≥4 columns, ≤40 rows, >50% multi-column rows, short cells (avg ≤40 chars), no prose content, no dominant single column, and no structural elements (≥6 rects or ≥4 lines) on the page. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/mod.rs | 2 + src/markdown/mod.rs | 29 +++++ src/tables/mod.rs | 274 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 0105e6b..e8a2a90 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -26,6 +26,8 @@ pub use crate::types::{ItemType, TextLine}; pub(crate) use layout::detect_columns; pub use layout::group_into_lines; pub(crate) use layout::group_into_lines_with_thresholds; +pub(crate) use layout::is_newspaper_layout; +pub(crate) use layout::ColumnRegion; // --------------------------------------------------------------------------- // Public API diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index bfc3a66..c51e1a7 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -800,6 +800,35 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .unzip(); run_heuristic(&unclaimed_items, &unclaimed_map, 6); } + + // 4. Column-based table detection: last resort for borderless tabular + // layouts (e.g. exam/reference grids) when ALL structural methods + // found nothing. Only runs when no rects/lines exist (truly borderless) + // and no other detection method found tables in this band. + let band_has_tables = band_items.iter().enumerate().any(|(idx, _)| { + band_index_map + .get(idx) + .and_then(|&page_idx| group.get(page_idx)) + .is_some_and(|&(global_idx, _)| table_items.contains(&global_idx)) + }); + let has_structural_elements = band_rects.len() >= 6 || band_lines.len() >= 4; + if !band_has_tables && !has_structural_elements { + if let Some(table) = crate::tables::try_build_table_from_columns(band_items, page) { + for &idx in &table.item_indices { + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } } } diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 1e9e0b8..7e6ec23 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -267,6 +267,280 @@ pub(crate) enum TableDetectionMode { BodyFont, } +/// Build a table from layout-detected column boundaries. +/// +/// When the layout engine detects multiple tabular columns (not newspaper), +/// this function uses those boundaries to construct a Table directly. This +/// handles borderless tables (no rects/lines) where columns are defined +/// purely by text alignment — common in exam/reference tables. +/// +/// Requires ≥3 columns, ≥3 rows, and ≥40% cell fill rate. +pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Option { + use crate::extractor::{ + detect_columns, group_into_lines_with_thresholds, is_newspaper_layout, ColumnRegion, + }; + use std::collections::HashMap; + + let mut columns = detect_columns(items, page); + if columns.len() < 4 { + return None; + } + + // Refine columns: look for header-like rows where multiple items share + // the same Y and are evenly spaced. If a wide column contains two header + // items, split it at the gap between them. + let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect(); + let y_tol = 3.0; + + // Find the top-most row with items in multiple columns (likely the header) + let mut ys: Vec = page_items.iter().map(|i| i.y).collect(); + ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + ys.dedup_by(|a, b| (*a - *b).abs() < y_tol); + + for &header_y in ys.iter().take(5) { + let row_items: Vec<&&TextItem> = page_items + .iter() + .filter(|i| (i.y - header_y).abs() < y_tol) + .collect(); + if row_items.len() < columns.len() { + continue; + } + // Check if any column contains 2+ items at this Y — needs splitting + let mut new_columns = Vec::new(); + let mut did_split = false; + for col in &columns { + let col_items: Vec<&&&TextItem> = row_items + .iter() + .filter(|i| i.x >= col.x_min && i.x < col.x_max) + .collect(); + if col_items.len() >= 2 { + // Sort by X and find the split point + let mut sorted: Vec = col_items.iter().map(|i| i.x).collect(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + // Split at the midpoint between the two items + let split_x = (sorted[0] + + col_items.iter().find(|i| i.x == sorted[0]).unwrap().width + + sorted[1]) + / 2.0; + new_columns.push(ColumnRegion { + x_min: col.x_min, + x_max: split_x, + }); + new_columns.push(ColumnRegion { + x_min: split_x, + x_max: col.x_max, + }); + did_split = true; + } else { + new_columns.push(col.clone()); + } + } + if did_split { + log::debug!( + "column refinement: {} -> {} columns from header row at y={:.1}", + columns.len(), + new_columns.len(), + header_y + ); + columns = new_columns; + break; + } + } + + // Group items into per-column lines to check newspaper vs tabular + let mut col_buckets: Vec> = vec![Vec::new(); columns.len()]; + let mut spanning_items: Vec = Vec::new(); + for item in items { + if item.page != page { + continue; + } + // Check if item spans multiple columns + let item_left = item.x; + let item_right = item.x + item.width; + let mut spans = 0; + for col in &columns { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > 0.0 { + spans += 1; + } + } + if spans > 1 { + spanning_items.push(item.clone()); + continue; + } + // Assign to best-overlap column + let mut best_col = 0; + let mut best_overlap = f32::NEG_INFINITY; + for (ci, col) in columns.iter().enumerate() { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > best_overlap { + best_overlap = overlap; + best_col = ci; + } + } + col_buckets[best_col].push(item.clone()); + } + + let thresholds = HashMap::new(); + let per_column_lines: Vec> = col_buckets + .iter() + .map(|bucket| group_into_lines_with_thresholds(bucket.clone(), &thresholds)) + .collect(); + + // Must be tabular (not newspaper) layout + if is_newspaper_layout(&per_column_lines, &columns) { + return None; + } + + // Collect all unique Y positions across all columns (row boundaries) + let y_tol = 5.0; + let mut row_ys: Vec = Vec::new(); + for col_lines in &per_column_lines { + for line in col_lines { + let y = line.y; + if !row_ys.iter().any(|&ry| (ry - y).abs() < y_tol) { + row_ys.push(y); + } + } + } + row_ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + if row_ys.len() < 3 || row_ys.len() > 40 { + return None; + } + + // Build cell grid + let col_xs: Vec = columns.iter().map(|c| c.x_min).collect(); + let mut cells: Vec> = vec![vec![String::new(); columns.len()]; row_ys.len()]; + let mut item_indices: Vec = Vec::new(); + + for (item_idx, item) in items.iter().enumerate() { + if item.page != page { + continue; + } + // Find column + let item_left = item.x; + let item_right = item.x + item.width; + let mut best_col = None; + let mut best_overlap = 0.0f32; + let mut span_count = 0; + for (ci, col) in columns.iter().enumerate() { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > 0.0 { + span_count += 1; + } + if overlap > best_overlap { + best_overlap = overlap; + best_col = Some(ci); + } + } + if span_count > 1 || best_col.is_none() { + continue; // spanning item, skip + } + let col = best_col.unwrap(); + + // Find row + let row = row_ys.iter().position(|&ry| (ry - item.y).abs() < y_tol); + if let Some(row) = row { + if !cells[row][col].is_empty() { + cells[row][col].push(' '); + } + cells[row][col].push_str(&item.text); + item_indices.push(item_idx); + } + } + + // Validate: need reasonable fill rate + let total_cells = row_ys.len() * columns.len(); + let filled_cells = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let fill_rate = filled_cells as f32 / total_cells as f32; + + if fill_rate < 0.15 { + return None; + } + + // Need at least 40% of rows to have content in 2+ columns + let multi_col_rows = cells + .iter() + .filter(|row| row.iter().filter(|c| !c.trim().is_empty()).count() >= 2) + .count(); + // Need majority (>50%) of rows with content in 2+ columns + if multi_col_rows * 2 < row_ys.len() { + return None; + } + + // Reject prose-like content: if cells are too long on average, this is + // a multi-column text layout, not a data table. Real table cells are + // typically short (≤ 40 chars). Prose paragraphs are much longer. + let cell_lengths: Vec = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| !c.trim().is_empty()) + .map(|c| c.trim().len()) + .collect(); + if !cell_lengths.is_empty() { + let avg_cell_len = cell_lengths.iter().sum::() as f32 / cell_lengths.len() as f32; + if avg_cell_len > 40.0 { + return None; + } + // Reject if any significant number of cells are long prose (> 80 chars) + let long_cells = cell_lengths.iter().filter(|&&len| len > 80).count(); + if long_cells as f32 / cell_lengths.len() as f32 > 0.10 { + return None; + } + } + + // Reject when cells look like prose sentences: if too many cells contain + // sentence-ending punctuation (.!?:) it's prose text, not table data. + let prose_cells = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| { + let t = c.trim(); + t.len() > 20 + && (t.ends_with('.') || t.ends_with('!') || t.ends_with('?') || t.ends_with(':')) + }) + .count(); + if filled_cells > 0 && prose_cells as f32 / filled_cells as f32 > 0.15 { + return None; + } + + // Reject when most content is in one column (newspaper-like asymmetry). + // Count items per column; if any column has >60% of items, it's likely + // a body text column with side annotations, not a data table. + let mut items_per_col: Vec = vec![0; columns.len()]; + for row in &cells { + for (ci, cell) in row.iter().enumerate() { + if !cell.trim().is_empty() { + items_per_col[ci] += 1; + } + } + } + let max_col_items = *items_per_col.iter().max().unwrap_or(&0); + if filled_cells > 0 && max_col_items as f32 / filled_cells as f32 > 0.60 { + return None; + } + + log::debug!( + "column-based table: {} cols x {} rows, fill={:.0}%, multi_col_rows={}", + columns.len(), + row_ys.len(), + fill_rate * 100.0, + multi_col_rows + ); + + Some(Table { + columns: col_xs, + rows: row_ys, + cells, + item_indices, + }) +} + /// A detected table. #[derive(Debug, Clone)] pub struct Table { From 587c4bed95838b1be93a70eb74c5f055f52bdabf Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 19 Mar 2026 15:40:48 -0700 Subject: [PATCH 20/46] feat(detect): flag Identity-H fonts without ToUnicode for OCR (#7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(detect): flag Identity-H fonts without ToUnicode for OCR Cyrillic (and other non-Latin) PDFs with Type0/Identity-H encoded fonts and no ToUnicode CMap produce garbage text from direct extraction. Two fixes: 1. Detector: new `page_has_identity_h_no_tounicode` check adds affected pages to `pages_needing_ocr` regardless of PDF classification. 2. Extraction: extend garbage-text safety net to TextBased PDFs — when extracted text is <50% alphanumeric, drop the markdown, set `has_encoding_issues`, and flag all pages for OCR. Co-Authored-By: Claude Opus 4.6 * refactor: replace integration tests with synthetic unit tests Remove PDF fixture dependencies from detector and lib tests. Use in-memory lopdf documents to test Identity-H/ToUnicode detection logic. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/bin/pdf2md.rs | 3 + src/detector.rs | 158 +++++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 34 ++++++++++ 3 files changed, 192 insertions(+), 3 deletions(-) diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index c930076..77baf31 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -268,6 +268,9 @@ fn main() { eprintln!("Pages: {}", result.page_count); eprintln!("Processing time: {}ms", result.processing_time_ms); print_layout_info(&result.layout); + if !result.pages_needing_ocr.is_empty() { + eprintln!("Pages needing OCR: {:?}", result.pages_needing_ocr); + } if let Some(markdown) = &result.markdown { if let Some(output) = output_file { diff --git a/src/detector.rs b/src/detector.rs index e170895..1ea5558 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -197,12 +197,12 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); pages_actually_sampled += 1; log::debug!( - "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={}", + "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={}", page_num, analysis.text_operator_count, analysis.has_images, analysis.image_count, analysis.has_template_image, analysis.unique_text_chars, analysis.unique_alphanum_chars, analysis.path_op_count, analysis.has_vector_text, - analysis.total_image_area + analysis.total_image_area, analysis.has_identity_h_no_tounicode ); let is_image_dominated = analysis.image_count > 10 && analysis.image_count > analysis.text_operator_count * 3; @@ -292,7 +292,7 @@ pub(crate) fn detect_from_document( }; // Phase 2: Build per-page OCR list - let pages_needing_ocr = match pdf_type { + let mut pages_needing_ocr = match pdf_type { PdfType::TextBased => Vec::new(), PdfType::Scanned | PdfType::ImageBased => (1..=total_pages).collect(), PdfType::Mixed => { @@ -319,6 +319,29 @@ pub(crate) fn detect_from_document( } }; + // Phase 3: Flag pages with Identity-H/V fonts lacking ToUnicode for OCR. + // These fonts produce garbage text (raw CID values) for non-Latin scripts. + for (&page_num, analysis) in &analysis_cache { + if analysis.has_identity_h_no_tounicode && !pages_needing_ocr.contains(&page_num) { + pages_needing_ocr.push(page_num); + } + } + // Check uncached pages too (when not all pages were sampled) + if pages_needing_ocr.len() < total_pages as usize { + for page_num in 1..=total_pages { + if analysis_cache.contains_key(&page_num) || pages_needing_ocr.contains(&page_num) { + continue; + } + if let Some(&page_id) = pages.get(&page_num) { + if page_has_identity_h_no_tounicode(doc, page_id) { + pages_needing_ocr.push(page_num); + } + } + } + } + pages_needing_ocr.sort(); + pages_needing_ocr.dedup(); + // Try to get title from metadata let title = get_document_title(doc); @@ -390,6 +413,9 @@ struct PageAnalysis { path_op_count: u32, /// Whether the page has vector-outlined text (massive path ops, minimal text ops) has_vector_text: bool, + /// Whether the page has Type0 fonts with Identity-H/V encoding but no ToUnicode CMap. + /// These fonts produce garbage text because CID values can't be mapped to Unicode. + has_identity_h_no_tounicode: bool, } /// Analyze a page's content stream for text operators and images @@ -461,6 +487,10 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { .filter(|b| b.is_ascii_alphanumeric()) .count() as u32; + // Check for Identity-H/V fonts without ToUnicode — these produce garbage text + let has_identity_h_no_tounicode = + text_ops > 0 && page_has_identity_h_no_tounicode(doc, page_id); + PageAnalysis { text_operator_count: text_ops, has_images, @@ -471,9 +501,52 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { unique_alphanum_chars, path_op_count: path_ops, has_vector_text, + has_identity_h_no_tounicode, } } +/// Check if a page has Type0 fonts with Identity-H/V encoding and no ToUnicode CMap. +/// These fonts encode text as raw CID values that can't be mapped to Unicode without +/// a ToUnicode CMap, producing garbage output for non-Latin scripts (e.g. Cyrillic). +fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool { + let fonts = match doc.get_page_fonts(page_id) { + Ok(f) => f, + Err(_) => return false, + }; + for font_dict in fonts.values() { + let subtype = font_dict + .get(b"Subtype") + .ok() + .and_then(|o| o.as_name().ok()); + if subtype != Some(b"Type0") { + continue; + } + let encoding = font_dict + .get(b"Encoding") + .ok() + .and_then(|o| o.as_name().ok()); + let is_identity = matches!(encoding, Some(b"Identity-H") | Some(b"Identity-V")); + if !is_identity { + continue; + } + // Has ToUnicode? Then the font is decodable. + if font_dict.get(b"ToUnicode").is_ok() { + continue; + } + // Identity-H/V without ToUnicode — flag it + log::debug!( + "page has Identity-H/V font without ToUnicode: {:?}", + font_dict + .get(b"BaseFont") + .ok() + .and_then(|o| o.as_name().ok()) + .map(|n| String::from_utf8_lossy(n).to_string()) + ); + return true; + } + false +} + fn scan_xobjects_in_resources( doc: &Document, resources: &lopdf::Dictionary, @@ -1131,4 +1204,83 @@ mod tests { ); assert!(result.ocr_recommended); } + + #[test] + fn test_page_has_identity_h_no_tounicode_positive() { + // Build a minimal PDF with a Type0 Identity-H font and no ToUnicode. + use lopdf::dictionary; + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let page_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => Object::Name(b"Type0".to_vec()), + "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()), + "Encoding" => Object::Name(b"Identity-H".to_vec()), + }); + let resources = dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(font_id), + }, + }; + doc.objects.insert( + page_id, + Object::Dictionary(dictionary! { + "Type" => "Page", + "Parent" => Object::Reference(pages_id), + "Resources" => resources, + }), + ); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => Object::Integer(1), + }), + ); + assert!(page_has_identity_h_no_tounicode(&doc, page_id)); + } + + #[test] + fn test_page_has_identity_h_with_tounicode_negative() { + // Type0 Identity-H font WITH ToUnicode — should NOT flag. + use lopdf::dictionary; + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let page_id = doc.new_object_id(); + let cmap_id = doc.add_object(Object::Stream(lopdf::Stream::new( + dictionary! {}, + b"fake cmap".to_vec(), + ))); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => Object::Name(b"Type0".to_vec()), + "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()), + "Encoding" => Object::Name(b"Identity-H".to_vec()), + "ToUnicode" => Object::Reference(cmap_id), + }); + let resources = dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(font_id), + }, + }; + doc.objects.insert( + page_id, + Object::Dictionary(dictionary! { + "Type" => "Page", + "Parent" => Object::Reference(pages_id), + "Resources" => resources, + }), + ); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => Object::Integer(1), + }), + ); + assert!(!page_has_identity_h_no_tounicode(&doc, page_id)); + } } diff --git a/src/lib.rs b/src/lib.rs index b6de2b9..e1c39e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -437,8 +437,23 @@ fn process_document( (pdf_type, markdown, confidence) }; + // If a TextBased PDF produces garbage text, the fonts are undecodable + // (e.g. Identity-H without ToUnicode for non-Latin scripts like Cyrillic). + // Drop the useless markdown and flag all pages for OCR. + let (markdown, has_encoding_issues, force_ocr_all) = if pdf_type == PdfType::TextBased + && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) + { + log::debug!("TextBased PDF has garbage text — flagging all pages for OCR"); + (None, true, true) + } else { + (markdown, has_encoding_issues, false) + }; + // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list let mut pages_needing_ocr = pages_needing_ocr; + if force_ocr_all { + pages_needing_ocr = (1..=page_count).collect(); + } if !gid_pages.is_empty() { log::debug!("pages with gid-encoded fonts (need OCR): {:?}", gid_pages); for page in gid_pages { @@ -827,4 +842,23 @@ mod tests { let text = "a$b c$d e$f"; assert!(!detect_encoding_issues(text)); } + + #[test] + fn test_garbage_text_detection() { + // Simulates garbage output from Identity-H fonts without ToUnicode. + // Needs >= 50 non-whitespace chars and < 50% alphanumeric. + let garbage = ",&/5 /5&(#(8-!5 *,(6--( *,%@/-A W"; + assert!(is_garbage_text(garbage)); + + // Normal text should not be garbage + let normal = "This is a normal paragraph with words and sentences that contains enough characters to pass the threshold."; + assert!(!is_garbage_text(normal)); + + // Cyrillic text should not be garbage + let cyrillic = + "Роботизированные технологии комплексы для производства металлургических предприятий"; + assert!(!is_garbage_text(cyrillic)); + } } From 967b70a788d8a07d83f2dfefa7d727df5352992f Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 19 Mar 2026 18:37:06 -0700 Subject: [PATCH 21/46] fix: suppress markdown when all pages have gid-encoded fonts When every page uses fonts with unresolvable gid-encoded glyphs, the extracted text is unreliable garbage. Suppress the markdown output so consumers know to use OCR instead. Only triggers for gid-encoded fonts specifically, not for other OCR signals (Identity-H without ToUnicode, template images) where the text layer may still be partially useful. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lib.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index e1c39e3..dcc2c81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -449,7 +449,9 @@ fn process_document( (markdown, has_encoding_issues, false) }; - // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list + // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list. + // When ALL pages have gid-encoded fonts, suppress unreliable markdown. + let all_gid = !gid_pages.is_empty() && gid_pages.len() as u32 >= page_count; let mut pages_needing_ocr = pages_needing_ocr; if force_ocr_all { pages_needing_ocr = (1..=page_count).collect(); @@ -464,6 +466,16 @@ fn process_document( pages_needing_ocr.sort_unstable(); } + let markdown = if all_gid { + log::debug!( + "all {} pages have gid-encoded fonts — suppressing markdown output", + page_count + ); + None + } else { + markdown + }; + Ok(PdfProcessResult { pdf_type, markdown, From 672be2e9e5a931469ae0dcda224e00d8cc621e7b Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 20 Mar 2026 16:13:57 -0700 Subject: [PATCH 22/46] fix(preprocess): keep first occurrence of repeated headers/footers Document titles, column headers, and newsletter mastheads that repeat on every page were being fully stripped by strip_repeated_lines. Now the first page's occurrence is preserved so the content appears once. Fixes missing titles like "VOICE OF SOUTH MARION" in tax certificate PDFs and column headers in IRS forms. --- src/markdown/preprocess.rs | 95 +++++++++++++++++++++++++++++++++-- tests/snapshots/p1244-1996.md | 5 ++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index d443f75..657440b 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -423,9 +423,13 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec // (a) its individual text matches a candidate, OR // (b) its Y-band's coalesced text matches a band candidate, OR // (c) any sibling in its Y-band was removed (propagation). + // + // The first occurrence (lowest page number) of each repeated header/footer + // is kept so that document titles, column headers, etc. appear once. let mut removal_set: HashSet = HashSet::new(); - // (a) Lines matching individual candidates at edge positions + // Track which page first shows each candidate (to preserve first occurrence) + let mut first_page_individual: HashMap = HashMap::new(); for (idx, line) in lines.iter().enumerate() { if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) { continue; @@ -433,11 +437,18 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec let text = line.text(); let normalized = normalize_for_comparison(&text); if candidates.contains(&normalized) { - removal_set.insert(idx); + let first = first_page_individual.entry(normalized).or_insert(line.page); + if line.page > *first { + removal_set.insert(idx); + } else if line.page == *first { + // Keep this occurrence (first page) + } } } - // (b) Lines in Y-bands whose coalesced text matches a band candidate + // Track first page for band candidates + let mut first_page_band: HashMap = HashMap::new(); + // First pass: find first page for each band candidate for (&(page, _), indices) in &y_bands { if indices.len() < 2 { continue; @@ -455,8 +466,35 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec .join(" "); let normalized = normalize_for_comparison(&coalesced); if band_candidates.contains(&normalized) { - for &idx in &sorted_indices { - removal_set.insert(idx); + let first = first_page_band.entry(normalized).or_insert(page); + if page < *first { + *first = page; + } + } + } + // Second pass: mark for removal (skip first page) + for (&(page, _), indices) in &y_bands { + if indices.len() < 2 { + continue; + } + let band_y = lines[indices[0]].y; + if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + let mut sorted_indices = indices.clone(); + sorted_indices.sort(); + let coalesced: String = sorted_indices + .iter() + .map(|&i| lines[i].text()) + .collect::>() + .join(" "); + let normalized = normalize_for_comparison(&coalesced); + if band_candidates.contains(&normalized) { + let first = first_page_band.get(&normalized).copied().unwrap_or(0); + if page > first { + for &idx in &sorted_indices { + removal_set.insert(idx); + } } } } @@ -598,4 +636,51 @@ mod tests { let result = merge_heading_lines(lines, 12.0, &heading_tiers, None); assert_eq!(result.len(), 2, "should merge font-based heading lines"); } + + #[test] + fn test_strip_repeated_keeps_first_occurrence() { + // Simulate a repeated page header on 10 pages. + // Each page has a running header at y=750 and many unique body lines. + let mut lines = Vec::new(); + for page in 1..=10u32 { + // Header at top + lines.push(make_line( + "VOICE OF SOUTH MARION May fifteen twenty twenty five", + 10.0, + page, + 750.0, + None, + )); + // Body content — unique text per line per page (no digits to strip) + for j in 0..20u32 { + lines.push(make_line( + &format!( + "parcel r-{:04}-{:03} owner smith address oak street", + page * 100 + j, + page + ), + 10.0, + page, + 600.0 - j as f32 * 15.0, + None, + )); + } + } + + let result = strip_repeated_lines(lines, 10); + + // The header should appear exactly once (page 1) + let header_count = result + .iter() + .filter(|l| l.text().contains("VOICE OF SOUTH MARION")) + .count(); + assert_eq!(header_count, 1, "repeated header should be kept once"); + + // First occurrence should be on page 1 + let first_header = result + .iter() + .find(|l| l.text().contains("VOICE OF SOUTH MARION")) + .unwrap(); + assert_eq!(first_header.page, 1, "first occurrence should be on page 1"); + } } diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index 4548660..5a05ca8 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -33,6 +33,11 @@ Date Date **a. Tips received** **b. Credit card tips c. Tips paid out to d. Names of employees to whom you** tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1** +Date Date **a. Tips received** + +**b. Credit card tips c. Tips paid out to d. Names of employees to whom you** +tips of directly from customers received other employees paid tips rec’d. entry and other employees + 7 8 9 10 11 12 13 14 15 **Subtotals** **Page 2** From 2a8e6283962b3784e8fd532929e60fde8f0ef099 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 20 Mar 2026 17:26:20 -0700 Subject: [PATCH 23/46] feat(detect): flag Type3-only pages without ToUnicode for OCR Type3 fonts render each glyph as a custom drawing/bitmap. Without a ToUnicode CMap the character codes can't be mapped to Unicode, so extracted text is garbage. Pages using only Type3 fonts are now excluded from pages_with_text and added to pages_needing_ocr. Fixes Korean/CJK PDFs using Type3 fonts (e.g. D2Coding) being classified as TextBased when no usable text can be extracted. --- src/detector.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/detector.rs b/src/detector.rs index 1ea5558..535f9e4 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -197,12 +197,13 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); pages_actually_sampled += 1; log::debug!( - "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={}", + "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={} type3_only={}", page_num, analysis.text_operator_count, analysis.has_images, analysis.image_count, analysis.has_template_image, analysis.unique_text_chars, analysis.unique_alphanum_chars, analysis.path_op_count, analysis.has_vector_text, - analysis.total_image_area, analysis.has_identity_h_no_tounicode + analysis.total_image_area, analysis.has_identity_h_no_tounicode, + analysis.has_only_type3_fonts ); let is_image_dominated = analysis.image_count > 10 && analysis.image_count > analysis.text_operator_count * 3; @@ -215,6 +216,7 @@ pub(crate) fn detect_from_document( && !is_image_dominated && analysis.unique_text_chars >= 5 && !analysis.has_vector_text + && !analysis.has_only_type3_fonts { pages_with_text += 1; } @@ -319,10 +321,13 @@ pub(crate) fn detect_from_document( } }; - // Phase 3: Flag pages with Identity-H/V fonts lacking ToUnicode for OCR. - // These fonts produce garbage text (raw CID values) for non-Latin scripts. + // Phase 3: Flag pages with undecodable fonts for OCR. + // - Identity-H/V without ToUnicode: raw CID values can't map to Unicode + // - Type3-only without ToUnicode: glyph bitmaps can't map to Unicode for (&page_num, analysis) in &analysis_cache { - if analysis.has_identity_h_no_tounicode && !pages_needing_ocr.contains(&page_num) { + if (analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts) + && !pages_needing_ocr.contains(&page_num) + { pages_needing_ocr.push(page_num); } } @@ -333,7 +338,9 @@ pub(crate) fn detect_from_document( continue; } if let Some(&page_id) = pages.get(&page_num) { - if page_has_identity_h_no_tounicode(doc, page_id) { + if page_has_identity_h_no_tounicode(doc, page_id) + || page_has_only_type3_fonts(doc, page_id) + { pages_needing_ocr.push(page_num); } } @@ -416,6 +423,10 @@ struct PageAnalysis { /// Whether the page has Type0 fonts with Identity-H/V encoding but no ToUnicode CMap. /// These fonts produce garbage text because CID values can't be mapped to Unicode. has_identity_h_no_tounicode: bool, + /// Whether the page uses only Type3 fonts (no normal text fonts). + /// Type3 fonts render each glyph as a custom drawing/bitmap — without a + /// ToUnicode CMap, the character codes can't be mapped to Unicode. + has_only_type3_fonts: bool, } /// Analyze a page's content stream for text operators and images @@ -491,6 +502,9 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { let has_identity_h_no_tounicode = text_ops > 0 && page_has_identity_h_no_tounicode(doc, page_id); + // Check for Type3-only fonts — glyph bitmaps without Unicode mapping + let has_only_type3_fonts = text_ops > 0 && page_has_only_type3_fonts(doc, page_id); + PageAnalysis { text_operator_count: text_ops, has_images, @@ -502,6 +516,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { path_op_count: path_ops, has_vector_text, has_identity_h_no_tounicode, + has_only_type3_fonts, } } @@ -547,6 +562,40 @@ fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool { false } +/// Returns true if every font on the page is Type3 (no normal text fonts). +/// Type3 fonts render glyphs as custom drawings/bitmaps. Without a ToUnicode +/// CMap, character codes can't be mapped to Unicode — the page needs OCR. +fn page_has_only_type3_fonts(doc: &Document, page_id: ObjectId) -> bool { + let fonts = match doc.get_page_fonts(page_id) { + Ok(f) => f, + Err(_) => return false, + }; + if fonts.is_empty() { + return false; + } + let mut has_type3 = false; + for font_dict in fonts.values() { + let subtype = font_dict + .get(b"Subtype") + .ok() + .and_then(|o| o.as_name().ok()); + if subtype == Some(b"Type3") { + // Type3 with a ToUnicode CMap can still produce usable text + if font_dict.get(b"ToUnicode").is_ok() { + return false; + } + has_type3 = true; + } else { + // Has a non-Type3 font — page has real text fonts + return false; + } + } + if has_type3 { + log::debug!("page has only Type3 fonts without ToUnicode — text is undecodable"); + } + has_type3 +} + fn scan_xobjects_in_resources( doc: &Document, resources: &lopdf::Dictionary, From 95b45d154e7f84a701bd908aa6362dbfe713decb Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 20 Mar 2026 19:42:11 -0700 Subject: [PATCH 24/46] fix(fonts): fall back to raw stream for uncompressed ToUnicode CMaps lopdf's decompressed_content() fails with DictKey("Filter") when a ToUnicode stream has no /Filter entry (uncompressed raw text). Now falls back to the raw stream.content when decompression fails. Fixes Identity-H fonts with valid uncompressed ToUnicode CMaps producing empty text (e.g. neoenergia tabela PDF). --- src/extractor/content_stream.rs | 13 +++++++------ src/extractor/xobjects.rs | 13 +++++++------ src/tounicode.rs | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ef7fe61..c1952b7 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -75,12 +75,13 @@ pub(crate) fn extract_page_text_items( if let Ok(obj_ref) = tounicode.as_reference() { font_tounicode_refs.insert(resource_name, obj_ref.0); } else if let Object::Stream(s) = tounicode { - if let Ok(data) = s.decompressed_content() { - if let Some(entry) = - crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) - { - inline_cmaps.insert(resource_name, entry); - } + let data = s + .decompressed_content() + .unwrap_or_else(|_| s.content.clone()); + if let Some(entry) = + crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) + { + inline_cmaps.insert(resource_name, entry); } } } diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 5491bb8..f780783 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -179,12 +179,13 @@ fn extract_form_xobject_text_inner( if let Ok(obj_ref) = tounicode.as_reference() { font_tounicode_refs.insert(resource_name, obj_ref.0); } else if let Object::Stream(s) = tounicode { - if let Ok(data) = s.decompressed_content() { - if let Some(entry) = - crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) - { - inline_cmaps.insert(resource_name, entry); - } + let data = s + .decompressed_content() + .unwrap_or_else(|_| s.content.clone()); + if let Some(entry) = + crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) + { + inline_cmaps.insert(resource_name, entry); } } } diff --git a/src/tounicode.rs b/src/tounicode.rs index 2c25668..2a757ab 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -1734,7 +1734,7 @@ impl FontCMaps { }; let data = match stream.decompressed_content() { Ok(d) => d, - Err(_) => continue, + Err(_) => stream.content.clone(), }; if let Some(cmap) = ToUnicodeCMap::parse(&data) { debug!( From 0c172302115622318d5a03e9b2dbe8fa51c4cf15 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 20 Mar 2026 21:07:43 -0700 Subject: [PATCH 25/46] fix(xobjects): fall back to raw bytes for uncompressed Form XObject streams lopdf's decompressed_content() fails on Form XObjects without a /Filter entry (uncompressed streams). This caused all body text to be lost in PDFs generated by pdfrw and similar tools that wrap page content in uncompressed Form XObjects (e.g. Cambridge University Press excerpts). Co-Authored-By: Claude Opus 4.6 --- src/extractor/xobjects.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index f780783..5a7a3bf 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -144,9 +144,10 @@ fn extract_form_xobject_text_inner( return items; }; - // Decompress the content stream - let Ok(content_data) = stream.decompressed_content() else { - return items; + // Decompress the content stream (fall back to raw bytes for uncompressed streams) + let content_data = match stream.decompressed_content() { + Ok(data) => data, + Err(_) => stream.content.clone(), }; // Decode the content stream From 7f2995ad0cef79dbb31dc4a11d939b4c1bfb32f8 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Fri, 20 Mar 2026 21:14:15 -0700 Subject: [PATCH 26/46] chore: update lopdf to firecrawl fork with uncompressed stream fix Points to firecrawl/lopdf@edf6279 which fixes decompressed_content() for streams without a /Filter entry. Our xobjects.rs workaround remains as a defensive fallback. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index eea0ced..32016c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector" [dependencies] # PDF parsing -lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "845cd3d4648d9cefb7b5def5fb387df61ba3a0e5", features = ["rayon"] } +lopdf = { git = "https://github.com/firecrawl/lopdf", rev = "edf6279", features = ["rayon"] } # Error handling thiserror = "2.0" From 6c005ef4c22afa1d74ae0bb6ba6623d7fa0456a9 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 09:41:10 -0700 Subject: [PATCH 27/46] fix(fonts): CID-as-Unicode passthrough and subscript merging (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(fonts): CID-as-Unicode passthrough and subscript/superscript merging Add smart CID-as-Unicode passthrough for Identity-H fonts without ToUnicode maps. Uses /W array median CID heuristic to distinguish Unicode-CID PDFs (Chromium-generated) from GID-based subsets. Add merge_subscript_items() pass that merges small-font items (<75% of dominant font size, ≤4 chars, tightly adjacent) into parent items. Fixes chemical formulas (NH3, H2O, KClO3), footnote references, and subscript notation (vf, Hfg, m3/kg) that were previously orphaned as separate text items. Co-Authored-By: Claude Opus 4.6 * fix(subscripts): restrict merge to purely numeric text only Tighten subscript merging to only merge items containing ASCII digits (0-9). This avoids false positives with ordinal indicators (º), letter subscripts (sol, vf), and small bullet characters (▶) that caused table restructuring regressions. Numeric-only keeps the primary wins: chemical formulas (NH3, H2O), footnote references, and unit notation (m2, m3). Co-Authored-By: Claude Opus 4.6 * fix(subscripts): restrict merge to parent text ending with a letter Only merge numeric subscripts when the parent item's text ends with an alphabetic character. Prevents false merges like "33" + "1" in fractions (33 1/3%), table credit numbers after spaces, and footnote refs after punctuation (land.1 → land. 1). Chemical formulas (NH3, H2O, KClO3) still merge correctly since parent ends with a letter. Reduces pdf-eval regressions from 13 to 2 (both are correct reversions of over-aggressive footnote merging from the prior commit). Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/extractor/content_stream.rs | 1 + src/extractor/mod.rs | 203 ++++++++++++++++++++++++++++++ src/tounicode.rs | 155 +++++++++++++++++++---- tests/snapshots/thermo-freon12.md | 4 +- 4 files changed, 339 insertions(+), 24 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index c1952b7..e703b79 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -880,6 +880,7 @@ pub(crate) fn extract_page_text_items( } let items = super::merge_text_items(items); + let items = super::merge_subscript_items(items); Ok(((items, rects, lines), has_gid_fonts)) } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index e8a2a90..15554db 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -363,6 +363,92 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { merged } +/// Merge subscript/superscript items into their adjacent parent items. +/// +/// Subscripts (e.g. "2" in H₂O) are rendered as separate text items with a +/// much smaller font size and a slight Y offset. This pass finds such items +/// and absorbs them into the preceding normal-sized item so that downstream +/// table detection and line grouping see complete text (e.g. "H2O" not "H"+"2"+"O"). +pub(crate) fn merge_subscript_items(items: Vec) -> Vec { + if items.len() < 2 { + return items; + } + + // Group items by (page, approximate Y) with generous tolerance to capture + // both the parent line and the subscript/superscript offset. + let y_tolerance = 5.0; + let mut line_groups: Vec<(u32, f32, Vec)> = Vec::new(); + + for item in items { + let found = line_groups + .iter_mut() + .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance); + if let Some((_, _, group)) = found { + group.push(item); + } else { + let page = item.page; + let y = item.y; + line_groups.push((page, y, vec![item])); + } + } + + let mut result = Vec::new(); + + for (_, _, mut group) in line_groups { + // Sort by X position + group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + + // Find the dominant (most common) font size in this group + let max_fs = group.iter().map(|i| i.font_size).fold(0.0_f32, f32::max); + + if max_fs < 1.0 { + result.extend(group); + continue; + } + + let sub_threshold = max_fs * 0.75; + + // Walk through items and merge subscripts into their preceding parent + let mut merged: Vec = Vec::new(); + for item in group { + if item.font_size < sub_threshold + && item.font_size > 0.0 + && item.text.len() <= 4 + && item.text.chars().all(|c| c.is_ascii_digit()) + { + // This is a candidate numeric subscript/superscript (e.g. "2" in H₂O). + // Only merge purely numeric text to avoid false positives with small + // bullets, ordinal indicators, or letter-based labels. + if let Some(parent) = merged.last_mut() { + // Only merge into a parent that is normal-sized, not another subscript, + // and whose text ends with a letter. This prevents merging into numbers + // (e.g. "33" + "1" in "33 1/3%") or punctuation, while preserving + // chemical formulas (NH + "3") and footnote refs (word + "2"). + let ends_with_letter = parent + .text + .chars() + .last() + .is_some_and(|c| c.is_alphabetic()); + if parent.font_size >= sub_threshold && ends_with_letter { + let parent_right = parent.x + parent.width; + let gap = item.x - parent_right; + // Subscripts must be tightly adjacent (within ~1pt) + if gap < parent.font_size * 0.2 && gap > -parent.font_size * 0.3 { + parent.text.push_str(&item.text); + parent.width = (item.x + item.width) - parent.x; + continue; + } + } + } + } + merged.push(item); + } + result.extend(merged); + } + + result +} + /// Helper to get f32 from Object pub(crate) fn get_number(obj: &Object) -> Option { match obj { @@ -1018,4 +1104,121 @@ mod tests { ]; assert!(!is_newspaper_layout(&[col1, col2], &cols)); } + + fn make_item_fs(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: font_size, + font: "F1".into(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn test_merge_subscript_items_chemical_formula() { + // NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7 + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 4.7), + make_item_fs("Cl", 100.0, 499.0, 7.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "NH3"); + assert_eq!(merged[1].text, "Cl"); + } + + #[test] + fn test_merge_subscript_items_h2o() { + // H₂O: "H" then subscript "2" then "O" + let items = vec![ + make_item_fs("H", 250.0, 499.0, 5.0, 8.0), + make_item_fs("2", 255.0, 496.0, 2.3, 4.7), + make_item_fs("O", 257.5, 499.0, 6.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "H2"); + assert_eq!(merged[1].text, "O"); + } + + #[test] + fn test_merge_subscript_items_no_merge_far_gap() { + // Subscript-sized item that's far from the parent should NOT merge + let items = vec![ + make_item_fs("Text", 78.0, 499.0, 20.0, 8.0), + make_item_fs("▶", 120.0, 498.0, 3.0, 3.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "Text"); + assert_eq!(merged[1].text, "▶"); + } + + #[test] + fn test_merge_subscript_items_no_merge_long_text() { + // Long subscript-sized text should NOT merge (not a true subscript) + let items = vec![ + make_item_fs("Title", 78.0, 499.0, 30.0, 8.0), + make_item_fs("footnote", 108.0, 496.0, 20.0, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_same_font_size() { + // Same font size items should NOT be treated as subscripts + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_non_numeric() { + // Non-numeric subscript text (e.g. "sol", "º", "vf") should NOT merge + let items = vec![ + make_item_fs("∆", 200.0, 639.0, 5.5, 8.0), + make_item_fs("sol", 205.8, 636.9, 5.7, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "∆"); + assert_eq!(merged[1].text, "sol"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_digit() { + // "33" + "1" in "33 1/3%" — parent ends with digit, should NOT merge + let items = vec![ + make_item_fs("33", 78.0, 499.0, 10.0, 8.0), + make_item_fs("1", 88.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "33"); + assert_eq!(merged[1].text, "1"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_space() { + // "Health " + "1" — parent ends with space (table credit), should NOT merge + let items = vec![ + make_item_fs("Health ", 78.0, 499.0, 30.0, 8.0), + make_item_fs("1", 108.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } } diff --git a/src/tounicode.rs b/src/tounicode.rs index 2a757ab..6e13f4a 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -18,6 +18,9 @@ pub struct ToUnicodeCMap { pub ranges: Vec<(u16, u16, u32)>, /// Byte width of source codes (1 or 2), determined from codespace and CMap entries pub code_byte_length: u8, + /// When true, unmapped CIDs are interpreted as Unicode codepoints directly. + /// Used as a last resort for Identity-H fonts without ToUnicode/cmap/glyph names. + pub cid_passthrough: bool, } pub(crate) fn build_cmap_entry_from_stream( @@ -467,10 +470,24 @@ impl ToUnicodeCMap { match self.lookup(cid) { Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s), _ => { - // Do NOT blindly interpret CIDs as Unicode codepoints. - // CIDs are font-internal indices, not Unicode values. - // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. - unmapped_count += 1; + if self.cid_passthrough { + // Last-resort: treat CID as Unicode codepoint. + // Valid for Identity-H fonts where the PDF generator + // used Unicode values as CIDs but stripped the cmap. + if let Some(ch) = char::from_u32(cid as u32) { + if !ch.is_control() || ch == '\t' || ch == '\n' { + result.push(ch); + } else { + unmapped_count += 1; + } + } else { + unmapped_count += 1; + } + } else { + // CIDs are font-internal indices, not Unicode values. + // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. + unmapped_count += 1; + } } } } @@ -1628,6 +1645,68 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap base } +/// Check if a CIDFont's /W (widths) array contains CID values that look like +/// Unicode codepoints rather than low-value GIDs. +/// +/// Returns true if the median CID is >= 0x41 (letter 'A'), indicating +/// the PDF generator likely used Unicode codepoints as CIDs. +fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool { + let w_arr = match cid_font_dict.get(b"W").ok() { + Some(Object::Array(arr)) => arr, + _ => return false, + }; + + // The /W array format: [cid [w1 w2 ...]] or [cid_start cid_end w] + // We extract all CID values (the first element of each group). + let mut cids: Vec = Vec::new(); + let mut i = 0; + while i < w_arr.len() { + if let Ok(cid) = w_arr[i].as_i64() { + cids.push(cid as u16); + // Skip the width data + if i + 1 < w_arr.len() { + match &w_arr[i + 1] { + Object::Array(widths) => { + // [cid [w1 w2 ...]] — CIDs are cid, cid+1, ..., cid+len-1 + for j in 1..widths.len() { + cids.push((cid as u16).wrapping_add(j as u16)); + } + i += 2; + } + _ => { + // [cid_start cid_end w] — range of CIDs + if i + 2 < w_arr.len() { + if let Ok(cid_end) = w_arr[i + 1].as_i64() { + for c in (cid as u16)..=(cid_end as u16) { + cids.push(c); + } + } + i += 3; + } else { + i += 1; + } + } + } + } else { + i += 1; + } + } else { + i += 1; + } + } + + if cids.is_empty() { + return false; + } + + cids.sort_unstable(); + let median = cids[cids.len() / 2]; + // Unicode text CIDs are typically >= 0x20 (space) with letters at 0x41+. + // GID-based subsets typically start at low values (0-based). + // Use median >= 0x41 as a heuristic for Unicode CIDs. + median >= 0x41 +} + /// Build a ToUnicodeCMap from predefined CID→Unicode mapping based on CIDSystemInfo. /// /// Supports Adobe-Korea1 (Korean) character collection. Can be extended for @@ -1874,23 +1953,25 @@ impl FontCMaps { // Try parsing embedded TrueType/OpenType cmap if let Some(ff_ref) = font_file_ref { if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) { - if let Ok(data) = stream.decompressed_content() { - if let Some(cmap) = build_cmap_from_truetype(&data) { - debug!( - "TrueType CMap obj={:<6} (embedded font) char_map={}", - lookup_key, - cmap.char_map.len() - ); - by_obj_num.insert( - lookup_key, - CMapEntry { - primary: cmap, - remapped: None, - fallback: None, - }, - ); - resolved = true; - } + let data = match stream.decompressed_content() { + Ok(d) => d, + Err(_) => stream.content.clone(), + }; + if let Some(cmap) = build_cmap_from_truetype(&data) { + debug!( + "TrueType CMap obj={:<6} (embedded font) char_map={}", + lookup_key, + cmap.char_map.len() + ); + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + resolved = true; } } } @@ -1911,6 +1992,38 @@ impl FontCMaps { fallback: None, }, ); + resolved = true; + } + } + + // Last resort: CID-as-Unicode passthrough. + // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H encoding where + // CID values ARE Unicode codepoints, but strip the cmap table and omit + // ToUnicode. We detect this by checking the /W (widths) array: if CID values + // fall in typical Unicode letter/digit ranges (0x41+), CIDs are likely Unicode. + // If CIDs are low values (< 0x41), they're GIDs in a subset font. + if !resolved { + if cid_values_look_like_unicode(cid_font_dict) { + debug!( + "Identity-H font obj={}: W array CIDs look like Unicode — using passthrough", + lookup_key + ); + let mut cmap = ToUnicodeCMap::new(); + cmap.code_byte_length = 2; + cmap.cid_passthrough = true; + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + } else { + debug!( + "Identity-H font obj={}: no decoding possible (stripped cmap, GID-based CIDs)", + lookup_key + ); } } } diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 08a217a..789ed1a 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -48,7 +48,7 @@ l |Temp|Pressure|Volume||Density|||Enthalpy||Entropy||Temp| |---|---|---|---|---|---|---|---|---|---|---|---| -|°C|[kPa]|[m Liquid v|/kg] Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| +|°C|[kPa]|[m3/kg] Liquid v|Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| |-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99| |-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98| @@ -104,7 +104,5 @@ l |-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48| |-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47| -**3** - **f g f g f** fg **g f g** From aa387503c3a24a2242a40326af5b688939919b75 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 10:35:45 -0700 Subject: [PATCH 28/46] fix(fonts): suppress garbage CID text on OCR-flagged pages (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Identity-H or Type3 fonts lack a ToUnicode CMap and the CID-as-Unicode passthrough doesn't produce valid text, the raw CID byte values appear as mojibake (random Latin Extended characters mixed with C1 control codes). The detector already flags these pages in pages_needing_ocr, but the markdown pipeline still emitted the garbage. Now, for TextBased PDFs, we check each OCR-flagged page's extracted text for CID garbage (C1 control characters U+0080–U+009F at ≥5% density) and strip items from pages that fail the check. This is scoped to TextBased PDFs only — Mixed PDFs flag pages for OCR due to template images, not font encoding issues. Adds test fixture (shinagawa_identity_h.pdf) and integration test. Co-authored-by: Claude Opus 4.6 --- src/lib.rs | 101 ++++++++++++++++++++++++ tests/fixtures/shinagawa_identity_h.pdf | Bin 0 -> 51459 bytes tests/integration_tests.rs | 24 ++++++ 3 files changed, 125 insertions(+) create mode 100644 tests/fixtures/shinagawa_identity_h.pdf diff --git a/src/lib.rs b/src/lib.rs index dcc2c81..6fcb54b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -401,6 +401,54 @@ fn process_document( let (markdown, layout, has_encoding_issues, gid_pages) = match extracted { Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { + // For TextBased PDFs with pages flagged for OCR (Identity-H or + // Type3 fonts without ToUnicode), check whether the CID-as-Unicode + // passthrough actually produced readable text. If a page's text + // is garbage, strip its items so we don't emit mojibake. + // Only applies to TextBased — for Mixed PDFs, OCR flags come from + // template images rather than font encoding issues. + let (items, rects, lines) = + if pages_needing_ocr.is_empty() || pdf_type != PdfType::TextBased { + (items, rects, lines) + } else { + let ocr_set: std::collections::HashSet = + pages_needing_ocr.iter().copied().collect(); + // Collect text per OCR-flagged page and check quality + let mut garbage_pages: std::collections::HashSet = + std::collections::HashSet::new(); + for &pg in &ocr_set { + let page_text: String = items + .iter() + .filter(|i| i.page == pg) + .map(|i| i.text.as_str()) + .collect(); + if is_cid_garbage(&page_text) { + garbage_pages.insert(pg); + } + } + if garbage_pages.is_empty() { + (items, rects, lines) + } else { + log::debug!( + "suppressing garbage text from OCR-flagged pages: {:?}", + garbage_pages + ); + let items: Vec<_> = items + .into_iter() + .filter(|i| !garbage_pages.contains(&i.page)) + .collect(); + let rects: Vec<_> = rects + .into_iter() + .filter(|r| !garbage_pages.contains(&r.page)) + .collect(); + let lines: Vec<_> = lines + .into_iter() + .filter(|l| !garbage_pages.contains(&l.page)) + .collect(); + (items, rects, lines) + } + }; + let layout = compute_layout_complexity(&items, &rects, &lines); let md = if options.mode == ProcessMode::Analyze { @@ -555,6 +603,33 @@ fn is_garbage_text(markdown: &str) -> bool { total >= 50 && alphanum * 2 < total } +/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts. +/// +/// When CID values don't correspond to Unicode codepoints, the raw bytes often +/// produce characters in the C1 control range (U+0080–U+009F) or Private Use +/// Area, mixed with random Latin Extended characters. Valid text in any +/// language almost never contains C1 controls. We also fall back to the +/// general `is_garbage_text` check for non-alphanumeric-heavy patterns. +fn is_cid_garbage(text: &str) -> bool { + if is_garbage_text(text) { + return true; + } + let mut total = 0usize; + let mut c1_control = 0usize; + for ch in text.chars() { + if ch.is_whitespace() { + continue; + } + total += 1; + // C1 control characters (U+0080–U+009F) — almost never in real text + if ('\u{0080}'..='\u{009F}').contains(&ch) { + c1_control += 1; + } + } + // If ≥5% of non-whitespace chars are C1 controls, it's garbage + total >= 20 && c1_control * 20 >= total +} + /// Analyse extracted items and rects for layout complexity. fn compute_layout_complexity( items: &[types::TextItem], @@ -873,4 +948,30 @@ mod tests { "Роботизированные технологии комплексы для производства металлургических предприятий"; assert!(!is_garbage_text(cyrillic)); } + + #[test] + fn test_cid_garbage_detection() { + // Simulates CID garbage from Identity-H fonts: Latin Extended chars + // mixed with C1 control characters (U+0080–U+009F). + let cid_garbage = "Ë>íÓ\tý\r\u{0088}æ&Ït\u{0094}äí;\ný;wAL¢©èåD\rü£\ + qq\u{0096}¶Í Æ\réá; Ô 7G\u{008B}ý;èÕç¢ £ ý;C"; + assert!( + is_cid_garbage(cid_garbage), + "CID garbage with C1 controls should be detected" + ); + + // Valid Korean text (CID-as-Unicode passthrough) should NOT be garbage + let korean = "본 가격표는 국내 거주 중인 외국인을 위한 한국어 가격표의 비공식 번역본입니다"; + assert!( + !is_cid_garbage(korean), + "Valid Korean text should not be flagged as garbage" + ); + + // Valid Japanese text should NOT be garbage + let japanese = "羽田空港新飛行経路に係る航空機騒音の測定結果"; + assert!( + !is_cid_garbage(japanese), + "Valid Japanese text should not be flagged as garbage" + ); + } } diff --git a/tests/fixtures/shinagawa_identity_h.pdf b/tests/fixtures/shinagawa_identity_h.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b6639f0b76213f045ef98a6ffdba93ea440cfd38 GIT binary patch literal 51459 zcmb@sWl&wwvMmfGK(Ih?XX6eVcXxMp*|@tCG&sTC-QC?KXmEFT5B`yJ&v{qQ``-KW zX4S6RtLI!jyGM`FvudqHDlaTb%Rt8hOS-)?IRncC00L|cEnvC1Vd->fQ5mXg#(se9H0eYW#j-bu&@AhVCm%mEU@$|XJze|-rt{B-FVR?CB zK{m#J-2wie3|Kb@kjY1g42-bAe+~u~z(4*0YykhVv9L4zn~jZ~`QL0D9E|_d#=yY9 z{%>ieSInrm_M-zJU9FoFvB_^ z5Q4BTAB@Camcm5e&KODm(6k!F-uoxRU=Cd&xDOs4SFW3ppJzx_$Sk-H%;OGr_%A>I z>GGdU{ovDoPvvh0fAeqv**N_(5qd!@5Xe~2)=dlev6`Kk5x~O1@*xFf2XpIxw<>}h zZJiyAK#ttpAC<8+QUW4f0bO0ai1_)qcX9X}YYcVsi02o==bpQ-(07g~-3kxfNiA9T*jY9{( z%Av)=#;)^Gi~oD@e}nj&fP=Z6ldZ$Y4*%FuSnXpc;0HP%(I~k)I)SYJD(j;Xat_8I zhYyG-Bn|8gY#1o%KTt6L;Ep?mz(FL&Ckz(D_Ar z{yT{r_^qI#?JZ1E=b`ESTX7#4d_Mqw5D9(&I7?j7(@MEvhaM85Z2no)P?I$3QtRU8mY377zCF=Mt&)tKqKfgkIq8DpJEFHQ+}H_5V{_g~C{ z8`nh5KsSO(Oub;FL7bqjMs8H8ad)n+l4!3Q;4B}4`ePL_o>smvJ};xFWVOC+*2sCj z(`djY6}b3KhHHFz@2g{d8YF|Kx;($9#&>TTiY1@AuhkTe&|EECvm-k8EMiLGMWtf~ z=(qY}>higH?1RYL>~M;*B(mgqD$6DMd@`k@84~FeoTb|w7-^GSWxjOD&eJzOBR8sC zNg9B!YU)#wIv~Fx7Ifp{1W1*ZOwJAQbQ>beCiS0bD>M{6c<89ZI_OTJp6d(G>kD4o z-9m}+N8jLZ@zdr8bLnO#{9$+MME*ohHiYw1;=ep_g3_mk^kb2(gzX60Uxyd%i0eTff)Y^;vbCtJEi(}>VN6^ zKUob+uPzQUH#KwmJ1_88tWE%SwvR*nFS|ib0ET~_7=OLW8d!t=|E2!Xo=qJAA5Hl` zH17nUWn}!2@825*|Mnx=zp4SyG66pVV_;?YXj&Y=4;e~)oCD@Y0yd^rpuZsq8QA?Z zp1)E2r$qjd%fE8{KWT&&$j0`sBY^?H%Erj@-;af{78o~_g%&oSiPKooylI(vO0gRE}QpL$O{zgKf2DGf`i3jY2nUz%c_V>6zI2f8`r2^;h(P+CYaJqb{;P0rQSDfd4f(vT?VG_V?(la{kVdkVa z*B!1Pv9ZrOkr~oKwTApV>al%zV&zix-P?=nH&Gxy^i$G<3x-d7CzydY%-vzZ8jdQ^ z_Ls;Pr5l5G&w)}%>hk5N$?UI_Mv6#%e4$lhG{~6!j|QS9hhMY4wj#I`8_6Hua0y3B z`*^#34r5iUvT<%32}=uiI}1q^co~{SW(zte^Oj~5i{T})ll`MOJmTY*n{1348Y||5 zxL|7cD&Vf^u8H^KC;WnJJx+z+g=#%mr;fZdH))|FrVA-&C5$Jp8{L@Ab6F3o9H1%BrU}2} z6T05Z>h^}as6?O%u>(CNaQnXa;}tn&8tVQls^5IAYjw2VI$V1czub2#Z&8b#cggKx zKK*k%ao?lq7RmQxsdHtjmFk^V=I^na z+ahq!=wlaWN9V_7I} zUs&M3PVfbr5!6HHMj148C!Sdc#npTK@OaUUZUTWv@Q?wfXBJhUNbTzb*Jcbv9we~t zy63otJduA$S6~?^V`NGe4#>&0YsN ztm^~6@YtrDlJC7|^K&q#`xKqAmKv7CLDK;_=8J2%t3Z6yUGe~9b1tsxCH~3&cH2t; zVGm$?GXai2*-THRswp=-0&FW-4~_YbwXH4KqxD7J=@(i6TA=HiF>@_mp4kS&*dIFv zs^B`u?Vs~)*-qjIw0I|!0;D&ZE@17f;T>IbfLUX(Em-&w?9% zK&+n=4r*aLM+g)E=DuNmRq%nId)#7X0Z`t(1-E2N?(UxV^q8L;fs=K%HKolKxqn9v z*rL0rH?Kf3iEo4c0kPJh1I%q#!GnK-h}>JZdGG>I;$oPwtiTO4vN9IcRDk5uFETD5 z{h=TN*6dCxTx~69^)0I;^^*~`ghnJzn-O5rkiyIgtC|qsWLp*FlA6e8`JE0&8-vtX zZ+nVEt=BEC+3%v}p7k4=ym%u3Y7Z}j*nz)xGB>be5ofb49^To%$YJh6WtN5`jt1qi z;8qGi`UD8U(O1!7m#O|NgDXx%j!hsW%N*7WH>JR2ncX-h!1>KnUopgUnAicSdtD@( zDO*`dhz`5r(H07`6>=&d#KPRH<-zY^&=XeKqYxTkF8#6?7o7M=uZ0XxU=t{&#Osir zV~ucLsp_hRQSV>smb)rMv%2QJCfpoiZ}lak2%Ls_wH4a5PkR;?F%o;bNA|&){?enn zP*BU#GUUlP8vgX_xDA$|LO{TJ4uxY-u8 zYMlg5lk7)@##RP5rb7(bxi>i{C{9f9S6_YUo`Z3`c>y1ZIY#<&HR1twl^ z86QRYEuz9b_aVuTp}KBpAXVO%D|J1zpKM@?Xd)(UKZ*wvfu6IED#jMG6n%m-*&-|% z{qF9f4`py?Y`5~Vuc60sJ2=Q*-G_uw_ejV4Bld$qV58MXQ@EOIHiuyH!D20aJjVj9 zy=a3?{5(p-<}INOpIz6_NY!RXVlc3~EnVD4=9VS#>5CDGm}U#0h2T&e?DKDi#J~9{ z9u^UIQQdt(NozQcR(p_s5)?C}C-+MBC=%7>CmeO}B_Lwjjc@DI#F}q|R1gZ+e7zB> zYp#ZL4G@9+wZ^41@YzuSryU>v6cd-Aew2rK1(8>XXXPA8rA%nvPn@*!G8gt|jvcsv z0O4*$|1P4Rg*){RisGn*_GqOqaO>O-Gj2Cg3dzuxauG4*7(G*n_b%gzElSypqv2FU zV8KdQ6$N4k$*1V zIlC_N>Y#$&PYnc<(}b`aqSf=(SmLj(1I+i=Hjw!p)S%qnd;B8_@k-gzy~x#y zVf(Pdh&pzefd)bml&kJ$*scZTfL44P@uNb}yLuCIf2Z%0I^*|iH2AN_qGx;z??qr_ zL}(DX^F7o!!Hr;i-xr<;F1}S-&6SNC(47pO=QEUlU9Kh6!Rn#!XEP50%LnsqYR7Km zph{2X$7~2K-tM3OBIxNeOG5JyJ%LxCl~wwCt3|Dzwb9B?Rj@=YlpL$q&UA-oH;gD> zDNEGYtA!6kF_pqInACS}hv0Ew9{5?Cv0Yc1V4nD3h5iavG6_0N?n(b?=541W@_PaVcT5hDiqRv9b``Dl3}>))(NVmQaW{UXHK z3mI}}XX~BDuF_Q+S5CdW1ltuR#zzWd-yKvzraLvK^d9?_S-z>BTia^yEaflflkaza z?<>A<_*!;vmu(k6=PV2w0xvC))Z^yHSlh-aZg8`H#-+v4lJ}Eay|=rbKdtM5;#f@i zCcLliT)WiXa@$#7COqGKU+*dmPTy|Q2zoUi6pm)yk9mcO*W#EP3bRB@s?DwLjFk?4 zOv*o0=*nce(Mt~gib2M2pyM7jy))U_GmURr7ep#N6()PK#SL;Z7MuSb5?JZe&*-4W zmnTO0hFn{^LgBe-XyiWeMQn0xc$}eEw}j{=pHZh|peRb3-ZI_iHD>g8;Snwi4I=R| z^4;0!^M$;wd}W*C0r3wzN?ng%bH-w}ImS;EWq@xSpK3+>LcCnNh7E&*4ni;6Bapi& zvwkie`qjBbP=SimYCz2pAUI z{ogxU_S4g+yGt`kI-`b-Zw@{)qmVyb=}y{=+}!pc(t~+i$IDJuH8?~#K5?ldqQ0QI z{0vouh#He7BkCRPe{Ox=*{P7GT8cnxG@gsL8MF=L+d*UnH3Y%xvSnDpebZP=XY(xo za(iGqp|{a=uv-c#qmp)>qSIh{#;~_Eo>rSyrmD^6_|$vWWh zBcrxwjd@Hl=er>WQJl*Qhu9L|(uf&cK0gPl9=bdQ|CCPGKh>eX3&$#LxEo3@o}3Y> zZ69n)+@A+i62@854i*wyliZf*RjJZq8|S3&b-H%$E*>fe2TyM-(ed3|i|i$3(_$t_ zm|GC;pGfUkAqG&^4XsQy0`M+f`6h8roz%KASlHX15cqnvE^Vu8u4D#-%SB9}SY`BW zXuM67Myy8_>@T#Up;w?23A|m;YCok-yxad_pIfQ6wsk%l3rnD{E)aPR_T*dc+AHuq z*Li-~af9_aS|v_WrQSGGZ@1{li*_DUE=iU4{i2K(v-O!gjB|=A*eYx)JeP9Wk}+*8 z)9R_O3B$G3R(iGda@UURc|Z^(?#|DV2fx|phVNqduKBsqfA|KHy#cNur(_rp*f!Jl zv}QwSsA%f@nR?gt4sHiJ7S^n!qO!E+{;HvO5_W!4X4Jh=w1g`62YsTv;39Mv=p9!)``n|eqnu!$6mPl;_FmL^IKQd%G1k+ zPwm#yP&@`5kE6c_hSG{!#fj%;&-qe!_#{6ypMOSbPbX=1^LLf9npTKcRcgkzPp>B6 z8W68`+0)kh2LUQq%~i~l_;u4vR1W$YYodyO;C(}6PCPLhR{xCpCvrjYDmH_kdRt1Y zv~$o?$nD^_TdpiGuv0C1+_ttjtLpUso_IY-bgsSm@jk2E^7hobx)s^SwllVZW4U$Z z856`i$q^HgPQjqhhIz{@a|Dt$zuSNE;h7Yjd#K}dP9i1U$!GSc6LL~U>$72&1?bax z`^=Ok>1lVE9sDMyFTdcc>B=c}-A~M;<~V?SoC@t@^A3JYW91P*(0%%qGESZ8?kn?f zk`3uiTvv>q9$7B#p_rHV^J}_QFawGigtnsjH)N#*=N67=OZW@kLYCbV4GG5!4h3v_K{_If1wpWz2yIpaH&=VP{a)uCIWl?THJ(9An4mPe*cRlk> z#8#low6WIbuH{;(76A9{Gu!S#nfPv)X=+vR4DsVt=7>F($kq9Xy++P8DnYykUpy=_ z_gnLrzuQAW5&83P18GBT)(Ar0CuXXx?ZvJfMs`aV4`B_B!gv@t(m}oeOy4ksZ{KdQ zI}NY`s_2RHzivUnMmzV|jbgAaxiE%S{%N?*Bmr6}j+sii4y7sLDB|XnDB5DTzbwVkOe>nvqndn`F>LQ0X^Qr7DxN(4 z9g=$}>24Yea1hMz%U-xnWZv`MTY0k-j+#*~Pe&)ukoj%#I7sSx-uy)8pcv#4Z+hr( z>9$Nx!bwqbIc#;~w%UTFS~dW;zR=gFkRX67b2c!+l`%p3BU<}uSI$MY@Rv-ww7qR01p%9Qq}d>4|EosvIS=I#>!+YOn>xwNABv8Kx%2{*3vY@W+X-POby3SSF0yU=b=c z#o=fhm@1fOP?+2*u0v%a7wv-H-Mw{_Vkaj#oD6RrN0yjF4r6h3BrlZ`Xxp4GokMwt zh>m5liD&@Z&g%I|zw#Xo+XQ{9-+(K#b|)wVBdR8e4i|#i%SU$UiIUo2KwACk=$#mQ zFe%62$$|N_uq1XjtGwmn&RNQWT8_&0{JXa;Za1yme!)bAPLUTAwQcBa(m1mZ5Czy7UuZ|aUF?B(q>nds<>lX&DUfp5PgvJ@uyf072 z<0luoi;l-&_q%B-Y#&D*y|a4Dx+arv>|SmzXUiNcT#c%zJ`ofz;PO`m@9*l#kDGOq ztY2;OsiZJhA{_Rn5Nj^C#=i%dM~G$~(Ag)LfouBD=RSQmlG(^zR_Qs;U4DZY%H1q) z4%4(ws$fWUp3TH{2N&*_*VyiEh|+|X9Qjg+q=&>vb>@@Dvk{*K!!-)rKX{J({3T*V zk9UG?)HWEB-(+$G|18ZcPbCgt(D-)m?6OL*Eq%56U0QxgKr{^;{;Kga+Khk}H?5jX zf^ZqY(hfeee~O57c(iD=4*j@+&8sE!o+5Hj4##6s``t7#=tlZ!J1Xalhi%P40!7U7 z^ooYquJ-PxPYl{MqiD~v$#h_eGAxC_N~3sg3ZbYelSkV1)o^;uoohpCi;Cr|Y!y>8 zGR`leLc>VX^&;P^!nz^!F(6funEWYj zripu8CbD+{xi0th)K;ew9}4EO@ihjdWL<48xeNrW@%L8Lb`pY6xG2Onx={_8&39 zkxpRm6;oxZDkNqgeJd+svM_BtJ1UN}XPDhJqy1&yY?sJOn3?13gw`f*df=2xX|&n) zuz{mqV>pqious?_=j5QC&+`|7Z}?4>_is{$2nPX}DfOYHkzc=*?4p>6$rqWl;Q*uD zy9k)HQKwd7l2cn#gEFv#=1n~YxTcCK6=PsYa@w$PZB3`O-Xl<^M*B|qDutJ4lFUGVm9FWGj@@|l{x@7#oqGg}^? zb-KIgNWE4L#kWx+NNMFF*E=wGUCs52YfvAr&F{}a*!xbdS=9NVGAy;UWqKAo+>dMO z<`-uj;c@yEGOYL2<|Y|woPRaZw;g;+@%V|+bC{~xxyBL1ipm6KT`BeWXP&r>tJ1r% zZhf@%Wj%2DNX@#G0?UFIrKWimY0^^0#%IkWO@x{B024Ol$H}q1AJ1rfAN+;3)+$pZ zi}Q@F=<_Z63I5l3+*8O0U}!ZT{$t|BuGe>XPH>z>Dw%?U!jT?#gcWIoLa2#vIMPy% zb|}rCX|NN1{j5ms&QWlOcYjBci!h)Uh;|4Vt{NGOl&IjeKut>}-&8qXo5e(_q-Q+D zZHvFz}Gt+TWVS>s^*L`&}K!Le7njqJ)uU4)a*VC4jDdj zY(0G5HqsEydu>kIukb;rGHp%H4QM;Iz-0}WA`aCY%7?aLo>O~OHh(+Kj;_l$KA`o0 zM%0c=v{GUoC5b7lfXdubM_B*S6&WiW4lqrS*F5`;Nle;ao{oZrU|U-&njVem1a@R@GJw`IIJiBMM}lDPE?T;ymNEs% z8UdhNK|~-bP(DD@zBiRjW@h#>GPl`~_7&8@PtvEyBk%{UaYcVq)Ck`3pzGv};YmyN z9nKuo5;`?75fDD1IcoK+pQLUO*n?KBPYAegH&T2PNuPG`|06?{j+|Zd=8z1gB9`1TZ#v zO~mV4atQy1)9x5?gtVE!wJ~wGBU6CrrAN8Ckq2Y_<`1zom@cDjbFO|nP-;IPXg225 zf|!bAW*W#b?7k)%J7u@BgMz8raL8M&YeD(LXAy~DuL_o((yBhC=1+Ndtp_$vmP)YW z3_bon7A>)&`unXXm+thmNw{dAR3{pu@1?Tbid>U#NA=Km4qEET6Rgh1Cz-0v;jFb5 zevT4tAOjJ?;&evJV~4{!tW)Q($BQKnqevS(VOmFVCjjyDI+hHRM`QJ7W9F%$WJFy} zeLrd;#RhlwH+8!8oEc$KY0A+v%wf6TC+72KHB*|}uKM@4txA)q$I|BC4A*O4&1*-} zy1^?FVwg@*w9(B%?P!SN0|){n#7SX#cLD|3PtTJdX5fUEE)-I;uGp_&+U1LC+&@QQ zB%nwd48*Top`>W5DmTvS8%HvX7P4`xhHJRXwunq#8wM&NT?wxKRQkzshvWmqVQ?cy zejI8+DQ!&5c8SNX9%AWd$)8PZ{48*`wn8$s`a3sX|C?}Bi7*qMFv}~}pxx)kEVDz3 zuQ5F5NKbWTW&y^&fPTo%=%?x$JKplP(-wk5{2|%d4meY1EKb%m#vRcJH_=ae&!l^; zTU_@Yq1G6azw5^wCZOZBNEL{MJ$YNK=ejYfB+^w?=z|uX5bquYyTPc8G zP2JPz%u#qO42kW4VJ@ zNNB{7%TUfc@J3)I@xji}*Of;xRI9u0r};e)=@1z;9X7PDC%<+&Y+;0J>2h7B1*~A9 zP**MIwtFAXPG?m4K5rDAKke8`c^c~?h0Sb}c@MAF=x-b=Cu8Q#?K+i@OJ#|e?-;2| z4vh^f`#cnh!@1eh@j~EsIGW4oU(q%^#ypN7)85m1U5B=yh5JPPdi5AQ9{YhsLY2no zkm(ukNQB6k@e_nf2&sb5jN5n}AA`GyS4kXAg=_-6SC-MTiMN_+E!soe%xKmy+L0a= zD0ptsXrN1VJT{IDlDb{x^D1)K+6xnGCpL}Mok~r3_UCzb=DDll{|&KyL4e0Gya2vH zCC?;@$c}sx027*`Ff$V{Ob6|F7CLiwe-z5RLpThNnUtDp5ttk-%#uPfhnt1RckWnj zpJ|s!mY5WJ0|V!<3R=?Wbws|6V4|Mz5^;$dDKNXaRV)(7#XRlTlD>a^Cg2qkp^zdL znMR68x7+m7ap>}7ifs92j&tHRAt+?u z_KYSw2Vmjpl`n5EGsWT!peu523VGco@XT|U9fX%?q+C{H7!jA|zzB&< zm)cE>ZQo`eo2Kr6*e?6#B@F6=o0$x45s$@18A?hnhyjBpw#pLV@kH9NfN1gGq8H6M ztQikh+w)q+Oqki(B^XnS@*Bq6>!Sw>+Z>gMYI+VD8NF?_qVlbc1 zruP?R6hWLU8f=UYaA5YtBW8Y#CQ$^gs_FgiQ|32{sVX_gQI{-sw}ZA5u1;SKcU>Ua zFj)_J_(KNQ%#7DrjnTrmbMjrRw$(SM?b%$TjU_WtJ7M3!c9xH4^Q>HAmIuZexM&+r zLek)*EncFH!#UiAFISBzGxJRsm;#@?VUi7LC8j4p;Tx;Us#9E!4J)AN*)YNUnP{x` z)rsVCB?*&D?|yr|>SqPc{!Wh~owxL?*_pn<##0%EyCV;DSb273?a*J~2fG_2!ETfQGoXc2E*HT6}7 zW?7TzqF(|#5WX{HC}^V6M9QV|c#0~D0?l9G_2K#^?)h$NdFQhKIJ{wqnPkzK2%aa% z{P;Hnr6V68 zX6|xblSKCW6l}lCZR?QJ#zZFB1d}g7Tbnc~`dZ$YzuuzOJOG@u&3`1XQZO=d#9Rf# z@_gTeroYXwbN)UU{$wIi;CF5VLECKEJK z-$6=dNU)SBMP9Yw1XG*E&eofRQVF6Y`Z@=9o4W&B&g1J2m=bey?2L@*UA>0OBc@L+ z91?@d@)dhBFz|VP^pg-@$I_ zpb;AFU&+?VKi3luPCmH~VvfECaG3a#Fm-!@K>3^%!L7p(yW5U|n0) z3hC~vENY|zL-g8i6x)=Hlv`%PX$`gy&LM3k<26q|1VP?NG5jqngT92M1(hruIz~7h z7+vLYp}uoNs`Cv>0qNCA^!Many0RK)yXd*i(o^uk%|Eg?VN!AK?KF(&tH8C2&k^{v zH-(Q;z5W6yXfZp}z!c{&Fucx}<2 zJ3ooZ4wFfU?LZ{W4?k#_3PrChD59m-zm?fK58}mhpJ$8qDA;^O2{A7=TXWfHU2>k0 z^4HtaZQm>)j$W^7A7L`d;P|?**73Y+ec{fg>t!Y)YbatOQSGQJ0&}9=sVvI7v6Bl$%dw@QTjXd$zp9{CnFNv9Q z%`-Nh2F;WtW+>CO3WQ*ZVa&Zpv>`x(3>#=qsBYS}a$(oz%8e(JQ2#~24 z686bHfmXyx$Q)?oV}~DWE+!@bB&CEp$BOzIbz)}>M7oX^;V8)qHSOAXGs3)aV2sES zk5z{HBcS6)NiND|RPv$xa*sfYUAeQwv=vW%O~X41{6!&`rqiHXQmubnN%Hq|P$YZ`yn z-_tb^IPp=A3+0*`3yY#WZf03)vFT=!=)$@oF1$XDXS$h1$-C@{9WMLNY`Pu7lN|YL zO?r2|b1|a?Nmi4SCz#9CfxQ~+^Cr{B9Md!91@)>uxdz@}J4A~b<|y8=UG{1_WGv?= z^}yW<9CzWHV#-8yNLW*6Ik+%qQh4)dlG*cSn2mx<;wGH!1fa2<4+zt;Ij=$qH}_(` zjs!%SP+Pk&Q6lTUd?(-9O&O3X9-rP4nI+8*x?ee9TfN#&O^`djtI_9hdYS(5@GhBR z===6vc;-*#1LH=EOm0bBmkSVLAs&O0V*pdf{q7W-mQ+o-VM4#QaN*XOA;K@$xsH>o zB#JAMM>&SA>TY64Rv-N4q3d>HP?x)U|0iV->q@$)QYM;h16fU?l^Q=U8o{un1am*X zyv?;VUTQClV(`GQox&)gYQS9yfCNzU=Po9xS*(TC80rj?9tQ>IOa8z1M4^>sA=+4i$ zoz;8cRn*y$S@pqbI+A36VFB&3d-5fPZ7V=->Qx-ZZ>B|e_aS5Cb&{w4F>%hlB9{aq z zU%x6Xp0+@Vg*Kf<#cQx=Q5LGtPYOJw*=vZ4mX_+pt`eHHxKkrYwps{7^3ZB}J%hpZ zb?#IVl1fi8$S~*3aM<=PIL;GGyu?l_Y;_wg-`MeczN~KmfIAc0+G}yZuw=WNNp>I5 z!mxs%c=r4Xg9|IkmbKPrmCp-knH*mo1AM-T%%IbmfrDz}Frcw=w1wJWBG9Lce#Oyr zQs;kO8A7%4m&2PYb*oCl!$3G>I!K;lb{3aNnYo~dcGE#$qF63(8M(Sj9xCr6g7eTy zlsS=3zV^FjXL}i%tkS*yy)!B%*N~8a?s|Sv9^P7JKVJyR2hM!$ViYaA`6!$+-O2N! z+tpCt>}h%K)1hMcyb!r#S*fx&sZ%u@Df!kEc<4Rr{ zJLhx-qxoZNEWR}qbCk~le3p``0xMfg7_ITq2^WR*VbLXNErdhXQ{ln%^iBqW11*!Q zZGT*OMw{N-b#gIhs%jjHuG2GF1|8C~``pzZ+T)eGG3y;fW9Q<+_r^)c?Y~lb5jnz; z5?k`0TtdYVD}LP;pHlWMO{HZHhEQM;`%=3bx+iovX>=Qf77C;07$QxyUjIc2-ZCte zdnYp`>0ZVAOuyxfU@PN-*CjF(q}nBy(^sW_y258t*>$TG>D`KFB#MJ(XMUvK6O&A5 zS15@imWy#QT=lrKb^3-Lo1r^ZU+Xnj-%LYTV@U%Z6Eb-76u9$F2!tiA9vTaKg*})^ z8=94sxxYExb4x57s;41eV5{lPBnVQ}<^%(?na;PE51~n90JfeY?-ezzg4qL#lJGmA zU1y`RwG7Im4rrk-l=NU{`7U?pA>+>%nid4vK_={C!jdD@K5fyl<&-qEzxUO<99Cuz z)g^25r|N?eJCqV`prflJGQr(z(2TH)WJALW#^V^r7)aehH?{_AYu1ru{ty#EU+DWr zO6%{1D=71uX2)$=(%;(Dm7PxrpSFALJ0u25e7t)(fX1Z3h#UkvO(yI&EEF|+Ag|FuC)O<2+VBMekK^} z<9>@zW^1nm5c8*;SX6nD#-EpqRG8oPcAD? zs`fBZNG}8U@*-AvI4ek6t^CK0TBN4MBslSko^J?4XGG2F)9)v$GEDr8>h-!T^{o}K z6B9CXcongI-6{P2qj$zC+}3~4J8)(lRq%0p#(%3}FenutNyprqH^0rDz6Nuas$Ak% zYjCVCpCh(EOUjgrLQM&33WuQXA2Mrjjl;a6q1#VeGF9P7k3xnhG@o$@8ziYVbEu~H zA+EnK4D*7xXUkmQvYl`yoIa9EJ+8^>o1)QXw-B+X))k+U8W-Lqe}rrl{XJ#-l6+LP zXwIbbEU4c&qj-9^rZ$Uk%V1c3?P29tUmsRz8EZ#ON-3^dlv#Z_SzpQCl>;}=Z>JRv z--3S9xWh{WC)%#y4)CAGFf|;*{Xn`nO4HvYYV4i{*kTJi?qx7;So{i&_S_fjuyl{V z>Zg(iPQA`@*ldlYBb?Th_+H%|0@t%=^3^#+tH(xa%xL?HFvx$U2ko=cc}?V_eAc^P zVkxsXpPUei#vrwH>u|c+EDVbImJ8$%ow6(#J}Lf0Fm7n9JCCJZl!c5?u0ihM5uc~m zn+r0fThMZ=VQA2}$G~ce+9FON|E$8? zfSqxF;I&rbxE>@#@D*4Fm^57z=h+n$Eb9ex_Y-Fb7w31O%b9MM_D{#kOSFK!qFKk9dq^0@`Pw}w9 zPwH^P{L)k+ZPX$Z>Ve!tqA)tx{O(#QB|9-#6UE_DN_NP?P|9ga)C|RKK*!hBt?r0V z9(dKI0p-Wtxu3WUu{}NX`KzHH*z-#>F%|TdgvcI*ALN}TJ6jUCT&&bPovAg9sLvO} zQhF;`v`Q+BhvyF~P&=FhzIYQ^waK0$Q5MSg6LG|1(kkYn473_i>Z_ZAwLxfoIszY+ z-=?CZY-tO$ANp)q9NU6v-ECvqEisVGFEO@?AbSwp!axd+O8v**yM-66Ij|0Wcc#)u zwXq3@Y321S2WmYRmX4ZuW)BR16VL8|Tl%v;ay|d(M{2K6$@Hdi(7pi)d~yj5(=Fwo+~6tZ%kS z&V}(Wr0mQ3^1OIHnvVe_Tg80hk0mY&uJ|~l5|3ruif<-BeGWY^)Fcc!jdt|lqMhHE zE2fB&$PJLTy;(mS5>!D)mmN@jM9179%udfg@CoimsRPyP$K+jPuNPUWm24j$w000s z9=u&+94Bg_lkQuRR*1$sk5H=tb0T{Q-h}Sw_UeOYzDx|iNsGcagpcuRa=nv%jfhA@ zL2SHYCPpgjv^W0FH!MO(km~0#z0D8eZAe}gpN^x%l`y~yqO5QUfTVBS*Fx@Hb+e}k+~_DcF=rD$7e07?ZslD&5olk+9UO&9}aY*K;Z=#o|Ve?kTW?c z$&8qVC8T`Ub#yrTbH&D}YK479tZ{KgBV1{o+ptCX@+lG7*S{+;XBdh}SiO%0V|9qnJDd86O9(ExuZPX%8}VosZG>oJ|MssS zAwn3kEq=Z}FLN^H@eOJF)H014a;lco;CI~Wo_1hRXsv69&k~)S@9u_Q!*Ns1jyQAU zFQe409UvhwP@?aAtjK_pa4bhFT_9q1L^E#gPmh9Oen2U@_YD}*Z9HYIA6Z&zAVMx; z1~fs&HU4^Ht$I#Thg7XE4Yw33dC($w1PoOjaud-jiDRCYh;xtdL#;U8{S~21%b+ zdG~C$SmFhCd~hSdJV}%8R{=v7WomLO|8;*r%KsW(I$f0Gyltn`Qid!#@L)6Tpdf{o zvqH9t{&9)r0)H&f+=Gs>@c}V{IkFh}MQ1foZHJOgIb3y9H?T}vql}w(UCP$jX|yUq z1-@ct#6Kc>Q#^6oFAZ0d@0!APUwgA-RDoJE%q4ea`jo#nzIXNXk9gbC@TS(7XUEEz zWrz1Z>jjNtkgJ6cuZ!%xM+HP$I-Smi6zg1bX`}>i5J;WGz zBof|+LkOaH8a9qFcNWVG`l%31ME3!7mhdtyMunIXt^+=;c}@Z_tf&` zW&lFP$72pv-sU3;8@|?C3Kdt>u~mil>E68CX?B^pY4LiU_49RETRko0azq!XBY>p! zWT0MQnCbG6fm{)+%<>Q~VRPQj$g| zyzXi(NjA6}@m}G%IyzpTEg|A|_(+U&pG=f07V|NQ`8kQ*<;2Glw?L zBs8bdM6?V+@=YQl3Lj5^e2&XGwm@Sfe`f)I-+V%!;qbi6E{};Yw&18ZtoKvKuUrH& za|bUg#U;g2%tjn~svGQ?Snifn(=Z|lIn#!i!Qky8KX$N1je0z&gR_QW@ijyX`h)U0mG=`j4DfJ=H)t} zs!B-k$M1d>^05<;eORcjK=%0GqKM03|CQaIaoh(oOl3+Hsz^jq792@R71E_1`VhH> z|134x?V=VR$1V`xk1sVV9|LP#)OF>tGFKR9kwim@54T}0^#1c=g7OdO)nR`|8Es6| zF+5*hEG-vBy#1MBvhb(plxqxzK78lr%4^?!$Luj3%w-r_FE|~WO-v}RAj%Ot*q)jt z8F-zf)5Pwt8t*xH6-Nt#%xX7153&&)S;^nYC8H|9<1Z0qYy4lU{BJ3){&u5*@qEMEcwIXX<7K55G zU6j?_oK-~AGZP^)O^k#E5;2nq3nanuWn}DO8K;+Qxl#rsZgUwu)OV=;ULi;jzc>~rE0W&Jmw6SLzWptDrC1wD;U-=IvqD{Bu2Qxo*Ku* z&S+Qd?wACjaF7v4;0d&YG|~`_njsoxY@$>vONY+h{gs8&NG2C{pE$Kg@hD#drj#h> z_aHh%%v{t`u!Y1dex!UxRbmO^!LNtE!6}r@Wsw&&FN! z9hWX$Ql3;4Op%L3at$ME`P1BSkc1wH6P^p~n7*+Xw-pDAsbZ3ODE8>8iiye>W4btZ zY44U^s+R~;FL?1@-0RX+^^#sF_Al?pbZ&oce`oZ@8eClytfBI2a8eC;*cz;_!8LJl zn^N%d6g(pZ7p7omN^%NiNWm$SC(rAQp+u8Nu_B7@il(B8&+Tj{Ax0?d3&(#U52;2S zoocc{%{GKQFO=H}M@@DcJDLc*JoFng0(d-d481@zjkA1|wG_o#z zm`Q%^A7`;8)+qC=arg!0t;2slt|%5(V=%|!gg?K&(QU|2j1rCboHAp$Y?>(QML{qp zrcJ4GmEJIG1S!9lW$dIs2pN^(n^vyE*W(-Ju4Bz!9mlfIzsVYW#J^<^+i=6TlJu=I z^o_y5EmFQ6y175v>Cix=`rL(6*YD6oM4LCi{=iG=g&Lhk*73r>Udb!q^rFSyLbDWu zBR(CEHIz@@i%wa)g=JXL$}z7h9{^4Ck&?EN@r+p*h$258SgC^8S{#ps5?wFz>o6$D(6gq`Z1TJk$NrC7*g(US-I#MVm1uS_A z&?be@CI!BacTPr|9E1tc%+chvUf9xQ;lAi38%t#p|l# z$kaGY$8AmA=pZRc2g!7XbCQL?v@r8%$6D6_bPtg30n7}L?g2cYb`M~B;M5vUGDXXq?M;1sL~8nQM<4LCucnXcgYG`^_&!YcRWYkJKY|~D)!QGz zb&p`?kyzOLBcvOew;#zQNIn~o{PM)DiBwu5UhcxkmFA+5UA8ya>E)|%<_(FiQ}$Iy zj;ykuawXo7Ddcs`Yb(sj8_iT)nXUAPe1)2Do$T2NpDcvPv*^(j4lz&coE>|0|{XB>JsM6(CP${w~2LW0Hlij z2PEm_{dXu^zx=ZHvr7m6^)I$V`HS*BK(mLGnc7{N)aP~B`|V=*x(bm z;{+}_QGbFu0q*z2tG5070n+b}GgR&ONLBUw@qV9JwLd{uwcn}TwF|oHcXjSszKh=F z6{~iE&)8+&xAgRu(^SW4TynbpG<6zQIsMpmhtTdVyK(*Q&fUv*)4Pf9*{#_b+_YsA z-b9?nCR`6Ca66k|_@=>aTh8LMB#JnTOU~AxrOra@**61u-AlW-bWy`>p9lLeY_J(OVKW@v6MYF~O}%Mq&M2q@K&nj_Q^ zXgu6fS@a2E!HsZ`8x^{491p=hPQf3>Rd@0N1PQ~^V2u>rxq(msKnv(W7kGT%8-~R)&F0p|2swTdHma- zrMAcxhw)h)%}DtMrs~$FfJN48*52~}apot0QIzK&I6lui*Y51>%+B7M-OXlml59eb zO-RD#*pLLWN!UPuD97gDk|UcC(QqUS0-_+GD8vI1Q?XVNDc*`!@vd#H?U!$Bt!=H} zR;{hJ)>3OBnf%_F-5en8|Nr|X?9MyyI~SSf`8=P`=Lyb{cv{cvg&@sYbgE%=(-KN7 z7pPmV&8c@6Xn3ub*XfKir>R%_=BcM&zu4WT988{5TZG0h3jXb#atv=G-2 z>xo|27yMo=W|WcIXf4XGt*&jUC2JK|pf+G3ao)F+JtQK7T73gadX#-nj|WjcPr%dU zA$gC>gGdjNq%5dxUvZh5lE`tNiB}f*o`lWR#FH)_NIq8^;@x>oz%;F0s~oVlXDvQp zZD4KFT9RMuT8qfFX-iwWyOp)>o^Grmx|_OD_j6;P2CvhFTv%Rq<673;Sgi{+uBjuS zfB=dx6X^t5mkb5TfDCw}WP8EV;8NsSnvcKME3Y&^ zrRXF7*!AoY<|xrh93uV%P0$6K!2cXnJc)z3`wUbN02NPIg%F;4B1x}oo^JcP9 zBa`?H%3wY556FRj{9Io{BmQV0GdTU-;o&3YJkKy6w;bs@bV!M@Lp_IZI3H>{gbpdP z_mB$@*6cqL)QJtx;|Ixn^2q-EN1lgFXbftpwhMpz;xDnyThD+EEMR)-=_OZt+Txnm zuj(29WH82+%tYwC*2oom z4OZLueT*h`RSC!9Qlklxc2hH|M#bwbb&Inyl6@p#N3okO-s;NL}pGryxC?n zjkDOaZ{PS^70q--t_oM2W%GU!E3six$}qg$SGeTbb*y01k|b@+3>BHQg*ahFX+dpK z^<0aG$x9VYj-p%J4(|1MNTXFp(QIfYDLS#nBBcv9A-BS&=Y-RebegHT-eTowO1o$} z(jpQGjRGma{5v>?S7_WZNoMZ{}!{)fh!hxZJ2M zP~d`}-&7JTcR0B0Oi{8GZCmwRvoo2NTpVW0H3dj?8c2K}==w+=u~0f*mki1OYMCw% zWSloJtl8qadrq0Kv{viTBDJlvvC%?Hv(lX$3;MY@LDG0${_Xj^s&Gk;-fuIQtZBiH zs@j0B&~De7Q=Fv5m(_GcSLvs9MvjAH4HU+xOWMywdrVDvnZ+&j9Qu`x2KD& zpfTIZQN{M}E~P0-v}tKEbN#w3pV6e_bUKZD|KgHG16eauOi4N|mX6`~*75d04hA%> zp4P6q4G%PehRcQPK7OQ@)R214ZNB3(N`c3}Ja3(5G{5>Ixl69<*;h*o0@;uJa9&<1 ze`zu$TfhUS)dS^SX$Fm6V@{ur7Tofks~D0N%{cjudN$ghm2EU>M4e7D-?pN(^{RsW zETdDW#qTWqgWXnLMIJ-ZT9MMMjbS#=>NT8ebm@uttkYqydEwaWa^#N&qlMf2=*=%G zwI7RHxC(xFd>+YC0!lN_OU{w~^24g?J=0rXH#0lQgav|@&zZNdv#g-D`k=#T(37V7 z17+lWg#m`5w3v+4`e=&Q3#`1jCajE-1#%^Zom}5jSW(fntKH_+= zcTf?lgV8}09)!VUJ~W8M?BJT+HjPGsOe}I0vkq(C03Kp&(910ha<~f3i9Z7z48ACeF9yZDN{fb1dr=TxOna!Cf#l=z`$107>$G<9s_tcIH_H4*qS!!k zS!q|{eYn60t{)bL3`pfsx31m=Y{q$TypoA-eHwK_F=b7P&?V*rCOvf052#=LxeRx0@d zWhlI6bie%g_vXwID4KTUq~iKXkx4WXoy3jAy}>op$_C3IQWh;k;WEfC zt1d%jg-frxYS!Q^h|G%4Lg87EKdX8cnw1q8+yRjt(H$tf1M+uN??5}c*7KQznGng0 zW}|fz#wWujQo}Sp(x_D@8IJF`NPqNDpG7VovmBmDlt0Ap8Vyu z^?5e3&ykW0Tt&sRuNS$stiCeK#&h=cJ8qknNk2Gm^-KlfG;YV7d6XfnG?r!gV?lU3 z&|#oP>gB5^`vdUsWTTN3B?Bj-WU*d;dw{li4HeO4w^!X{6L^u~w&&+(S<_rIx9_bY zwG_U#ZOyeUdPkZUGot*lP3#h8m{?7$BYKEyiGJcP;3dbjqHf!?NG)Qcy3Bbv4S>-x9$qdt*gmNV|h(VDs0d|@3r?|i>#@dYuBu~R+DO_cOAZM;rtf+?Yrmg zf?Z0F=~5<@pAhhosha$znle2XO6Ys0D@U_6xYhWz0h0`)E>eDKNarAsG$N@qvb_Os&8frv*4Y3)ynjc|rIJ%73n2;SKQOONIa3 z{007L{_~&D4jq&vw-H+4p5_21Fgj7k2T$HL!pS(KVK(v}!-!Ofb;4%mVPZ3J5Ahgr zmY5sNF1`_px4?~C3JxymjjUh4YSqDOBAX&8($)2YW5tKz@WCfe+z=b^PcN9lwIeF0 zjw6V%yj=OHF>om>MC^==|MbEcdg2u`mvtVM-)|9b#Ys`vYQ|-xGd{|geBY&43<%R@ z29I(?Mq*1&-irLm_KHU~+AY(knJ^&z{inZxhy05v$*p53_SQqB=rOP~n^W+h{2C?V z67iJ$k6Ot9r{$v*n^ z`#k?XqIiMiG-l5%7L=c)T~F*$SWb$D$tk}6z%)CqiOvjzd!xab<}@&vS&M-*ol&pl zb-eGcITuF~v6S`DJ&)~_B##jmz!MJvY{aI`&o;n~|BsSR5Mj^`vk@A&gZqseTt z8ja6_mLc)I8O@lYR{Jmpo@5>(I*5bB%f#!%`^4XfkAj0;eS>`v?SpV%dmq+?ebA?x z`+Y$6eY)GHJsk2pC6*`}(( zqEehVAo(vV&3yM>GLcj=QR!5aWL=g?@U7=mo*tfj z_zp%e>9LxKTs_Cb*oG8aGn=#l?%`boQ6gXz|?ATng(Y#;^q z-f~i?7pSd0MJ`1XYM7#Q!D82UP$~Zc2QJW`j{aQ!1Q-6K-w%w4;>dtlo`;F${Ueez zGWt+A^z+AvIb(#Xa~eMRK%2>wK+7dh}iS1D=Nrl+ClqpmC(n zpaBMCR^B_=M1xIu!&9C22C<;gbJpy>)ruEIQPqmt!NQ7RFmvTK{LG&}xU(zClcedn zd4Csc^O)z~@houvJpAxS7dl^gX=b`ptJUHfVlRH@l%1Dgl+^bBks@(uY z1WRlm+sK~6Y(#NF>xhNKO5!@=Mqq<4Mw4Pms67d)lUkBc63&$*BFe{j)XqaS-@+qa zNgsDK6bqsD5LAa+LMVi%LalYVgSimNjpm|oF68G{=c3$7e>4({p!Ntob7ZHl5Oo{O$7AYi*Y?q6fsV$ zHExVd1%@U+D~6G7>>#>f6nut zL(S)czzpD!^ldq%T>RiR`457yFbwsm6`puup%Q|uD|c89DQ!P$biF);@qaD43$q~u zCBIfF(zIB~BLsF;a4TOZpNjjaRf7B(WeBQXssW>Z&sZOOGmf=RVk@zYc$|2P_$`jK z#|H->Isokh__Yr-4WI!P(g%PX=o*YdGzyml1Rr%p5gFaq;oItiwLb9iI-$Ys0|5_z z^j!T`H#ptkp8xpq!Q&7;4(-SBxyPH1qvPsTj{|x9#NN9G?}F%E(0&(QyQ}FgbeD47 zyIgp%W0>dyVjBccK>i7+9)_-M^G^&9pP0X`i3m&Qkz)`tJ?=^-}#KURnTRve~=viIAeJRYr63WZuKCc=86qc-kJDutl< z|IbQr^4MKk(X6LQjkkPtai(^9j@e)_w69)LQ?WYFP?Tjb*b2`)3V~pH7HP9$?aeHi zMJCyejD|GZ-l3Fk^U*oQE{PHtEnSdqG}^cbdNZCEUqRr!vYpM3?m*i|cL4E>{5Kq> z(ldNTu9>8O{}4<#MZWm($VE}SDDRc=&c%0{+*%0OFmci*2Q@5XcA!~2E<&@i!RoRw z3`(Ab?BrLk+{Dtf0VBSrPBS<#5#o6F1Bcq17e5{@H#&5@PHzm$@BPlgwAImw z(Qz8?u2)zRQ$h_tXNGn3L;1%zzefk%V8-p|UK+kMD@5 zh|kxNz~s0EhO@;x8Q(KZGo%tS;XcGWKzM$N?3BES#Z_FQzd!Pmltpv)8d%oiO*`KZ zP!t?I4m_5GCDV_13LG@e@Rn$A+sLPsp3E8f6#4%4!0uZl(a!z`w7-HkKLwrIr}^hd ztNhC)`O|==Z24}k6P3dDF=9Ks2OH%B1VIx6W82tgm~tYBYia}0OzeY=!Lh+?h-62z zQB`(08+o$xvyu8254ie4-w*wRgYKJvzX@*IIamXcnrICQ*8t_K3Dlq(waEfxO$s3n z)8Uub>hwd%M98eVvE{P*oLoR3)qlNxrPmF9nC;A1KI54;Ax%%ievCA z$8S?<6^^6jxB7=uSazZ@sDf+9CFnXnJ7roue!O@qQG(dXCdZ6YcEV05O*j=qEWGqSi!X%ZJE-TnsY5zZLRs2gmn{i^l{C@_TGurIXc5sNb{WGfEgiStH7QM@HYr zTX|l-L83Gsd(HGtoY8t(zq%khsZcPjfe+>ZGw=Moe_lL-xHv8S1uo?(Ma6aBr%uW4 z+4a^8o;Hhp-N`g>u+0`TSwWBEKM(7>eC7d+H)TXC(MD_{eib|%T@YJ3papQx_`n6{p5hAvH-1wWM~WqSSO9-;i3Swa4sW$CK@L zsJ6G*ksV*cjy%!&n34_c^;j|1Lp`3X*JD7_x8?h)eW;}j@;%Vo2IamoPg|R(%vVk? z-O$s%cz(FKx?<@ROJiy+S9{lSrl|I!6Z9~#P0Yk8BK&uW6%UPg(>ZOTSxqV>*D~3G zp1`h&f=pl8&?uRRQR;NZXQKF72VXMHqT$wmDhQvh&wo~VLC>CgUfDd~3%AJq^6w|? z(*UH1s*gF6MZQ0hB|nq!9@Z;N4DT<2WuvXz1pzgn9(YSd$NJIzqh|%7ua#bTangJ`df)eMkfc;S zw87nrr<)~Cr`Jp6k11%mZ0vgWK^!eXqLZhHw2rPg=`TlLuZLw{reH&2QlJ%j$HrR&W{kGEwb{Fi1^4;aTvo^8h#&zGl zXL(lqEh}Hfa{Q82*oc>$aTn7Um+Oj^<^Pr$KJhM%|1WyB|BcU0itBNBGF@`N%Tzpj z889h`0^{S!yTBQtHp9$u8E*)}8=eKX7#$qVXs;_k8bpH9_~gj8*Yym;8697&nyn}S z^5*_Eq$Nd1v(%#Nnx;>uOERoBt*2qx>u`hDCTVH2{LnM6%0pI4DT%Nsl-z)}S>jYeYA7O&_6O$?301FkrJs?pd+wYPYlGYg+ z{=>uFy&FQJS;uRg`q0C%U+%GZ?V6#te}7PGOBZ)jBYpQ(DVjy&c5UyQo2q96E?d|yz|#I$qEKe^E<_r+BCMCeT&&mEFi8TZY3Tdo+EyW`O>bz z4G`H7-GIUyAb&&k2DD-4-~osnh#o-U1CW29`T#nxdHs|sn%`NCi|FEo*~B0L5h6;U zFah~QHGzm%y=HA?VdG#UL>i-wDBOt4Wn-WbHNJDQEoVW&df2cS3Kqc5^^3I&7HAi* z-$@tWf4pW!aa?g+;>i0ayE=X;E-w<}aV?Uli7H4<(D;eiBfV3bGZXAzN(?S(23;vQ z(@0FIz&oW9#zD@jv~s-THts~^dJ`=lRedPZ!IW4U?=+geB^s2dCa*Y4zxyC$;1K-a z&Z7{!Tsp!uGuT)4{$_fw8s@OUy^U*^sqvhQDD>YK}Zrs z`IF$)@_i7K-+e_8UODrI5-V@W1IB4k`GfiO&Q!gQak!I_p>B>m^!ufuBrI#+fgKl7 zarTNuhBO<~*q4k0nzm+b*|CQa?Gi&VvAr6DU0|qdzq6U)+(wpS28J|(gVRvt=ad2K z<1cRIv#i|x;FSOT3&CnLuU)fp7uL)M3IvMBfx7%5KMr4BkgR)FF8^K;(j>+>im?Lt z3IzEpd4sw`@q0V>2-LAPmJC6clw9mvY!wB8=Jy3>={bAOp?#}xiiKw_cni|-Ur5JT zK9NUj-8SCFyt?34j5|h`i8;7T@qUv;nOc55etKP+nYnSXGLS1sp}`@D4ncUReF)_bRS%(|Tk93x8f}AcTYDQy@K)5{xfVLl!o2>@{tf-) znYG@txAynnde*!44Ac15V56cNCTL?K+$Y6$0@f!I;__s;vP5Mk`Vm-SPeED}GX;5z zaQu}hYQyHg6w{RiAXW0G?6c!CBrZZ`C?Z6f0sjr;m7I`@vnGrtDbW!%P2MBHO1&1t z_zu!=6vfu&Ad^F47&5bxGJ)YCN|-57_pv#d%k>&oGG_PZq-nHfQCp+Ol9bU2Rm;7I z63$TRdJ9vSZxRL0?Ok|Aqyh_Z!LTo<{CWEoDWtOpJFDcrw@6N`%C%_tkATBT^R?V4 z{{ttMUhetP%mM)Z)bRI3%HXJ-<#i!zjsvuU&Q{oWO=h|La9Q~?N==Zc4;XIcZkojR z=5W}ucp%S7kx!oz#Zyp|B>&*}&|xz#8TOqV0w<6pUj2t6XJtbrmOMr-0rnau!neU{!&w(BN1}u3sW&o_J!PUNRFDu?w@= z`7DCEUPVHC!HWf**CA=FqM<>PB!t2mpaL4u zV3n*<6p;o<4J=p4%yL++tS<*@xuTBoZn~ChSiaN&OP7L!%$bunEoX|2DaHAUPa!Ib zB916u2~IjLi4%e3onk~PQ8E#QIG9S{3bZnPYM+d}oNHj^L|^JhX3_7+JLHdX3^>uW zxEmj${6){}oJ!Q_ZS&VRSj;0!D83o9%bpu07FSp| z3W*Y|Pc}mYtMf%bH7;7RXx$>xwHjGAkFwRGYoPN<)9r4Wgi#dnmCf@TpS`fTWj559YpG)btqhi z7wZCbs7|?L9n>`u_7wX(d#9b6SJGL6u#if@a%wRc7K6ALQlJ2WK&IFVY(X38u>sL! zXhMDVlEq+aYO=xN5_=z89w{gCuV|Q0O7Tlnw?dtLlNP3G)cD3lc*Nfk8Miqj6QQXJ zA2m2to{)hhwJA=ZOQ*Q&ws8f7S1+L;m!qZX?06?2d-B>-{U7QbxldO0E>LhAP^$8M z9{qDv*Bz&Z@10i-xGKZwH%K%3%{xLM-yv#{FHUeT_QeS^3By)Z2mXxio@v_$V%AA67~RUN#zpL9K@mNyVL z!Y_kkI~|5l&rYCrCJUj!&ZeDcCsskOoj~ry5w&x(c6#>o^6BLCRF0sgxlGEzCX7j@ zCeu`W@*w(a-bZrB28@|l89X+e?YWu35Rk%X03an~c zMe?g$s}Q-WBQ{%EncXuRugnh2ZkkQHW&=4JpEbKMq+ButR0vvECR}sI+lsb7DhEeL+3))B{luVC|PN;);>ZphvPaWTcq2*JYhj{+PPb(S>xb)S%Y?5ih|md zy+VRzVBW6e5l|CORGgJd?4^ZbJc~>vUW=kE@p-iJIdR(uW||Wm)~|>5T-z=PzlBW( zy@?bY9zjf+xAEb>I}C!BmOhvFFWb@IGBR8KwVEE8=yoMNaB=-$8SzPr}>L;2TmSnh_yhi+V@wWaEo?Y-$! zB+S5K3YSh?rJXHJOJmk&PJc<+UqDyM5X zjU{tB2Q*2cpHB3rsL@&9;9Hj-eosq#(9p?~=fD4bT;(^8oo4T*KgCr(K!k~Q;+o)` zrd(;QW{U z;w2SE;8kE3*>E5!L4k{T#2+6R%kx#B^!u++xV={@+!>eYsrVbs7#DLBE`Sj(yBhi4 z=$GvBoLM%n+4wKWrI1F)Yc%qkq#I&R5DST` ziDg75aWBjbeth`u*>}U;D>rn{?uPD_TlYry#`dD{-uAtS3hafwYGZFNkb4i(YhX>+ zj4q(Mpljnn=!ka2I#9Tyy#pWE0UZk1b+|eZ*}*LhN5io&3WwXncr6TJ^&(*)!|e;M z##J_^)Y~kOz*S{Fu_DD_ z(UfK+!G|Tz6ne%qlZA5>xF7~z9EcyvTOb=e27~;O{NBW9$LIybjDMyPd1-!eoDSu z&gOY|U%qgNb|#x+e^E7wW^I<1qOnBh8hLf%jzuly#tav;hahysQ)5x~023oBi0iNl zA12NaFA=X2ZxX*Iz61sq2k(x)9(x_NzYf)}x4e#C&(wup|IXs7&C$)V&8U4dRBvwC zj5g!h&Gjv@vv5{1(9c5k*_N~DES^34T2EKBE7pbDyP&$Or3-c8$*!f9{e-^MAI*$q zqV`Ov&TPp#>)D{SY(&`JI;6`VB81 zKKIf~=MKMY=zp!{JKE5ucVXqbkoGRn@5Vabg11UTYRgcqr&Ah7YDt#HEqh#2{qfGw zWPT`xHeQ<}ahF#`;z-=rSChGJ zGWk)%LY8%FaIt8a)sX%3@J2nZ2{!ASEqbyr&CHuPtMIVE^IC(~Dwu@p<(R~3M6>%J z1`uLPDBEv82mS zNgf@|fRi*QVJ*r|PeDenL~2M6_*%;-nqoaHSz2}QrK8(L(a4_tCM&4pwK}cA?kx8* zOqx~F=|sOQ$}(?E(%puxlh>-=O0)bmh7GP+*#QJFgv4@zlJ*5ime!F;#w`{~C>JSt zExB&atwqH;J)`FYUGm0d_`l~nDhRK5VraXWqRbwVGiur$;7KxD+xSnz%UF_S1cq;IV=0!!xPkXsEycC2rAaNV zqxdL65`AO8Wn)Z$NG6Ji5HTz0D$QSt)}sMLHY2D+FdboGivXo-L&&C3XZ(L}u+^pM zJG7uxN)qNQiB6cpRxmS-vk_@Ba4#V%V4UJ5B$^@M2U(lp6pZ?vpBot3>}EdlxlF7^*VnVE0$dp)%Tl$mmGgLuwUT=9 zp?XqF>NRfXfstkM=htPXF|<7wFtdl~{JD~W)ANEcTiz=F$tYL~q$EkY|J>`}qe)ID zu=*dp_ZWehUktO3hnX|Pt;8L~oy1Y%cfmW2Rt>qYCv_j}+tRh*nyaoke0%J6xZNp+ zZg&fz7CgjK=5`ETw_i2i-E&VvtN|JnRBV9ihL#4@fL9t0WgeUrEsK?*_A-oiWld$M z3{RIGHBjhgYD+;+um|0L5DE?k52Avj&~xzU(StqY^c^NcXx$FjVHH9Q{6_NqKKv8$#j!6TKN862PoP@7O{t81idBS(gQP+p<`cH^g2K{t3~du= zP^xH*i9S?Dn#v2VWDT*GDUeB}k@lY(T$yAVGv2;RyM)jdXC*iSC`X>HDhWk$-jSixUVhus$-#& z#=3ht?KDYi?VSs{^RmqHHrZ)nI6RdtJ4~+X z+0D_JGj@e&M~bQ88A{lHHm_b|br>H8LshNb$cnscywswi1?jB#NqB1{UI%WWI-Fkk7nX+*duLA0(x&0xf%zh2xF_gY$M=Cy?wq4lCQ z>wbAPB{XA{`~xL~l^2P>l)l9JvAfv2nJvUp;xO?ragsOk(~S1ISM0&>+r`ym-dC5-vl6hx%xc#w{Cm(Ac4m)|))3FxT5*|>* zNd6Rd<%{%nB#_1zR~E{TYCiiWtqe6Tl%2RMJ#KI)2l!Ooq8?jRq)HTDf|~gg;zd15 zwF)thnc8rf01Q*wc)v2;ls3hlHcjxd^eKJJsRCnMFw;`nIPsc{RwZ8j6WmMf-%P!| zDvQ*IN_02b1p6|1F&V?CMIkh6R05;HYb(f|+g51BJc%={U)Ji#FrIu!zTHA=jYba~ zJn(LG_l9F8tgCHKSb5~rdtR{68p)6Wn$sDAS^fyez6b3EFVD*#jy~wKnnbICGV!Ck zXP*_tv$KV=A{#{`-g%bi&pHP-X<5^2f1kziI-NF2w`v(x{Go>Q4j@nKMPri3rxz_3Tp2Ym^^vHA15vtO^mOa57BZ9>BIEx=A-h$lVyQVcb z|JDG#X8H2Fr+c>INKC?b<-v8((zyNm8`<^ zO*J!UB*u#eo;{y&W2OI|S@at&@jz1oI93ID)Xn!22COpK)Jf5Zt zJ+2({IFOG&zxBj|ZlKe;{oRN$cLTyZ-R*=uXh$6fdJiDt01zh(Cs4bC4>{VQy%F%P=8EIwEm};*0lk5*0gQWE|WgvNtU>&;@B5m ze$(+q_KNBPZNh;h)eAI@HpwQ2)md%eJP^Ae2p5j(iwe!0ru!Q~_$C|-SN0}}T9!3@ zkNlo$#LK7rfnogU25aV%)!9Oz^jH5=P?Vj)Im{*U1$hMC`>P=Q_2S2~GDNc6YY|Mm zufhX_AdC@mOvV-GL9sg5&U4-81)MQYel3b$UqjkmTAF1_vXR+iqFE{naZisv;1dMT z&)$Ce??2O9GQ?N0?hq>l%C>(7(vUjNV%lt@gi4X>oK-x_>^AY6Q2uD1{FW#}IXWk= zd;Jg3n?#f0$+5>_A5agh;~gb^j(m5XpwpAc7ORm@tG4IWH(r;YrRP~uE0|nsH%w+9 z{Jv&Ii3TT{-O^AqIsFPlGCWUmS8ZoVjRrH>Jp{oLE5`=eXPBcnr}kkc`$OWV#2<-o zh%>=~ZMCslXs_+5MO0HQ)MD;htCY4{S1qdD)Uf}_BkP8?4xu5OFGEU|8iFA;)rNo^ zdV6M0EC+HFa+?EGj#e z`7a;ucr=~^%1?r^+EnfBPt*(Fgni{xD29|+QEP)5<>R4wX}r^u=sb@2{;3RPJTl2d z9Z(J)=b?!UPEcNj{;G4TJh|kOmm03D>a>sN0wO0_?qp@cnkRAnsY(cwXS{2IE6Q%T zg0G8S-Fn<*(le~-P!}+?Fk2usaD#ji*AKyB|6(9H<@zxk%RrnIil>`7E*6jMuRsC{ zOKnU2vZjt`NW)u14W(JzG$fxhGQ7dygxp6C&&}Wkp5?t}##iX~G4cq$4wdh7Xo}N~ z27M|U&@H{CyK5%MunWufr>A{?H&i9rRJQ!h!5_%aVkIV-JrH~Go?uNQ&(bDq@~?WF zRE67N)|=DnouON9%ee4tb1^a|NeoSAhiQ{jN73-7reZVWPcrZ(!Cxh6Sd)c1ws)!I zlEjUBukByP2!byE*1`XMR+OClp(80plu#;BVC^S!pAqCod! z=o2-MWV`&kvtJsgS@ggDk^JQ;&gwQi16~jyxNE)9VB)|8FLo{y^jZ;l#QEFbsgV!L zV`K0n9ACDDB3Vx0_;o8a6dthh$dZ-QQcVUKKM z-!+A=#WU{8Htm0*^mt}+G$r^;CPEr9@Q$H0VAk-kE8*70AR{sQ02vd_)?OJSHW-mx z9+5x9BQH=gcl8c|Wz0$LPp>!6^qCEUR6?D6=?AR}`#|2@;n(6#qgfQr5d1PHn-TCQ zbqZd3?uaCsSsX2!>T-C+u9fr!`!H7S!_QX14L*Ds1O)d)&U++=>trJt(0|V z)PlL0eW&HD%xtB$nx#7vVUx$40lvUvF$YW~54grNKD3`z-m}d^@Q{^IxW!;0U?|)T zsK_V$Cje$L3lN6cJ_MWHl3U43If1a002I*~#u@4~VJkbpg33b3)LQA3#j;h_S!p$e zF|B6#F?Y+`22E(>(B}t)dLo`3%faN8-jaNH^L zAUWA$c_m3XsrbGi;K8<@T$@2O1EY*qFS1xcw$JQ@6*Y!lTD`Kb(x0}+TP$OL;DSTX zGTYU0(&CeM7AU+WsPYv#ckJ`#)4e9F{OrO`9LYHkIgA1LFq@C>>Y!$gYRREk7-2MO z-EaCBf6v~Fi3Vm|Ju}vW(V0rr8fD#SO)A}}6^LtF+)}b@%uAKePabs^z%7g5H4W?5 zY|j)cg1ivac#hzR?nv**Zw7G|As?v-wkms_pRY5o!6S1E3S1q{Ed=w?7HWoDlgACht?*W{boJT8-16X#l>dr zt9{%5c)W3GuIIz!K4ZZ>rM#;1_G$P)3f+C&Ev>sdRu=U5V^j=X4Oeb-$s~ghxjT!_r}6>iRr% zo7aRAS&TJ!i2qfb9tYo0r+edPw1XVfhMum09I!UO_rBXt!ms=UeZmvhBGO1EJRSw_ zj4b6qJlJ~BN*=P7p|md~E%g6zjywMn?5q=hh}>yAT{}JoxwVo6?FRm(SfG^#v`m-* zIul1)`+~`{Vj~C(^NBB*gn>`1B^o6RginLJX249d1nm+jq5PdKl~Is_1aIFXRQRY~ zId(LwDGuRo!$+y`huYPwgel?_tlscLOSa4v9db`yJGPd=vUclCV-_a%IDtRH-=9?u zYf|Fu@UA5)v|ajVv@iBbn6`PF{ctx5@>Dp<;#cUGAHGlT2e=E*@AKV;8bhpV!`<+4 z)&T^G3RG&y)YFr91QoB_{$XX^m8%)K4G;g%W4}n|!j0e1j8o13UJlAJWtf4c;m1p+ zxRHUT7V|rXT(a(x5sM=>?Br%A#*_c3sAG48#fYUxt>@bcwn(#7zr*pLUt8*iXa)rK z0SQk?z6GfoBJl9luzMu)aPk1HI_zw~c+5ErN*>@FbCr1kCaA1DExwEJFs^HH1NQyS zirw+*l^njxy2NteAl$gS3U|T~D~Cl`1gDjlA@gx|^~|`<4K4{m&?Pw(vcOveKp=#6 zJ#-M8#bW0_w(Ol!>murS-j*(<6`*8ZyNUW|)Rf)Fl$jJ;pMc-t)y~w?<6bRc8D(u9 zWvdyEpYQnCs2UGD=Do=rYIM|l{`4$LA6-QvhQ#$WNLyW*Hon&sBtm)K*On&Bu-N;+ zS96knD*FPwqZTp`0OO?bNJ*FX&oHcP+3^hHCFhD|_KX;4*cmImyLHq;?<>`m0pmY{ zeO4vj4#c!tq*-QayTcG{d8FJgHpUz>9h9fj3{kq@C>nFtbbAat+W_BHeCHmNSFL#-0Re>v3`Yti@8f0|~~p*s9ZwLHR2Hk~61TWZFw@J#1F zZ9C{B?IfE^gUKv|g%dJ8fjc}Xp-9>yiYzB;u|1bfRxen%d?$Byk|qdrUv%N0tYtnd z<>a;U@xLTSqi_+N`lJ(vrH#ASl}-f*1v?_EBLWSk2u9JWF``HWG2$5vr3YcCA~B8$ zCxPQBoio6r#vXX`&UPj`K|yOhYg20@ENvms5gK-2L%4C=kV!K0tO!%V9y&B)E6B=F zZZN99OE=BbylFr~BkgJ=uSQ3=2qVF229q22vlpR_mze1&B~$)JkxIsDrQng1?u&Tt zD`*wMevaeZza?@m$A0uI$LO>R-e1QGqM6`wdCkGCnxJE5fnK4JJIy57m@Wr8$@8w6*?Yg+I z%342>;eN$^{<5uugv+7zdFP}Zdsv(EfV31@eT~C6crlEvB3BI@f&$*xauvSByp-OK z^qs0ms~=0snx>N|_2{_ege!|CPr(D%ow#=B4##=h-n+H)?Iv=&4?f&|P0TooX_3F5 zwX)S`6Mlkz7|>o-NCejs&h|l*Qf~2gVo)c}j`Q_-Df%DWyw}rC=5=@4eo|m|>90SI zxj)`Hy0`B69ghF@S7;f(boP`;z*dry)=qU^PB(B#B48P+dPt8`bKt9s9mG%@KUCj)YLXHpgkFciq)Ph}{J^Xj5-1m3i9V^u z_z~of*=LGlHi4@WIk~uG{tJvxBfO8Rp6m^is$Z~b0m**E8a?cI9a~W?$tZPrPqL8nNalX>`hwY+V9UBO-c7(v$` zw~yJ!f9*KSs*=i9tX{)j7Bwl9zrvp#j*VrL|&W@fN#1orqfHEmyIGx}IP zu?!%D8fUKyPXl+Ds{Cdw|7k`Y$obRtXB7a7;sFq1 z{|z7yUd1^IRerSBqegLxcYk6zl*h#EUEoe&2O#s1{iRL$g zG&em4$Z*j#RJNRfK=fM`EDr|>e-gh#*4_s&S;Zm#F+E>gyJb+Z`B8rKiB%7iG+QLi zR3>ghAAe*H=Zp!{I@L7=l`6KtaqIpt`n$)mS|eC~-!BVxWxQ)N=aMyBG7pV4X``BQ zNo79`(o z7xFOI;jGd4Iy`Wd4+fsUgUwnR&1zaJDYS{9i?i8BGhz;iX=(-;TV0X5IyM<{pw4bl)#%*gG?|Q1Qlss zZi1YUw=Br>`yISIJx#Ya#_g>3(tmzu@5V2IggQ1%p@c~*a?r|gFkb0zwTKU?3Y`L0 zAs6jlg8{g-djf*J+sA{ilm%s(I1g9py%SFY3S0ujV%%NbbVqP*gq|d)oaOB+(xaO0 zARrb`jrF&XD zY3|`gA)4?!t=6&-q(G4f*Ua+r*-45PWt4p<9hra7LyC~X^RRs@r2bl{MV?H`Hl;YOw%O81 z`Pr37Av|)`j$mkN3VR-0m~W!^O=vISCFs}9JZgrfymw9h__{c|L7Ba^5OeONq;mbR zzL9ZK^^-*@K>GF;&rI#u38o)Iq1n0{C=FTI(`KGZtwC#MvKNxyLnHD>k4DhVg0{1N z?Sg8c{PH6D`xxG#Hp|1<5zKF88&zD&`N-lcl#9is&Anq`YK^DE>Q>wo8O+{FX}TAO ze2i@9x8`!lq>#_I0Zj+M!L$iBfezB1(Mr~XS7=MD5a@7f0xe)etT!34KCFOPE5snr zu?VbPf>RM{LmW)KN%g8=C>s9`x>hl9Xn-v+L8P~)AA~juU>;8Cm`2c4GZYJgD1sO+ z$K4`95rm{n5q@?7z2ynQi|`^mw|FC@UQxaf#0!n_xHA8vMjsTyd6aj02itKrW(WD= zc#UQ>Fy1D+Eh{PjLIe~hAQeD*hIMOEK(B#VzY2^hup>z}Vcx~@E|&vTq{Ta=YOehVWFDeFMr)~wbc{^G>_qt8neh*PKcN&aNJTNl z-`z11>3dlb)X@AZKd9jcoc>F(z|LA>>gQjI0W5Jt&!`Qhjk-D3v8nyI|HlFR$=qD7 zaBmcGVY}HLvlH1hLewF?#1>>sb@ymogF5qAgji9$2f2p>msUI&!YQq@B1j@$N_~%& zEyP`Lt3Yja=q5?;{C+j;a*-A((5;BEl;A*`k_C;fQdeEdV=9=GVr7#GRH>@^)03fF zXC3@|j!*?W`U4+5{)O~X=z9wB?ITqYV<{q{Ds~k@Bh>3?m1=brmgnRq7rhNTvIfam zHk5K?G0SeH8fKAJ@>gUc{bHiW7?km}LV*oV-y(hDLB_JaQrq1)S?tTfsJ{882=S)0 z-fo$KFaKv9X+4VnSrtX`r@YXDu(%8innv4XZ1KydpQ^3d^3KMh4yHA>Pg_V+Q*at{pz{S^HRWGp3{9<)DD{e7M}H2NwDN>=%7^<9Zk zDjY-|ePl|@&j)Dm#zg$U&8Jy&=0daAtdZ;X{N3wzC+D#qs=_k;o0_XI-t8{;@6=ZA zsO|1a=__gM4Hbw|Vd9ON-jFiXWDfSFF*=iYq>&Nkf@BRz8xlU~&1EEAJgxY_$<-7h zss`NDir16qwJv??`G>8te`Q7V+JwtQP&)lc^uQnmT~ zCgwlpW^Cvwem}ai)7)n{nXt~g4?S}qJv);8n5u>f!}*Bb>j}=#qEWbYWn@FeI0Q4q zu4N09Y|w3rq+tslg$Vpqn5xi(l(;(_l?5+@q5XF(swb3GdEFj;!G##B&+9O$jz;sQ zbn*3hTN8$<$?qRc4 zCoyVAEurfDT$7F+U|a{Dae4jiR7EGMBnD1_<<^qB&05=t{eK*RUjaLC{KZ)Y)6wDm zN{ldeix)mmLDR@Wlw|KC!;xMp%bLU&G*Y6g6uG-#cS7UTo^4}WPvK33?b(K#3VFII zvppo(Or*aPVO84TG{I};sV)5#>};Eo6sa{z6{!tD3IhQi!3fX-?gf+sI)n&N{s10A z1vXNopbvJG@Oq-EB zsIUl%%C_s4mou;3ky=K%Jj6ad{_NvvjjJdKd`f>-yygV~rFTjac5gxiPgI#H3@`H^ zk)%p56w6~wMqc%}p6t1nr(snNB8*~UaoX3B9MEDq>R zl5t?9+4+Y!a(Emvfndq?)|WrSs!(CPBKcH>!CWr)fAo=bafls z6E85jkSm-H0dM2^R+CB+7*Qk1m5nvPvi!fL{DXlEfS`@eiKW_+@BMr@?vdrJaSt z+!ZT>(BIb$QH7LKR1;ouwj_nIBuc4vq%`41LQ8VWa|%dTrL)Lb8LUk!6hdPpH#0r; zLRnln+2bdS1vfGA8%6B}C*NHp%&6lwL~6}iTAZloM8nI`P426pF~ZzphctF2rhPLF zF>C#0C8K7YFLvXabY22nufrc=J;K72AvEKX(jn_2oJ^D%YP%=3SYZpC@^ zun-00z4~9Tz6KL-7auXh7kS9_N!c4`*J;qL?^Tk7lPC*sO7B-AB%7FK6V| zFj}~J)z12J`;|j{+>SK#=QE;vz+z#Ti&<4pjEvkIp0$9BB zt_FVrseB3!0r+kuiZ%z=)VIc=Yr@K`#w+QnM^{S6{+TGo2`)9(8FTp59Afaf86~)| z!n(oI;g$>rqXv824lNxUmhX(dmo4d*OIq;a^2W;N@2r1F=rP+6aQC%0q1$_ae8wO1 z20cg(_!HfLl&P`5@F{s64mF$e^C-Sc*gsggp7yl;t^c(kQ(ig|s(Y!~sndC?*|DIX z#a+#Y$Y9!?xJp^s4{_!Q%^i1G({l|d|X;Bi;fgm_Nw4^Cc-3fMJ-7#?9xrS7faJjyAK~-PWq}*mQcqB87>Gb ziqQPzTVr_rbJaQMmhWs-@jcQTu+PG+Pw&m*>ZobUdhoCtyHre zUz|lg0&$~QhozBcEc3BetT2#k%uR|i*};U>I_bCUDH z(i$npY}R%N-nE*dl)UmBjSN3>;(#umu1PoO5#`y0uI&Vq+wYvjLbicvdXjo!>-=3oW1|xg%l13rb+Y1EN~4 z(bI-*J^t2$?6VH6BSp4?{9ct>l(nUUsvam8lej9zL6+K~ge&!9o-RbS)JZfnF~K`j zoJ=h$mTvSF!NDX=_Gwo^R-EJ_j#PU;7qQmxPo%);jWeh%Sc|?PMbH7uBbog{5pL;A zEI)dQ_#u&=kSBr2hoV|=|M((HW8?(gzea26B;~oeEjdwoNM7^^&X`#r0dB$}qDw12 z2E6+X3)O-jM^|u2!b$r)25erb`~kbalyadPEn&})B4eLm*Zn}mK|v=T+vmB{aa@)GF98sZOY>%(4mL_bWQv;WERUk(_9GCRE zbOp3_Ug=VzNF|n_oG-O}e^>gut0YfJhl?&e<2j?ThZD_uNx3ErkFzNm^iR(6UJgm7 zh}*C{$2a+*O>{>az&4N;Z%pASIe`^2;$#Feb~Y@jM0Qa@tYc$d$ZbrF!7ar@hoXLY zUJEsT-@2rtAwjjYauFyBB}a$TGRra}NK%0kkW@nzse)=wQhhm-5P<$0E})#O)ikL> zn!~z3@67i`OLSwYH)XPn?+}*n{e3{||p{&F(n(K=*ZA&DzKdr0u;fQ(IcK zj@7i|ngJ_85cl&b=Nbxp=jn{csfgi3E2mfm3jJa-=xl+Z?P2B*m-|>7yB;EcTLV_| z0QmFI#YWwdXA8{CZtLl~&>ebu%&4@1?{bAyKFajXKbP|3@{5_c7|Lohw(W&nD_W;p z%Cwr!DLG(UwhLIAb|Uh*T*`HzZ>Dw9Yn|O9HxAt;^=JJAL!+rt27`q0%;EV6N~*8J zJhW_LR^%~uyTFLOBR9*JpI37I@0-eUcS{}0wpRI?bCSyZ+3Hg6Rld(8Af-0V_LP`< z{l9b0o~>MQs-b4nw%BW}B+fEfBLz#98#SHGABYnh>WFC%WZa5o9P&lYzm!k+$A?MR zIk*;Kdv2Ci_tql^Z&7YLeT7qdtu2tovOcoyZIBaL1iPTJB=CK}aF+VMn24|#sgCg3kVv?(X>QH+&}4 zn&VS>ta6t=14C7^^e?3;lO>lsdJt4)^Zo8sZ56IrgU}vyCLBZVo0f9CCp#aSmF z-XE_itDw;u!J1=>7g^k_T+vdZNAMzQJ}IA@HXUA1sq*+|wR&F*d%e%<8+XoWJe-v3 zCoWptH%}nE!_Kf)t=)e-JtZ-q=GGRlvWz2`&+yUwMIZRq+Ccd=yE_VdDk6@Dz<7;0 z{ikAR&kA7uYi%I;GB&s`Qv<7D)H(Wh43{r89;gc)K2BcasvwPd-KzD9)vR|CDeT+m zBjrpjCoS5mw4*G;PLw#_J61+2Te*rGJMx%IF|01O5AB`pjWAaKu~d<%C2ICa$3ukX zr+3J@hVcen3*xKuHCfC|vqrD(zq2-+wvBMNRZ&wx#I&o^)crwKl2bq=4MQoT4j`KF zt?g`3m@I|*fN~^+d^{M0GD>h1CPcZA;w2dc-oewt87x#_VIdVE74{idYj9Xv9eJGG zvN~AR5@_|Q?N7U)>ur4mFO5{nq9^T%55Z6YXe|W=Ep1Ktd9F+SYw1Ti3m+P8CyEq3 z;VPzSeWO;?L(?`T>BDqLjb0`ZSXBH+qRfx@QjBRcwgy|^OQNR$Xm~6-s77bABy&N# zyJDTitD6SRIq9#b#HYk3GvGIiL-b3eKVv3@vIf<}6&UTm0D!kTw|xPx0izXW^lNdvUY>EU>E4+@<9%^_$D*1o; zJoS#2@)sKY$;Pq2$D68EdSknO<<{p(>;m0QaxkJRy!LQI73u6JF5v2+VWVaR?#g7 zZRdwN(kbw?qsl4cDaSV_2>S37)=O?9Y2!=djOzpzzurPR?ZsDVLUn8%^pLv!MFPRQ z{S<;(I3QG;8=EKLSp2BZefcHjIL%eOODpOXgU|$4s}VInUpw+S_tU!ao6VDR?G`&k zJFVr`2$T5eqUhVA>+XHJ$i{I1ewDZ6g`9P$HtNt`#2aiUDNhxoR|%DfiDl8r+^AF= zJCLJb$#AF=NsRgu&u5d^cdlJVC=>Bd=|Lw+Sp^e)>4k`0gQKg{g@D``$k0kMy-Che z>5EQMeTl-)XwosiVa6W8qYzz#%V`)+8oj9!RQMM^x)aU(mpY$Z_$5j>-ibK7+A*{G zP(uoH5goE!Mf-UxSgq_f`=?p&5&u6z4F1_~S`q{~>;ZcL1mfXjdn%@lZ_0miQu!>s zB>mg~pRW1@uH|D{rnn6fR{1+#1mnK(>mj`%>h15mcHp8{jh?&f7jQ5amkvFh%gmXi z8MT0mc^`pelr_I`jGcS?ctMbg#i*!_?%?eFZW0lE3(<{JK zIuutb2VE+mwyO|4ZU1I@q2`J7tqxxQcAhc}D)JLsN5Q3i=KlsW?Wy3;J^|J~bo#D` z*O1J#Dyr&ccolB9bsCu@X8-XmX7tEnnY{87QZ+!i+;KS(1}Z71#@8 z*wy&Y89i`rLE3*Xe_)MmPS0NpUP(0(h~c7?N8lkpy({7eq{)14?~HNDW<=^o2D8(4 z43!$yZd!xExf@AMR4muCuM4jL2QM?#>qji>0u`uv#vOwRCecwusZjx(BJeq0K5CG- z>lx1 z9}4aHrrnceeE1|WBQfV!y3d01G#YomK5~7&{L&l!`s1tfoA95@FE3xr6KWj#&H2lo z?U~wIB&2MhU0MNlh!SKPK(&r#5_0Uj0bYR=(D6b@TW5vCNnCFcTyTNj6IkvLaESU{ zB#dB&Sy<&Ipz9%p*KKRyllP*+%0nDACi#c(j6M%Kv0eoIlKqiS(_ho~Pt62oe3<*5 z2!?+Fn|vM(o)cvlLp^8{9zjNR#)wLe0s~itASwrEQ7&7&ISyhbV*J~r>Bq+|s+Dsg zCKK{NPl)!3(N4rngA|JXEJGN+cV9^YRW_B!@4^JMkbpx^;g@+OpEib7w3@;gUpi5r zvkGNAI^y5)I$!(@uf>NZTKcuWSR1l$9iQ}9s{(2INTvN1=Y@n}Wv|DhX646Cp<9Hc zd&2`gp!XaOie2|#dP!_^Q@h2L91v<&%0-+_u^`S5mU@P%nssB}I>qU&D9#M3NwV+f zBHno?TI>5&RgtY_23y5UXP?As)L-ix!x%Ya*=Fw5O3(7ZQsa-d2Ce$g(hXYoYEE*( zyH};pz~7xbMaiNEx_)y{yom$NB9e(ak-Ydcw;l2}!D)`xJWlkLM)1X*pWo6h4|_13$^= z*<#HHB+!BhMv{s>4ml$_F=0h1B*yTn&9hPU(ZR$1K?o<4&beT`G(iM+8G?6y|3Xj> zne(IaeDlBYDEcP~%xdrChx>tron>)^gJ0djLN3^|dJF)T(4uxNVY-=E{g; zl_DX-7jIO!hGFk5Zyy&*@#=+vX>z`sUw>2YanOSxH2mpj5&9Q5lxwl|{O@acl&i{X zHz3s-EufKPQ}mwmcCmqj{ZD%;#;h8V^{%>_M|lxB{m z(44oifZr@rTVDtS=qXh|El>th1c;MR1`+@@ZDXtG9TXeuiad2pX{u{#BBj{^lP;+| zSx}SJ{z|)kSXuY1QSnKh@A0|+-HW|Or^oVx){>_mzb3Bm{m&Xt5 zqdg;Z(8o&QxovI4KUmI3<`ZoBQ1Onc;WhN;83ZOJY}PT5cj^z;Fw2d%pv z@)a6ur|cDZWmzz>85T3+->5XzB}~^8WV&7Uc&WS62AHkJ^fphAEJLV~;3%#(V~zB3 z!_;linXljcRNWt}dDRoJaII=~j8WHB$|d+gqtezWZiCcCI>Vuhy(NvQh74|3e&I@0 z5D`vXLb>`Mftd{t)FD2PpjAAtI0c5)0;ZmZ`29XDk6Uz_+QrPru4Z{Alv3s@h<>fy z^%AWyJW=HZmSwQGNqu$1)1WU;LRS_!c&9@`8n>xtGxj(lj{PJuJ2;?4x= zL!Y|iyH0_Vwk%*kX2Di~c|dY?{DkiS01zeDrwL!=`PdWddn>lpf0c!b(0fU1PTEs> zXke{ElCsj5>y5RvtfVo$j!wNxbcQ>wfqoQq;a%e3j-(~jt$>}I>?RURC=w(eM&l<7 z5iw#!76-6B`izvE7Vr;lNG+xNIV9kMMnIMwH{;@HYD3&W4QJ_x+=r+@CO9l+>ph~L za>#Sgf|C^{Z)>|a`oZ1=BN9u7_IDjyxiCXkI_&gWo>$159GMJ#-`4pGZi{VxUW*3z zw*}!3OlePSbTLgis|8YxsE^aB-ED~V$o+`ieVwF3m1@$3bh^x!&B%KW+w6yD`9mQu zyTmpQUPk!dsjx*tDnXO;BrKS`ik(e9gCOC7_0X~ra(|FZo-zQCT@-Ych+fh}q#Yuc zwSh=g7sQLni#c$5S)&M{7Uo}9_$+Q)KhQoZ66mEeI#?FK`%Wizk{PpLlFdcEh^mQ~ z*Ci}V2!;LlWV4wscD6MTk5VPi7rbsSHu-P>Ege#y* zX)*&=)7TKfOp>H}!;Rvnce;G4dTYvcCz{gKF?x@oBE2J*To6RhXC{psd!>hqIHnf6 z77%`O#7SOh^U_7?55JW_>PN>Xq2_))nZs|l1(x`aD%*bnwh2R+Oz3=`w3;6Bn}r3L zfR2__nk%UrHh0viZUbaj`9tP5Wd55C99CUXa|6FGpxDU&N<|i4U{%yW3LUjDM@bhI zj0h{{kZLn!Y1O&GycH-fUkfhxD%b{g83)Fz^g61f$2V_Qq&>334j;>v?77bo(`T>C zCXqd!2@_^a`jk*|m6(t+Eu~2*898>{Ww<482gncy1(5M|Une@tNi&=tbt04MHL1b8 zoVPxpy+uUv`_WSHi5F-MxP+&fHo#)4HLEr?z~_PVI(*gl#a0UVq{Mv-t&IhVloXRu zwI9(7F`Lc(Fyz8q$|^WARVRN}wQuAX*0g}S5U8fsUL+-u3fJH&>4teiJw9fX{2K!)qhz`$ za!+VPbaf%7u?-PhQm;tmhS*Fi4~=K5oxLi&&`P_~T)oDaN&yjyHiyBnh4<08?E={GlR1#dL#C94C6Q+inRE>G95>^x?l=c`VG#=pAR8`tgk zbEIDMrd1oZ)>QTCmz7Vk!=0E;#rqXmS|#{wndxsUv^8&hj)9`0Cnu_^VrWo+P=+=N z<>*Ps5Nf9DYsf*3Fb)lj3dipw0>vP{51|=w^j=lH=RZwoxf#a?Zs#NbdaxMc%Kd3wU09FRRaIEj zm03fUslm$!T7N_C{E_=xivOnk^wQ+$yR~mlJG!;y>z5Q-zH}Y!3$vFh zTHWZFa;f2xKO21Ucp`p$@rvYn#l|*A^z;2>eYU16VsZdF{deY9I{wrc&9JTIvu~c? zv*fiqVxn;zFjuaKfsdpI$gpa?sfeQ)ALJUu?` z-{+O#;gG?t=;U>=p}s2>Qnpf~sw2G1;;^llicBF+i!Qi zUMtW*!eTZSB33d;8O+vHEI1Wu*t8i)-H|qv78s=RGIc39<^Ewi?4x`p?4AFa9MBTX97HTE+aN4ufMaVc!7C&*82>3 zHD#uA&_PYw&VNH{$!qO_keA#ld(AuVTR>rb=kB2PiF+Y=*U;fGn+BBza1p=HyiQMt zyvL*&WF`?j)WRrr{x!{me&ST-G3C_k%h$eUz_PfG3L}*%uq@&Kl(l4kG&kz+o&xaK zOau02>R*BAzZiQ~Ecoey8}X3%S~0c3rQY!vkd$~b1U|M>rZJ00;NuTi%G6#k;XZyP zwGHgZ;}1`$$}iB=UcVfDy^ZX8GoRGn7%D6c^ zQwdRsN;w?L0F{18ucx+SED$AHGET7Asc8s>Vi_E}N=q*{!?h}HJgyx^h6azHu!+=K zR*?9;0Jn92<*Ui?GTGhb7i7n?F!h(#i#aNR0(B`uqr4RUY(;DxvQTX#L7Ujg0g9ar&_!#H;~vMtHb|9}8xSyF|9wmems3(wlON*YbI#eYVrJ z=f>}fEy~}~Xa#&$xLolfi~Q&_qVVbOrE< z`-|8QM4vlT2=664gLFac+*0d+$)2OSKmrwr2cKKIAo9Yh&zm07A}>(21c}U!RD|-> zquG+_J-|g3n4Tk|kUcZC0Md%m7XzjQb0vn*WG@)HP+y2QFHkad!SP0(cfuz%vaN9rySU(XN~+Sl`4v@lD$G- z>NC-s8(mI2>hDHh_^?y3+ubyIR=)6n-2RiSPW}gDpK8YL zAbEUm>l^NDi!Uz6{$uMcAi<@DX@Sf~<3&*W75LTpc&CkjklWTjqG##Znq%!*>{uuJI_G0M#Toa^EzkQL zhub8FpWP$QHv;E23`tKz{}5p{^%{Bg*UA&Pcr=)mWB;;_{b;&9n$OCb&oVy^cm8? zz)narzCkfPCkgT36bIdYYy#ojj69e#4{GQ}LFf`52d%3)6{?{obdHmaz%@#TvN=W) zqKOF|nu!T{a2f+W2xHW4YKNl5j65iti5~KIAcTRIm#_r{E9CUg=;R)hY+wij>pTrh z8EFV*<(rVOl~9E|FWqt}9ST`F8$|0jrcmrphr*RSL~_W{phzH1sDog|BWVcKjacD! zHo|1jM+oFD7U6{?L@BAL&;o}b1g8M)_OC=M*`6DWThv4p8?%=H?;A6s7HhB${rMjT~ep zwpMMb8QU_DJ+s|e`xk7Wx7IKg->uAHeQH*iPJ_jBwl7Q!f2u~|{#-s_&(yGCIInK~ z^i=(7>|#~2s_{@On61&=KqaQctvX&{(cyVJ^ErWm${^f-ZZ2R=rP>Q_{b4+6$z^JG zrQ@KfVOz`Q1~{F-+xcW}`&X)B81Zewc$TFlsEKLvdrxm7hL`=0B; zTEfb+6-Z0vIV$ZgIU!f%!R(S_la^QH8&~9t+TLT+<^b2Zt*&cy$GIFQ$9dUieA0wQ zvtD3Fl1pZM!pbkRLWMVerT&ZHvG43owHir)#eGg|4TQObp3z1yJDELLeKno7cIkJRZ2Igk_BCC$KvKj0W~*RxQjb|%g)PNs&o|GBa^vVvk{Bw!%; z&jk+;6up?Gjf<%hy_k)mi>ZjIvAu~Y6upe8owLT4)|lpPy|Y@NY@0)hd8qk(~e&46(Lk_y5l1z1QW zAP`stRKNpA1_Bcew8DyIA7I!9#z36`koOD_(MI4L*RE$khv2 zX+01X;t$Io$^-}CAqtEG0YHG2Peii+hXzdlO9LTsCVCN5GecJ!7btoaPX|+aQ4be! zWtU$`py*YFmHyAuUpf9qMCSkXf&Y()OiXP5w~AMq($$x22)k$MCh8P8?j;Mzbbw=S zP&;hhi@Nv<=Vf0%^O$d>nn_pN&YL4wwiB_3iQ-8cRrmpA_U*Ku1I3#kZ5o4LRs3u~iS2}iLc~^{@TYXmUSU50JT;Wmh$5v3V2i$lC zX^NYz^SCufykRG@{3XU4BOK!o(!6eh8bU6%M#oq!qIL+$Xp( z6EWl)fCD~h{>z@>MItlH*CC}c&exK)iT)?d7p7BFI|6xH+LCEuDpn|Ov0~Pu9Eqd8 zIpHd)V@WPfdQ+C`2m8q3W?FzP5hR!$Sf?C7Hvbk-P!61hsl^Y)DA4TDM|0S}p$g`> z0CLd-$=30ZO}7wJO?rH$HTZc09PqVAys!5lmVS?gl;522YMB>MjBvh)0MXn$a~#*3 z*!k^+*G3OM%mP2Pg1ypJ)lVB&d6s_z6Y(7^Tc8Gcy&_PR6chzJuYjPx2%(2s$PhU$2TkZE zj5s@tSclZ8o{u`5`1y?zr#A`ew1|PaM@G`mh17a!lY?q{SSuq|U0%*cT+uS*b{*yd zl=;=FV+IOwT3IcD^-U&T#$NOAWRt|SkhW=lJ85&cXMC`C+D4x} zK-+MaWxNQ$kQrEHq_7gpz)I0U#G_Gk0gr&5`B-_XlIWgFRBxM)=DTYxu)chtf-1cp zxZYm!Gxi<1d%qim-#r2B3-DXblk*hBA_0PGv3YN?1X1D^d_;)2 zivu2nlBe4;bsS8=pJ=fJdThe`5DF~&tWi&vw7-(%#0)!&WsE~Ct(B*zubA2;wiP`T za>+)!EONF7(UJ?bW(y8g_Wp|y9r(#l=(tJbFw5#Ni`@&9<}pM?m14s?d3R+jQr@FZ zuA}Ne##y=de=@kLXL1;4Z`?kr;>S(l4m^}SgA1xqr9Is3D|{F(jGHgT-s!l z-TpmsFT`IgohuO-l8mLKjV7t(STE%;27O_{#@R_RBKlf!N>mYKOS#cYouCD3M|Mn1 zbche9SSWre+o<5`rcC_mOrC9{d&@Ln+{7U{zcx;=Nf zV;e%9(d*2^=@KLzw;v0mue(@1CL%_mvv?bTq+=e)l+#oRPm^rG3WRheh48UeD9D~q z`2mFBl|1dw59(@K;PYDa0K>sGtUb;!cy?riQ^2#l0In16;}Um5I140lOBtmi_yn-| z%rC})WFplTw`(_iUV(nK8GwVDoe#`MO-(-bs6*6V1GvsXoeMu$l)LYnw^++ld#i5K z_VeeTUTfe3RC#C6U-=dy59{O%V89I+JOMLKy7(#4rxhDkL*VewT5EHaX`K>nK*mzr z{e2rddAqVo6ki|id5sBedyKVhA~;$=WO82HLKtaK$N(IIjg)M&_U60pQIwst{EsQ`*`Yf5 zhadaSypt-m`!7vNOr2c8GeNFUd!gRT&HSR*UfpKj5x2BTQYWf@nWOj$cuS4Ik7m`wFR{tHfeBidl0#5 zqlQW9>b9EGk1dwJ*68AuyQ1)*@!-dWi2^aJx*eBX5r3z!#yj}g%pc3aORr>i>Gpab70sDK2Q%!q%OU za!P5bt-WUF8e!gPu9NO5b%yksZ5CL_c--mJoDXr4T61z(EE%7=Z_WME`yU~Xh- zXli6(WT9uw78x;av+!P?~H=wDGr6n0aHv(B9nI$=? zKn9}4humTWHxdI9(^HGN^u04vila?|J`IWmx<0=Y=rcnkZ52hSXlHzx}x y15-CwOH)@P0|Q5A3nybY10Z%WwJ>+FBdj78(#B6LDgnFH(A=0yRn^tsjSB#!QpITi literal 0 HcmV?d00001 diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index bd599c9..8355f32 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1037,3 +1037,27 @@ fn test_firecrawl_tagged_pdf_struct_tree() { // Fences come in open/close pairs assert_eq!(fence_count % 2, 0, "Code fences should be balanced"); } + +#[test] +fn test_identity_h_no_tounicode_suppresses_garbage() { + // shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no + // ToUnicode CMap. The raw CID values look like random Latin characters. + // We should suppress the garbage and flag the page for OCR. + let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap(); + let result = pdf_inspector::process_pdf_mem(&buf).unwrap(); + + // Page 1 should be flagged for OCR + assert!( + result.pages_needing_ocr.contains(&1), + "Page with Identity-H font without ToUnicode should be flagged for OCR" + ); + + // Markdown should be empty (garbage suppressed) + let md = result.markdown.unwrap_or_default(); + assert!( + md.trim().is_empty(), + "Garbage CID text should be suppressed, got {} chars: {:?}", + md.len(), + &md[..md.len().min(100)] + ); +} From a199768c4e1e31a2a1c261d9512bf16cc2a9c4a5 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 12:14:44 -0700 Subject: [PATCH 29/46] fix(layout): detect and correct rotated page text (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PDFs that embed landscape content in portrait pages via a rotated text matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW) produced garbled output because the layout engine assumed x=horizontal, y=vertical. Track the dominant text direction from combined matrices during extraction. When ≥67% of text operators are rotated, swap x↔y coordinates (with y-negation for correct reading order) for all text items, rects, and lines. Also estimate text widths from char count × font size since scale_x ≈ 0 for rotated text. Co-authored-by: Claude Opus 4.6 --- src/extractor/content_stream.rs | 113 +++++++++++++++++++++++++ tests/fixtures/tnagriculture_06_12.pdf | Bin 0 -> 84091 bytes tests/integration_tests.rs | 50 ++++++++++- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/tnagriculture_06_12.pdf diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index e703b79..795185e 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -128,6 +128,14 @@ pub(crate) fn extract_page_text_items( let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; + // Track text direction votes: (horizontal_count, rotated_count). + // For each text item, if |combined[0]| > |combined[1]| the text runs + // horizontally (normal); otherwise it's rotated ~90°. + let mut rotation_votes = RotationVotes { + horizontal: 0, + rotated: 0, + }; + // Marked content tracking: (ActualText, MCID) per nesting level struct MarkedContentEntry { actual_text: Option, @@ -280,6 +288,11 @@ pub(crate) fn extract_page_text_items( let combined = multiply_matrices(&text_matrix, &ctm); let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let width = if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -422,6 +435,11 @@ pub(crate) fn extract_page_text_items( // Emit one TextItem per sub-item if !sub_items.is_empty() { let combined = multiply_matrices(&text_matrix, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let rendered_size = effective_font_size(current_font_size, &combined); let base_font = font_base_names .get(¤t_font) @@ -495,6 +513,11 @@ pub(crate) fn extract_page_text_items( ) { if !text.trim().is_empty() { let combined = multiply_matrices(&text_matrix, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); let base_font = font_base_names @@ -589,6 +612,11 @@ pub(crate) fn extract_page_text_items( // Compute width from text matrix advancement during BDC..EMC if let Some(start_tm) = actual_text_start_tm.take() { let combined = multiply_matrices(&start_tm, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); // Width in device space from text matrix delta @@ -879,11 +907,96 @@ pub(crate) fn extract_page_text_items( } } + // Detect dominant text rotation and transform coordinates if needed. + // Some PDFs embed landscape content in portrait pages using a rotated text + // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW). The layout engine + // assumes x=horizontal, y=vertical — so we swap coordinates to match. + let (items, rects, lines) = correct_rotated_page(items, rects, lines, &rotation_votes); + let items = super::merge_text_items(items); let items = super::merge_subscript_items(items); Ok(((items, rects, lines), has_gid_fonts)) } +/// Counts of text operators with horizontal vs rotated combined matrices. +struct RotationVotes { + horizontal: u32, + rotated: u32, +} + +/// Detect if most text items on a page are rotated 90° or 270°, and if so, +/// swap x↔y coordinates (plus widths/heights) so the layout engine sees +/// them as horizontal text on a landscape page. +fn correct_rotated_page( + mut items: Vec, + mut rects: Vec, + mut lines: Vec, + votes: &RotationVotes, +) -> (Vec, Vec, Vec) { + if items.len() < 2 { + return (items, rects, lines); + } + + // Use the combined-matrix direction votes collected during extraction. + // For normal text, combined[0] (the x-component of the text x-axis) is + // large; for 90° rotated text, combined[1] dominates instead. + let total_votes = votes.horizontal + votes.rotated; + if total_votes == 0 || votes.rotated * 3 < total_votes * 2 { + // Less than ~67% of text operators are rotated → not a rotated page + return (items, rects, lines); + } + + log::debug!( + "detected rotated page text: {}/{} text ops are rotated — swapping coordinates", + votes.rotated, + total_votes + ); + + // For 90° CCW rotation (the common case: Tm = [0, b, -b, 0, tx, ty]): + // device x increases = visual "down" → negate when mapping to y + // device y increases = visual "right" → use directly as x + // The layout engine sorts by y descending (highest = top of page), so + // we negate old_x so that visual-top (low device x) gets high new_y. + for item in &mut items { + let new_x = item.y; + let new_y = -item.x; + item.x = new_x; + item.y = new_y; + // For rotated text, the "width" along the reading direction was + // lost (computed as 0 due to scale_x ≈ 0). Estimate from text + // length × approximate char width. font_size is the rendered + // height in device space, which for 90° rotation corresponds to + // the horizontal extent of one em. + if item.width < 0.5 { + let char_count = item.text.chars().count() as f32; + item.width = char_count * item.font_size * 0.5; + } + } + + // Transform rectangles + for rect in &mut rects { + let new_x = rect.y; + let new_y = -(rect.x + rect.width.abs()); + rect.x = new_x; + rect.y = new_y; + std::mem::swap(&mut rect.width, &mut rect.height); + } + + // Transform lines + for line in &mut lines { + let new_x1 = line.y1; + let new_y1 = -line.x1; + let new_x2 = line.y2; + let new_y2 = -line.x2; + line.x1 = new_x1; + line.y1 = new_y1; + line.x2 = new_x2; + line.y2 = new_y2; + } + + (items, rects, lines) +} + /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance). /// Some PDFs emit a full-page clip path for every text block, producing /// thousands of identical rects. After dedup these collapse to one rect, diff --git a/tests/fixtures/tnagriculture_06_12.pdf b/tests/fixtures/tnagriculture_06_12.pdf new file mode 100644 index 0000000000000000000000000000000000000000..51bf3f2359ec4f4c35293e829fc7a2151faf6465 GIT binary patch literal 84091 zcmcG#1z1$y7B>vi4HBY+gc3>(IV0U5T>{dbGc-zzq?Dkflypf63=JZsG?FrebUB1{ z`kgWE{nvBveZS{@hv#ti*=w!6Vz1vmd(Jv@TU|zu6U47{2%n~&U!#{k8@E^NcS)(-Z3KHS-{Q3Z*KUxg1-+dsSf9M18gZ|3L3*zPd zd!NAH#^dAXgZ!0`pYQME34kCxf8`V85fu7cA4K49eL`Rm_^*6mUV*>a3>Jj&{gn>_ z`I{{u-oM8H#4ji)@E2JSzmU*h^@8}pfAa$*z{B%bTlo0|A%E|K@bmvg2S4!nYfQmF z7W`Ef%)`g`S6je5;J=N>!-Gnff6ckOtC^jnl`9cGka$`^&N1-ua%(y{xibJUX3*qT zgjqW=0B)n2G6o_LAsIe?UVdIFIf#IiG`|2C%nz0ULm<)uvQj)ULi}PP(jXZDK^cBt zK50Q&5D$>Lau7)nAHS3gNLo@>mQPm9h+Ey&$`h5n{JcQI;=8$$;xDL0Q+$(U>uE z4(=DO6HD~y2;h|k+Wf!)`FZA(DuMF;x$fyI{(bqyRp+Q2ocA0@*N@c-xo#{1*AKia?PC^?k8pcuEZ znVY+clck-tot5P;lTkzbFb$;!HOy~ys5VLs>WmUcoqzYCu7B|fNQ!Z*dO*#sT%D{@ zi3Q{ox0IR0=7C5ti? zi21+bh>A1HVN`5@aZzLN0eVn+`GNSL9MDmeQMPk)2c{QDTwVdN*w1cVJ4<)le-nOY zG%Ec8x~7$zlZWe1ct|@rI=N~&n^{-^BS>rUF#JeJfJyeTyS$b=u&4n2@<2cDPwdHS zff!J?Iskg|TD%OrKXMyjmjfUGaB6GwFrZdc6f(6z44@wiC(0yOCkrhrcLQ$VPS34v z_1N8rTTjj09zZ4_BnRbT;QN7`pXNEC)H3jJYrEP(|5MKYz&yxz`{g-Xz%By0zSo=U-#o{1aM3$baF#1}+3eRS zD%w6W@kn6(>-0CLFHWzTHMh$H*eu?ReHOOT8in{+MZv{NNc_hYFU~aP5c&p>0}5CN z!0Em!b%k1!GWU~66<>OCvE7W$h}Ocs5gZy=o+Xb_o+^>y5l+F-F-X(bEho?x|1@)g zoiT|msED81_i6I5c7;;-@#|)xJKi}e{P?F4^r2Ku#CP-faotAI$D5wzVy3DyC%oMx zXJTxBAWJDiNI5&CP1O^@&gs0SlK+H#v6?$F^tEM9OuvF{h$nNYh1$pU`jj%YDOgvG z(X12KyO)zx+SNt8I|5@`UPJLCIijTlWvyrZ(gMm=I$ZkDS1*X%QrHC8jK*Jz+(r=i+Bux1 znRS}t4v9HV1@Kh3Mj8^6$xv^aL2C*5Aaq|vI%xSO(p)>Fd%faiaLlPbfGQ-uLM#vZ zu-_(buxSW7z0`a3n7_G(A|V#b*EZJ) zG4S~?s;KD(u8Jz|GZ#`im`$E(y{Mt%^N!)M!Gzzuvp@=w-wBZl&(KgM^G((6VsslY z-3-GO{hoGpa;qp@dg)4Al8%+Tqes`5)Ckh<4_Hr2;Sp- z4xZX5#iE7D+*C*0EQQ!Ve*_wz>DMEmf0Q5KMdRS*OinX5i#yudViY@hB=lZ^%bBuW zh~MR2Xv-2Ve>CmK&T?1GG8Xmju_T6;3Nq$;o(F_;QyuGiXRnymg#t0Jsa(N3k50>I zEr<9%BbRrUyiGi}6uzE({_6c0MDqCP;nSV;nE05HFp3^rSbOY8*)fq?VH~Z%DlQjc zDH)i2><6U6LL(iBsg^u++jBu-I)*k75y$K*?1BknjqO0<{)wmx5b_xx-eWqGwnI%um zXVWpaS`cJ_2D4)rYfK0716ds%1W+&k#q)~dbPU6vh#h2g=@Nhjg)g2<;nS+LEr@5# zF2b&WhNupt%zQ&S!wY$HqApCJb#fU zbxk2C_n7oK*%Iz&7J|6|3u2y`Z-fTsV=TB=bj5maoNrZxAipy^)+^pcw0389Z+A~n zb+?A@+Vtsr6GI(}?nk?vot;muF6a`zaJ?DOI$LcNgAa>XTFt+Pcd*LhSfrGSvmEOh zzX)MFpi|UztCu$K`6L!k?a97g7rH{mkms(jvFPBha8l=!;T6bJdjGrW;YN$PTRz^G zuP#Z6bLqP`S7>Z97*v(6iU&AKdVdufsz!e;($YwM6xEsuNC@=owsFi3ib$r3jUOz! zyp8ufk{H2$a-Uv2gm%_s$p1_-nKj%^n_YCYmSlP_5SC_l1uP){Y+F*VDbp^Htu%Vj zxkKD>1Wh7e(*$lrS%6H6fUdwzoEn04i!0$1T-_NZ6NkMf3=!&3-470Dn(q;1Q4@y( zLA5iq1xTeZ+s1pwhza6h&!99oH)9jB=P}fT?=hm^Wvaz998R-T?T1GO)$MjTn`%SA zVnz=0f@+Pf7a$$5ppA@fh$P%uPo3n(8NrU3*{5I2wJo6HFRGbY%aIxlXJ^txcaR&8 z?Mx&ef7aRUY0=)Ppjiy|>m)VO?b_pwG>LH7-7PJ@ zoR(?5=dq#*@8jWUmxvbgo!2D`n10V*iwaq6o%FnOD+_;j)7N62um)(jw4YYZY*+X*ksWQ@8Pq#vNDn{*vdosKP1CY)~=0AaolVu|#-a&;zY|rZ35SP={ zsOjt8#Xj0qWSoNGj(LV8!&8HS;O8S*QInUATp@+(hS{C1lR|~UZ5t&YBNEJ}{CR=7 zW1RB8Cxakjnu3uIdwL|pGl>C}`5zBC-=W=NO62$_6ry%f{}qKoLVx(q2JAxs6e4)g zg6EsnvIv41*jwDAvgOy{M*EkknuRv%&!Ty&Kvz0m4#qFtJW&8GxjMR>ot$2t9-cPm zow%iyL&k^XHw^Bw=O)5$wi>TG4NNqL7*ELBXKhuMMUqIai_)%V$CI66xZcE7804&` z`E>L7TMl{(*eQMJYr&vFf=TG`pRvQ~B>)iIdt zt7P9aHYRGx`x?}fXWzS6h=A#L7+I6)pRjqp7*>F>tJYDPAy&3i02)9 z#4~CrPIfI_#E2&>3O=afSMbomCq1CJ)~tpue9Tkg0fKQIiepie9yRNEPZcpw4GjY> zVun4XR1xTZq^wgzS(mEmVuwA!4-o5X(1Jxk-K-~%Dx#s1xt_;Z{Qejr-&7ugsDrAmizD#mnq_ITVdJ~q?U}EG zKM=Arwmmi<_!M!6y)Z)wG8Qliuutx%eLs!;W%(Z|?w9NPhw@L*(eQiYpZpV`P~S!V zE1iX8YXc2XaTa|9Se<5VS?`!prv1sZwmCAbCt)}-* z{%0*GoHn`-5tp+quvRB=18TLiUFxH+zRE7Wevk6VmR{CjdlY}tV=i-9PE$f{pr>BhF{_f|y?05##>1r< z5n^On$j~8Z7*n+$>=@2E*X4Bay=Y~MzU!>CnGLX_W`-dT8G8(;$*K0^1$5v-!ZHh3MCq5~FKDItRns=Mk`|p~E7;rT4XT zxO1Lb@o)>~YSE}{QO*7uKatJ$j9fSkN>34zWfv-hvKtVLho_?S6rqN|&;%C!IZumR zc+fn8Sl&*UHVgR#@aWc0hGTfU25=(+7yysV5X3UJjf}ns6^*Hw=%K?(_UGDe(3>U) zdIqsmu<%jOpj>!r2XKD^BKK1;X2{bm7v7F?$F5P*7ZDvdMI7oR47m79HQ*77;fHEe zoTGu5hCqc$fxzuRg=Fo71*H((`9HidoHsu8A3a>MSF=H>ikb3f2V7byL{6Xrgad7L z4~D0@jUGD4+X}xf=s#3Lfvqj^=|6!D^)&0hgN>Kx&tMaFr2$dM45yWR!H&AZbIjGA zcN<)DWm_C%xpxqb30_mx#vk?AaFNsf_9)mv%jLh7 zCEnPwJ6FJ!Z}@)NbP~WEpw>U;758$pzP9tJ*?nhZ2qdE9v`PXw96PZRX?xH!!H{Ir(Rg)}@~Zib%=W6Q z=kuKdY3nKt3wqvfPUNbFKXOB?`>D%e>JkRLXm52Jys zP4hrJ0ghYsBUMu!oU zKlJaxx3%+-5&-aXtLd)(K<=+41Xy5%YiL*cOnDFMU|FrFq0$r$tf;CsO<2cM`#ip;;RFUd|>E#E0k(FARiv;GNr zs0YaZ6?sAs(4Uc~1fv0=1p)g;(6z$Xwbn5j`?xdk$k_m`>txTS?&mMD@LVC!v?B5c z)9)T%gv+m+RWD8~b}ybS4#uqnES?*<5!*aF_>dE}S&TeB@!fO##@QLWDPZE16E7_? z0=|F8Y&1ZA*Vd*`Xh2LOEl#?@IQ>R@LK%%M0#`jflPgctW;l!PbWy_xoqGDL7M64|S{`8Uu_SZZ^ay$n9@UYL)GA?% zx9DQx_B-`tS#z$@MUlJncn>8l?5BlKxlzjvn5{uf!nNF#qmlA0Oq_?5>Nxs!qc97P^cZ?z zBRo03W`(mDpC0kblSytWa19!^P@PVbgS|i$Bx^^&0n48_duI21p-Q1;Z&0JB%LzFmBez9=1b9Q%fW#Ii;JVcc>esF+50^C|2 z=I*E(mbR;h)sF|-+Dz%FyzNWeK!MMRGsxte|&s=45$Oui=v|1 zKe+^e8ub5i{FeSbuF#K%{HXt^LlCGu3j&XgP!(R(^=}JMePBS>4?ci~;^Xi*e0eyd*IaGlE>wOFa z68h`+H7A4|>63ZkfKK|npz$-sEvis-*=!@KVR_kn(OaU_st>DGmhx4fT%L1T4e(sO z?#+Vr^y%o%@{4Jl6OL;2b#YEZvOMlI$|1udS zsOEJWA>)#fH@<3mD}(R}BcVK7daTk-9sk3&8iH4AgmSp)({@FJl8@&v2wB(?vK8hk z@&g=$(4}&D>if8vI+rxD$;m0x*xwL#EEl8<4ajhhg*33laLnCJPkeFcePc)4h4FnQ zoiqm`O9oEI$AJJz!<)CA9m9xcJik^rb`#GWigi@dNm(L5?QiM$ED>kK-JV~Epxe^4 z2u7J&I$;h(om@){bgKF*&=j+-xI7X9oEM04qO4JP~a+jfZy_8w10K}V)-Qi zuz-~-^;RJU%LCV(TrvCykuuQJo(}kSrFfNK$wCz2fJL zve^6d@bRZ$7c`b(+{IKc`|n}9evq1k_BDDR7tS;~D!JB+y_s*t5V{9aBBHcaC5=i9ibv#A$*tG%+>bbM>R0=$b1`_adwObr zva`H-(Y?|hU{dDPi!jL3ZmO>*D<8ib=V?A=BY2H~d4#}z)`kz{blKCRaxaO#u12;2 z;eL!%PW$d9g!q1Te96|CDL>p7$xxFlO8@+3#%#aZZb&U}tzfCH{hsgg!;G?K_4T&} zeON@9MrOJkYee28rOxCF)9-B^XDO2`bZ;DN7QaEb;U&Lc z-&!)NX}(J`$Xri6GwfGC*RHY2IMf7j?9qljm$xGz*i?&j|CA@+qEBCpJoSp>@*~|s z2*9}=d>%e`NO{uOD1Le2tE}I*$lctx*hzw1FR449pS(TNTANN&pDXA|IwAW=Ww#aB zN`P9*>Hldf@w*m{s&N1PC7oY|W>i)Gj~n$gU8Uy;LDH60-QzW^>N^nMn`a;LWJj1N zz45%&Eo>$#CmNlFlHLW+|V%r&)u))CsBqb-6 zc7aN1L^oQ(ogr~&3~F{6UrP^rB3tjrxED;TM&1;RIB_vqr?(|@n;qv&Qs^b}qMk3; z#L^$uxUoRDMRUy*9W39_QDG1+b*jIiPU_DW&3#xg;1K5-N5uwPcXDVhhrUvI*~0rk z@9jp)m+K6|7z$bv;-c_^g2QFkDNZKWK*qPSGG&4Tan{eFZo9Z$x~ z^h&jU4y!3zO5Y@&M4svH@o@>6y=}tyy88}S?=$Ws&%4G&>mz4Kg&DTI5_FBHq{Eo> z-c4>J5;w#(Q)+{ui)uSQh`lkH9d)Z#oCHa`R>kFnrKj|)V%OXgP8-?Q{HY`8^JzsC?0Dr^bVxoa$Xwi2 zpW=)5o~FM!)!uv96x;=|RBc_Cr|V^9nvPG3GUIhuR$U6`%xM0D&vwWU*c%52uz}H*c~nJPKtd^YW1}`4P$+8 zS%xiQ@;#@)fDMT0d(9=O>G83z$v8DxhK~|5DUlipCS6>f_!_gk=5K=b9tl9CAL^V* zj^gJYtP2gYv5R&(VGSNzW4ER*1U%RBbsT(-bbRNc|ALPa_u5D9_(3wRh>cqxKVQzl2 zn^=lbt@lv{Xn5lN7Cz_G{zi$nVzpn4MxU^T<>+NZ%MSMDMw`)*FUNFBFWbVqCpr4A z=}T={tUWflLBS1T*{M@q98f;{#zXM+@}r|oXo_tI1o4kDP=h%RcB%afXP?kP#$d5HZHMy^ivUAGGJ{^gn~WMyXkTa9Sn4TmkM zr^Q!sH3JH@N%?U;$mU~<0oA?T*`9~`kQ56e8TXFY6ZDDAI-Bkm)NdO&Ki_B7hq=}@ z$p>OH6~3_!{Gb_#)xQ{&npML7KCRh5dgG<0Qoh&*Vod&_U`XP#Y7A_fF19vJ@ZjKT zckks}eSX5G3U6!bjeIs9bP8&Q&NOkhm2B7zw25rTgY0ku9{%I z$8@m6D?^V=)yMmM#|xw(3j5M_G}UFX8`hef$5*~SKW;r-#?rBh^S_^`)FUBYi&@|5 zXIWHM{-Ody`fipFYu|!!vqsMyg%v9QfeUH$br+xGhG;f|WjTz>PlMwI#IM9VIBrPs z4Mcoz(`PAp_cTF}WAUT*Q`;+qHL zr5P{aujPBKwVw_O2Zb4z)`qq3+4)$llpNY}*2pW$>+9cOQXZyp6O+cGFMNr)aSN=Z z?7cdrU_N+uSm*OOBPkhw9OSzIPww7nhR&W3F)b3AfQB<)vuSwHe<`@U;G}# zk!*@@ZKZEv)}OSjUmBLq8Q7Sbhq}wD$)16NZ`#H3#X%crZ_J^p&| zepDwFC53q|9vTKd!xb_#UawLvPGLrzFE6^3DjrVbokwOE6ys#x+gYrL0(JP5JP8npa)|OQ;6<<}RiSoWIgOqj}YL1v1rvAN?S(irpq% zDu-v+E=_mhdIFfa2>fWQDB%EX()l^Br%+?@e43&}qsJayVx6T?lAXKzGl@-u7^~9L zy(?%R+{c6sPl;*biLgiVZjCj_H9ypWy_;U9PiEuSj=IghSrU&O#3Rj89=}eRocl26 zr5z1=a_e3iIDg9gaa@pWR$H4?z|CRdkkj(iR`vYZfsvPe^%;`q1Wr)#Rb5g1Cy%Mh z`%i1(pE%K?$}!)4TQNxOJwxN!ZJ}#99`HNP_hJ{Kb~kHjIXE~Fv(KCE)*cc%%_MG> z*lrr^cArlDX6n+zn^M{HBxQr8ysTJoqw-<8a#=^PINgE?-SM8+V;lC~c<(%_=O5Y& zluLEyk>htpu{~<2v>d)`kq1~!s_~0Y)YVy(qnA(ORB(Mm`&LVUw)_cAvzllwaCV`cpfKr` z0MaY(CHcc~J~GfPJ-Jdr@j((YgeF>u!}*%dJ&?YF-tkJisW3H0W1GJ-X;jejHoP5q zXL?_}vOojVtI9!Prr07_kI8;yf;rB zzOdAh8{?3)TNl70IJ8sdyaj9O@j+qw@Ivt*a+8=%C`aMc}CHl+={`X_R{kzlyhNejMG0%Oy-#Mt!9%2H>6@D}LLAWsbQu zyMnDOp6C8OSxLLxeT^=*ZnkUeBk{W0f;tk>uUFLX79vL(H~crQZ$LN5Z$z)i5ad~X z8m(TgW}Mo#%^b7 zbd;pzz?>$!T*ST)tZ&4o+K6)j;g!fp} z7^^(&$Y2)JsVmKb-ZsN#)>O>=^4ZC)oAZskdFOZhZ+>U^e(!r;NV;rP$S3pYPXEYW zszDAYM-FE~;!GkAN4b7W_4_vqrFTjN^`Z@CD&$5bg-t z>Uui@g!h#7XF-I&sZX2d_`odJu&PaM=T`g-vrmg(wO`1^lZ%2w652ZAIAZ%S^sqtN z%ZL;UkRGLAaC($gU2c7K z-B*wL_4_WAjfssgPtncOJ{90CPS?+_o>$Tyuc?JzArQ|?VF+CiT{ zZy?&TA1oRq(km)2!tLE=+-7WFUTU1wgg2Wt`*2%hn?|aEM^2A*`fl2{`LoDpPeyiK-+($QeTW`F>Rlv0+0AJmKzSe!s z>um5PGpJ6|PO{{wD)t34_?F(yCh=p*a3uxB4?c`S9U?BFb-L$XY>fPZ)tvi_L)XmO z%t9-I%mPTRh3bZGgzRIG{0-+D89x+$JeFnw@iDvI+T+&Gev|m7>htv1g|DKY)!fY8PF-Qk4XZcY z)K)-y5#KylMSGw3ZVzS}E;ZVI&fdcs$J%(x9FqIK>3w=gRpjC|r-vd&6xyRY?tafq z72999DU~KY;JnC9*5|k@rY*(xk^8gOw;&E9!8@ry3byFpV@~=Y)O+(zHYG*`P7khC z>o;S48U29IemcSSIUAB*+9#$b%r!EiZ!Y0XMdm)8RXzJXBzFVe^pxLw&FFBSU%rMG zns?}4Od4 z(ZilYagSq<{YSp7OaYyhs*MB7>5p_-%BB%k<9XW+m3Mr{4#(E)4>q0&pgX|dc+Hfq zU1oQe+f=NE8GCJCbC7ii8^5e8bDWVeah`)8bR0|_ObE$9C&vc2oHkk;LD0o{bjQ~| zU(QrIQwNB!iD7ffGmg_DkoZ$Iv|Mz9%jCM0TPd+A=dWyEjipMZmZ#CCJxe=Ew@m+( zA(2s*NtYR!dHUM!^<tW3B!pmla`ZfpB+E%PI*q9 zA)ZVFd*K<?Ooq1*yrDWf1rLadk8%| zJBm2Ieq4A0KIuNyJzYI}bdGbLe8GOvaH(`T{T=%K`xQd8qu{->f7)20$|S!wmXeBE z4Ay3jZdTkvf!hmWN zswDNVpMY>%nEjA*a{ejz@84Shb%6g-;SUWceLN_AKMng|6r$w*pzjawc>QMw|E%wS z_U|w1{-?KGc)@}^{J(x_2VxN97la7@anJ6X=;NzwbE7+8Fb%RD_3_pVQpOh$8x}~8 zwuWPb%2=~!twNIh@Qd)eB`MO0#N0@xr*$Euw_uAmT$I9n$R!Q0J~hve%d0V33@XlZ zVBzz#S+b}NSo0AZ+-n)!n{8e7%XE`K^Nk}$AI89jLvQk1J!U+btn28w-JKr}$x=g) zxN@VrLq#QFCeKF<^QkMkO5a*M=;Lk9D{%BI)h8ZwFZnI@OZhzMNN2HZGl8EKdB*$c zFqMA~CSdAzmH+ZJmTdYk>Ly4K;&D$^O$80xt!i*67tf_q>f15B#Fjd&`&PHAnq6XN zDEcF%#~5C|j;`nD3;N9UeyM{2Vpi{$v>%ncK6-+aDtf^FazQH?Jq~M{3+u`yhGbp- zJ#4avE~S%!+u0{2hILPeizP}6zYQ1aWaO5~(j2iXukvM~UnAU@(83OwR;`>m0C^0# z=UvQ5z-npKQ)Rd9Njqxr>HTSmSG$!c-iKt(esoLd8Cf}cri*VMXID++F6M308hVKy z7<~gr01r0Bk?UofEcqfje+Rd7dZ9G2bQX4@nhHnGLg%s;GnwW#N9%htb7r?sC2O}t zy$?5LR@++wPB!PnHYNfj_PYaag4bTQW%>(Ru2l@SQj0OMP%$scbv~!(^E8w)*Zg*j z$4<_wqMFX@F|~i4ul3x1{A=}o`FD`bR+G~Tw_b83o7?V-JpJCpa=rRHJ_oPTYCagX znpqfM=Jy-GVO5TEm2OEDK5A{V@BPk5XhpQOeUBAB4qJ4qrHOdl$g{HP8O#tiahYGp z>rK$9bFDd@I-r@GABUD8&t3_u3vILuO+`BC1Q{o*z!Kiv$ev;jCcp@`lX?})Xo-I! zmA7MvETLzBQ{qH2cbu4q)@h5}6(h~8R9_j-wIi{A?l5D9XZFl2&y3OxCjH7RXWoWg z0sp19(nYRX9^)sTbppFYclA3ji*J|g>v^-(FNMBU6zoMV#kgLpFPT=N=PLNXx5V$2 z=d0;&P7xg))h73H*wNAAs}rpyEiBE!&}tf{nUY+T#Xdw~i4RiDPt31X@l&bHKX%rs z3#iiw%+}nGX#J#TR~&qY?7l-@&egnI?CU*-R)H^y-l%^|nCbD)lY!*NvtLWNznthg zM{BiZy=4{y>Q0eQCGRJHSWtrpr{>(_#7Z=e^wB-%UNVg_mCt4F86F#+8SZtNn)mF2 zWRCf%A60MZ8>Qc!PT08b2ANSEtY2j1Qo~Bn5YX0Pkii{(uWZIuP$x6Vp^e8q>c3&T zYvdA6eV2>kT4Zm>on1T(KHKaXuPG*E4Ucn#qM4N$7Y5;as7Sk3QPC|Wzq~c?2=H#|Q$Dqm>K6(NaHnbH$G7h^ZM`Z>=n8P< zndOsn$O|U>hew9HHITGhz7MZ0=N*5#-Kc4(WwcHovE_4qDYoKDNjewcL1(lm^3bSL zsT6I-!>DGfcq^jps`CQ~_xbluKMFjN2jW{elOs>g>@$Sk;!M&^`60aR#@Fyb2^DV%u`U za>B7#A9M7vIm;I2gz}qtvZ{O|mD*Rt0%UM8_$-{wz{ViAG^jMaG_T*%0p{SyyvbZL zbNcc?e$z;e+oAp-_aL|?IcY$z_TkY(9|Iq~Mg!(IEG5JC$BbdmA5l;z_1w0b7T)#9 zBJooG9&R#j*nhA8p~^tX1@^#vW+(!k6a69xjeblP*JYNio4NYSLf?xzr`S5h>Ha;Y zxNi6HjNaAn%=rgL8|zb2Gq;QqwsddO)A)sY`-%l^T0t&(PU4TJmrm<5UD|pEkd0HyCdSu`<5caCF&oFf7b1c27P@ zKgLqqB}}cLSyhh_rG5ssG^4mf`n(GKytnR2WL&xX@~6Ew0i6qUyMBY?(J=V|*3VQf>J%7O|jNrPa|K z)#;u?s#CI!b|jQT#&>f4sxyhd=o%IfK9s>(C#BQ?(Xp6(%VE}*BKnwlg^jf4m{lOV z%RN3ugH#|(`JO84+O72Lk$p}qEv>Jzi~>2Nck+1cRkJAMuDe6((y?qdUJYjZPV8*eeKXxjod^A8 zRWx6PJzvh4*9PR&tG2#vCOqFNQw}kBEo>KY4OLv!>w`=c`cbH6Op0 zJ0?b&SjFm;MZLv(MjE#_iW~CFN;EyBXS8dzQyNOQ3LD4uH`6?38%&L=zd@vYrY6Yl zY09sAH?+iz6S+sha%nd8=n57+$Y-=!oo~szhzc5ZB-JOEJL8!wJNTw6orLavC~_}O z>sGDNS{%KnmR2ypS1>?Smq>6=Luf?to_@ia_KacekR9){O}A>}H%V)G-YLdqx@!sE z3C5)pYqbZ$-lYTMoBQ6)l1`(Ld%8}ayaKBNnVKQa#!kZq2k;#=1lc9DaUa$!-&!&^ zvb{;NKl?7^U8jWqYaVAV_18!17pKyG5;sS~_RaicZjN>DyIjPHWr>exuBja;!rperHpIdfq>83mu$dy*I#VXArst)lSqQ560|CspRe83J( zG6}&KmB$2B@0s)nl}y<=f|e_;w1y;9;J3DK@8X=31-#GPrMP@ru&XN)WV1_~(M|A} zY;X4}l_?|DycI{+UA?EeyElxxHJbu!D+rwy>6#*!ACuhOX+e*5nJ0CQfPiY)H_pLpEg6*S$8FURvRM!#SRB&{p1FC4n^P0zyeuiJ;F%v%Rvx06d5wvM{)(B8}<ls2W2m5EMul`3H6q`HcqpDT7sRFP2VN?!2FH*#7@sW zv$wS;-~;!2mBaLN#f+<*%XVI>E4@tmOjA9Y`YgT682YyztkSREzxzP$NQ>vKvb1iwk;43%yH}$Kf1|Xku+l#5n)c!eX27@ae!-l2;O_`|6?LsqZ2{^2rI*Cu zp33$Yk7-@?u<3Tv2?Tu#6kMxXjUlmWGR|-qx)`?X4UO?9DW@k>U|noz;w2nD2^qR!WpZoMA1 zR^hqifxSNQ5uc2+Tdm6(do9~ut(kKt`N@(jV_X(VHNBsgk+_XdjfTrCb>Ym&JHVYD z+C8a*wD{x_?iIZ1m#mQ6 z(><_UgU)EzoEhrcVuSwDt*!=v=~pmLp^v#KZ7XUp;o4?ITeREa%oL1n&xWsqP@uif zneTlQOs>IW#vA4*)P~&fbbC3o1tSp}_G?3~d%6qPr_XQ&76jV#1)9D)`e32<_>)$~ zAqRH{!NL8<@GuII34)`jYda$4xw<`fkqNs!-pftjjDvOCO4jnQcx}a(F|DzzaaJ&S zODo604o|^LzF6Wlh|h~(Jq{sf>9ng)jvvrc%{^mY@Nv@>wOJ8z!J5&cKQMv@um-mg{Bqo+2B z8<>xGqX)*q-o@seCW{f%M-N*ZfMf0A=}YHg1uRwcmAV4eoi-J{dDr-9qI%GBc-!8% zU*~oR(k8H?)7Kq;d0_QaIiC9iHp3vsh5Gj^_%p+@r#DU2(DvE&n^-2v?T9b}LJgRg zpk4$=bQSqnwj(`#;~ssqMiZ7B-qvMhPO~G=tTR%|$@{KU4RKtV!QWRq#ShmZm@Y<# zb#9J7@VkpnTS6QnlA+gLVF*(mV#cPg3=x!g5b)LS}gH(YP>Ag68hw+qy z6^klK!FcYTvu}Zp%!r;_pH<Y``)pg*<=Dbt#Px2vDCo;#=yN5$fHU3AmY?bmrAUgQIU?Ra<6ov&>P_qEGsG0nIjSv1 z!>!JwK;Cz6ZO1jQ~kC6=M;u` z5>Sy&SZ5GT2b;d5f6_gX?x#~*O2*2$aFBBHR9mb{u_3S=fNJrjSITU8O0oJB^{z_QQKPY7(3f(_ zXafpu(p&UFeHU6xvqoDt;0{5_|LxNes?daW5bkamrVhycB-)CbFj-aB) z5pe-Mz%EqaXs&^{U*gu>o;z|c?#F|98fSp`%XuZQ+*&y!mgE5k!qZp zs8*~QVr_(7>RwnZdYU-~Oic4@3-z7vqQ&k$hM!$RUOLUQL2G=N8 zO#+{G(lYGmXLOlAr6;WYJD{5cjZ#3Fx#;(qR7}18&{;b47y*Keu<|IIEuxS{*2`)l z-KBrgO}z^p)edp!GuYx5KS2#VN=Cz{e>jQMn)Goiyazn|y-yMk$yL$|m* zSLa0Np-7L%ui<8#&-dwLRsyO$gI1!Wub`W4WRObvbc-HwDE20TYEuKm+ZeIGN+U#X zxtCgT1sV;_)&$2}{+7uY%rXttTLnGug*NVj6a8o(onyh}kWnhu55+v~QRe~p_9xiE z>AW59q=DLTbQ^hxb7L_U>%D@RmVsJ1i0R_L`L6chCXO?{yLXcAWZ63TA?sn`mCqIg z*|ovgmK=c(>oG7M0|w(rOjP6-Hjj27DUhW9@JA@lK_<-5lBO^n+6g#e^2f;#BGb~E zPG(w~b_Pn4(&-;%rk*gFsnSm4q?v+G{q|%#c4yktY43aOe!u;`@Ah`lClKv|fvXYL zg%}=yvHpdgK_lRy9qqtb$iNL0hLttO{SM+?k0sQN>+qY93feMtxf$ zMD}yH;2To_`6c*zdC^9E1W?#v9`mRQZ%1wVPQY*NqiIp}Q z%@}5MrKJLv`~J&PDOxH^mRO<@6lrRtahYnrkI6KL_w*%!el;eisFpYAUV}bs(4|28 z{m_BNUE?vT;S7yyiDTnxJQIWAvL)82938bbHKMGw1jrJgv}*Zq7FTb=20^Ohomql} zQUIlSWIX2e`%_Jg8jcRjLmHC1HG9x7K~ckvW{zqW!wucf zW`j=CY~zf2CMThxOwcwWj|@JW)VRSE^Wg}B-VXWT#-nOz=ZpR+k4Kp2hLpNoP=oIiq@ZE=3kIMDkWB2#pR?K$4VUP9fHehLAcO0yDT$7HhMn`urgYL>cZWT@op2)2&frS^=@tA6bT*X(ynKN4^zTts>k3^079dqk z$TXp5)WKZg#E?uSS)C@6WKumG&nzZ1SGalFt0m5)U=riF6N;6f?yQU_`mze1*tI7) zEg?iF`jb-xlW3+pm8}O|avJt4iXkOT%7l!ej6ppZ>U@d_hP`)MLFj~``-E7H`E~NsQc%4I53$J737R)A{a6iWZB*M1ag(Mm{FWqGxejNX3Z((u5`Gp~$oY?JlANBEzg!seJfyrtjMpZV-f{jCf8 z7?LO;Q4Azo71wIA+B!zYtgl0jyj5lq1qs1jE-_39W^7=(E_`mfU}I>?4@4Q7{QT@n zFcq7)whe@9b(U9ES_l{473-|o_sTC{pX}~=NAH{Z#aDkh`Wb#3H~&#z_0_$<>vQ^J zhUBO|jW1)i@9lb56iUoiOAfD6yylBIRARM0hXa;+JE}z#wLnDOwy|TbAcQ^LJ39}) zd-HSi*bzpKaCluyg}&wDSz>x|UYxe~ukkS#N>{3Xk|Q(+uo{C_1O~ykMQM zPU5SUtKwwYd*W|IVZf1craS|+V~%m>xM!?Z2$Qh6U2HGiO17HgVxn|X{D9mszc0RD z`X};d^M~Sxr4GqOUBs0u%qR`z&JC_BLTRnN*-q>VaP}+6N>NH;<5f75@lGB8Ps>e@V+iSw69Qo#Zf@s3XCZeC=B*YApv{Gt9d@WCT zjpuo{^(LN#-Fh+5kzV#*!VB3InM>!PC{Wt(cbF}0?e!h4TtI*HqkG@Rq-B@^$oZ$@IP=}<*(ETW)DSc z%pIOBeAE?nM?FHltKMDj;hX_}fS+((^j@sF=(}2T)hCoApX8&yb-rW1mwjh_ANvF! z%ZszJYJ-nRHoK3N8ez4fKmmd38srG&$WN{jY_sR^fFjr0nr+0!a%a0zVG7ARQ;}Rf`FnGcp*7^s+Z{K6qpnmZV_%NW@Vd#*Wrt8oo@^$XA$|a)cR%DZBAd z%A_zaNMV!UfTtrIG_F+E%$7@~MD)77#9M*c2@?i_JDp;c(bJc_g}ktHs)4_+@Guk# zrSN~~t`?vu?mN8yecax@?sjjFdx0FsTh4L_$19TDI@M}4cpx-D93IBtgCs_6^dgAb zj!0}8ai$Na2KuOYIs^EdfM{2d2I@0LqX~t?DU+sjHPN~((E&c6sGNzr*&HeAc zd%JhD-|zc=-}hTz3w)701(rhKM!r8i1;~gIDf}rw~9sAOf#|e*jJ{oF;oRgaiBur~z*iBiq~^ z?k;z~`?#BO-=rhKRFvxS+9LK4ieS#HX)%q*3(9)ldac z0Y0o&PSdreqemCoreS;z&&$<@7up63mXe%WibP>Uc~xk`(t^eA4R2LHzi=g|M&n=q z?dA)ut1fQsRPTOqM*TNx^Q>OGV_nmhxI4s}szueWG|X!2TDIf0w@-dA{%G4tb?gr4 z3BV=i0j?}2U^kRvK1nEMcqi}WEBIIWTfAXZKnBuArjeP#LSdQED;yS13k<@I7;g}0 zisb|v<9R^{qmLCAL0SoDqBu_qgn*MQMk~SzVFV72piFRR932|OND^+q!l?QX)xjcG zD}+U)yXfOILA%TWd>cdyr)e0yfEMbizP%o-uN1Vc2|ECm8?7zTAhZX_P(G9D1W zoT&SkdKFJZ|5Q0(+%jNXKGg$#K~!cMO-X#AWxnNk=X=60Or4f%CdMMk7LSx-X|q5( zLIMjYSR^TocPloNU^STp3u{e6n#B-hXb&{JZ_w+o&aQX?EcGQtaI-!ZXEh?SJywl; zoYm$}vgTM7D`5?z4;8B97fAl4=|a^41#>Kmk2x3-WR?U?_M+>N4cuHBis$Da1~ zS3US+_0c1rI(Btn_h;9Di&cPl7I4vyrC^^bbDFq~#&%<;vzO{M_L)AAhfIUgr}7Eu zf-J;Q1=2jR**1WGA&yw-BiIGFokX<5BE}~{)6yVJ1c>5$%!1qF^WYv0Fi%f_K?*a% zOfm!$M!|#q2q6uuWH;%9vM72@;wX@FC@~V`5${TKL@dtK|MJQ4sU|yK3joF(QzIl6 z(=t#EQR%vL0TqibSujuxdLgn-<7Obu>KI%3?V1*G&HD!)s}C>yNxg-#zV7`px^HXw zlGW=f%h!<=iIrvD(Jkuur9Z1Bb|Lj5_W^f|JI`I^SPzFdg2vpOkHdXjfD3X2r#&$@ti>Vp(P11% zByHr4AkBk58bJ0uN zI2n5IeWd=K9z9L&L)0HOs*6Pu7S$Bv;I+n#3U~~BIU@!1%$rVOOMRB2w_KZ$puVcT^=je0wB=%y9>#Zrm7P#To5Q~@ zVS57Joq!S&43NED3dgK9Yh!`-1r9<-XuyfK0S@O#-ay3G|v{Y`84NfX#d(VDdykVtQ%VS5UF3ubV1(_SnSWKMVQ{sHek>jWIDL5iQ3$nPR8#`2^rt2P=erh3ET0nwmgUlGBlnWt@&{EfU$v`BlwQQ`fH_Bj)@Kl&D_*+eer1 zy4IyjR)Id>KoyA`UxC2q@yG%hr+3T}1={07+Ic0`~so~kfz2IVl@8Jro$Q!x^* zV3Hu|JGr_QPlhEY)D29^5nVFBao$~MGXP4Uq;wSO=Yw49EzxYLz4cUkwnO`%#gdZt zQ>Ui9{}AJ61V850On>nKu43_l#PXLQ{1TJuZ;T{6JgEXw6Q3B!qM}%Col3nP4*@Ua zkqLr2t{&SS9o(#*#^<2otaGPPsd|tap7`m4qKO!F(tP#xyXYh=vp&{)S9IEO9GQ&xC)*+7pw9pQPE2!|>{y@^-yf_0Xp|P{W zN$3z__V_@BPBOmm&(prp+@&IM5U=dgK8`YYl*hqS4dCwnyv~+~mH%lIcrjYUZi9AT zE?2X1j{^4v!FQX*#{c1_V`Fmvd z>UTc=W8dmMIB(Id&(Y7(Li8fq*8AOkO&uGqJHyWP8`~9T-vG>h4=`(LQEi=F9RnDC zGqjX0^|G>1U1&zmf}G_!tS&S;@726F*>;0PGmDXigMs1RVI@ zol>IqNsxn*lG2i*H*mq>wpRw)Dr%pfwf?zXsWT{c@yE3@7B@7!(DLL#Hke;>%DMW< z!S#LLZ!U|{Qx9tVfq5V7*>mXmfL~!`8`R}nkg1_g;wA^IMOegi}= zwGO)xqW7?3Zi%=!A^s(3rPwO45L!9~3Gp|2W_z&b83G0$M=T?5VB_)l^{Q0r^yu*pO;3Z*a?#OF&dpm-Tz)}zyK-rdx|!Y#i= z%g$UuOBT<(@~7GtesbqO&VA?p%;X^C33Oy-E@zuQp^Pe(E5eCik*Or5|_4L9LeQpyW=13Z2MIT7t8mk0fc zWRfYtC@OYmyD=-2)R@iYur4;lSd_J~mFyOyT?~{H6HQ=p0cqMkZAc?0l^(04{sOedn;?#zz@FXgeT`{P*pWG0txZAqH9=W~WAZ35 zJXJ>?C5EHcos}A0+RZw9C*(WC-oHQdu0lUh0Ib!Z=EY&#Fi43gF$je-1ESk9e`->< zrq(Crz{pVl=xp!Gp9D`~@JOLwk~efprLG5YJ~d@jFTxY4B`fu1csX4vSLkc-TDnW_ z(GSW&{f->gb9(7k*{gpjKhn?P^Ym}>f_@cWqgUj0-M2=r)z{;0x?XncTQQ$+EX6D6 zGPzt|jekh_blgm*%Vzyq@vNM$^RQm&PvWUGDeLt{KPT8rJyPn~I6^~`r;E28g^kiB z^Kzc>_`L~DBPPLOj_@T7HMm*&4PR2&{?eqO26*(gZK@kX5M?tl*EOMl)(C_O)upR@ zE2_L{p1r9aKyO%YSHQnmR7ja1G>Z_54P8$V47V7TAzq5HR|f~gMbwY{1IX9UJ&X?G zDK4xQwWi?|YMGm432yUs0*T}6;1rG-gIdK(LU~UMqN#?1$ZMyhV~nlvM{ag$Hif} zR^F~)gOFkhf|B++I*N3}(J|+ybJ4lv{1c2!m|PofqCdUgsV>0O4$y)krS|9(+b@Yn zCLRH38WlR&E)8OiqX5`w& z9HY%3D~&D#_ZcS*oURZ;jxMTZ+uzcXl!?bwM1Mp@!1RHRj0p++a`~rReVqo^tCLjn zutRAaDODM=C7?oBX^fmaEEQ^8YpB$vDn8A9tLuT1a!r7*$y}SgiEa^F zyqo+R16#5-XKxN|4sVKhc>s_Q$qq!aBH`=^4_5eOZUrYfrF(Ux>85U_tD$Yh3+#fn zf|Uhb1$_k`tKe1vE-*{`G?W2mFz%jtcYl88?;e&+*SWR2&du$I0O;*nYcN?md9tdB zX?v=nEL9R~+`etWw8XEMZRtnT(MG4!Iqn>BI#JbU`}beEc=+%TK0CBy<$?04(4xKN z>~vlNle+v%XJlmL!M*!xACzOe2P{zgu*NR-up?PV!vCYY?1Q5y?>N5CzQ5mI?sado za<_YVfeXn2mp~A3DG(9@A=L14wub=?fpnY@3Id%1!LfjE6-9#;Ml!xYl@1~S4Y7;? zTM++&p&CGB5Q|{xL}o05IFL(!yLTaBI?d&t=bpWtxqW`m@Av(EzQlA2HBvnkp5xAw zWEs;Gn|WLq>;E4b^$*kI+5J(v;u@;*x%UIGxKD%X6bNoW$^hvilB)AB3wTQVhwCDAKMqD%Bi0e_xH z@uGTfyBGI*v2er=7X;3Sv9fIt3ouEMaOn%Jpq<`Nchje7lHR=BL}4>-;%3A*G-}4E z#|ES3wjMiLzpkkx{YKAg0hOdv)4!<&;a|oO0fIyX0}mZhOrqa9NM+{#g`_|OdS59e)EPK z?;FzIuib?Y`(l@yIb`%kTUz90J}eLdZSF6O?pW$m0qm6lWd;Jmf_mVdhB$Iqpf z$4ZUzL|(~~CyqR~{_Ec$Wan2AXwqS*VFrAdS;EE1I91Cn=UTa~977`t&m#$(!B{R3 z2$37?ev3w0Jk5kr9NS=55iH$=T#GNqTk)+piK}eiU$Un%XTb+JoQmVa`X<0Itzqhw ztR_ux@DmzGsqMnhrSvTFg>(&h?Dp+Dli;bgLC1)~Q>oaq);NY`I7R^x;ij|GIc7dL zPia%OyI=F|@b6JR@}Kp6N8h3aNfZ%=GkG3P6vEOe8>$>Tq19^j8nIk!)$p(u*V?si zjnoiucUX<9-72BlLJ<1DcB0o20|&!B5jPx(-zjbAd5VmiM!tP++fbq4dGYt_TSKTY zzVY0?Gw0WPb!dWDkBocdiN>~lMC@)lefz7nKTn>gC>wOwP}@-({wMM^Y8T5Am_V1XHcZ#Es}ZAOetI~TPgh~n z>G^Pq1dfLh=7qzh36cy`w3ni2q7z?i2{bq@5;zLsR2RMoBcTu>wFMGwAX~{xJ{2k~bu*QU1$*yNW$*n= zIY><19ZH6k4gn_B0ZgYHP_kt-g~bq%MFiPoiEyFsCYp?XiyG1RzW-2NcRnR2g7JaO zU@(k%&UPoL-^z&*F*+=a5l9H6U_sz$h+qfI#E|)bS_=Gk&Ei6Of>&V>2UeK9qHH}( z`t<}EhJgXYK#L*F#|_8wIObpcHSs6zXZ|OVIzgS}Pl#V(Uw}X95wBz4aok?=Eov|S zj`%UzL4C|05VjPm>3?C0X$7n zqXph82s}qKOqk`oEX$FCC}vFw{0JdpI3h)Y;su6dX_jHAY?3(k1ez*TehIjbP87HJ zFnvTgVwKo#5SC&XWB|OUDh=dANS)P}3I+T6LTX$y>Er3I(-)x%-NcQ% z7>R)&sl3w(l2z6JJy}dn!c1&5YO$6wAvTxN{h=9JWo}j81?7^PD^n}f`T0xK#{5nB zFRR-^yF-Vy6QR#Ek)|b|pH}?^^l)ENU5#(TchiUH*QahHgL7wiW+XvzMB7@D0x ziF&A@NB_sDODZA|zK$Ek9iSYUZ z5xeivG{~sk`^J`y%|JG?+0^F+rG-mnrKq7lcIMPyJF5vTFMVB57^3p+)Z0hq?Rfd` zGi#fx>(Iio>(Q|brp}x;MiKCDi(h{w`OHV@&gcIyGk2`YR#Y6=^2iI-xq0E-Iny3Y z|Ci}%0dC^DqWf0+*RHf$eRd_wl6Ni3ADP;4km@r zq!cG{Lin`gXKY-iIQM#w0wfmSUO9%T3P zH+gNI1?PmnK6T=R=Fg z2bm^ed1OSIU?xH*gp=Z@!e7Ok>{e^lk(p!?N2I1z;Yh zoQj6wjyYLEM-_g3Iq-a_BGMjtB62W7Mcg`PO70{8K9|)AjCgiB3(1U|JP>}SPcut= zY0Zqd;R0Jp!+XxwZRr_X7%H#`n4#dsz$9gE%V9fqcj~pdVr1Vyz#mE-Ui6rX&FDBU zEU<_5dfF~mnR9q2WbN0xOhCznU+TiIcbNc|UD7hprX>;}PoYFs+lmAD5z&wV1t2nH zqPmK3=(hgw<`kkz-a^{^FOFFMd7%sWKDvYyX2x_pw`Sq$V#n6jZ>LfSt*97!ZD zLM|k%6V`9H?Ouew_w@FLh9_}9`Wg`GVaR<4u#TX?dUALZZWPxF^~ zI?8wO+xcPmfp#z~K~oOMbdxwud=7VJNl+Rz!%_mt#t~&xn;uJ2K`xz3`9w}frEJ{? z-IRltv7*WqgiQ%AC@BQzYba=4KtXB(QP!Q?OQ`tmGZ6copibLig-+tXSU;Q{mpo~k zKDPzuC0U2;@G`}GCv!x2M>xr!63z(1N=Ju-lvs^#VjkhPF!Tif8Fh)eLtSU+W$ZGp zli5b?p?*popof?tc8C+Al)@ya1YO3Iv1MF^)J!$g0@NR9a5#Y$cn3o{X^H_a38#}| z1&1IwsY&v2GelQ#bx{_v>m)rb9_GEfZDl$IM81QA zPLfOAlSU`Ucs)tnrMezs6I9P?jE${_SkF!rM2pdCs~^2!{l&WeJP_M7+G721<~!)@ zPU|qd`W3(`13NoF95gl7t9n;+>p5zYLP;(uH*$^cFJzj*n$i!^jr zpq($tGpS`*K_{l|A7<`3ez(%;x+jS(6NV_k?nRJZv12}lHb_ea^G zfH=v2LjIlp2mgt9fxf_8l;lI)De_Ou``jNydNa3^>Ej$8T+Rij ziXrk*tgns@b>wZ#BlBE32*tMguN3g5xR%)N^kiY1LgnVW+bIF@9y458ha44%Q|uSs`Y-~?8whX zMurcJj9|^XgCHEW;IB-Km?Z-=;$3JZ-AQkz9TiHOYrSiuLJ7QEjEdx5aaJVL;&PD` zC&{PGQkI1nJ4i+-A$XaugvO5IL)(>u3fZn~SB@yB6-tqbIF@szrjsOl(Jx1mg6mma9tJ7Rv>kvc~o85JX-V6>b9do)qMbfZpRf6P+2?B9yE-j z+OVOqBlF0E-*{j}1r;CI&{%W*YxQqhSHZ_hh^Y)dmXYt7?=l{ykSkU_YT=+V=o={R zFXLHXlaEx6OH-~B`uW0}(zL;pN~@%GQonORIb=+UY<-~_Ym7f;tc~wbcKBX2o{#ah z@ozFs&Lz@vcay%*V2xODyjHByYmA!0ni$Inw1?Llot;FVb zQKAQrfEF$hxpHCFFbFXyfa1n{y-9xMWzG?p6&L-yFrgD)E-G~Z}4 z2Bp`H-y0trjBbb$MTM{*Zvnd#)tKkU)p=<&M}+p+C`w{$=7)gYp=mHt3!*wuFS>%D z&yiu&fhc>XHvlscGM5q*YNxJHBz7TS!lMJ#stM0kO?aqk*3>3dth80Ls1z>nT(=sv zMJ=VS3Yk#B+#%EwnhlY2&i7#XY9Hqkcz6wq+}tn+M()pCKT7n#k=-t;mZlBPV|X&`0JQh2MGQ9sI$(da0K^5P4E^4Z1Gx?1bquh2kI6yuS(}=+Yc&^ zQUiV;7QU4zq;!Ax@!BGvf3bD=(I1^Tf9B&7>xQSj?TN~Qa2)+E({}Ch-)2xnV#TVG za7BUN=V@Nhx_{SiezN=P3l>HLg%N+ab4l}yuYNd+wGuEn@yF4EXWQMW>s*i{@bUA1 zrt-1tYt)l{6Fr?Jvgi43gG9v&k(JEEfhi#3Ih@i z4Mc<@DnSIXRzyf5ucA_52B-riBO`R6l&J*@vr$xFD$ZphisDV z_I{b_l*+X*&~Gav+}$!V^-md1-DK!*x0P==o28ickCHD%F0*H%u+%1t6)UEP*%UCUEu0@Q|k@RZH*L#~} z3F&g`M`&9kwSS3xhx$Cy%iNM6@XN%fOh})lye0E5Rh^NYnZML24Nj82fwAwVejc(I zSrqEq%A}8IAG20n;{r{yKeH)c0z+XTjADPZe>eKIBVR`PD)efCyc(`?L^{ysR_iv| zg6zut{lj&OSvTvbUlaNpoi0qV>K6x{bX~`x65fdEmZa=FO65-bCUbte=C42>t5WV@ zU(V^e+@ry11-pBPb?pS#87JMdB-n(Wecf$RPQCzcHIz!qStnEMQ=Gvzndz;RUs5hs zcleKVvObg?uaop*|AsUE12CR3Up4kHPIU-AM7DE|N)BT^1e2J1Z?sRkH{=I#uES|t zr}u$<{V7aj56WSh>c{-~y62qx=~T(N=V*I~dG1k|vz>vpXU@38LwMlGiQ}7A$tB?bgute<*9)jN5hWWE;pKa_Fn^D_T`%v3( zcN<$Brn+a=VD7@g@XkC!|54Byx_FyqbVwK4>i&#Xo7MXgjjgt&HWNu>)02(e@6mOt zO~vGLH!b(84rX3#E0TKe7w8SEUC!S-u1*qwn~%KCmw8S8PX@L@N~eOYPA6uRJD|GUO)3&NJslPzLB}Cw}}s+<0J2l+0l76m$aow zM)sPNYBTD0kuI_|_ihL5go7dffWz41Ah=6ogZgI^AMnvo7TW8adi&ff>yS@G_L;EL zA0_P5PugWtYHU&eZek1fl)1{H^{anWAE!Q2{hY?WsC`n#`mexu{MGp3N*Nj6>s;st z`HZnE^dmd%?ZGMT=rQljV5i?V*lF(zc6zIWXZ*>*9&cN)j=jjOkE!X5FVp)J#ZuiT zjf+tnbvw(1(4Va_ZJFQ1(Guea#GAqWnAhv%8{)o0`(MbctZZ46bv^#6Msj?7 ziJ5DwJdjmr_T4e@`eW`#H8xf(^WConpBek#90YbPcc_~CrKG59A+6nNS<79TZ}ul6 z7wY}7tEGk72kFRNOf^4d!`xuh@HoRKLWn8E9e zF$>wt8ZUd$!Xlm0yWl*>-CHaD>|aZ^GlBQR37Ko<%RGB9c6XAPVaqJdJ9*N@J}LLu zpPKkH&&`%zrd>@70*b&dW1=3rw^6=?-=hvC` z8>ENx7-!Qh*rC|dC0p^Xkz7AZa;?LZm(VYfce}_cX9{hHirtt_p9f#}*gh7<*BG4# zvgIZ4ZcMgU$^Gi8@%fABcO`JX&7qsw|Bdo3V?)Qpv;5#dYL|>j9SUmk=bnAPMWO*{5JlDqZj5{$-WCsy`ffOpvd$fmSo_J8Gcxq#L4fFN$ zUW_8?!9ERiiqPe7!@So_@)c_;`Eu$%kn5HDe39fT+E!t2J1Xq;4#hew-&E|O)D1Q5 zZ&zabI$ZbFPHSJ&-<(fm=gIS-Z!_nQ(#(FJH*=BXBxU^0Chu*sEb=86YLmDd-7hxg z{}bGueXtw;^go>kJ5LF=3eAK!-J85Ow(wScivO(rQYLbCzQ~zZMCu;I58{%7XF zTGFpW5$RWv%aJv--$K6;$k~+5MfM;+1h9v?Ka;;5i2HT7z;Qs&I-W6g`Sjla)Ac=a zf&b7iS6<}*<5DF$PJ-{5Hz)cg{~<0}M(5dxO#W*`=iiu&{v{g7a36T{?Neg^KA-!K z6G_hVvyfBpk+UE;%bU_S?`Yn#=FP3|$55L0c9nTgrp%i%lB^iq^m&RiE z702?6#OQdO{-XIP&PO!4T?!Px5WNcl-&pL}=3@>`Y9sW989PV30;@({@+|xUpDZ$^O`x@zxyvOzTEzEmAmOEiUBva2y7yL*rECT*j zgN{gIU@%kTTUMPEWIZnh^W(- z`aU=BL-x$*zL>K(CHc??u7$1eNc~y!56RqzY$C7z=i2WFHH=krvA5vGB>#ES^nZ_h zQ}Rvm#cjN;a-Ex&wWQ}Evys{O_lfW@KD*f69h~)EGHpgvSL7U&!T9+Jt}RoXBf(~T z`dGJ>SpGCws<9J%`mUz$0@7`eP5qB#CF49#|0gsC_^tlSdielZRn>9)xA)$6@7o1I z2#`Oquz?I1ptB(c1PQAo;6S_VE)ouu5eQ5Z1BztnG>C$@;DC%R!mddP#Dd{C5n*F! zIDjOk18D>#Y=VZF?lNT{d;0v&J$Lu?&3xu}&UxqmJ?HoP9^zoVl!YTDpR&g^ z4#RZnewV%dmUmq8@$)_Sud}lQ!r6>NybAl-7}JKbPJytlS{VwHgTZng4B@}8z!d*1 zeXqA|AM=LGHvgP#3${u*{q4xxEUR<($m%rf$t{)#`l#>!exEMdPG3J-cSo*nvbbq_I)iN;XXUBWe#%f}Asx|$zwsW_8 zoIRIhEcWGmb6P0TCUC1gH?^`z&lveZ(7~0JT3s+wZs&VVfX=BU(m7ZrCEjMUyRi>O+070$7d z687q36Wpyv%*mjJu@v@8=i^u#6x;LJ#huM=!KX4V*ksR4d)A+8=eiPf@8)tw2Rr8D z&$v^GwR?4%eP1RM>>ANjehufA{qd{OSG6)Fshj?~ltzzoUVp{eS;V-brAJr^XQXp@ zBl>U|>%Nt?i)5+yi4-b1E6fQmAk9P+U!*WVnd5sT=ak&(@0YpkS0&sFt65LIqV;Uu zEi%nn8_Ly=%7*e$mzYO$^_rVUbLD-qp}fx(m`8K{ z_R5CxUMfL)L*-}zYzO8b~yG2 z@EmF0y=I2X?{)R=ZbE;ihvOEQMjhg@`4HAYaT7Y0=@_?3#xZk9*I~aAkELxJUW~`K zxG@_xxVioY`%*?;=g!y-C*tw!*vzA0XRKq)Q`Gq_`C)F{jq$u2uEE~p#*=uBw%x`i z4A;-#$It89CN`mSZ`elq8K;?cxV>B-k9(eXqUizy-2Y3P{DSP|oj4PJn5VHi?eSQ{ zs;{*^cb0U>h*kGle^@I&Reu*Uo8W8GJw(INN1o~i_2KqCiQl*H$+yubXa!4e;>bbr z$LZr|oWnNw&BfAJ#|S<_uh6%~UDX`^K`B8;eyt>7MM9rOcE>=4nvVF%k>0>i`=CrKkOmxlG zHHnUD)HiQ%CeQ+nW4QIG)!3P_erq%IhTZqn^D-%zBVP^9I(?XBlQ~6)q$)gQ&%jix zzedQ^=ze1BRv8`cmZDS~cr=30O?0!9Ks`avMhOWM(ZBD;7=~#Eq5*zF4CC%8$iN3y^ZPee* z*ham6Ia{3FV~{KD-!J+#wr$(CvBtJ-+qP}b8r#+y+qP}{%zZ!4-sgW#Jui0c`lc)C zT%B}xr7KC}`h9Go-pU3<$4L}mGSrTTHQ3Zk#xu8MS6xszFVp7h%D0DWugBXQ_caBl zvk{jv+iQ8}_lWwwi5T7xgI*WL(ZX|>oM#=SRkkB&5O`%ehQsN2q_o?A7bn9;ct!@S z!glPUdx6vS(((;2ymCY}%q`?EGEvtV7J26;X+_jV*jJ#JMBTQI&hj30#JXe}2}tIx zk`O|dT-RGxE~~8&f}w*o#rXu;5aVU&Y{U=%A=17w6C8Q=;}Xwq@~;yzn5WN!d~>;xu!QoUQ|e3%RTWtlAhX` zJrTbESet0JC0^9|-CE&(wvBT94(Xsv=CcbW_Pn5$yEU|m`Y}DuxGy+qO3I1Xnd1ZO zJIm`OAn^&Bj=Xf!Y#lB8*1%Clzhp4XJ3OZLp02tJcgwqer)#nYm*f@vFvBjBL0QtG zJUkU?&W@7GHc55c1t5{uQeE48ebcqB@mU|zRuyk!mIJv6#VXpE%+Lp@?q6(G_$Kri z&;@mG4BuWmAm7`a1X?i@-NZP12k>Yy!6CvD|0&a?fc=!Rx1-(LqtO$V`|u6w-1rGU zsZJ{STGM5LcPaFkcShKg;yqCEa_+ugkoB2sIBo!l_ijirksd0_BkF;5p!h{{;awZ& zgm>{t57OLvhbOLI4&+@VMg_94j;k4ow%>)Y|7-+3*0o&Pq6X^joU{q*PCu0I>=QaP z|1G-e5xfN!W#4h|x~bJ3_6;kfG4O=OpZ!i(##H`usLU^Fsv>E2OY}kzBOBA)<;V;F zRiI33lC2B!B?0CQckUUMZP-q861bG~yY4w*Lg52sLSWU{}Cx)T?Fii{R+C`q16fHN-c;zwN~FE46XSd8q=x zN+vxz`|oN3wvMI((sR1U^vCaOjRMC)+wr>-jK%{d#=%#@MGZ{JHFGU89^q-3^4zEb zvVzz?*1Y4Wt5&Ey1dufae#w#`mvI>tZla@D*#RwI+g73yEknGslY`tw`qCY=9 zExaflSF1FXw#}Eo4%G!8na%l9ne$;S6Wu*F*7T#ANG8@ zV$nkzyR|R6Xi`7E<#lL#m6c4fzqdCVI^1uLAP)rGBm)TG0A>9A0+``JqT+zT^-uKh zHQ=4d;p&7KaNKpw;KRl7lj2ht^Plvck&Tj?RfYnB;{y`v!nnaIn;RDp{jI*Y+d6g~ zL;|0uY`b|(r*rPJm!IRfMqCOtB2znG=3*JP!)aSk(ds8XiF0}yR#vAtW9|yuDbb2 zui*Ilx$fE|HW#X(Fr*5+`iLkE|wiv({C$m8IM#s%?Q zwl^ky>MgPMFTis6lFP$vUN9}1 zM-YnzfbI^dwuC?S-oNuE&Vn#~G!1bDGTQ&XIW4S|c`b);64!yS$VFo;km~=bw$-SL5q*TeY|#9E;;C>>dL4V~ zIlD{eHO(>JdZ)8ACmwprJ-}+YdW6wTAsWzrb^6XxRP$ZJy?rrWZG55f7kysS@^jJ> z-0YLf{ru{j_MNW1LDn*}q@(|>MSkj|e291I=^aC;`)@njr~vXx?wNN4%Xpa~P)OV< zvHzb0nLjP6xk7W|#i~mR+nZb>S1;KPg6sZbf>i{py@sV^4MbP=)E5o9OuwEOrZK;Wu!>rvI+wAFyY2v416$58N`V`#DPvEx7MaaJ)8Kq09RX2 z4UN=f6ObFKqf1K$J11(>u>ghZAWv>cU#;N^oa73scZC5TeB2n65gC zlovj>KjVIb^5NjDiI|#ySX~)vWxlHvPMXXYn|XOQqGbfh$wy|SrOu?2Q!{sSkG^!_q^tyO9u>x&_9^p7 zlswobSFUNO=3!-gTD<-f&47)SP2jaXQr~Epz1I7pnZwt|m?AV}=UUQ!aHaCuI^|{S zBErTqbe}f;BG?^&D=0&HsGjm2s?ZSeDUtrHq%Bm|71*}tjVEz|8aOHp7wYy72v?O0QgbVde7fNh9X{V7sykcoX|Bi2j# zVD*{YlddhD0ZtMqn*8{Sx(N2rN5>SesOFPte*$GfKgfRKJZzC5>HuZ0#L!nrb0sQZ zg~jil9nv61;V7>+9y|ZyJmHgWZv(f$N+^BjIM!!1=64rIbeq(QjfM=XLggtqXI)hE zlD@)*iB#mNpLksrv)( zZG|4`j+uR)@K-g7+jNdyajy_d={1FNa(o9_!4^@7t)@ViUdZWmdPvVM9Jy>6n{{JT zhMi?@$BJ>oU+Z->qZ1r0V_c7*?;Pj$phP654XOqeENP~5mWIjqS$u<@1n=}H6fSY+ zUfN#Ko)I2gf7G5rJ#kFwj<*yeYWBQols(&MOCESK4pPg69KNmh$CW*DM3<)lZzb~?6gX#WFSn&u>2L>b z)@50nHq@x^p>fp6S%E-5rhE3)(JNDMPcTrQhu2mk68-c>&o~`nswP-WAHA+|fSHQp zl$y;F0C_f}OeY=5fjV>Dpc;C5(5_fObU>Tl2fUJmwO&v|#)P@z`f%Y9N9+Qoingo; z!7;xMIJ-WepI+l-TRuBFQg~9`pN;V#!AxZsDHWU{-`x=8mK;Xsqy9@8^PS|#|Ex~T z5$pPUuJHPUEqN07ttcqdUcYhLVv_7`DhAnRNN6&2G_|uwy*FCY%O_wOE-q#O!qE`a zjwdwg*b#UACTw#B@yUG><;d9!K{I^zBh>f48wi`DmN388BNr0;P8Yy*E+Qt&4IQvPI8 zCJS54H_hv;8SvMtARAOa<3oi#FqEw|>N{tQkxD=Ez1u204?i6zc~4K18qQHR&EsY` zh4CIga~Hf}62VLA}J9Kk;b%f5Z3c&WB5b1kGxr-ve*=AISM&vVYz-V=#lBhCi^kt1L8N_=hepzH3TqZog4 z#mmBcR9=|wRBXq2ULKgP#hMt2l92vDd-zA;Ozmz!ji4r`&7JKzDRZSrLldrph{IJ2 z#$@v$&IQz27~lxnyW}_L&-WGLGWU4xAgQ4`P+9(a_wP z9Yg#w6HGM8J>}yLdtrJOXB@UT$$nju#Y7k{hE(}ox&iGKVHE0+A%`l;JOo<_qz7Is zT$#VohV=2KH|2Hv7!S)HDWZ*F(ODJc?$Lgmk&12Hu~=)c8e{S{iyg6t)lM%gS!z^K zJDib1c%cwpRk&kUV;bdw0mQQswX1@$DBp+L_c-1g>X6sXw=VAskW)|rdq2R1-7rv% zzs&+u7(}H-*!3esCCA|Tx7k!r!*XCv$@j>3*%ydR;{%3A?AIutDGJOe$0ym-oJswG zvpWnQB(;0h-nNxAXVsl^2GC2epMpPPoYD9)mH$$hHvA4tC_OL@I!E=CsfM_fDPH>YxoSor- z>bZA42(j1l&J&z^`oG4|$WBY6<1gC!g`H3-egkK8hIFZ?fufQ~D$Zqdof5LX;Z@Oh zh?V$b@M*w~JG5#77)cF3uQx)oza)fkm)E%wR2ebvv(16PKX4^KxsMN??NNrfUG?ff z)WWLEj{5)=@z1O0mWbrnTS?995(eCib=&NC1&>wISR*8O$Q>#usf#n0Qs2Wsxy-}DyZ=~vz zGeW$7EcEQrl7-u^CtAPdtv@iVPh@pyDOgs?Ejh9FB@V#BOTyV8A=fmA+mpb!AxzPT zW5xTrPd<`8dHgBGxAfa){Mg8EDSw&R!@4%0#Te?!3hLUZbZ!GP-Ckvsz{(MP*g!v; zH_IUdOu`d=M4hGwpG}^tIy))ToM(@qEqEX90E9E>rAuO&)Y4v{dtzz%F)z)>_j7hl z*W5ainsd^DuR7%rsuh6l=71@=`Zv*m)958KD4;2~zf}O<8-_vw zgk$?d?ze{#rljP3W+0RJ^I?mwP@wE%qw}Yc&W}GAy?sHbc1gUAE-D%84`ZC2ES7DW zzm1hO|9m%3>~l;TN@=1}VsR3tgS>*<8k-gw7NXYU;x4Z{2z@T`TP?R7G;ah2&VV(j z+uSZ`8V?I<^Npas4or(E<0ZT_y+n6!?$Mrceib12u7>rL4;P8ZSNW0;bZ{)f|<&c0a`SyrHE z^}6xet0Q+#E@764Xx;FfzB6$qpX9_eifTQ)J>WBK(HKugVlWuGKCG@^*t|zT+d{d9 z)`&GpJ6G^Ji*OL!0_xy50Osm4L?ia70_WiB1!CB5S)NgG`3~khbOTElP}{RcF^*&8 zL(}MfBAI^fabN-i8fF0@=^=7Qe$!5dG3`*hVRHA=fXD&wyJZ^b|C`m3zyGE_{%|#t zAFH1}L?5~z-NX=mh(1amz2EBO5%y=42=d>Ii6Z}-q-gS}pJ1^%u}KP3auAwvv=`hLc-2+;oxt04Wqu?l+m zLiT61p#L{k3wr-%rL^xS;kQ`zXCKfKC_hO~rcC-bRLbPavEG&g z4c%m_dmSoLZ@5S70=@@sF<16pAvShi8P#^&lDda(A#dz4ifF&1hU@s3Byvq3L9$&s z0YUrh;{Y1?zs1^t0JAnht^-NxmVHVo(9xkMK#u@j0jZ7voBY7vz&;zkIKFYdBYw)r z(Iqv=H{N+7*PwdDk3%Hk{qhRWyEv}Yz3A)oyvf;oXPc|QFODXPn5x8av8M+JD8-U6 zopkBisS@HRfF=BEIV`hql@E|;#s&`&7MGw%v?{XzB~2t^p+q85u2fd8Gz%4xp@LC^ zQZG}A3Gt|QC}GQEMM_H%)lal%)^ zpV-8QapK8l$zyZ1jZ>wB6urHLI!;7ID=g}4`r@6b2R*}Qnj;b!hFlv^daHE54n5@` zM2<_(yq2LBISqCwhz`PQ7}M&{1~60ao*tz{TsN0`%t% zCYfI^IkyPCiqmQat1j&V&|XvpvU-QO0K88tutC0%JHKOjz}!yi5`U?3@3Z%2nSKn_ z%X`Gv(!*w3>mm5C?L`gK0$8sZ_Wvq2C63DmCcr7^Um)P}^DtcS^iMdd6ZaE{mB!a3 z0Hx)&CZb^I64DFH$CNs8&gRs&z%i*=QMEz~`GyD%zP#-?3kA?Q7Km+pgWe5ExVK^~O8$mKAfmh&hvttt`A?4m$Ev0)D z+K7F~!&+zR_|a(oezyfJo5`9=H^HGSf!DfoICsoT(hwv2JmcJw`zR-!mJWQoql{6L*AdX zJUO=-q;PZ;1Fr+W0ygsf_ays#lmakpR6CE*jI1q!owVGjaQ5F5AG8^@A zciRO_8na0EmK2VCqwgYTTitPvdBRl;siQm<(sjbDK|9c<`qE^Ps;FcHjo|avYdUu2 z;De2Zdw``s{Rk`lP@nc}%gfG&kZexgjJTSozNn-R1RPfFokHWcR(?PC-(=(VQo-sXqt5z5OoN#iGt7FUjna< z^>5r0#U*HJ94aal%93i!(ntiYa;Von7MLf5Qv@sst^=;!=FuP_bep=mcCNep@*ooP zjFb9v#HCB~Q2Z5BsjHf*MiFsQ#+nR)4Dr69Wp8BrcCg7d??gohDfU#X<%aJi+gKTK z@v&7Ax(sg_j{i>^J|jIn{l9OZ{co$N zzO}iPJGp>^xxN*}e?4PvWz2}r_>V)V@b42c`qsvDQj&5CQi4B+1AnHY5wx{3l2-b+ zQPIi4*vZiB$8dD`hkN{IhN}O)RP^+8;!gTj=7s_`rdG!PD2z_V)++c6OsxNfJ^nN8 zKa1^PZs%m{K&STKYs138@gMpk>;Jp*qUKgk#t#3ioRhJzv7xQe&r%uNm^zu^Gcz&% zCx57wxM>l{4_u+CH;l@ZcY5+zaW-8i-!LzPR`0IAR*m7CQ*7y28A>fXxIS*auuppN zURf^nm4Yg3*qK&gyng+_Ij(C^Q3nO)_*l25lj-G&8$!opfj&7wjat0uy>?QDn{!^6 z;YHc8+GyT|MaNclbRGRXo>!cfbDU)UrXnk}X;#@U&|r}g6I_NnUpKd8>nW4>L#NTlrIhsQS-{ME;c*ycqc&1k7hL>m|E|*i=OP$ zPJ&jzsmZR%KSSz)v4MUyzG@oD2%-r@eUN$pwSEe|YoBkc9aHqapIui>E z3&;OPFxl7{IavO4EAaY(>ycGaLw#E+&Yjnp$-(yo6&GO^he#|I^cM(*yt9ZYF>sJr zD8tAJ$tmTTy>$_QQ;9bLNJ(Q1Jr~epF|pc@S?sb#@s6?` zuRET*tF}}huc+XExe^QVkA%f*cX;};^M?Pf)DrAB{A(~M)wBHz60}pO|9EKBy~JYf zSFV)bWEvtDOY!oQO7<^4oE;T*7LTta8K|Oh{&xkv3|2-$tNP2rS!4b;bzWULVmP-) zGp_jenpQk?3?wu@&O358)mH$aA<=1QxOp-6M86KlHatuuRJ`+!0(F29yHM8z)1X0N z^1=e(tDaxX6^5o$;f^j3J^%^{+Lv_=L`Q+J-;|uC^9Wv0#-Nf# zNDd;~7rBXhk@K4ROq7Yw;LhdDaKgxkoQ61tRDhQF27I%A>k{LQ;=`nbi04We{;{8P zGtYMtkK-1INj@HjxYKw|`dA>x)Wr0TCM{fD z8D!sJA7k%JGtx5Mu>J?aY#SXvFto5tkiwD5mJ*WIFAn>q%pduy5KyG(BpEfLWlrN( z5iog5LY71=Tm>!KGPCl1tyUYH8Fs-gkarIzXLNiO!N-|Ad zWKQ-nEhn@yGIKQ(&Nt9*saDfBmvN2YnxC7VTj1(*>$uMUp64w4%6(UUxqEjJ^&%XL z*)4FKYi;9v@6{{GVJI6azY6BT6H>z`un#ipcgPWRDtVR)4LlDp7^JmIX%SQ-QVoFk~XCnnU^BZ3ELMWk)ueW6>-hM%`u$@L$5|E zUXZ({dXD$OAQv96%P?q9Srn|4hwstvi|zl6e!_^z%G5=V_JkXhnmYiAywV^GZOKWg z)+cq@lIt8I_Z%X>cjlw($Lt{safobdDcU&L=wFR{Z?cQ5Xs*FL>0}x3z4O1rdn#`R zwVOWv%te>W=y#=IHPNidtb1(kHklSB!0qSI{z=-=+!>^eGq6*OvvRw{rnEMPoT|L* z}T@wK3ew$EVl=BYl%z2JkoL#AM3KJ3!URN=Jk+_O%{wrj75&casM;z>MfS;#fEbC zp~@?j&7VGQ&UwyZ&b2HnYwMUsm-?3`mwGJf>*`%qM($R1>bPn?%^xS2#%*RC#vJAp zyGtPBGLrtrwZ?u$JtQ**_2@KTeAWl|x7e&SF_I6Bu0&^_xVKqf41AU88Iu{(8PU=2 zj~Wv-A=f-yH(GBqiKcD{f^I@7SiMYXflQG{b@B!=t+~sCggs1ZuC%$|v1!dBficUd zXC~*APBj~DHcnXQ5pB^n35zfNdn#v7G|O}k+>vLC)nPhSwrR@S<#nMxFY7~Ro^8V0 zC8JPp`^i3RbA|TNpqk$)WS*PsB|ayaIb~XmTW%$rkv!bLVndD}%r^7ZSy>=gVu+~` zNb!N5Z(-(Yu{CSKxMlz;ux|}2`xHL{P}E|L2z~JQ`cy6kYurMPNBVrUhC~kZ5&y== zjJ8IAYyCyp=w;FRjqw#t(Hiccgpa3qi(X*|m0T)F`~`%SLI62WkHZi)1Z|{GYC?e8 zzyPL;59`PP2L~Suk`_x>2d|qCyeig9_%DXJhY+IxT1ECRhUu~}3%VxuLtdWk;)`O= z{S{Ad3~rud9V@W15ao0r;yIjg>FkwU6W}sOJDjsw6Y@02yDZ@zD*>eZ1EeGjb_>}; z5u7qlz$rDTB=~zIoBM=3S;5glAm<=h<0Dcdfeoi{|3IjUJXvN&oKvLa6grzk*CHAa zSpmyJ^7w?iiHMIej!!!;@&%^mmgv7R%+wP`?feR-=xRlb&AeZb#6K9O42n}CWm$wx ze$v`J*MdLi9@&Rfi#nv{l)@@x%8*LEQ*%U>iRFq$bBuBAqg5dK2y4Cc${=flRCBPk z@wyu7xCw_9*Lt5zy*9NfPEpR(kAi7Mw?6#T@WW(6su6JYMyv?9h~ZSl=!dE^Q0f0( zkyjausoTE*<{Fi$o4Z%F8IBx1LMgp4UxNoX1y=O8kJt{+tQFW}!g?>frG` zt_KGI0D&)7*i8Ugi_tP^h3RPhPSMxhNT-O?2kLNK4W$qS!3p@8351aZ!4Id5?f{lD zzav?CerGEOpJh=h94ptg=SeJxD3{ZgF*>Vj%d0B4{Uc&xofWqiQ7%}YH#!e-TB2wi zR}#j|?lj@J?y;c@f-sjYq^dL;4Z?u0&4VepN4o> z(BAcZvHgJT;bDge*pWkb?Xdlc?2%)J7}!yVcipo5=ynG50N9S9rw*uX`e@ovOz)k$ z``BJlRqZTo`mOCzs)txwQI~f;eoCmQ`C-{3{Vxx5ev;ePLCSUGhsCTu_3zNy1=Yp~ zd)}(0tm0Bh{;^0cHyf@r;i@doBO;seULJtTMnL$hA!|o9fRCXup*AZ(}Rk6kyH3b`q z`74P9I+?*@#F=|IaM?_Yc;!UCvZ~(URR66ZfYRXq(J)tMFxU7s&A={qF;4ux`)u`K zCxY2_q|?A``@GbCLFpq``z>o8C~P?r~86 zy;qAus|BVrpwRit*cqjILz%MA(&^83i>k6?T{*y|7VNr--5L6P3&1;I;T6R4f}YY3 zc`A-)c^e)T7q1p7Yn&970%IgpS@KIAm{D{nArD_%;SWUk2r21r)`VzdBn+~BW570K zLZKWh%FI#WbYWf*Ibsp{9|$u9S*UIy3{w#)qR>quuXrH}kz!OiNw6$Q^mHMMkvy^9 zs9q``bs<4Rp7nomm>waVa=$%NI7gMzMYGGGK115)92cT)$e?c|cP<%N#N^$BrqBbOMqg)>DLb>dG8WI*OGN9?DjF7Tg&KuaZoq-C)b1VNyW^v7Ep^D-SzIz6@}NwKwfchqs^qV#OX$f z1`7FqGbdtqo5ceq>iDS~GO-*=}WT zi(W)O^cbb7=9ir4vv*_I31q?r<*6kGQS7yq!X)HDlg=nuZF|j zBk+zjeFUAqP%Lu+++#2NT*(wnCI=>)uLB%J!Viw{04O--lup7mPFr7bf!aMC-&S8T zk?4K!i-UMW!B99L3tu-$@xbXd!ArTDcJ2V1Lv|_^>d>G~i54z(Nuo`oI{Ew%v`yY7 zMf1=lp2Bs^_0XnG7B>NJRJOn1UF%EvOUMVorvCas_AbbbKz5Ajr#jIH(?4DZul zH$FB$#=VTYS++Crx-hn+u8nO|*w)`0TE;YwWuHo3nLbrN)W58JtG`ov7Ics3o>gCL z->+Y&K9#>Le4~4obdPJE?>}L5{9jkI=zG!zm4t`0p}ZO7CSU6L~`MLBw=->RfLbG*>2;dmieLh!;aWqd1WAMNsFt7G#WYhT5B zI^KU(Yb2DCq`)H(mk$Y16eEX=8Dzo=l8`(h2O_*9c1XxCFw#V1Fj)M}OOewIrA}F5 zyjPQyTjy3}q49C;PEY4EamaaYlfAm$dgi%ypMKrA$I)Y);Aa}?v;)N5q>^^~(Gb2639;5MVk5KODVc6e$)sE#^c!Wv4qL znG(0r=e);Rm3&I^HquR}^$h7j^CADRb2D+%e~G`L<}!MRomHfDTtV2Fi;qDJwxGnZ zoB3b|AUko>AJiw&FVZ)aMw?%pv;)h43;g*p3rz15+0v_htw4N1!Q&MQ~iJc!G^UHb`;X^b*NVGZjYo^Ccr( z#!fXAe!o}#BVa~i?G?9@bR)G4;>d&UbprhTBlP%~&}3wdL38fM6zE($1?Z{Z1eXnY zXLRE40dKY-{qwivWFwP=9DplG@aaD?RM0i!J8}#lYFZT>jARp{4-v{^!SRhy8&@zCB zrAmKi+RifQLuYH)?blKxm^Ej{E9VyVjP!)y<^uKHp zXeF8+oqi&RBNVmB{fQtT7wduEw|Qkry4+g3;%9U$_ zT?;Me6j?k;ZA7)O;W&&Jx!#%bk_a^4(4C6+0bmK9N<;Ly;D7fzVP)2nvu^CDkMpXXp!&Hd!i=W10RpY zq!}0X%=3km=oEd;G+5?MYS@_~RK35B`XDF|Vj?k|TdaW;UkFIS&Y{50%YI0IAw~Qd zS^#;c$TOf#%1>nyyo6_nCLdIw`e4j}ofW4H2IVFcJ*UqOyFR!5gi=aJEJf?w3}Cyy zVRFhn;X8y_9KGh>4I8h&Gn=58i>xPh`$u zU1F*g?%P-PVTn7|mIWqS72-p4*iq1RLJ?Bvi9zXjAg~CKQax0zURoZ!wE5GEwQgp0 zhKX-BSF&s*6ywGRq-uETNgjfE*0^1iO}(Liw{{JCvdfLpiW2;T!uHTw%THP2m2^8W z!fk9i9~GQ3-^0BGu7YX{4QVF33Pf_u{Eq*FxMGyEbRV@o`Xd<|rsnPdtakG*Btn5i zt|GiP*#pQU_VTUN#~>KQL?sW>RL9C|_2h67KKLkDCB!8d@&ii=#1uW4e4&8##s}UC zeae%B@ha9Sryst9+-AU<10sHOM?b3{31M`#GlcDX{TavWmwfw3>}~^8w!)Po(n%yN zAqXqKQ8>0!+G_#77nM&6zyqqN%1?#hj!)whY%;RZWQP*&!0M~w5BYAt*D;kQB^;E( zJtfoR{Gp1+a;fA~AN(!r@n%sC$EvQSZJ-*kHBb*=1)$G-rDtGTC3Ll~$^O*q=?6Yv zFC3m@&7aa02$RahZdy@ZQ+ZP9)Ny47RMSX`t^5n~NL5P(EUqPFl%RFI1gv?vRy2ui zSS9oH@i)1vm1Ad)T5F{>C;6OyT_}b};VS5|D6v)aYlFlZ`jh+h=@E$~rxBe)0FMx# z%Bxz@co8)9VD&qB78UObjargiMzFH^n|m~b@9x1OBl}k$9IZ2nNUynbWEAa0ks4!~ z%#k6Hk}nZ5dF~GgrsV{UK^cy;$+M=V5o$b;N>zV#BdwbFJO6?ZFR)%zqbwViQO#Rj z(JiaR4jN<1l8hBs+&$lP3tz;Io`uWo=a>3XFsC|vU@(h6D<8Xouk~yx>1u;*Iv5v3 z3^`+#&cR8o2cB)g!8WFjwj%jYhzsgG01_SOk;xZuz1|x5w^Gjy zj(zj#3U~1Sp7CCR{g_wr6^B|7ktlKbBK*cOautGiVd;|8KIOW&Lr`(kL{+$#bG;4T z7LV!ksgW=58LSy(g?aof2Z@t62_PK^XwSZ%?FIXuEu~I4iAUBTxEj{4AiOIrY{~t? zq*wcfsC#E2uwFZ)s?e*)d$$yXXA~%erj&)ivRwi6~UFDMi?E zK7S|>=Fz_Cik})2BGe+#AG1d`{}0c&BP)`1^G(Yw6dEV|u{-|3GrU!wU(z}qQ5wzQ zijO!DG1`~N+zoMbIkB=j-07*Asrr<#xnBceHYNkWmz|%cl$GR5LQt**JE$Ea1SIu( zCwnh&le)^m6&-~nhl#jKAs7l8YOd+Hp&2i=gn`~HpsxI5h-qbvWDVYmv?0D>WD!WpfB5QBz-dOq| zM=h>GB;(5Lc*H)7BlSgLXJ|%?(w`~aSW;`2hStP6=irEYVo*BK%wc2d*Pl3_)cIke zY5}}h>2xegf{WYMwsJWV!)MK_^Ij%4bb^pSlwu-;gwP0#!UQZ85>-jdtk8lf#LILL zh;+yl@r`Sc(Fua`w`S$XrP7>?J8J6WZ#*&b@lQZiONBJ0Z zZGJx$g<4F~2CQ_17#aTM>?zNbAH={6gcLSum4v$1 zXHr5=ip0CYBeQogNZ=C#jE4f<#0=7M=w5Wd2IYfQr;y6n3Zq`zf=r>VW7dpc)W|1w znI$hBYuB(G#8r5FN{ncE?!y9<8O$dC_-zZ{mA7n{)B?TDBi(w8=VCrx#&jtqz(YEy z@1zw2Gy_zX()cT$J276ajRR#zs@BgkTt6|8b(F4s%vig-%!>=n-36BT#$Wmw*{hN` zvq3Imzo75$ca5M^EW8czTx+3=D!s-~=izyn>r-K*GIJaESGBryAt}f$)kgC=@n0nt$(vI$dC^JeQjT<$(Z9gVq!A z*8FpNP!4~-h(AGAj2fm%^PtrUffH~-6@xsGZS${TvYg>=hKos4^bEevyD9VE?s za6vxJk!eS(YQXc&ao(Ym(36ogim)SgYzuHfsY8DBiI#Bwb^Pv~?!x;I?wkwM8$Gsb z91eV~bezUwWA`~g1QHTQi15H*)<~hgKF9s2Dk0K6nXdjR&e|Q*Jk{5A$VAo$mg0Ks z#9YoA08nvs;R!&Y96j>jfIPZu*4yfoVqVMs(a5mo+`tg0;zdJ^oQ?{x-sSi_zNa_k9uX&MPuH<{r$+iDGBzvySzS? z6%P*@&2lTv%nj)~ody!&v}%;sjV+j=za-Ef1Ota~0-0Cc(LJ|*folc?s228lg5Ky( zbFVA1c`n^S{_U*k-6XyLXex`n%nJib?s9X~ouXiR`T}u&P?i`C2GqwUlgoRrZvJwn zX)60JQPeSRor?Hs2|wuweu^NxPh?7uS*R{>%#$;&3;zu{Qu$J}Y-?iD+(*E!UeQj@ z<)%ZKoY$2Yp3V2V&yp!j5_;NVHm%$VQ+OEGbF>q$R!O0>@oLdkEtAgQD>h*8Yfv1S z;y{iW(h%AFkK*zpA~eYjxp%w2Z_5K9HOR~y-m)PB#K)kRFd+bQ8DVhgEP0GrMaD*x zxz-aHE=4R+?0Gl491^5xMzU4^GZq~wUb@!r(d=2#Cf<4iPqB+4 zQ!@k@Ya@^}8O!g5`>=}V;J!{W`tYT_MRmgv1u(gisL6n2w@I4lxgrkLw&ChQG@-!y z*rA7iV17Aa2bAFEaJseM6x{T#&26mo71TI>s!$)<0_Z)a^njKNUcZy=AL6YKxDVZW zZgnZ7#5zb%_&$DlOU-qA;@P43(_Q^fb$KJLl5Cs z7Se!I>eaG_CDGW2CC(%_V^qe^EP)U>YqGI|EaZnUQW(zTPvtyc++&ydQiop-x8YsC z^Ijjn`s79epyH})BiB5=E9XX8$KP?t$6my$(kUF6{n4sragM^>Q9Z@9?V|#?rE3SgAO<3J!tGJ;W51h ze~n)dH;HVoj1gPtS8r-%1~Vw4nD|V#egZ=h4?3fs4QyuLJuESYSvA(8u<|Vmx%Y^h zx^UfUkOuh!dv~e^clXC*k8;&9E8m9l+f<=pRc#1poC^SB$v%gPwF%W&{s{5O?MenX z4Y>$I;ChHq3c*}grwkMezwC^&htjf^op9<2k za&o;o(KlWVX-V{Bup}jvOdw3ymVhezYAl(>z)UB)PrNQyyp0UVkooXT{Fe>-hELX6!8o77F7J?aJZM)8f% zonq-d=b#7cqHMR>#I6S&oSaThC@SRv-gdhd#9UkC(X8I*O4e|MWKAxSb=mf-nLCUCg=&XS2fVC$t#4tH zDET5(E!|oY5(I+vG#gKKV#!O)>hf+YX8>>5BJ-XNP+fQ!w zt~hhy)DNHi3Xs@5bCdG}i7|Ayz$KhTL4)pZP-74+nV1A$P-R3UZpp2SVx>>lIK5Ki zD@lY%!6L+{VVM?TScpqyQ29oE@J45%pH2O&nuGplBS5Ew=V#&M*Oi$1< zf=+_6mPIX(S05i{p*jJh7Ai;R1qu;^5Eqe%TLg9nI^Axql7nclvyv1AgPPAIN;8S7 zu8LErs$%i#1boD)JJk898<ezPfv{oVZ+ zGyl9v`;%f&K$#zg-iu(3l6QfTX8w~+d#ah(at6X%PGYkHN8VekfVaHGW(AJC+gO2M z&z)?y0>N@;yFGtS&YJ92z*~XM3LI9D^D(2YSzRHmk&Z|`(nk`%lwC@G%CRa)79-hM zfft>u0B%Ug7|VHCmSYKnCBg~p2p)mP4Wr|R$Z#B-!i04g3?dliks4upS*_6R#BaOK2qwT*-m6B^{3vT1VAJKt?k5|p<@V)ud=2g>GlyTSIpzH-G7LuUTmrzXHBt{TpGf-Rq) zV7Zc#26~GY01?hHyprKeNZX%INJcavm22=hKViCMOU^ZC{V=i6d5Lq_Sc21R^&o9s z7>Tp2G!~CnXT4M@t3`1o?u#g_g79n8JMpnsFI`GyYEkQH_Wb?FPiIbmavz%R070N# zZyEn{@KwR=NIVuJsug&o+-|E@;E{4$szNYC3q31>Lkxnf5}HP&xkTOP(^P7*PB+Kt z8gU^;2to^Dw$Nr^3MREbDtNSzVHwk-AaYdWYE;vv-RAof#T!fZmb~V9%k!ahP5NWV z?(hU%4;Hf?(WiPmV=lMX`}!YG{bm|%m-qf?^yHDgWsQFPkF%K#D>HXLzI?FnIxPQ5Ak7E9 zS;9Ps{$AjmrEXM;S`mvvb9I9tLZF=r@#VUgh&o~_WZGz%&swq`kSRI=(>DX=vsNJY zV07i`75Uc|Ldi*aG)WPf-%*5?2!?M-(US1D)#c$1woPcWx2tX8_4b#vSM0mAYxb*t zMF`V9oRgIRTT_HKb0Hd)nFr{!o+cP37DD5+7fn;gv?htDcQFGMv-qhf;HRR1pNe&| z$&ZW}8Ker9 zBX_yY9tk1jFyw=T24ZszRR$rD3JE5Xp-2l11WO3DEkXf{1R;!}PPKx=AT0u7T9OnLJtVJf1~$%w*4is91#dv}{0cAV+W?e5!m@9ypW|Nq~|-zV;e*l%Oi z7IXGFumEo&3krw|Bc@`41x0hTQc74_W^-3dNzf&?D*r~ap(+;;S`C*Ull6N<}<>b&h}p?Jc+M+|K5qO<8v<|isk@nvH&%Hl#7bV*%92tH54+6 zn2bSxL+Xl(JP|_@3k;9LV1iHT6adaqh7l#|f;!`@>#`aWHsBw+0+fBobdy@E=rH7H zo>ybbz5Exw9CF?iSjda^S5~rSiFaqO^9PZv2a%EwawX1MO_wFJsr}g;35$$p5o8gT3(h)B{)tLw!xT50CXt_f5~gs$DA;&k%nu-!u8R5qxF=6 zl_?&w1Vz(UN7GhE(^w$d>WZ1b0xpbPx{J7a@s4hcx^Ac}I$vI1FxWKS8tJ}2OR3w{tD4B~%qk&OFm%Fpg zAdo<}X^v-?`%Licm%GNAoE4I~)3mAuwLo9i1CcA@v=9nOo&rt_9)}d+v>>I?LM3x3 zI6uF9LaG_FtTKX`jLbAPNT&w8Y-RJ+Q>U*lU(vQbF>vu>VqnK(o0l(r`kBQ`woIDd zF=yj}&ZpMzr=hXGS+?iHOM4c-Fm_zumXkwZ^G`qfCZ4-=(=Qh+YTGn8RMRp0l?_k5 zdVu-qx)ibuR7e^*u98)2&jEEG?FMZh;AP1~p6db(Zo~>$T?X8|Fx@)4aR=PvqV@vA z<;-!;c2cJ^6V+nOn?Q%|mg+FG8lV&+k9J?QsgyHrDGxZ*ECZ3w^y{NPG3bzQJj94y ziWPFdl<5*Pv0muZBlTiFP8hLZ(i;EuV8yCZrJ3P1u= z2uRicu;m(&z!PW;c&pr@%RLcON)~$qYHGnv4O{>PyE7?(5c6SE#g(wTIrD!7Flnd0 zOUfsVVaf+r1cmQ$>x8)}%*)%$>waaYZ|%Z4>uM`-#fcS12eIw!v;9x5`C{*DACkBB zt^Qf}!L{pN#&h&FkI!8H!AdPqzZ}aST*CU!#K(y*6Q3q}et(Ki*mbn;rFM`%M27An zBsM_5XhTl2mDTr@F!=M^!;x2q28YK=+Ps)H7OZhv8z>3G{h+43k^0J=WD;SSWVgSO z`tB`jl4#Z>9+{L>tBdrd2FPkK>{c_Ys#eEp4HT#K7}ks}IrDlwcDi)lzgHOO+q!TWE*=XYs6bTEC&&Ww8O*lUjYL{jh#hyXm;;R0K_M z2u^BO6;Tj0hf|hpHVwwIq}eb+uuaUxAc;(ymI)sbO4*#3%~4a(GT~c_A&RmgQK^@# zj44RguEhu@IBcIYZK(y ztyz{47G(INn85RtRaY~g83jD#;xe2|BQzsI^T$XwO3MC99{K3d-CZwV#9zGl;Hc~} z@%Wtw@yW#ZNF#p!*iW}_1CCid^r`rS_$~_L@vM@MEF#Oo1SidmR0k>eEVh7+OjPPv zgjS-};SFe0xC8AJ57B*&V>IqK=lBb{622LBJ3V1{I7|zr0(W7ynKR8%pE=K4pS4t6 z9)8lZ&9jrf=-ioo0RN60aKGowK$$40XX-%#is+x+1(lrE&4NnZ1%EG;VQ4gD2#TKX zdI;s4z`wy9KS|Z4-y$c!nsn%Dy+7ZSF`$OuI_hPOJ+qu;ESk?fCD`;DMoZu-VUV)u zZa?#r@KI9>g4p9#p_mF)C%AN76GFYTG~NszHUkFT%^FS8kB9Yha4@O$OG56bJfIy< zURkN&x8*a!A>K?6qZ}cA=KjPVKj}|g*!2fI^~^u;xXGu=&OCSUo@V^fBOX2bzOezzCFL%oA`XkiNv+7Cz*oo28L+_hH*h&W^s@0F>~-#IXi4Xal3WH zg=9bYr^GOy_|sIu56rJ|N0h??qu`tvcdI7zqKe8?PYLiDhTo)<4CaJ&E3CT8IUfc| zIe-pGH?axT!hZoJ^*c3SzXO9iaSqhpp|9fX_c(S{#u2E=m^|NVmLS+wci~)>x=nc{ zF!<~!DW3ZCSOpE)WJwlfK^CN}KrlcgyQ--Ul}cV;rZ2-sr4aQ;u*V6ffSet{KGhvT zkiTfO5MCQ`Q>Y8+&>h5^k&MI1Hi0D52WcyFM@s!ZzYqE);v~7FBc&A+E0`IAW-789 z|L0KSFB(?2&RVnMou?CBxN^t7k_T#DXr9%Xcw0Q~4bOZmalUVVB5`nGX=g>r1J_=; z`c~_6+ zfl~=kRIBq;{_nWoi7U$R3{SufD*@W%M^h+BrJ#oAfP>-Im67;EaaZXq4WTYkhPWMR z7DbE{E0n@Lp>!un2g2!%q>jBp-r#N;9ej^&5|4K#s$WYuI$3O77o&{pNuvL6jxGsyTfXjvV1p*a&Ii`3b z6R62fKpj0=rH9S-z`dNh{+ID;jg8{E!gKGu?(DqB_Ie-ode{4~on^r`2C~$Vhev36 z1|{_d2UbNwL{$S7PD&K0BHSiGh*GuXheK3U74=7{)Ca-F7}Hi(R4M68ij<1-tD+T3 zqO?XRvJk4V-E;2j*qfp%@$TID?#y_0=AQ3-=Q|SaTls!^%!}gjzx$iw@SFMkV@!+3 zACn%5#WoHqimF947Li3sO3RT(S%w))s|pkwB@z*2DiNycAhr>a>O_zx>3ZMpmdQ`# zOY*dQOP1tq43FvEDoI1}xTK2eBt12zhauI&km`|8oAmS-j%Y2ApDqXTA{M6Na_-Dm zhi9Nd%{cu|e}6wjvcDfUh$L1g3#*e1nH!)v~Q2^YVEMbE8=AdxAtUNhIOBa z0*saw41w;IS$kty@*B$-k}G+DP35{OH-vX4!_QZX0+z!h+)_~_5kf(r>@kUQ_6-Cz zZ?#u;S91!kc1ec7S9M3?0$dD^pM3fgmQ9%Y#YuTYi_^5^kj-gDqAu7nkNJqRrn-4P7 zJ7R%iF_12Nal9J`{3h3FMeqcNR(w<7KO`uj==0kO1m9(L)hrixAY|||)n&RA0}3Vm z69TnaBTX{tMdOt59xSD?)!1r_T~gj$VeJw3N-vr(S*J~=Q<;^`wbm1KySQ2P*-mq# z6*))W5=Yfhc2WG9s(944tZtb?8dC;sWVg%!GUG}6N#cVgWK08NY?_wkV4U`QBOdjp z=tZcMJ(uM)n3n~SpqM+1{!7| zZ69%nGf68~)AD|KL>A>qdU4GC(on1z2WL53?T^i48RWhaW{JA#?J9D-zu&3*B*1-Q zMxTC}+fCT@bPG$ia?fuCV{r}`djqa zEOF0~9NF6)jjthJBl7iy%C8sp$WwEl{_rcie; ztQ2Em1H=|T4i@!t#nZ8TMjD#>Sj>n*2Y@cM24?-N>uwxm1|OkMNUID)oKaO2nI=)H zsZ0{VG0iPGCb7;jiQ(}p6)~+SkmCR587gB!>6oH&kn7-Y`~*+9YFa00eYd_}KcbK5 zvd-$OcdeeMiGYu6!Xtl<<_Ah6NHx>_u!)KwbwU$WtQPw@U!XlHay!KfI?pxeK?pV> z5+_Tj>f++9t03kq4TFHn4a$O+379a>{4Hfz!s&@EW#;z=u(zy&mZLSFhy&~mARf(f z=vfb-dj27s*rYz5ea@c!FN;!f(aJ|L1ILLa04uc(&;JtrXPK%n#QigfDT z2MfTO??@+rF(Y#$fjgGwfi0#GBP#<(anOCPk#2XkH|}-zHcGmY0x1xhV`#ku=6%K~ zdv$U57VpDdz$uSayaKxp3Cw3;yjTQ7!0|CF4uG&h9*PcIeK+9~NFC-~P>yt2=sLT{t68S>C~k!`Bz)=YJuRH%{z( zrOCwp)#IRX*l`R@~Wu#LRMFgg_60 zH@W!FF)z^v`@3W99epm&TRZxk+TO11;rz|9)khiwAY)e0-?`t{S z^1Swvb<92)dCh*){JA}8e{6kZJ5UnSuHERmwrd-jm!z3QG@^K}V;XWy)1u9Zc&hn# zi_>*I)wJJ);c9LcGVPpKv6yXJEcF?3rRvC)3fr(Nl~lnxub^BZ6O@n%3YytCzPO^` z5-HVm?nrJVC+6B?A-!TDy<$uBihWUfDgReZvS=h9?g*TKUh&(pT3?2xr^YhynlLG^ zf=W)@=JIK4Wl;uKiOeh3|6z(((G5nJZR6y1iFnxsb@5=c{Fbr< zme6T$gKbGJhLV@Hc=M##>+6T&ae>&N6QmvHOMK=Iuvo59IvQN(zk5 za&{xVvzZI@_22*b_#Z#m**UmlarT|TZ$4ehZ2t$jaB_6ln?GIXmZzS0@7Vbpt@+%p zqYJ~N=hZhJ)z$f<;_7wBzWnu5u+sY$Ka&0>e;{;=8<6W8T%jY>4G43g4(CG*@p5Vr zjE5irU_%?f&V;HK(!{94dL3@5!^sejSHxJ25HtiN0DfoXfOtT9LHw2^oXVR7IVE&N)4f8qc*~WvhWBPIPxc#zoH1|UOl=!-S#(drW zo^vwyO8%@lYL7OhYDTw|DR{|3LMwEUf*^DyJgK**ARGkgF-ymiXOc9TkD8@aN1o*6 zsEqo|jjfz2X{l6H6b5DnW{NXQD!hU>)Zcz#(a}o9rh?$Pb7@Cdnn=D5cU34I2mXhhe6@HR zhgn=HNOCO51mF;u&Fb)Sj+6Qc&-mn3k|d>Mv?N2(89(%2s;dOJiR%pSdwWWIXk|-U zNnTsFEX%TGkg;rxh=~`%8AxzS5uu=BQV6ke0YjklNP)Ca2yLJ>cxgIO7kR7Rr>_2b7UVSo)W8Z8eCUXS793_8z3c{mbX+@mge>zUOD z=Cv$-V5sA}Z{tt84!w5x<PHKvteZM* zMY*vw-uTM$)xTfX_SY@i&gZu*nww0nj6Kl3Vg31aPhVxAEw$hxPJ)Js$e^B6uqjux z3qj0vT!dnBcEZu{?L5PO+n!2+SF$lugkogr-7NpprMVjTo5v*pHU==qbB{j|d}pL)U@q)dig|M(*)@=zjd`q%?0{qVUn6`g z*)_y9kE;GIcioTdgD9eAw*}jUGiWCZvW(seS(e*z48`l@>=8_9sDK9}MLndG;T;l> zFrT=5PQQH4olG)W$2zXI$ciL*RL#WL3THy8Ra%uD(hkRNrB4}DWWM0tPKTpZ;yseYGqCNL_h8IB;G?dVIw7bom(X%75&n^5sTK)smUc_< z8X-%?RxGsyqg{9}#+V#&m+~zHGa4O_GA?KD@PCdv3J`gcJmBp=nLvL?(2T+A@sE*&$qe^R zSh$c_0tk(ng?R|er-~CiBT2UDPZFG=WHvY^5uSYit+&xLV?*IF=!a+TzX{I&-OXz@ zZse-&Ob7CRc=%(^P6q4;A^TkMjRONM3rQ&0;GC1a3FRfa4L2ta6b07_i{{{>O&=je%)lK3aIJl zskAf@W}i!{kP-67YpUy0T&iXkH?t-!EsQ@YwZ%6mKT!Up+*WVL-Hr7c;`BtM-l!|{ z_?J|zuEJGjLTlD`X?wL{jo+(vX*V=hI|buFq280$6f)r&(Eu{=8o3FNt_$-ug^aZ( zxLoi*nuD?Cr-Ps2W zr2pp8*ynjOc`p}%w6jpWPZ$d-dczT$ zpCY|@k)>5yOjM7SMr)&8Q9eZy0Z}e65*M?@DWHU09*d46=Pk9QyuLQom%{r}C}qIM z^pJ}i68-v?BW+ap!wD(Zm4vO8i7ane?sOvZhHzfMA)#5og2!elPwovu?mA=Bf<~>m zLVdZPdUAz)Gl6C_L8Ja$C*MREH(?~05q9m&YG^#J!(fpUnd`Q%j?>&09~dBhq#q;} zUiMStI{N0$tV2mrcS3ZQ3T!7lF`Y#{3>%LtidNOjjv=wmtaP$ge$^eRS*rO3foA38|o{G#)_2>Vk)^luh-)<%A>3xYLE$iAk46c zZCdtvd)Mh1>t;5rytD!(r)}T#&EmuURTp+_KhUB(jPPmGc;d|J#dTkAd+O)W;_uC$ z{_CxeZh6$BsliBvbJdth3p@RtJ7-%EP#=eae%twdHh?T=iLkuF@hiu_I8DpGIz+1ED($!V_tCk}1d`l;P+}s}L$G_#D;t zBx@s_&0^jPq}d9L`w+O%edsV6L|h0pqq&Gh#1N37h2ox7OvVdQbR(Ty5ET$~BZ_WV z@%eEwR&=S4+%oc$`$Zn|sVGn~qcJ}&QsAy_dPD~}(_>&O_4{8O2!eR{bEl|8G8vGB zwE*bj$f;{yKy7!2CUZg&RL4665i>*#;wjsBlmmjn6_#MH&Xk8MSullwO%mq90vhwn z<7-vcb@ypg|p|t zue1z*%w7lG2;!v#ZKoMyc+eJqr<7}{!!BHQq1y%7qI_RauWfQ67opD7BSm$+8DHn~_A0vWKP zwG6qbR=BO*%s%D{GYDqQ<&v4vdojB7?9l1SvVAEH&AK3r>_UF-7qvt#-M$Ur>Z|QouXl2JA z{&w-rXMriF19JZXOz8%uETl~N9Vh103bkIHs`3q<26F-avhz!xdFBedj9>2fny1z5 z3%$etz3_5ipm4x*!}xb#fHI;l6p9B4jm{1dP8G+3Cm!pYh#S<|c$zxhGs|4yOsgx@ z0pXv%JLs0CBQMx*-33f3i*Dxsbe9io6W1Nb-@AMF-}c?v=eztdah%`;u*W}#gtS(3 z83WX{Nk~^LsG2qvrD84H2m(q=(`DL5=tifgQnw;aBV9Md{&Wq)0t(Z}RD`sN37FPN zquXE_nxGpr>beX5`C^{#_j%v%?*~>1nFBynTIDT%h^lfJoto?0 zQQ9`|a#izx;x@lvrY-A&jV;@^`|Tkce3c%Rz+LSqwJ)1>Fw|#lnI0^gxh<@I2)6*zYd&$x1`k(IH^~1w^cE7&&d;9OaaS)5&=qoMJ;1t9Ii>i)zNfQ3w=LM+s-6Tx~F(Ue5~N+%6VY%xztj5PFwR8fBvEt&I`w^ zWK({opww3&lwb%2rj}svxC|sf=R7FX=oC!GVAKgPY>kT zI7uPGeXNVb1KVr;d5A^S5Tt>$v?RvS1tdctST1EOywyV1QcH-5mQG?Nd(&%~z3C-; zi?%=hSl^P2omqoiDl98NW{_ELQ8WcKFU*JnP!&Y#}J5&ZEr{pVlb zw14`N&>LD=c4Zf0?;$)xULe3x0L)sx5~ofuEk~72gK}>(C-$ zAtSKgFeE2yq@4$i)sC#qtDcuNnw)H7yV2@=)>z{dlnurfV@m&bJhHek+k9W+ea#Ox zk7P%)N^7chVMlh2u_m>8;lrti7k*yZl-jhgKRcAY)O;;`}oGB@FtOlt`HR>V`x}cjLKqP6ztth|@uH63Z3|{-&p^sxElg@_et~ z8}dljH=z7k=5HSJH?Qn(9`iTPWuF0}fxpqaWO^O=n@0yKii!tTJXJyeRO$Z=6}%lL zPNRk-XEe#ZFBw4Tdxhlu)8={evN>y#q}gHiK=;6bHA^12xL`71%^C`<*+6$?Cg_e# z=6NRbJhM8J-I1cUm+78kcAzVmiL6TJW7fx5p3DQhS1G?&>3g0MI{=XjIg1c>MTBAr ziS>x^#6u0Bj&FshF2`dF7?rVL-U%8;pdZ)-C_T^hf(fP>;1mK!^j73-hfP7T3R`^Y zEo1qr9fSL-B6!!>dlO&0`0_8FKeYM1(ONcF1-_2_rKhDUP9s?n=r~1yv!QntpR3l-B%QYs;*L$2IJ-m~?## zkhXxmIusu5#OyzPo0dvrV}n*h?}OFJ04B|$Fv0b$U<}c8+U(QMkI`ys-7a-RucftR z%$;zBZSJT$=FYmr6&wzc!$EO4Urrh$`eznwZvq|)s)s{!1rb=BV%|-K@6r90h8dOa z3K6;^_8A*FkU+P91UBe&$Jef_s_fndGI`2Q-U9W}-Be`)JIn{;Mmwe!xav7SB1e>T zL=M&Buxi#~l!0#UD+mk~XKIfbI?_eSJRvSEZj}_w#}7JunuU&2W+9_HgmV z@m)WApt$pybuZ^((D?9`z8`#d`f=g);XR-G*5LFhaIJj+1bhV3fFkIWhNFUBdO{)> zG8YPn&T3X<3<<*vjw-ZNp$XT;f?)6okPDd$1w{Evrz`sQIW1La!Zmy;DOFeGLgqpN zQL$5wDXP#?g(h74Z3V5B*11O;(MGi~ZCtyoO=t?zlG-+HNIO_sx}wc$dQyW?LLq{v z$>OQm@zOUeDu^#)Bug^UWhE^ka*&LYF)~iB5P6(T5CM^TasjRg2_{%!Jt?C`7-~eP zTObZajnLSq{DYWb9!luEt5fVqxB5xow&$n8rzln#maO!bI@R`p%>VmWdz|Q!jC{sZ z&2Je_u@$+v2uSTK7K`K`=g;4XlcqbDkTCoGnO=Mk<0Xps2E6bSC2|qsBFsgEA3uQ$ zk&6%);qpEF(287yxCoaTmvn(hY3W{)llDrIr%IASh(M$m!eK)Yoe;4lL#e1qBddua zD+${iaj@fgkOSegu8$ZvX>=GphG@_lw9y(E7PTV7qLna0v=T-gt%MPxWiV9w`2Gqv`UIg;dl||`vB_by3ql}fI39m%N4P6xnWDq2Pa;&%qG>+Yq1tl zYEf|Fao^n~I$|qu>V0jiZ5oxO%9Lt_+gMIO#T*54V{dV0OG9h2t+klz{LcO4`o)X4 z_Z*1a{|b5Z&ghx0C&BLE+>5uMj)oXzRiwr1jIqO@Rd~5tp~>8&RBjYyE~HXQCt!*7 zgmOWCL=(;Mzobc7)It<<8JdP7eX2Sa8W)f=*L;iq;d=2|U5Lu{u~dtCVs_$aw7CWD znkd41RALJ$w%~saR%AjXBDZO40SEb0`Ud@3ai@Ms{F|&Cl5wNlq@>k$d8O77?g?)s z8|4kkMs1KhCmqnv%D*Fjkgv+umH)`M)p%6bB~c`TEGwF(!iA=)X+?GvMHUH3OS&UT zx(>&js2I+#Br7VU0n*6{Z2Fo+C=ybGO0VMe&`S$+IHMER2(;)1KLrNI*a#lh@q&>? z<>OPvzH~H(*Q+x34?z-eYgqd*?guWhW-Y*axm1%B^_>1i>VMS{UvELZ&UoZZJ5MHc3mG5VuX0 z5K=@`wGAW|wB!f)Y=J`))vBOUfvBz$`cMTVszXyzBL6@rO=4eXW^XTJLoDm&yYueN zdghz&F~rkL*Ke@Bt3?nKpJQmz@M>9TPko(GWR~VP*pfIm+9C?k=pwk=SBudWIJ{qs z<6&q{3x1GJ*cD-8r0VP89dA*5X{

    1u|f)sH5R(QElO!0l!o5FeE!S*LTA6fo`<@ zS6CnfbvRovxCp<%yKWZ^bn}0QEa!4i=%fl|RF*(EuVq(BP_7oR0X`JACa-2Z!y8L#?p+Ygs&I7dT_Xd}mCg@o1-E=Bwgj+-ux5 zj$I31COIa??d0}xC60wlBGHT!lF&EduoP-qkbR8;W7H%hB$1dt{W38{iNxfP6WbUi zAn~MJ4}+kDn*miZGpe1PX=cdHV0OZOhQs!*yLi_yB{`c{-nM5)xU-)wu4lF5Hqp{`3bBMI7+N=guoVos|k#*cQ-FWV2i-j1XfDZ73?LjkHA#~4iI%s zC9t2s8i4~uKqtexpTHV{y-rh+(N-sBuFGk8o;g%i56~dvv&(ZBvdKBy(o*gQ1vRi|Ds7ic@3oFqo zrrL4n*aa1R(IS`vq28-Ep{4^IKb*Ls1Ou6%qFKQ24Y>sa6tGBwuZ)Pcq)OFLDgiHt zU=TUoBYQ-KuYs`G!#l;cF;|Zy)H#Qtg{X{#(Th_^P$QX9O^!Am>)vxX_Ja$*J~5ok zFW&j{kquoBJ=4l&4z+LIy5a1}(^IwduX;AO9vYlFOc(d=efSqIPyNGfuh-$Z*1>gj zuI&KF@B#Wa>ZtlJ=HG!y=5~N*Q9ejO-hNL7ud3t52S&+Y<3i9E)cvr%0$%q@UZ3Kd zQDJ?=$@f(%Y^1}NGH_!xaC?<;Zy;VlN z7F)G!!|qv<0sNKQ4WzHtsKXkQ2EEfbVw^I@4Ax-irjYJ(Pj}gP@vsvqW<9rF+NAU_+qi8~k0OO5tVe?i8Ehj>C+JcP<1!6x zwIi6aY(yyw)6E0hAKER54D_i-Eq^*Wf;JNa9Jdibn~DL>o@hsXsdTo%11OK`iF+8& z@8Fzxp3@zk`PZOQqHE-uWVQifpG&ezS@St;v)kYqpcG#f3Yu8e6iliJCY2A9$`LOt z-jt4MWy7hFD$YdY}4rb+yXo;-}(tEPjuYj5dZEqf4SY;|JnGYq-@~6<%d^3Y(OC zIB#_eJCyBeZ+M3_7Jn~z#kf-aUd=>sqUM8mDXu5kdbK{ZfNfPDWLK#h)sN&G(P>rI ze2gB6AfweIKAG~FGYDl8gfc6HigcK%aS5oBEp3tZNo*X`Dvl`zYuI&LM&y!=^83%+ ze)b9G7cOduRuVcTRcK?Seh_G)oBS?Y{H`SV%f4uK_|p_M2Ac~w0#1QRz{WrhtOX1} zQ4gCVz~%_BIReaRprGjhQZmFLe5sBm1`3)KdKKgsFk>s44FIdlEMHWSF5Q~IL-|vO zE|4!sjF1*l1(YDEK5K;XX+9lt<4liFhLXcaMw$C6E|$yHw;<68xE`j5f;6N{rdDGJ z!VCg`tuhaYMuI=8GQHL z#mg5j{0-{reu|?12w9^659Z7R_XmK=f+U+|m$K{FZ?gR?FKU7)h~9uEdMQQ#GTv@V z66+2KKuE*`AV4QHcgZz(6Eyc|X-;6b)!9-`Wb{vMjj#qDpW+#*1TQ?r6K%5Hzv3Jj z&7t-$a>7XA)~*S-&S;fe;4caqEtLAF0pHW-(3_Lm_ zfvUd$^~KwBn;u)dWXS`M1#8&M(Y}?f$7@&Qw(OdEAGv?7^f5C8*RX-D!F?-S34ty! zA!uvH$Docl614=*5I9ZX6oHcjP7rtwf#a2!@Kgtz2qs#^RpPQ#M`ByzDe*<|xzw@1 zn{)rbc*StF5pGyL_pf2jqQ61YY9o-0ypR|3QeMt0d2hGSEp|)Ya<|g$9m$N;R@G)w zwW)g+rZ!5Qa#yCSu0PqI+LwAo`lWKP?(p104TI8el{ab!>xMJuGkTpH*Ap&Cx*+8O zd=1*rgbR``NVy<7TDofcYg#r6wP{6StK*pvE63R6qk%C4+hvV21j%ECBQJ z(Vi$Bjf6ZbdJJ?E1lKVa5wHV@i|qf|uGZM7jw?Jf`?&Yc-rf7y_1)dOYrL#qn_XfP z@7iV+CwqY)h*DmGY_W@zv?w44Lold364PSUts#Q8X(S3#ng{KVG)2YK=A8sok+k`N ziYlcKB$`NADnhu5BJ!gk+dXHl*WIXHduGne&dko^oAZ4q*%GL;sofBF!m&ad4sS!Z z6_2?&dXuLTPQFsj0TbD?E2gAa2_=wSwVmT#wS4 zFXfD)kvEF1d34~Nmg}yYo0?l#d!eJ#b?4w}gno=fK~F~KF;@*Vm42rkdS!5Ld*kaL zA78zAQTDZU`#0`;QJ70L>vHDnJ+ml19mha2d z*+O#pUl?DGBWp-VBtnUzfk>6ysKb?>C?~-3 z&ni$?VoMj`6DK&?oM7p}Omu>O#6U@_GFe5PB8;mpJVK#%>$$ z_XL37JZ+BBPkAtCgt#KeoBI(0`L?#!S*h0;#Y=5!H|pE-MklaLy%9k?Fx~z}*LV6C z3=fZ;ITOxh=DqWl{`dYqMxc9{|sM?rnl)pTnEGKx{a4r6yWn zsjyYhaI0}MQpkmAgB-S08nyyJ(*~SB$y;BGE3GPxW>v+NR%0xR0iZ3S%czJeqb8nl z6ywSO9&%K0baXDFjEaadiX1K?%BYIR9mR+;h(|sXrB|Xfy2?g~SQt#&U)XGi{kA=7 z&)8mDt$;$UfI@YZ!&q5uROW<(LWz=6t}CA_UZo-&lsVy`h(#$J7o^}%3%#I-Xi{Og zEM--E`I@S`2Hzt@{r`lGu86>kFLk-%LAV*)tA~PGkohIqFZ*N&9ldIT1Z5*ZFmTLf zhd|9CEXF}vctG$kJPXLM`~n6gRB-@-`RxJ%%>aVD0k0jsK?mBN(%`4h{&KY*7!MfR z*RK8XqvLOkuh_P_eHZ)L)Yy?nmakp+lY^}I;0B-opy0U;DBx@pqTqdarn8qhUS_B> z+$$zCcjm5JuGr86sAsOny{f1Fg=<3(pr{Mjc$%@L>o+)2pG>&yr+!X+itMAz*MP6j z*P6R|O}}{!o&{cGLQ)+ zucR9b#DWZJH=OTfS_&iunWjEOGD;IKl6Jm=EazofX5I1zWi#E(`s6-kFWE!)uzm7g zWe*>rBkV_>L()O{Rpl4tnDQDwNlx$&$T{gW|Ac(RZ;(&AaU%E*&jQ;-5P+v5TA9zM1^_XwH*;_{ zkgnyfzXB^)FAM=6bC@JR9bl}dP5zdo#e5!Y<*N#n2^rFy3ty}6sE-QiC8u7K{M)qGF zX^COc4e}S*YoIEGg@~lLn^x?^S)Y5?0<%b2^JzY8MLRmfc}jmf-T&c4V_htF_rmmc zZ`0K9mY3G=W(Nf-=TEQo{0mTNvyX%Z)?%}-I-<@RH5J3E7@9X2{7}Hp1Srk|G*tOE zSTWSO{*{O#R55}T16@XCs=3#I4y78&wLGGE0-j`CGyMU7*fF(Kz)@40P;6SfHD}+n zW7q7sj)e#ygvLmWX-TRfq<1BYnMheX#XU~Y(U_LXEG+1lNvdL6!I;?`XjYqp9co9g zJ#^d%WXw!>h1G3#hr4U~%)W46%|8F`;6CH!$jdc948Cd{HV=mnMUL@z1MlmXjEj*w z{GXApf>ZjxBQwc)W;hlKEqP`ZiB>p}On9{?v|-Jo#pgERhB;=jR|Gi#Filf+!!!Y1 z@klt_Xz~bDG*vUy#sCKa%)_RsssTSvK$3crwI(koS@Jw9ozdV#oXB~$-U*aU$7Ii$ zmrZ7#r%TRiG)Nts0yVD~%^ogFvWogi`|``T&$* zPR8_m6LEc_cVN65|&d`-G2MTE-x^{Y_Eav?AX>|`XQcXi2EpBfFVdulYcwr7+r z9d*|9Y`6qk~bZ0RHp5#nT(MZ=65zpP|*$LpC z_IBsK_94LEanE^A-r@amP2*ggra-+Q>u@vr=1%c4_CwRARW>jk|NzA-QD#rdcEFzfA{x) z|DX4LK0I)kGxI%X=FH5Qhj@-l1fqJqto!uYTG@OODo5&=VNR2J#liE`I}rK$fIcre z7i5a1(L~idB?x>4)t%aWBd7)-&v&lzUeAlncP}e>aTRNxt6Qde9+*Rqq&#M}7 z79yD@llDFgNrCq0NV?jeEa!4IE(6*(bFX&PV5@M=>jw(oY>cAMDtP#k!5Tv~dBKDLdf@m8oLuJBl&;b{Xx7oIZ2PDBT;M5%%YtBd zZbJvHmvxvnjy`}8QY1kLh+OR`t91#DDNx-?WV$zbmL#}vqWfUz#l3b~N$TU2b-Jvl zCJ#v3uXG`)wgSi|-uv=4Qk3}&o6_!Cg+}Ks8Y#T63+TOu8s|SZQa6+u4i*43fJDxQ6QU~u_>B4S5Cj}G29Wtanc#Pits|4jq_`4QAz=j@3C8+suFTW?8`4B zSBl*FjFq2%y8ds29zq5%j)22CCn3 z1GG-IeJhD_iBGw{X9O^w4TF}VpMuL?9neiE4`|V@=})IN^96-b&2x9XJWWmDbPrMu zptq!fvZ7wNxXF5AqfdaJjvHZJE+y+OPG=;D)*Qlrl*@ChL_{P#pj} zl=L|Ug|FVcZb>SepA@www%b#vD9Dm{J+{#gk8yXv)gR$qwWiAT<)@nC*(hVODMN0X%8*G~2(@B1d}Hxi5k z*$0n}2HT?hr7DRVVIP^mtyHFYayv3<;UzNBe4ooL&0Ss}3&d1Z#BEl+_{5%F6=zQy zpIRI^aK%~wd)e21HXb|A+q6XrFt=l)VUt^raot@V3~}r=Sj1+^-OqCFZ9gVIj~Umj z#+if#g*s@Y;GFaLob_qmnY8u4H%iwvE+2iT3sIJJ%eXM{!FFOPyb$$hAaeyB<+Vq* zR}(U#G#b*5wl19znP5gG<_G46N+li^G)@GppDK(9kfJazD^6}uJOZ5&ZR7bShL_os zIardP148i^O$UbN71j6-IU56Iy?S`W#Z4^5FM_@ePL--G@T*%Yz8dWnc6?Qttb=1{diuiO z%wt8+e^(OXaEMQSVCYq^%|W@D9u6aYb1EAc&}yT&*6Y4#8gQOlfX;e7Z0em|g{E!wqCrgldIsin-aCs_0#h}Fx0)0lm!IB)UI{G>t?=F> z>h{}PkKb!O)SNywDRwFnVwLJHW*>HzpUYs}c|qZ!0hqq2KyoUgqCgY&j9^{9x@uU< z-oD7I3>jY~ZC8(vU-Y`lobwTES8R}u-&+7*Yi7Ff${V-N0|id$6bE(K>U0&JR&_|E zpMSf3)Kd#}7v(J%N9a_UcsKs3w?NB`jpHRjyd>OPFB6;kHe>Kpyty|k&xc}Hq%b%tpprO1~a31d-`IWCExTu7k)V}#IK zb49?C690VbnWAp(>K+=W8xS6W+LJ3e{dC{EE=7>{=(whuu{lhlIcg+>6*_y*Khn${ zUR!OJE@4bsrdJVP{;Fp8h#IMFw-(gaQ$FoI`&jpS+tZ>Wy4|Y0q{TZzTGz@PcnRFJ z`HlgDXDK8-E@Q|^Y)Wc*IoxLk2BBh>wZeXsRhJ-b`XQAMpFJ(Okt+j6;xYh^Aa68k z>QW7(7zdoruWvbo2?$8ZPeQQ#N4MtRPg=OyEQwi}eE+unp@s-oA4|7ZFON^q(oL*j z@0dbUeX^`e=R5&LByLd)c1jb()00pTQ@BfE>bTgrsxw@7u4?P@Nodmz&oM>=I{tAV z;G`H2#dU2*C55_vKS@= z)X##{3W(7q$V06%$}n4C8dc;Mhd4}r&l709JhwQX5FY61QzpOVImZT&<+togwZ10! zWrM|H+KCfu1~{qF$zx9oMaJdN(6fn=3@guu7Ru6@Rtv3KrkCqhsWAE92juH~sMN|V zD1A(6t^K|GGX)|qa+wCE*Pn3vEF;Fh$t5?R0~XlqGiRq+pp(rA|b-4If9C z6o*j5nWna#bymrpmRzi+l?JbFsX=9G&Ehje_(Q@MG&M!{=Wl%$-0654Pc_;VLAl2{ z;TG3xcdGvF#uoiRJ1)3=%YS;_C6}FuEtU^ue)$TMo?_#r#g3NoE;F3Pqo<1o{Gs@@480{6!1W!3psVqaOeaD@1Iy4LVyjwcl zgL;GrE1vlTpwA=5%c0%^nd{Q#8)UZL=p=&qtefwg5K11z4m%WU1g1a@KNE{c36iA; z)bbVaaurXX66@4eE}0L;R1Q8D{35!g_k>NBI6HbRd5~>5C~>~3L!%+BHG9%^u1M$I z<-s>T;%ypIr)%K`471MAj7!I)!n0NNt`&tQA0@x!JZ_Fbh$|n?`)%lZdBiTU4RrPg zIY_GtKHnZu{ZPNEG&Sk(euz_@!|THP+KoN@_|zd?@Svn)vqCm8JX40mB-rTblK=jl zS%jhb#smXb)gYW=x%isR^K@5-h^hNU*tYsMs1DW2=8m$HbBYezZnqUKc!P1qoz>K* z;JHUtGrDG5bg&hf%DvLM|Zg*Xm+H(o+l)if!+z^#`ZHNDrAPF)WAFAa@2UA` zq%&2<-#`TuESCU;%f9l@p(1Rzt2TmP)@>_MRRaoNXd6e?+|U)I@@T(Z2bi^3)jln; zAo1|i!HwYdp5o=92yU1MqlaNT?ZbVGgOZUA&9>NNhoOR5pU&y{?YO87cMhP-3iH!CTzf4q#r1ddePIlp5LXM zCTp>|di))7-G?jULD6XWH@wtaE|;~_8qM$x)d$|WakqW2O?fykmk^z6sYDyqe=}ux zdITM~vTwlSZc;NjOj89LNZiU7$tNNSfCwxEg9kcUI9|*5>AqeXc1#Woa$Awi455pR z+h83?&**FHmJV?&ihK_ZFiKsv8MsaB?oIJA>};j;F+;{<_bX#H#@-dv+ru@3Q^z{Z znmI)1?ueYo!Sa%m zEz#6PbSbK4fE7JTOm|68lFlZ(wjY>AN1}$PzdE2NM#p-5%w0&tGlRQI``m5@Axn!T zsXvT=TTyl%P+6WG&CaoDDap`Bp|h(@GmFcQyaWZ>htD#r*E&pg9oXXUt(|6P>#cVv zM2V&fBnHvf)4vr_UTzCOTJ=5fXe%~RY;@pP+dbA8aFfoXU?0J9UW;6Sc0ICu$9hIk zwk9Voe*J!RCXd2=sq7AFL_}oLHT(tvs(LV7aE)*6An3cPc%*>@^z+phb2g?7&f(8e z41DiH4M0aVI2t&9erdQufkE69_E%WN{DhGYWu>p**WE|c_S#L225*h?8qX&LuOFSh zJwA}wpKU){3!Y0Nbl7rxY@L?yY+@y3>ZQc#4LcWp#sNEkl~*(k8~K_H<^EpHnNHV@ znG$mp2^}M^>R82u#C_4>ru%zzp?BTP!YEC(k{xq|JT@;?D8M3gmWu|DU8Q(;*ZsHM z$X`VmCvM^@r{^9~e9xKKCVVa|<7U|z-DhhfvuksWNo>i16cNq3K0A|Xiv^vUo@v^k zE4l<6U~Iu^(aJgvpM(K;mIdYPjdosMEmBs#Dn=XWJ~)g= zuI(FW-s~$-coWyU((7*MIWUPBiH6khJ!#qUcu9Zmv$Gql9CB_}s&Rb9p%uGd0!k^t z7ls1GmPbUWpQ;X7v}D*4ZYQ$xItLj0>I1~8#1;|_+dPO$w{)JfKd0QdJa*?++}4Ax z&+~ow!rx8-Lr<0r-tWg6l}Vg;tH>mlDldI$wA(RVrUhF+4C{lR88?5}W?Kw#ri%>^ zdAkuXj%qw7{bEC3A9TjqVy!<{gL!QC4EaB`eKpL#b(ZqGRQ^M=7Yw5ei6 zUDL|0NVKro#1288)@P~-{jHLAt0v(GM#NbWLT8t3!LC|1M1bf!Q|3E$WLy9_mH8)9Sye%Fuh*ID$orH{+88{FONw2_hh4az;xX&t z8bu~foziWU3$zj!TRY9KPmnBT#B77#tO`u-=-xjbPpop7wi0okC@?LrT2D@XFPX!b zvl2iR(NMYmocv&LSB62l-lHN>Py0LOG@wZZ5`Ib6Y|<9qF3j8T)t&MZ3Vqf}Po))I zmD7(TPfl2Z$T-e;z91r&lxi-Y=T;MaTsRsO%+wmq>S|dJdOCT_UD=|4@Pi@`2#5WH z6V=%}1-Af)M;u!Un)Hznt82Rw?hdX}_fgs01A;Cu5)?BdMb{at;pt5P8Is${=-0cN z)n6vLmb!g0=j!Zkwr8Jj=@mj-T>|)7n5M)i+jJcV43UMM55R6Vr)it74$R#K#+qgo zC7kg$Z>ihAi%uvR3pd?=SHh*cFk$BFLg3%V?)~;@w& zu*$OupE*=g;?PT-Hzo8&*d)azM@Rk3`wtV7>Gn48=~T{2E#7x$^SfL$`&zE2##LKXQw;%Mo2> zjk-<|Mxsp7go{VO%&Mks>peh1!A#b}8OeyAn%^IrTz}PqLrv3iZo`@Eu(nsRKejrC zdK+W>pjY^I3+j5^wYkDrIz#r1+=+nMr$=99fk_3eE46mMTh@WB=loJ_+2mG(A;;9S zX8^G*tMT7SRK-mtKH4W_>Ah*pEZ?ox|q3n%k4_9xeUWB$X- z92J9^$~?2~NboyZMwt*t@sP&q>9U#S2w)O}C0R&zfE}?WO^x?Sc%^)zcR1^De#Iag zT~BdX$ohcxM;MW2ySTBn(aTf2YSMzW&G$~xlfyoepYRHpEC45525WOe;jf)!|pd;+u ziDPAaZY)`mw|-IrEHCP`h;_M_K-xhg`JL#UCq}Z9Swg3MrRhbmm3GSRBXqUuEzJi- z+e?vWXO&cT(Wke@hcm{%_9j*vv#&3hW$+%$7#$;eawEBv#DnjCuY9-1I=tegHS%ID zO>PByQF7!Mu4#O8Q4H_EDQvu7$oU%+QOsCRJUz?bEbK5tz?|iM0AZDDR*uwjS-{7I7yt9`xumAK!}p53kTczmeH%uE`wN$A_-9wX(UMsEYfB_1pdM*_|! zyW*$avR(ZrT|m)gHz@aDM~oG&=Xd?DH>W#Avc;@l9}6;O!p-c`sI)pKP_3hVjO=0@ z-&a-h8F&NFBUXy2cQsY)Atlp)=Zhpf%{>&%+R4P7^>AppQ%K)ECo&Qxv6KE_LpM{5 zi=19Y@PNZDrr7a$^`1mKfU-ncnx_L#FbI)BSaMz5tkjeK!NdjECUt6MP~gxRLq zw!ZG0cV+_w#p%m~40e&{_gmvU$OcM#lz{*w>GW6m=77<)2cY1>k{*T7W>-<2sM{Gl zPEsTrYghAKl)QawxgzHdqZtwUZ(YN;C}yEbkLeU2w`=C^r;WbYxz_{*-2W0{eJ*!+ z<#X`izT7%zA!TXtoY3)Ii65v6$DWBrz5Ky#8)^OG_elTY9Isv7lYBG4YRlrX3Q*TuDU0n`u8GeD zf(HpaOd#pwr9NJac#Z#{To#*pOg#{thTm20^0Djo)Jm<)QIzTc#Ziv{Gh3_GOgz0( z!5GKha*Tp9ZYm!I^!2`hYQ|S0mH0H=% znD1$s(e-r^1I_`;TqFN$N5tL)U7)0Mqx}&vqsukgKDmt(#Cs0@_7-p4>V>SL%CzIG zu|)fem?VmXq9WY|=kvx`-h|u`p>EmCOEQpiB5p|VOEA596hl1&J=znhn~=3?K2*DG zpYjsPI6I`F(^7pr3SM`joPLl(-JO!x!ZdSOA}u!F{UVj6Mht?5nr5O$4#DQ&gP=0_qI>?Nc1U3o?NCq64RH|E37ecw!N{Xa?GqH zko)GNZp*HYTzq9@qhSM=WAj<0={vfrPS>NZ(LSNJSqEu~F~ab4dZ-iRVO;E+qV7EDJkwpL5V zirGG1%1-8_xat1hu>NVnB&#XK`mA*DbZI^pFkl&|uveEYshxojMs|O2q=2SOXt4^?1yhL>4#+K`WaWbk=5+OJi+U%`-XVLc{SCo^ zXCVZi9dY+{@=UfKR;Qlq$h3>3m%XqwxZ8QczHciu*?nUzWGEydndEKsS9!xatg&9a zeQ&#%1QQ*s(nPzsZ}vFz0C)G-!&F>m`n}l61vR33$9PGo^6!vfVINcNqle*qQXg># zA8dA=XH>@4AHPPn04iJbEtce6Bdcl4rb)*q8Y*PBr%B-DTuw=gG@PDATL&iKr!$b)CG);lYNMqNogh}lJD&s;oa&vB9+FK-FSRoe!(T+_L9kLMEmd$^;(7yhWq z7BRjnkL6Dd`SmsyDsMhcZHZ0t@F=fZ-Q3W5qyACKL@K0OC&irxP?6f=8BLeqb-(so z#3K&%XA8sz2ZkY)YmZ*$BF>E8FyP$gB6SR(zL|vMU;k*6$e0>@8yt_B8g zWqlRtFyMHzb)|?~?ZEs7Z2Q=K#L&4a**<1sF5^Wjct0OrD`S{!K8Nql@7WbRn?d|C zF~@zx*y(k0NestEFG+9V3Hn-ZLe!8~POFHAu@QsvGzg!IrNsa`7bcxj;K@#5-_BOKdvJy-I?m zcl1YA8;(+B7k%tpx7_zfE{B@&sE(vbZ}+Yzpt=voUAlzUt4)t@&nbqc(U^as8e+QB z5U%+=>qaWfk%tpku~f&6#%X;P;@PhVTb0gk$hRS~lze=}{)#hnWyc_@SL)(SPmPdvif>8D%0d=Q2e=qr>ip=ZVd5`& znFTgC=updzTLQD3?f2f^Dboj0)r}~}#D^wH=hYXWXwF>rjp!||kNdScO&64QBC8)z zQbD{D-uA*+oT}ft=y04$JBpuw&vMu}gNnOitBM1;{5THoah0aJR9tC`YrJ};J@D-u zsH`FkfOK|m9+%e^szj(v;~P!q`zb#QS`WJaJzw0G@jCTny;~3SdJqkD;^Qqn_RRYV z7E|n%X=%#T3TDcc0y#nZeh~cU8HN!z1gQilu5Lbg=+1g)&Nkn*7rb_}t+&?YMb82A zI#IBXVG77`zeTdeY2KKRg>yGZGF%llH^sbNomZQGP}es@?Dn;+b`Lu_Ge z$QtwIV8&5HcH?S(h?^Vcu#2=Hrec$%KJJ{v+XMA5_Ks8tfYkEy6UOVes3RfuBjUj8 z@-|n|$1bD?y4B+MZVL_VJKfC`XBUBnUGXlyQP+4oB*@jC{=8k$DZlYNdFMQeNyxKe zT8fnPQ8}IC8tL2X&wN5szP`^s+K$oZ-uvbubn zks3$d?jXjmX7O3f#YY}}Q-+L}0Hj6}Q?197P>9X>op;^|CH-Y~DFL!$gKlscO0cGP z{d8#LX}AXSeb%zHY$g^@mRC+8>iOxGiEFXA>7GwsvD{>O)TyWb{;AT^9(mzI?K|MN z1vP~sNV2`B&eoYO&85k1k2r7$J?o*hyHJGMwrv?Cl~e~Z(N6YY7^QyI@3mrome;*w3+gIdC!OD6Wc zkK%|bV0@<^asyA3PU9j|r)!Kbuel}Oi|ea_*MuM0nc?qTcj>f!-&l(G>^Vfuj17JF zdq}>1nx(bHfO;Fn7Xk7DGBezAFA)POW@EzaktcMgs&1BdUx~14mwRq-6oh4JjZ@`Ct>oC?&32)j_eiFoqL$()O6lUTRO^@ z_g@xPkWz?ypU%t|$Ezsr-=2jM#*0S3`;yLrHut}o$MP(!0{3kRJ@xMH!jMF+C93L! ze##Nghkd0}i4T@49^Ol)@2h>5q%zmumM1Qed~nHhk6IIFLod1Qrc7}laQma*pAqy& zs5mH91uqq=%=b-q=CK-0f-^_bWsV!g@K22iC!iE+uY%R4I@G5U^Da5zw6=xax~G?T z7h!uvH!%m%8C1Gr-?DM=(C|B>O#`D7DRV8cRwGEOkugA{7JHb02z$_roPqG>bKRi& zjP#;MVC;{z47OAnSOjng+oiktu71&Q*xQ9>*QxPGSlW%ML0SH)PHZ3Tv)V^8V2nqh zzAEO|54e3#&M$Yjqn6_LP)o}}lw5*?36unOuVj}h-7=2a!qbvOzP(L>-t9{`vUc&| zH;Kk#zr`2V*lPLIB~hwRA$!n-Uad+x8`dZ^E%YqU!pnxNz_;QEJ38b# zmN|#vxrFD^pUXwFo*c+t)nu?{yPC_g&MOw7p%WEqX$`c$s;E(x*It>g^7gz$Ic6_eKY@HTn1ywV(L zjW`!fDcV^7elJ|@yR!G@-Y6-S!}G~mDsF~g`Y?EN`^WBkTy|B1rwzO zw_u?b7R_qRjn}ff?#^A)F45cR_2p58S}#)0MQi_*llaixdRoS4shKTb9oh+!*nrm5iC$sL2WPbtUI_|?JWYHhV$@mK zXPlOm5=m2Y9@0+bZVe|qUv z=kh`=Zy)ct_PuY=%F3gj%c0R=jk?B9Wx5V0!k5VUa0s-#F2}}B5r`Oq89)jO??pQf z0I$Awz0vtnTjau(`hwW_K1{%%Yuud zp~I73TRR@B-t^KTrMIk1%0n}a$VcYB3U6K;YJ}EeG3OOc-BH5^Q_v)gfXISfJ4`x` zYIA%93$a26aKDy68oBbA8ni5MCG#^ES8O{98PkqJm0=!n%32%bg8J~T051^*pH&hQ zz-^3CXe1nFO^j(F3@wN``51r<7Y#u{fC$P6Zi8k30l9$yF_`URxVfnrnt>Mt28hFd zcCmAD@&F{QV5TSrE>3_X!UioWig46s2ZMPSF8UZafk5C70No$Gk}zv?D<@_Vq&dur zoRg)PGKh z8wdnQqhVI&Mj|$*R&WL&Kp73UR%77f1E@IJ!hf({Oc-fyi$)*;8h_})1A;)If2s}n zt+u4O6&jAbPzwzghZ`Y`;Q%?fjVaoUftw2qh5lhLg2#S35#urqk8ZwP-@fURc`F<| zER=PPpjdU`m9U1JCpY9HR>jrPowtZ}1<9L4MThlkawbC&uEm-NQQo){p1n>rsls(p z>fOF%qg0a&9}eT~`XXJfGePNamJ|imSkfdxAvz&V9hRSj~rWS;xjZDx}@JPvxPXZNS!A>&@-*Ur$yYYltlT z7oEC*@{>#E@2P|A_{`KgzZL`gjmjxvz4s=|$=mFY=L&;0RSmYSUfcMo&yXK@orHdx zNxJUR25}FWNx1#6Zg4;11zXgwVljSHkMMllMzpqnj!op0b~?|8cJh9nw`k z$gNv3>}kpm5WCfKLUKrZN_+C-<`OWbm?N7Zv%u9#QF}v<`2qurfMNO~`oC_?@Mr^j zGBjCIz&D+jRhO$S?wPOzlB%thg`OU+3@L@~1UiTi%6Vp|V==6^Fjq>y+iKBUwlJ`CTF5(J?OZsX;hF&5E`s)+M~2p`L$ncrM`YmH*+qKn@@S zA2*1D55m9$;o;!q1v6ks82|mRZUF(v!_Y``M{N*B^g{gN{!3dIpae%D?2txql%OCK z3Q)E)ME`V5r1f9@37hB==EC}yNrHaQQNhj{r43}@ya*WGzc}!34rPEiCIT=z{O;xY z-7AK$yNGFkthq5t`y$wtbbp4)6PPI+^;?Adx5yvf&&dtu{B!Oa@^U}LAnU)*6Gx;Am}SgW`7-VuWE*5}Y5?z90h_8Gf*!Erl3GE;<-ABdLJ2S}L8P$>zmmA* zAH!k36Jj(&qiy*C00##L4hJp{1kw}$fUaCLxde#h>`JPpg*ra^s=`7b08ZO#}6Gjj1blUN0==D<28W)A|x;WKZ{vg z|EnYltpxwA*}o2m`jIAo`6!0{?!Qd|g_f|#sPu=4{ue+J_L$w{U!jbR{=~7hLt6bX zw6PHYZUwi-h@dc50R5{jwtrcpzfAzsff3>Y{JAI?n+syxiC+w}w_&n4=7wS%0>lpF zVF!X#fM9-35axpo$ivSC6a@T(^Y^0t!1(3i{~MIQ@%|GBVQg;V^uI#+&G`?EANnAW zDhPxX^k)!A+hBap+|-5v#KQq%U{*KB?Bfn7mcJJTN|XZXk%8osY*D)8Yd0Z~?iE!A8a=KVbhZ`yaT9NOMfs!>q&+ zMs^qLiNVJhn8O$|gd|X05+n+Qh;U1YN$`O{d_3YH5nc&VK5opbDZ!GGzu^A`;y;G@ z2RsJd*qGA zwczE#9Ly%d$HNT~;krN;<&@wP7X@OdTzujpBH&-he-ZsRWW@i^YX6-3zd-!kNXWl0 zi!s~;W@m-|HO+tY{s#E9YJXXkANl`hTE2*Ij6Xj%M`3D1r@wSu*zO{eE5o1to%*Z8 z%KU%1xPJ`vBY*P4j4m7tYJ{+{ha=H&Y{F7ia)Sc1+a&%mQe{-+F#{?SBCL_fb#?6SM;i2^TTN1n&PW!~DekyU2fs zHZp_Rn8J;rfL|*ABK&s|2Q#?M|CG0XPT&XEKS2Zm7mH_Q{`a-}N&Wkh{_>CibOAwQ zBYu;M#|<>}uh#xy)8AdcM;65N@IOH!jO~oz$Ultt1XEzhnHwTuNT-VojfvO4G5teW z1&J95VMc=Cyn<%X&_Ea1^jnDE-?5nd}t&L^X!TwCcr!?E1NrGs!)KmxV9jKSDcHR zlbcf%lek62xOw<6dz&~P1R};GfjNVdmm4Z51`_81i*s{wiGd|9o~ Date: Mon, 23 Mar 2026 17:30:57 -0700 Subject: [PATCH 30/46] fix(text): handle Tc/Tw character and word spacing (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(text): handle Tc/Tw character and word spacing in text width computation PDFs using Tc (character spacing) and Tw (word spacing) operators for text justification had words incorrectly split across TextItems. The computed advance width didn't account for these spacing parameters, causing spurious spaces mid-word (e.g. "deve lopers" instead of "developers"). - Add Tc/Tw operator handling and graphics state save/restore - Incorporate char_spacing and word_spacing into compute_string_width_ts - Add adaptive merge threshold: tighter for lowercase→lowercase junctions, wider before joining punctuation - Add unit tests for Tc/Tw width computation and merge behavior Co-Authored-By: Claude Opus 4.6 * test(fonts): add large Tc width computation test Verifies that large character spacing values are applied in full without any artificial cap, matching PDF spec behavior. Co-Authored-By: Claude Opus 4.6 * fix(text): guard against Tc/Tw-inflated widths in merge and join paths Two targeted fixes to prevent character-spacing (Tc) and word-spacing (Tw) inflation from causing data quality regressions: 1. should_join_items: reject large negative gaps (< -font_size) that arise when Tc/Tw inflate item widths past adjacent items. Fixes FY_2015 merged numbers (e.g. "239.696.0" → "239.69 6.0"). 2. merge_text_items: cap effective width for gap computation when Tw inflates space-containing items beyond 0.85× font_size per char. Prevents column-level gaps from collapsing into merge range, recovering table detection for Baldwin-Edwards and similar PDFs. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/extractor/content_stream.rs | 42 +++++++++-- src/extractor/fonts.rs | 111 +++++++++++++++++++++++++++- src/extractor/mod.rs | 119 +++++++++++++++++++++++++++++-- src/extractor/xobjects.rs | 13 +++- src/text_utils.rs | 13 ++-- tests/snapshots/nexo-price-en.md | 2 +- tests/snapshots/p1244-1996.md | 6 +- tests/snapshots/td9264.md | 28 +++----- 8 files changed, 292 insertions(+), 42 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 795185e..4048076 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -118,12 +118,14 @@ pub(crate) fn extract_page_text_items( // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible - let mut gstate_stack: Vec<([f32; 6], i32)> = Vec::new(); + let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new(); // Text state tracking let mut current_font = String::new(); let mut current_font_size: f32 = 12.0; let mut text_leading: f32 = 0.0; // TL parameter (in text-space units) + let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled) + let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled) let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; @@ -154,13 +156,15 @@ pub(crate) fn extract_page_text_items( match op.operator.as_str() { "q" => { // Save graphics state - gstate_stack.push((ctm, text_rendering_mode)); + gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing)); } "Q" => { // Restore graphics state - if let Some((saved_ctm, saved_tr)) = gstate_stack.pop() { + if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() { ctm = saved_ctm; text_rendering_mode = saved_tr; + char_spacing = saved_tc; + word_spacing = saved_tw; } } "cm" => { @@ -213,6 +217,18 @@ pub(crate) fn extract_page_text_items( text_rendering_mode = mode as i32; } } + "Tc" => { + // Set character spacing (extra space added after each character) + if let Some(tc) = op.operands.first().and_then(get_number) { + char_spacing = tc; + } + } + "Tw" => { + // Set word spacing (extra space added for each space character) + if let Some(tw) = op.operands.first().and_then(get_number) { + word_spacing = tw; + } + } "Td" | "TD" => { // Move text position: TLM = T(tx,ty) × TLM; Tm = TLM // tx,ty are in text space — must be scaled by the text line matrix @@ -253,8 +269,15 @@ pub(crate) fn extract_page_text_items( if in_text_block && !op.operands.is_empty() { // Advance text matrix regardless of visibility let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| { - get_operand_bytes(&op.operands[0]) - .map(|raw| compute_string_width_ts(raw, fi, current_font_size)) + get_operand_bytes(&op.operands[0]).map(|raw| { + compute_string_width_ts( + raw, + fi, + current_font_size, + char_spacing, + word_spacing, + ) + }) }); // ActualText: suppress glyph extraction, just advance text matrix if suppress_glyph_extraction { @@ -408,8 +431,13 @@ pub(crate) fn extract_page_text_items( } if let Some(fi) = font_info { if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); + total_width_ts += compute_string_width_ts( + raw_bytes, + fi, + current_font_size, + char_spacing, + word_spacing, + ); } } if !is_invisible { diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index 5f9a3f3..e4f6b84 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -429,15 +429,23 @@ pub(crate) fn parse_cid_w_array( /// Compute the width of a string in text space units, /// given raw bytes and font width info. /// Returns width in text space units (font_units * units_scale * font_size). +/// +/// `char_spacing` (Tc) is added per character and `word_spacing` (Tw) is added +/// per space character (byte 0x20), both in unscaled text-space units. +/// Per the PDF spec: tx = (w0 × Tfs + Tc + Tw_if_space) per glyph. pub(crate) fn compute_string_width_ts( bytes: &[u8], font_info: &FontWidthInfo, font_size: f32, + char_spacing: f32, + word_spacing: f32, ) -> f32 { let mut total: f32 = 0.0; - if font_info.is_cid { + let mut num_spaces: usize = 0; + let num_chars = if font_info.is_cid { // 2-byte (big-endian) character codes let mut j = 0; + let mut count = 0usize; while j + 1 < bytes.len() { let cid = u16::from_be_bytes([bytes[j], bytes[j + 1]]); let w = font_info @@ -446,8 +454,14 @@ pub(crate) fn compute_string_width_ts( .copied() .unwrap_or(font_info.default_width); total += w as f32; + // CID 32 = space in most CID fonts + if cid == 32 { + num_spaces += 1; + } + count += 1; j += 2; } + count } else { // 1-byte character codes for &b in bytes { @@ -458,10 +472,17 @@ pub(crate) fn compute_string_width_ts( .copied() .unwrap_or(font_info.default_width); total += w as f32; + if b == 0x20 { + num_spaces += 1; + } } - } + bytes.len() + }; // Convert from font units to text space using the font's scale factor + // Then add Tc per character and Tw per space character total * font_info.units_scale * font_size + + num_chars as f32 * char_spacing + + num_spaces as f32 * word_spacing } /// Extract raw bytes from a PDF operand (String object) @@ -1070,6 +1091,92 @@ fn score_text(text: &str) -> i32 { mod tests { use super::*; + fn make_font_info(widths: &[(u16, u16)], default_width: u16, is_cid: bool) -> FontWidthInfo { + FontWidthInfo { + widths: widths.iter().copied().collect(), + default_width, + space_width: widths + .iter() + .find(|(k, _)| *k == 32) + .map(|(_, v)| *v) + .unwrap_or(default_width), + is_cid, + units_scale: 0.001, + wmode: 0, + } + } + + #[test] + fn compute_string_width_ts_no_tc_tw() { + // Without Tc/Tw (both 0), width = glyph widths only + let fi = make_font_info(&[(72, 500), (101, 400), (108, 300)], 600, false); + let bytes = b"Hello"; // H=500, e=400, l=300, l=300, o=600(default) + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.0); + // (500+400+300+300+600) * 0.001 * 10 = 21.0 + assert!((w - 21.0).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_positive_tc() { + // Positive Tc adds char_spacing per character + let fi = make_font_info(&[], 500, false); + let bytes = b"ab"; // 2 chars, each 500 default + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.5, 0.0); + // glyph: (500+500)*0.001*10 = 10.0, Tc: 2*0.5 = 1.0, total = 11.0 + assert!((w - 11.0).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_negative_tc() { + // Negative Tc (tight tracking) reduces width + let fi = make_font_info(&[], 500, false); + let bytes = b"ab"; + let w = compute_string_width_ts(bytes, &fi, 10.0, -0.3, 0.0); + // glyph: 10.0, Tc: 2*(-0.3) = -0.6, total = 9.4 + assert!((w - 9.4).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_tw() { + // Tw applies only to space characters (byte 0x20) + let fi = make_font_info(&[(32, 250)], 500, false); + let bytes = b"a b"; // 'a'=500, ' '=250, 'b'=500 + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.8); + // glyph: (500+250+500)*0.001*10 = 12.5, Tw: 1*0.8 = 0.8, total = 13.3 + assert!((w - 13.3).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_tc_and_tw() { + // Both Tc and Tw + let fi = make_font_info(&[(32, 250)], 500, false); + let bytes = b"a b"; // 3 chars, 1 space + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.1, 0.5); + // glyph: 12.5, Tc: 3*0.1 = 0.3, Tw: 1*0.5 = 0.5, total = 13.3 + assert!((w - 13.3).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_cid_font() { + // CID font: 2-byte codes, space is CID 32 + let fi = make_font_info(&[(65, 500), (32, 250)], 600, true); + // "A " in CID: [0,65, 0,32] + let bytes = &[0u8, 65, 0, 32]; + let w = compute_string_width_ts(bytes, &fi, 12.0, 0.2, 0.3); + // glyph: (500+250)*0.001*12 = 9.0, Tc: 2*0.2 = 0.4, Tw: 1*0.3 = 0.3 + assert!((w - 9.7).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_large_tc() { + // Large Tc (character-spreading) is applied in full + let fi = make_font_info(&[], 500, false); + let bytes = b"abc"; // 3 chars + let w = compute_string_width_ts(bytes, &fi, 10.0, 5.0, 0.0); + // glyph: (500*3)*0.001*10 = 15.0, Tc: 3*5.0 = 15.0, total = 30.0 + assert!((w - 30.0).abs() < 0.01); + } + #[test] fn score_text_cjk() { // Correct Japanese text should score well diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 15554db..50cb258 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -272,6 +272,44 @@ pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] { /// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each /// group by X, then merges consecutive items that share a similar font size /// and are close horizontally. +/// Cap item width for merge-gap computation to guard against Tw inflation. +/// +/// When PDF word-spacing (Tw) is large (used for text justification), the +/// advance width of strings containing spaces extends far past the visible +/// glyph extent. This inflated width collapses inter-column gaps, making +/// `merge_text_items` incorrectly merge items from different table columns. +/// +/// Only applies to non-CJK items whose text contains spaces (where Tw +/// contributes) and whose average width-per-character is abnormally high. +fn effective_merge_width(item: &TextItem) -> f32 { + use crate::text_utils::is_cjk_char; + + if item.width <= 0.0 || item.font_size <= 0.0 { + return item.width; + } + // Tw only inflates strings that contain space characters. + if !item.text.contains(' ') { + return item.width; + } + // CJK characters are naturally ~1.0× font_size wide; skip the cap. + if item.text.chars().any(is_cjk_char) { + return item.width; + } + let char_count = item.text.chars().count(); + if char_count == 0 { + return item.width; + } + let avg = item.width / char_count as f32; + // Normal proportional text: ~0.5× font_size per char. + // Monospace: ~0.6×. Threshold at 0.85× catches Tw inflation. + if avg > item.font_size * 0.85 { + let capped = char_count as f32 * item.font_size * 0.6; + capped.min(item.width) + } else { + item.width + } +} + pub(crate) fn merge_text_items(items: Vec) -> Vec { if items.is_empty() { return items; @@ -315,7 +353,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { while i < group.len() { let first = group[i]; let mut text = first.text.clone(); - let mut end_x = first.x + first.width; + let mut end_x = first.x + effective_merge_width(first); let x_gap_max = first.font_size * 0.5; let mut j = i + 1; @@ -332,12 +370,30 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { if gap < -first.font_size * 0.5 { break; } - // Insert space at word boundaries - if gap > first.font_size * 0.08 { + // Insert space at word boundaries. + // Base threshold 0.08; raised to 0.13 for lowercase→lowercase + // junctions to accommodate Tc/Tw character-spacing adjustments + // that shift advance widths relative to Td positioning. + let threshold = { + let prev_last = text.trim_end().chars().last(); + let next_first = next.text.trim_start().chars().next(); + // Never insert space before joining punctuation + if next_first.is_some_and(|c| matches!(c, '.' | ',' | ';' | ')' | ']' | '}')) { + first.font_size * 0.25 + } else if prev_last.is_some_and(|c| c.is_lowercase()) + && next_first.is_some_and(|c| c.is_lowercase()) + { + // Lowercase→lowercase: likely mid-word, use wider threshold + first.font_size * 0.13 + } else { + first.font_size * 0.08 + } + }; + if gap > threshold { text.push(' '); } text.push_str(&next.text); - end_x = next.x + next.width; + end_x = next.x + effective_merge_width(next); j += 1; } @@ -465,6 +521,61 @@ mod tests { use crate::types::{ItemType, TextLine}; use layout::{detect_columns, is_newspaper_layout, ColumnRegion}; + fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y: 700.0, + width, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn merge_items_no_space_before_period() { + // Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap + // to "." due to negative Tc, but period should still join without space. + let items = vec![ + make_merge_item("date", 227.25, 89.25), // end = 316.50 + make_merge_item(".", 318.00, 3.0), // gap = 1.50 (0.125 × fs) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "date."); + } + + #[test] + fn merge_items_lowercase_join_with_tc() { + // Lowercase→lowercase junction: "deve" + "lopers" with Tc-affected gap + // Gap of 0.12 × font_size should merge without space + let items = vec![ + make_merge_item("deve", 100.0, 30.0), // end = 130.0 + make_merge_item("lopers", 131.44, 40.0), // gap = 1.44 (0.12 × 12) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "developers"); + } + + #[test] + fn merge_items_space_at_word_boundary() { + // Word boundary gap (> 0.13 × font_size) should insert space + let items = vec![ + make_merge_item("hello", 100.0, 30.0), + make_merge_item("world", 132.0, 30.0), // gap = 2.0 (0.167 × 12) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "hello world"); + } + #[test] fn test_group_into_lines() { let items = vec![ diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 5a7a3bf..2cab02f 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -354,6 +354,8 @@ fn extract_form_xobject_text_inner( raw_bytes, font_info, current_font_size, + 0.0, + 0.0, ); text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -381,6 +383,8 @@ fn extract_form_xobject_text_inner( raw_bytes, font_info, current_font_size, + 0.0, + 0.0, ); text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -493,8 +497,13 @@ fn extract_form_xobject_text_inner( } if let Some(fi) = font_info { if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); + total_width_ts += compute_string_width_ts( + raw_bytes, + fi, + current_font_size, + 0.0, + 0.0, + ); } } if !fill_is_white { diff --git a/src/text_utils.rs b/src/text_utils.rs index 3be4d12..702c4bd 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -579,8 +579,10 @@ pub(crate) fn should_join_items( }; let font_size = prev_item.font_size; - // Never join across column-scale gaps - if gap > font_size * 3.0 { + // Never join across column-scale gaps or large overlaps. + // Large negative gaps arise when Tc/Tw inflate item widths past + // where adjacent items actually start. + if gap > font_size * 3.0 || gap < -font_size { return false; } @@ -612,15 +614,18 @@ pub(crate) fn should_join_items( // are positioned close together are almost always a single number. // e.g., "34,20" + "8" → "34,208", "+13." + "0" + "%" → "+13.0%" // Use a generous threshold since word spaces in numbers are rare. + // The lower bound (-font_size) rejects large overlaps caused by + // Tc/Tw–inflated item widths that make adjacent items appear to + // occupy the same space. if let (Some(p), Some(c)) = (prev_last, curr_first) { let prev_is_numeric = p.is_ascii_digit() || p == ',' || p == '.'; let curr_is_numeric = c.is_ascii_digit() || c == '%' || c == '.'; if prev_is_numeric && curr_is_numeric { - return gap < font_size * 0.3; + return gap > -font_size && gap < font_size * 0.3; } // Sign characters (+/-) followed by digits if (p == '+' || p == '-') && c.is_ascii_digit() { - return gap < font_size * 0.3; + return gap > -font_size && gap < font_size * 0.3; } } diff --git a/tests/snapshots/nexo-price-en.md b/tests/snapshots/nexo-price-en.md index 57c887c..f261102 100644 --- a/tests/snapshots/nexo-price-en.md +++ b/tests/snapshots/nexo-price-en.md @@ -4,7 +4,7 @@ |---|---|---|---|---| ||with 3.5% individual consumption tax applied 79,287,000|with 3.5% individual consumption tax applied 76,435,000|and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather-|| |Special|83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|79,275,000 with 3.5% individual consumption tax applied 79,275,000|upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| -|Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(fron t and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| +|Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(front and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| **Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index 5a05ca8..1b64aed 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -20,7 +20,7 @@ Name and address of employee **Publication 1244 (Rev. 7-96)** Cat. No. 44472W -**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A , Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.— If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. +**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A, Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.—If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. *(continued on inside of back cover)* @@ -64,9 +64,9 @@ Employer’s name and address (include establishment name, if different) **1** C **3** Tips paid out -Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3 ) from, 19, to, 19 Signature Date +Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3) from, 19, to, 19 Signature Date -**Paperwork Reduction Act Notice.— We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law —each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these +**Paperwork Reduction Act Notice.—We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; Learning **about the law—each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676). diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index f53d803..43201d8 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -206,25 +206,15 @@ CFR part or section where Current OMB identified or described control No. 1.302-2T………………………………………………………………… 1545-2019 1.302-4T………………………………………………………………… 1545-2019 -1.331-1T………………………………………………………………… 1545-2019 -1.332-6T………………………………………………………………... 1545-2019 -1.338-10T………………………………………………………………. 1545-2019 -1.351-3T………………………………………………………………… 1545-2019 -1.355-5T………………………………………………………………… 1545-2019 -1.368-3T………………………………………………………………… 1545-2019 -1.381(b)-1T…………………………………………………………….. 1545-2019 -1.382-8T………………………………………………………………… 1545-2019 -1.382-11T………………………………………………………………. 1545-2019 -1.1081-11T……………………………………………………………… 1545-2019 -1.1221-2T……………………………………………………………….. 1545-2019 -1.1502-13T……………………………………………………………… 1545-2019 -1.1502-31T……………………………………………………………… 1545-2019 -1.1502-32T……………………………………………………………… 1545-2019 -1.1502-33T……………………………………………………………… 1545-2019 -1.1502-35T……………………………………………………………… 1545-2019 -1.1502-76T……………………………………………………………… 1545-2019 -1.1502-95T……………………………………………………………… 1545-2019 -1.1563-1T……………………………………………………………….. 1545-2019 +|1.331-1T………………………………………………………………… 1545|-2019| +|---|---| +||1.332-6T………………………………………………………………... 1545-2019 1.338-10T………………………………………………………………. 1545-2019| +|1.351-3T………………………………………………………………… 1545|-2019| +|1.355-5T………………………………………………………………… 1545|-2019| +|1.368-3T………………………………………………………………… 1545|-2019 1.381(b)-1T…………………………………………………………….. 1545-2019| +|1.382-8T………………………………………………………………… 1545|-2019 1.382-11T………………………………………………………………. 1545-2019 1.1081-11T……………………………………………………………… 1545-2019 1.1221-2T……………………………………………………………….. 1545-2019| +|1.1502-13T………………………………………………………………|1545-2019 1.1502-31T……………………………………………………………… 1545-2019 1.1502-32T……………………………………………………………… 1545-2019 1.1502-33T……………………………………………………………… 1545-2019 1.1502-35T……………………………………………………………… 1545-2019 1.1502-76T……………………………………………………………… 1545-2019| +|1.1502-95T……………………………………………………………… 1545|-2019 1.1563-1T……………………………………………………………….. 1545-2019| 1.1563-3T……………………………………………………………….. 1545-2019 1.6012-2T……………………………………………………………….. 1545-2019 From fa94314760a549baceef7b8001ce058567dfc903 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 24 Mar 2026 01:37:36 +0100 Subject: [PATCH 31/46] fix: skip page extraction when operation count exceeds 1M (#12) * fix: skip page extraction when operation count exceeds 1M Vector-heavy architectural PDFs can have 10-26M path operators per page, causing ~60 GB allocation during per-op processing. The decode itself is tolerable but the subsequent loop amplifies memory 2-5x with text items, rects, paths, and state tracking. Check operation count after Content::decode() and return empty extraction for pages exceeding the limit, with a warning log. * test: add unit test for excessive operations guard Constructs a synthetic PDF with 1.1M path operators to verify that pages exceeding the operation limit return empty extraction. --- src/extractor/content_stream.rs | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 4048076..0829440 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -115,6 +115,17 @@ pub(crate) fn extract_page_text_items( let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; + const MAX_OPERATIONS: usize = 1_000_000; + if content.operations.len() > MAX_OPERATIONS { + log::warn!( + "page {}: skipping extraction — {} operations exceeds limit ({})", + page_num, + content.operations.len(), + MAX_OPERATIONS + ); + return Ok(((Vec::new(), Vec::new(), Vec::new()), false)); + } + // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible @@ -1122,4 +1133,45 @@ mod tests { dedup_rects(&mut single); assert_eq!(single.len(), 1); } + + #[test] + fn test_skip_excessive_operations() { + use crate::tounicode::FontCMaps; + use lopdf::{dictionary, Object, Stream}; + + let mut doc = lopdf::Document::new(); + + // "0 0 m\n" = 6 bytes per op, 1_100_000 ops → ~6.6 MB content stream + let ops_bytes = "0 0 m\n".repeat(1_100_000).into_bytes(); + let stream = Stream::new(dictionary! {}, ops_bytes); + let content_id = doc.add_object(Object::Stream(stream)); + + let page_dict = dictionary! { + "Type" => "Page", + "Contents" => Object::Reference(content_id), + "Resources" => dictionary! {}, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }; + let page_id = doc.add_object(page_dict); + + // Register the page so get_page_content can find it + let pages_dict = dictionary! { + "Type" => "Pages", + "Count" => Object::Integer(1), + "Kids" => vec![Object::Reference(page_id)], + }; + let pages_id = doc.add_object(pages_dict); + let catalog = dictionary! { + "Type" => "Catalog", + "Pages" => Object::Reference(pages_id), + }; + doc.add_object(catalog); + + let font_cmaps = FontCMaps::from_doc(&doc); + let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let ((items, rects, lines), _has_gid) = result; + assert!(items.is_empty()); + assert!(rects.is_empty()); + assert!(lines.is_empty()); + } } From ad10f670bffa0c5cd2520cbc84ddfc95101a4edc Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 17:40:58 -0700 Subject: [PATCH 32/46] chore: migrate from deprecated load_mem_with_password to load_mem_with_options Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.toml | 2 +- src/detector.rs | 2 +- src/extractor/mod.rs | 4 ++-- src/lib.rs | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 32016c3..912cf0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector" [dependencies] # PDF parsing -lopdf = { git = "https://github.com/firecrawl/lopdf", rev = "edf6279", features = ["rayon"] } +lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "052674053814a9f4897af94f0b8e46a545c9b329", features = ["rayon"] } # Error handling thiserror = "2.0" diff --git a/src/detector.rs b/src/detector.rs index 535f9e4..57f5bfc 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -144,7 +144,7 @@ pub fn detect_pdf_type_mem_with_config( let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 50cb258..1f63a13 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -52,7 +52,7 @@ pub fn extract_text_mem(buffer: &[u8]) -> Result { let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; @@ -127,7 +127,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; diff --git a/src/lib.rs b/src/lib.rs index 6fcb54b..90bc633 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -288,7 +288,9 @@ fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> { let doc = match Document::load_mem(buf) { Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buf, "")?, + Err(ref e) if is_encrypted_lopdf_error(e) => { + Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))? + } Err(e) => return Err(e.into()), }; let page_count = doc.get_pages().len() as u32; From e43bbb7ff7adefc7845a98edb486bda0d7388f80 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 18:29:57 -0700 Subject: [PATCH 33/46] fix(tables): stop rejecting data tables with dot-leader labels as TOC (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The is_table_of_contents heuristic falsely rejected wide statistical tables (e.g. ERP appendix tables) where the first column has year labels with dot leaders ("1973..........") and other columns have small numeric values. Add column-aware analysis: if dots are confined to ≤1 column and ≥3 columns contain numbers, it's a data table. Co-authored-by: Claude Opus 4.6 --- src/tables/detect_heuristic.rs | 110 ++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index aa1f7ba..80ecb07 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -803,12 +803,18 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { return false; } + let num_cols = cells[0].len(); let mut dot_cells = 0; let mut page_number_cells = 0; let mut total_cells = 0; + // Track which columns contain dots vs numbers to distinguish + // TOC (dots span middle, page number at end) from data tables + // (dots only in label column, many number columns). + let mut dot_cols = vec![0u32; num_cols]; + let mut numeric_cols = vec![0u32; num_cols]; for row in cells { - for cell in row { + for (ci, cell) in row.iter().enumerate() { let trimmed = cell.trim(); if trimmed.is_empty() { continue; @@ -821,6 +827,9 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3; if is_mostly_dots { dot_cells += 1; + if ci < num_cols { + dot_cols[ci] += 1; + } } // Check for standalone page numbers (1-4 digits, possibly with spaces) @@ -830,6 +839,9 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { && digits_only.chars().all(|c| c.is_ascii_digit()) { page_number_cells += 1; + if ci < num_cols { + numeric_cols[ci] += 1; + } } } } @@ -838,6 +850,17 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { return false; } + // Data tables with dot leaders (e.g. "1973....") have dots concentrated + // in one column (the label column) while many other columns contain numbers. + // True TOCs have dots spanning the middle and one page-number column at the end. + // If dots are confined to ≤1 column AND there are ≥3 columns with numbers, + // this is a data table, not a TOC. + let cols_with_dots = dot_cols.iter().filter(|&&c| c >= 2).count(); + let cols_with_numbers = numeric_cols.iter().filter(|&&c| c >= 2).count(); + if cols_with_dots <= 1 && cols_with_numbers >= 3 { + return false; + } + // If a significant portion of cells are dots or page numbers, it's likely a TOC let dot_ratio = dot_cells as f32 / total_cells as f32; let page_num_ratio = page_number_cells as f32 / total_cells as f32; @@ -1189,3 +1212,88 @@ fn try_add_label_column( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_table_of_contents_rejects_toc() { + // TOC with separate dot-leader cells and page number cells + let cells = vec![ + vec![ + "Chapter 1".to_string(), + "....................".to_string(), + "1".to_string(), + ], + vec![ + "Chapter 2".to_string(), + "....................".to_string(), + "15".to_string(), + ], + vec![ + "Chapter 3".to_string(), + "....................".to_string(), + "42".to_string(), + ], + vec![ + "Appendix".to_string(), + "....................".to_string(), + "100".to_string(), + ], + ]; + assert!(is_table_of_contents(&cells)); + } + + #[test] + fn is_table_of_contents_allows_data_table_with_dot_leaders() { + // Simulates ERP appendix tables where the first column has year + dots + // (e.g. "1973..........") and other columns have numeric data. + let cells = vec![ + vec![ + "1973..........".to_string(), + "0.80".to_string(), + "1.08".to_string(), + "1.05".to_string(), + "0.02".to_string(), + "-0.28".to_string(), + "-0.33".to_string(), + "5.16".to_string(), + ], + vec![ + "1974..........".to_string(), + "73".to_string(), + "56".to_string(), + "49".to_string(), + "08".to_string(), + "17".to_string(), + "17".to_string(), + "-.28".to_string(), + ], + vec![ + "1975..........".to_string(), + "86".to_string(), + "-.05".to_string(), + "-.14".to_string(), + "09".to_string(), + "91".to_string(), + "85".to_string(), + "1.03".to_string(), + ], + vec![ + "1976..........".to_string(), + "-1.05".to_string(), + "36".to_string(), + "34".to_string(), + "02".to_string(), + "-1.41".to_string(), + "-1.31".to_string(), + "4.01".to_string(), + ], + ]; + assert!( + !is_table_of_contents(&cells), + "data table with dot-leader labels should not be rejected as TOC" + ); + } +} From 3a15235244b5b8ae44962c14b41d9af256923f61 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Mar 2026 19:17:04 -0700 Subject: [PATCH 34/46] feat(detector): recommend OCR for newspaper-style layouts (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dense multi-column newspapers (WSJ, NYT) have extractable text but produce poor output due to complex interleaved article layouts that defeat column-based reading order. Detect these by counting Tf (font change) operators alongside existing Tj/TJ counts, then flagging TextBased PDFs where most sampled pages show high text density (>=1500 ops), moderate font switches (>=50 Tf), and a low Tf/Tj ratio (<0.15) — the ratio distinguishes newspapers from richly-styled legal/business docs that have high Tf counts due to per-character styling. Calibrated against 108 TextBased PDFs in pdf-evals with zero false positives. WSJ 50-page newspaper correctly flagged (5/8 sampled pages match). Co-authored-by: Claude Opus 4.6 --- src/detector.rs | 138 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 117 insertions(+), 21 deletions(-) diff --git a/src/detector.rs b/src/detector.rs index 57f5bfc..4e84516 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -197,13 +197,13 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); pages_actually_sampled += 1; log::debug!( - "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={} type3_only={}", + "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={} type3_only={} font_changes={}", page_num, analysis.text_operator_count, analysis.has_images, analysis.image_count, analysis.has_template_image, analysis.unique_text_chars, analysis.unique_alphanum_chars, analysis.path_op_count, analysis.has_vector_text, analysis.total_image_area, analysis.has_identity_h_no_tounicode, - analysis.has_only_type3_fonts + analysis.has_only_type3_fonts, analysis.font_change_count ); let is_image_dominated = analysis.image_count > 10 && analysis.image_count > analysis.text_operator_count * 3; @@ -293,6 +293,49 @@ pub(crate) fn detect_from_document( (PdfType::TextBased, text_ratio.max(0.5)) }; + // Phase 1b: Newspaper-style layout detection. + // Dense multi-column newspapers (WSJ, NYT) have extractable text but produce + // poor output due to complex interleaved article layouts. Detect via consistently + // high text density combined with moderate font switches and a low Tf/Tj ratio. + // + // The Tf/Tj ratio distinguishes newspapers from styled legal/business documents: + // - Newspapers: ratio 0.02-0.06 (dense prose with occasional font switches) + // - Rich-styled docs (DPA, contracts): ratio 0.25-0.35 (per-character styling) + // + // Thresholds calibrated against: + // - WSJ 50-page newspaper: text_ops 1500-3800, font_changes 50-194, ratio 0.02-0.06 + // - DPA/contracts: text_ops 1300-2260, font_changes 327-630, ratio 0.25-0.32 + // - SEC filings: text_ops 1-1800, font_changes 1-65 (only 1-2 dense pages) + // - Normal docs: text_ops < 700, font_changes < 55 + let ocr_recommended = if pdf_type == PdfType::TextBased && pages_sampled >= 3 { + let mut newspaper_pages = 0u32; + for analysis in analysis_cache.values() { + let ratio = if analysis.text_operator_count > 0 { + analysis.font_change_count as f32 / analysis.text_operator_count as f32 + } else { + 1.0 + }; + if analysis.text_operator_count >= 1500 + && analysis.font_change_count >= 50 + && ratio < 0.15 + { + newspaper_pages += 1; + } + } + let newspaper_ratio = newspaper_pages as f32 / pages_sampled as f32; + if newspaper_ratio >= 0.5 { + log::debug!( + "newspaper layout detected: {}/{} pages with high text_ops + font_changes → OCR recommended", + newspaper_pages, pages_sampled + ); + true + } else { + ocr_recommended + } + } else { + ocr_recommended + }; + // Phase 2: Build per-page OCR list let mut pages_needing_ocr = match pdf_type { PdfType::TextBased => Vec::new(), @@ -427,6 +470,8 @@ struct PageAnalysis { /// Type3 fonts render each glyph as a custom drawing/bitmap — without a /// ToUnicode CMap, the character codes can't be mapped to Unicode. has_only_type3_fonts: bool, + /// Number of Tf (set font) operators — high count indicates many font switches + font_change_count: u32, } /// Analyze a page's content stream for text operators and images @@ -435,6 +480,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { let mut has_images = false; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; let mut all_unique_chars: HashSet = HashSet::new(); // Get content streams for this page @@ -448,12 +494,13 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { Err(_) => stream.content.clone(), }; - // Scan for text operators (Tj, TJ), image operators (Do), and path ops - let (ops, imgs, paths) = + // Scan for text operators (Tj, TJ), font changes (Tf), image operators (Do), and path ops + let (ops, imgs, paths, fonts) = scan_content_for_text_operators(&content, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } } @@ -462,20 +509,22 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { if let Ok((resource_dict, resource_ids)) = doc.get_page_resources(page_id) { let mut visited = HashSet::new(); if let Some(resources) = resource_dict { - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_xobjects_in_resources(doc, resources, &mut visited, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } for resource_id in resource_ids { if let Ok(resources) = doc.get_dictionary(resource_id) { - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_xobjects_in_resources(doc, resources, &mut visited, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } } @@ -517,6 +566,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { has_vector_text, has_identity_h_no_tounicode, has_only_type3_fonts, + font_change_count: font_changes, } } @@ -601,10 +651,11 @@ fn scan_xobjects_in_resources( resources: &lopdf::Dictionary, visited: &mut HashSet, unique_chars: &mut HashSet, -) -> (u32, u32, u32) { +) -> (u32, u32, u32, u32) { let mut text_ops = 0u32; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; let xobjects = match resources.get(b"XObject").ok() { Some(Object::Dictionary(d)) => Some(d.clone()), @@ -633,22 +684,24 @@ fn scan_xobjects_in_resources( let content = stream .decompressed_content() .unwrap_or_else(|_| stream.content.clone()); - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_content_for_text_operators(&content, unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; if let Some(res) = stream .dict .get(b"Resources") .ok() .and_then(|o| o.as_dict().ok()) { - let (ops2, imgs2, paths2) = + let (ops2, imgs2, paths2, fonts2) = scan_xobjects_in_resources(doc, res, visited, unique_chars); text_ops += ops2; image_count += imgs2; path_ops += paths2; + font_changes += fonts2; } } Some(b"Image") => { @@ -659,7 +712,7 @@ fn scan_xobjects_in_resources( } } - (text_ops, image_count, path_ops) + (text_ops, image_count, path_ops, font_changes) } /// Fast scan of content stream bytes for text operators @@ -670,15 +723,16 @@ fn scan_xobjects_in_resources( /// - "'" - move to next line and show text /// - "\"" - set word/char spacing, move to next line, show text /// -/// Returns (text_op_count, image_count, path_op_count). +/// Returns (text_op_count, image_count, path_op_count, font_change_count). /// Unique non-whitespace text characters are collected into `unique_chars`. fn scan_content_for_text_operators( content: &[u8], unique_chars: &mut HashSet, -) -> (u32, u32, u32) { +) -> (u32, u32, u32, u32) { let mut text_ops = 0u32; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; // Helper: check if position is a word boundary (start of content or preceded by whitespace) let is_word_start = |pos: usize| -> bool { pos == 0 || content[pos - 1].is_ascii_whitespace() }; @@ -691,7 +745,7 @@ fn scan_content_for_text_operators( while i < content.len() { let b = content[i]; - // Look for 'T' followed by 'j' or 'J' + // Look for 'T' followed by 'j', 'J', or 'f' if b == b'T' && i + 1 < content.len() { let next = content[i + 1]; if next == b'j' || next == b'J' { @@ -705,6 +759,15 @@ fn scan_content_for_text_operators( // Scan backward for text string operand to collect unique chars collect_text_chars_before(content, i, unique_chars); } + } else if next == b'f' { + // Tf = set font operator + if i + 2 >= content.len() + || content[i + 2].is_ascii_whitespace() + || content[i + 2] == b'\n' + || content[i + 2] == b'\r' + { + font_changes += 1; + } } } @@ -749,7 +812,7 @@ fn scan_content_for_text_operators( i += 1; } - (text_ops, image_count, path_ops) + (text_ops, image_count, path_ops, font_changes) } /// Scan backward from a Tj/TJ operator to find the preceding string operand @@ -1108,7 +1171,7 @@ mod tests { // Sample PDF content stream with text operators let content = b"BT /F1 12 Tf 100 700 Td (Hello World) Tj ET"; - let (ops, imgs, _) = scan_content_for_text_operators(content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars); assert_eq!(ops, 1); assert_eq!(imgs, 0); // "Hello World" without space: H, e, l, o, W, r, d = 7 unique @@ -1117,7 +1180,7 @@ mod tests { // Content with TJ array uchars.clear(); let content2 = b"BT /F1 12 Tf 100 700 Td [(H) 10 (ello)] TJ ET"; - let (ops2, _, _) = scan_content_for_text_operators(content2, &mut uchars); + let (ops2, _, _, _) = scan_content_for_text_operators(content2, &mut uchars); assert_eq!(ops2, 1); // H, e, l, o = 4 unique assert!(uchars.len() >= 4); @@ -1125,7 +1188,7 @@ mod tests { // Content with Do (image) uchars.clear(); let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q"; - let (ops3, imgs3, _) = scan_content_for_text_operators(content3, &mut uchars); + let (ops3, imgs3, _, _) = scan_content_for_text_operators(content3, &mut uchars); assert_eq!(ops3, 0); assert_eq!(imgs3, 1); } @@ -1144,7 +1207,7 @@ mod tests { content.extend_from_slice(b"BT (x) Tj ET\n"); let mut uchars = HashSet::new(); - let (ops, imgs, _) = scan_content_for_text_operators(&content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(ops, 3); assert_eq!(imgs, 50); // Only 'x' unique char @@ -1162,7 +1225,7 @@ mod tests { let content = b"BT /F1 12 Tf (The quick brown fox jumps over the lazy dog) Tj ET\n\ /Img1 Do\n/Img2 Do\n"; let mut uchars = HashSet::new(); - let (ops, imgs, _) = scan_content_for_text_operators(content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars); assert_eq!(ops, 1); assert_eq!(imgs, 2); // Many unique chars from the sentence @@ -1185,7 +1248,7 @@ mod tests { content.extend_from_slice(b"f\n"); let mut uchars = HashSet::new(); - let (text, imgs, paths) = scan_content_for_text_operators(&content, &mut uchars); + let (text, imgs, paths, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(text, 1); assert_eq!(imgs, 0); // 500 * (m + l + c + h) + 1 f = 2001 @@ -1210,7 +1273,7 @@ mod tests { } let mut uchars = HashSet::new(); - let (text, _, paths) = scan_content_for_text_operators(&content, &mut uchars); + let (text, _, paths, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(text, 20); assert!(paths >= 40, "expected >= 40 path ops, got {paths}"); @@ -1332,4 +1395,37 @@ mod tests { ); assert!(!page_has_identity_h_no_tounicode(&doc, page_id)); } + + #[test] + fn test_scan_content_counts_tf_operators() { + let mut uchars = HashSet::new(); + let content = b"BT /F1 12 Tf (Hello) Tj /F2 10 Tf (World) Tj ET"; + let (ops, _, _, fonts) = scan_content_for_text_operators(content, &mut uchars); + assert_eq!(ops, 2); + assert_eq!(fonts, 2); + } + + #[test] + fn test_newspaper_heuristic_thresholds() { + // Newspaper page: high text ops, moderate font changes, low ratio + let text_ops = 3500u32; + let font_changes = 150u32; + let ratio = font_changes as f32 / text_ops as f32; + assert!(text_ops >= 1500); + assert!(font_changes >= 50); + assert!(ratio < 0.15); // 0.043 + + // Dense styled doc (DPA/contract): high text ops, very high font changes, high ratio + let text_ops = 1800u32; + let font_changes = 540u32; + let ratio = font_changes as f32 / text_ops as f32; + assert!(text_ops >= 1500); + assert!(font_changes >= 50); + assert!(ratio >= 0.15); // 0.30 — should NOT trigger newspaper heuristic + + // Normal doc: low text ops — doesn't qualify at all + let text_ops = 300u32; + let font_changes = 50u32; + assert!(text_ops < 1500); + } } From 95aac6a7cdcdea498d2c807deb5f11bcf630c5a2 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 24 Mar 2026 13:11:58 -0700 Subject: [PATCH 35/46] feat(layout): relative valley column detection for justified text (#15) * feat(layout): relative valley column detection for justified text Add fallback column detection using relative valley analysis for PDFs with justified text where item widths extend past gutter boundaries. The absolute valley detector fails on these layouts because gutter bins are at ~40% of peak (well above the 15% noise threshold). The relative valley detector smooths the histogram with a 5-bin moving average, finds local minima where contrast < 0.60 of surrounding peaks, and validates with peak balance >= 0.40. Limited to single best valley (max 2 columns) and requires >= 100 items per page. Tested on IRS Publication 17 (2002), a 289-page 2-column justified text document: column detection went from ~40 pages to 165 pages. 190 passed, 0 regressions across 191 eval PDFs. Co-Authored-By: Claude Opus 4.6 * fix(layout): tighten relative valley thresholds to reduce false positives Reduce PEAK_WINDOW from 40 to 25 bins (50pt) so valleys are only validated against nearby peaks, not distant ones. Add MIN_PEAK_HEIGHT of 20 (smoothed) to reject sparse pages where histogram peaks are too low to indicate dense two-column text. Previous thresholds caused 13 regressions across the eval suite by splitting tables, TOCs, checklists, and forms. Now: 188 passed, 0 regressions (2 minor metadata-only diffs on IRS P17 and 9978293). Co-Authored-By: Claude Opus 4.6 * fix(layout): skip relative valley detection on pages with tables Table column gaps in the histogram look identical to text column gutters but the table pipeline already handles reading order for those pages. Pass page_has_table flag through detect_columns to suppress the relative valley fallback on pages where tables were detected. This eliminates all remaining regressions from relative valley detection: 190 passed, 0 regressions across 191 eval PDFs. Co-Authored-By: Claude Opus 4.6 * feat(layout): prose density validation for relative valley detection Add columns_have_prose() to validate relative valley column splits. Checks that both sides of a proposed split contain paragraph-like content (fill ratio >= 40%, avg items/line <= 3.5) before committing to a column split. Combined with the table-page guard, this prevents false column splits on financial statements, forms, and tabular layouts where long labels or dot leaders fill the column width. Also tightens find_relative_valleys() thresholds (PEAK_WINDOW 40->25, MIN_PEAK_HEIGHT 5->20) to reduce false positive valley candidates. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/extractor/layout.rs | 497 ++++++++++++++++++++++++++++++++++++++-- src/extractor/mod.rs | 8 +- src/lib.rs | 2 +- src/markdown/mod.rs | 10 +- src/tables/mod.rs | 10 +- 5 files changed, 495 insertions(+), 32 deletions(-) diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index f91900f..60cf114 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -1,6 +1,6 @@ //! Column detection, line grouping, and reading-order layout. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use crate::text_utils::{effective_width, sort_line_items}; use crate::types::{TextItem, TextLine}; @@ -18,7 +18,11 @@ pub(crate) struct ColumnRegion { /// Builds an occupancy histogram across the page width and finds empty valleys /// (gutters) where no text exists. Validates valleys with vertical consistency /// checks to avoid false positives. -pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec { +pub(crate) fn detect_columns( + items: &[TextItem], + page: u32, + page_has_table: bool, +) -> Vec { const BIN_WIDTH: f32 = 2.0; const MIN_GUTTER_WIDTH: f32 = 8.0; const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30; @@ -110,10 +114,358 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec }) .collect(); - if valleys.is_empty() { + // Fallback: if no absolute valleys found, try relative valley detection. + // Justified text can leave gutter bins non-empty because item widths extend + // to the column edge. Look for local minima that are significantly lower + // than the peaks on either side. + // Only attempt this for dense pages (>=100 items) — sparse pages with shallow + // histogram dips are likely not multi-column. + // Skip on pages with detected tables — table column gaps look like gutters + // in the histogram but the table pipeline already handles reading order. + if valleys.is_empty() && page_items.len() >= 100 && !page_has_table { + let rel_valleys = find_relative_valleys( + &histogram, + num_bins, + x_min, + BIN_WIDTH, + page_width, + margin_threshold, + ); + if !rel_valleys.is_empty() { + let result = validate_and_build_columns( + &rel_valleys, + &page_items, + x_min, + BIN_WIDTH, + x_max, + MIN_ITEMS_PER_COLUMN, + MIN_VERTICAL_SPAN_RATIO, + page, + true, // center-based assignment for relative valleys + ); + if result.len() > 1 { + // Validate that both sides contain paragraph-like content. + // Tables, forms, and checklists have short scattered items + // that create false gutter signals. Only commit to relative + // valley columns when both sides look like flowing prose. + if columns_have_prose(&result, &page_items) { + debug!( + "page {}: relative valley detection found {} columns", + page, + result.len() + ); + return result; + } else { + debug!( + "page {}: relative valley rejected — columns lack prose density", + page, + ); + } + } + } return vec![ColumnRegion { x_min, x_max }]; } + return validate_and_build_columns( + &valleys, + &page_items, + x_min, + BIN_WIDTH, + x_max, + MIN_ITEMS_PER_COLUMN, + MIN_VERTICAL_SPAN_RATIO, + page, + false, // edge-based assignment for absolute valleys + ); +} + +/// Check whether each proposed column contains paragraph-like content. +/// +/// Groups items per column into rough lines by Y-proximity, then measures +/// what fraction of those lines span a significant portion of the column +/// width. Two-column prose (justified or ragged-right) produces lines that +/// fill most of the column width. Tables, forms, and checklists produce +/// short scattered items that don't. +/// +/// Returns true only when *every* column passes a minimum prose density. +fn columns_have_prose(columns: &[ColumnRegion], items: &[&TextItem]) -> bool { + const Y_TOL: f32 = 3.0; // y-proximity to group items into the same line + const LINE_FILL_THRESHOLD: f32 = 0.45; // line must span ≥45% of column width + const MIN_PROSE_RATIO: f32 = 0.40; // ≥40% of lines must be "full" + const MIN_LINES: usize = 8; // need enough lines to judge + const MIN_COL_WIDTH: f32 = 120.0; // columns must be ≥120pt (not narrow sidebars/fragments) + const MAX_AVG_ITEMS_PER_LINE: f32 = 3.5; // prose has 1-3 items/line; tables/forms have 4+ + + for col in columns { + let col_width = col.x_max - col.x_min; + if col_width < MIN_COL_WIDTH { + return false; + } + + // Collect items whose center falls within this column + let col_items: Vec<&TextItem> = items + .iter() + .filter(|i| { + let center = i.x + effective_width(i) / 2.0; + center >= col.x_min && center <= col.x_max + }) + .copied() + .collect(); + + if col_items.len() < MIN_LINES { + return false; + } + + // Sort by Y descending (top of page = higher Y in PDF coords) + let mut sorted: Vec<&TextItem> = col_items; + sorted.sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + + // Group into lines by Y-proximity and measure fill + item count + let mut full_lines = 0usize; + let mut total_lines = 0usize; + let mut total_items_in_lines = 0usize; + let mut line_items: Vec<&TextItem> = Vec::new(); + let mut line_y = f32::NAN; + + let flush_line = |line_items: &[&TextItem], + full: &mut usize, + total: &mut usize, + total_items: &mut usize| { + if line_items.is_empty() { + return; + } + *total += 1; + *total_items += line_items.len(); + // Compute the span of text on this line within the column + let left = line_items + .iter() + .map(|i| i.x.max(col.x_min)) + .fold(f32::INFINITY, f32::min); + let right = line_items + .iter() + .map(|i| (i.x + effective_width(i)).min(col.x_max)) + .fold(f32::NEG_INFINITY, f32::max); + let span = (right - left).max(0.0); + if span >= col_width * LINE_FILL_THRESHOLD { + *full += 1; + } + }; + + for item in &sorted { + if line_items.is_empty() || (line_y - item.y).abs() < Y_TOL { + if line_items.is_empty() { + line_y = item.y; + } + line_items.push(item); + } else { + flush_line( + &line_items, + &mut full_lines, + &mut total_lines, + &mut total_items_in_lines, + ); + line_items.clear(); + line_y = item.y; + line_items.push(item); + } + } + flush_line( + &line_items, + &mut full_lines, + &mut total_lines, + &mut total_items_in_lines, + ); + + if total_lines < MIN_LINES { + return false; + } + + let ratio = full_lines as f32 / total_lines as f32; + let avg_items = total_items_in_lines as f32 / total_lines as f32; + debug!( + "columns_have_prose: col [{:.0}..{:.0}] lines={} full={} ratio={:.2} avg_items={:.1}", + col.x_min, col.x_max, total_lines, full_lines, ratio, avg_items + ); + if ratio < MIN_PROSE_RATIO { + return false; + } + // Tables and forms tend to have many small items per line (one per cell), + // while prose has few items per line (one per word-run or phrase). + if avg_items > MAX_AVG_ITEMS_PER_LINE { + return false; + } + } + + true +} + +/// Find relative valleys (local minima) in the histogram. +/// +/// When justified text fills gutters, the absolute noise threshold fails. +/// This finds local minima where the count drops significantly below +/// the peaks on either side — indicating a gutter even when not empty. +fn find_relative_valleys( + histogram: &[u32], + num_bins: usize, + _x_min: f32, + bin_width: f32, + page_width: f32, + margin_threshold: f32, +) -> Vec<(usize, usize)> { + const MIN_GUTTER_BINS: usize = 2; // minimum 4pt gutter + const CONTRAST_THRESHOLD: f32 = 0.60; // valley must be < 60% of surrounding peaks + const PEAK_WINDOW: usize = 25; // look 50pt on each side for peaks + const MIN_PEAK_HEIGHT: f32 = 20.0; // peaks must be ≥20 (dense text columns) + + if num_bins < 10 { + return vec![]; + } + + // Smooth histogram with a 5-bin moving average to reduce noise + let mut smoothed = vec![0.0f32; num_bins]; + let half_win = 2usize; + for (i, s) in smoothed.iter_mut().enumerate().take(num_bins) { + let lo = i.saturating_sub(half_win); + let hi = (i + half_win + 1).min(num_bins); + let sum: u32 = histogram[lo..hi].iter().sum(); + *s = sum as f32 / (hi - lo) as f32; + } + + // Find local minima: positions where smoothed value is lower than + // both sides within a search window + let mut candidates: Vec<(usize, f32, f32)> = Vec::new(); // (bin, valley_val, contrast) + + for i in PEAK_WINDOW..num_bins.saturating_sub(PEAK_WINDOW) { + let val = smoothed[i]; + if val < 1.0 { + continue; // skip empty margins + } + + // Check this is a local minimum within a small window + let local_lo = i.saturating_sub(3); + let local_hi = (i + 4).min(num_bins); + let is_local_min = (local_lo..local_hi).all(|j| smoothed[j] >= val - 0.5); + if !is_local_min { + continue; + } + + // Find peak values on each side + let left_peak = smoothed[i.saturating_sub(PEAK_WINDOW)..i] + .iter() + .cloned() + .fold(0.0f32, f32::max); + let right_peak = smoothed[(i + 1)..(i + 1 + PEAK_WINDOW).min(num_bins)] + .iter() + .cloned() + .fold(0.0f32, f32::max); + + if left_peak < MIN_PEAK_HEIGHT || right_peak < MIN_PEAK_HEIGHT { + continue; + } + + // Both peaks must be substantial — prevents detecting margin drop-offs + // as gutters in single-column layouts with ragged text. + let peak_balance = left_peak.min(right_peak) / left_peak.max(right_peak); + if peak_balance < 0.40 { + continue; + } + + // Contrast: ratio of valley to the smaller of the two peaks + let ref_peak = left_peak.min(right_peak); + let contrast = val / ref_peak; + + if contrast < CONTRAST_THRESHOLD { + // Check margin constraint + let center_pts = i as f32 * bin_width; + if center_pts > margin_threshold && center_pts < (page_width - margin_threshold) { + candidates.push((i, val, contrast)); + } + } + } + + if candidates.is_empty() { + return vec![]; + } + + // Group adjacent candidates into valley ranges and pick the deepest point + let mut valleys: Vec<(usize, usize)> = Vec::new(); + let mut best_bin = candidates[0].0; + let mut best_contrast = candidates[0].2; + + for window in candidates.windows(2) { + let (prev_bin, _, _) = window[0]; + let (next_bin, _, next_contrast) = window[1]; + + if next_bin - prev_bin <= 5 { + // Same group + if next_contrast < best_contrast { + best_bin = next_bin; + best_contrast = next_contrast; + } + } else { + // End current group + let half = MIN_GUTTER_BINS; + valleys.push(( + best_bin.saturating_sub(half), + (best_bin + half + 1).min(num_bins), + )); + best_bin = next_bin; + best_contrast = next_contrast; + } + } + // Close last group + let half = MIN_GUTTER_BINS; + valleys.push(( + best_bin.saturating_sub(half), + (best_bin + half + 1).min(num_bins), + )); + + // Limit to the single best valley (deepest contrast). + // Multi-column layouts with 3+ columns typically have clear gutters that + // the absolute valley detection handles. The relative fallback is designed + // for 2-column layouts where justified text fills the gutter. + if valleys.len() > 1 { + // Keep only the valley with the best (lowest) contrast in the candidates + let mut best_idx = 0; + let mut best_c = f32::MAX; + for (vi, v) in valleys.iter().enumerate() { + let mid = (v.0 + v.1) / 2; + // Find the candidate closest to this valley's midpoint + if let Some(c) = candidates + .iter() + .filter(|(b, _, _)| (*b as isize - mid as isize).unsigned_abs() <= 5) + .map(|(_, _, c)| *c) + .reduce(f32::min) + { + if c < best_c { + best_c = c; + best_idx = vi; + } + } + } + return vec![valleys[best_idx]]; + } + + valleys +} + +/// Validate valley candidates with vertical consistency checks and build column regions. +/// +/// When `center_assign` is true, items are assigned to columns based on their +/// center point rather than their right edge. This helps when justified text +/// items extend past the gutter. +#[allow(clippy::too_many_arguments)] +fn validate_and_build_columns( + valleys: &[(usize, usize)], + page_items: &[&TextItem], + x_min: f32, + bin_width: f32, + x_max: f32, + min_items: usize, + min_vertical_span: f32, + page: u32, + center_assign: bool, +) -> Vec { // Compute Y range of the page let y_min = page_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); let y_max = page_items @@ -123,22 +475,37 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let y_range = y_max - y_min; // Validate each valley with vertical consistency - // Each entry: (start_bin, end_bin, left_count, right_count) let mut valid_valleys: Vec<(usize, usize, usize, usize)> = Vec::new(); - for &(start, end) in &valleys { - let gutter_left = x_min + start as f32 * BIN_WIDTH; - let gutter_right = x_min + end as f32 * BIN_WIDTH; + for &(start, end) in valleys { + let gutter_left = x_min + start as f32 * bin_width; + let gutter_right = x_min + end as f32 * bin_width; let gutter_center = (gutter_left + gutter_right) / 2.0; - // Collect items on each side of the gutter + // Collect items on each side of the gutter. + // Center-based: use item midpoint (better for justified text). + // Edge-based: use item right edge (original behavior). let left_items: Vec<&&TextItem> = page_items .iter() - .filter(|i| i.x + effective_width(i) <= gutter_center) + .filter(|i| { + if center_assign { + i.x + effective_width(i) / 2.0 <= gutter_center + } else { + i.x + effective_width(i) <= gutter_center + } + }) + .collect(); + let right_items: Vec<&&TextItem> = page_items + .iter() + .filter(|i| { + if center_assign { + i.x + effective_width(i) / 2.0 > gutter_center + } else { + i.x >= gutter_center + } + }) .collect(); - let right_items: Vec<&&TextItem> = - page_items.iter().filter(|i| i.x >= gutter_center).collect(); - if left_items.len() < MIN_ITEMS_PER_COLUMN || right_items.len() < MIN_ITEMS_PER_COLUMN { + if left_items.len() < min_items || right_items.len() < min_items { continue; } @@ -162,7 +529,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let overlap_max = left_y_max.min(right_y_max); let overlap = (overlap_max - overlap_min).max(0.0); - if overlap / y_range < MIN_VERTICAL_SPAN_RATIO { + if overlap / y_range < min_vertical_span { continue; } } @@ -171,6 +538,11 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec } if valid_valleys.is_empty() { + debug!( + "page {}: {} valleys found but none passed validation", + page, + valleys.len() + ); return vec![ColumnRegion { x_min, x_max }]; } @@ -180,14 +552,12 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec valid_valleys.len() + 1, valid_valleys .iter() - .map(|(s, e, _, _)| x_min + ((*s + *e) as f32 / 2.0) * BIN_WIDTH) + .map(|(s, e, _, _)| x_min + ((*s + *e) as f32 / 2.0) * bin_width) .collect::>() ); // Limit to at most 3 gutters (4 columns). // Score = width_in_bins * min(left_count, right_count) - // This prefers gutters that separate substantial content on both sides, - // rather than just the physically widest gaps (which may be intra-column). if valid_valleys.len() > 3 { valid_valleys.sort_by(|a, b| { let score_a = (a.1 - a.0) as f32 * (a.2.min(a.3) as f32); @@ -197,7 +567,6 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec .unwrap_or(std::cmp::Ordering::Equal) }); valid_valleys.truncate(3); - // Re-sort by position (left to right) valid_valleys.sort_by_key(|v| v.0); } @@ -205,7 +574,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let mut columns = Vec::new(); let mut col_start = x_min; for &(start, end, _, _) in &valid_valleys { - let gutter_center = x_min + ((start + end) as f32 / 2.0) * BIN_WIDTH; + let gutter_center = x_min + ((start + end) as f32 / 2.0) * bin_width; columns.push(ColumnRegion { x_min: col_start, x_max: gutter_center, @@ -525,7 +894,7 @@ fn split_column_stragglers(lines: Vec) -> (Vec, Vec) -> Vec { - group_into_lines_with_thresholds(items, &HashMap::new()) + group_into_lines_with_thresholds(items, &HashMap::new(), &HashSet::new()) } /// Group text items into lines, using pre-computed per-page adaptive thresholds @@ -534,6 +903,7 @@ pub fn group_into_lines(items: Vec) -> Vec { pub(crate) fn group_into_lines_with_thresholds( items: Vec, page_thresholds: &HashMap, + table_pages: &HashSet, ) -> Vec { if items.is_empty() { return Vec::new(); @@ -561,7 +931,7 @@ pub(crate) fn group_into_lines_with_thresholds( let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10); // Detect columns for this page - let columns = detect_columns(&page_items, page); + let columns = detect_columns(&page_items, page, table_pages.contains(&page)); if columns.len() <= 1 { // Single column - use simple sorting @@ -939,7 +1309,7 @@ mod tests { items.extend(fill_zone(1, 345.0, 660.0, 750.0, 50.0)); items.extend(fill_zone(1, 675.0, 800.0, 750.0, 50.0)); - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 3, "Expected 3 columns, got {}", cols.len()); // Gutter 1 should be in the gap between left and middle zones @@ -964,7 +1334,7 @@ mod tests { items.extend(fill_zone(1, 30.0, 280.0, 750.0, 50.0)); items.extend(fill_zone(1, 320.0, 570.0, 750.0, 50.0)); - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 2, "Expected 2 columns, got {}", cols.len()); let gutter = cols[0].x_max; @@ -995,7 +1365,7 @@ mod tests { )); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); // Should detect the gutters between the 3 dense zones, not the wide gap // before the sparse zone assert!( @@ -1005,6 +1375,87 @@ mod tests { ); } + /// Helper: create items that fill a zone but with widths that extend past + /// the zone boundary (simulating justified text). Items start within the zone + /// but their reported width extends `overshoot` points past the zone end. + fn fill_zone_justified( + page: u32, + x_start: f32, + x_end: f32, + overshoot: f32, + y_start: f32, + y_end: f32, + ) -> Vec { + let mut items = Vec::new(); + let mut y = y_start; + while y >= y_end { + // Each line: 3-4 items that together span x_start to x_end+overshoot + let item_width = (x_end - x_start + overshoot) / 3.0; + for i in 0..3 { + let x = x_start + i as f32 * (x_end - x_start) / 3.0; + let text_len = (item_width / 6.0).ceil() as usize; + let text: String = "W".repeat(text_len); + items.push(TextItem { + text, + x, + y, + width: item_width, + height: 12.0, + font_size: 12.0, + font: String::new(), + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + }); + } + y -= 14.0; + } + items + } + + #[test] + fn relative_valley_detects_justified_text_columns() { + // Two columns of justified text where item widths overshoot the gutter + // by a few points, preventing absolute valley detection from finding + // an empty gutter. + let mut items = Vec::new(); + // Left column: x=40..290, items extend to ~297 (7pt overshoot) + items.extend(fill_zone_justified(1, 40.0, 290.0, 7.0, 750.0, 50.0)); + // Right column: x=300..550, items extend to ~557 + items.extend(fill_zone_justified(1, 300.0, 550.0, 7.0, 750.0, 50.0)); + + let cols = detect_columns(&items, 1, false); + assert_eq!( + cols.len(), + 2, + "Expected 2 columns for justified text, got {}", + cols.len() + ); + + let gutter = cols[0].x_max; + assert!( + (280.0..=310.0).contains(&gutter), + "Gutter at {gutter}, expected ~295" + ); + } + + #[test] + fn relative_valley_rejects_single_column_margin() { + // Single column of text — the right margin drop-off should NOT be + // detected as a column gutter. + let items = fill_zone_justified(1, 40.0, 350.0, 0.0, 750.0, 50.0); + + let cols = detect_columns(&items, 1, false); + assert_eq!( + cols.len(), + 1, + "Expected 1 column for single-column text, got {}", + cols.len() + ); + } + /// Helper: build a Vec with `n` lines at given X, starting at Y=700. fn make_lines(n: usize, x: f32) -> Vec { (0..n) diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 1f63a13..9a2395c 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -913,7 +913,7 @@ mod tests { items.push(make_item("Left text here", 72.0, y, 200.0)); items.push(make_item("Right text here", 350.0, y, 200.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols); assert!(cols[0].x_min < cols[1].x_min); } @@ -928,7 +928,7 @@ mod tests { items.push(make_item("Col two", 220.0, y, 140.0)); items.push(make_item("Col three", 390.0, y, 140.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols); } @@ -946,7 +946,7 @@ mod tests { let y = 700.0 - (i as f32) * 14.0; items.push(make_item("wide", 72.0, y, 320.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert!( cols.len() >= 2, "Width bleed should not prevent column detection, got {:?}", @@ -967,7 +967,7 @@ mod tests { 468.0, )); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert!( cols.len() <= 1, "Full-width text should not be split into columns, got {:?}", diff --git a/src/lib.rs b/src/lib.rs index 90bc633..56368a2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -712,7 +712,7 @@ fn compute_layout_complexity( let mut pages_with_columns: Vec = Vec::new(); for page in seen_pages { - let cols = extractor::detect_columns(items, page); + let cols = extractor::detect_columns(items, page, pages_with_tables.contains(&page)); if cols.len() >= 2 { pages_with_columns.push(page); } diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index c51e1a7..0a1ab1c 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -887,11 +887,15 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // Merge continuation tables across page breaks, but only for table-only pages merge_continuation_tables(&mut page_tables, &table_only_pages); + // Collect pages that have detected tables — used to suppress relative valley + // column detection on pages where table column gaps would be misidentified. + let table_page_set: HashSet = page_tables.keys().copied().collect(); + // Split non-table items by band boundaries before line grouping so that // items from different side-by-side zones (e.g. left/right month columns // in a calendar) don't merge into the same line. let lines = if page_band_splits.is_empty() { - group_into_lines_with_thresholds(non_table_items, page_thresholds) + group_into_lines_with_thresholds(non_table_items, page_thresholds, &table_page_set) } else { // Separate items into band-split pages and non-split pages let mut split_page_items: HashMap> = HashMap::new(); @@ -904,7 +908,8 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } // Process unsplit pages normally - let mut all_lines = group_into_lines_with_thresholds(unsplit_items, page_thresholds); + let mut all_lines = + group_into_lines_with_thresholds(unsplit_items, page_thresholds, &table_page_set); // Process each split page's bands independently, then interleave // by Y position so paired zones (e.g. left/right months) appear together. let mut split_pages: Vec = split_page_items.keys().copied().collect(); @@ -924,6 +929,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( page_lines.extend(group_into_lines_with_thresholds( band_items, page_thresholds, + &table_page_set, )); } } diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 7e6ec23..4b7c79f 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -281,7 +281,7 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt }; use std::collections::HashMap; - let mut columns = detect_columns(items, page); + let mut columns = detect_columns(items, page, false); if columns.len() < 4 { return None; } @@ -384,7 +384,13 @@ pub(crate) fn try_build_table_from_columns(items: &[TextItem], page: u32) -> Opt let thresholds = HashMap::new(); let per_column_lines: Vec> = col_buckets .iter() - .map(|bucket| group_into_lines_with_thresholds(bucket.clone(), &thresholds)) + .map(|bucket| { + group_into_lines_with_thresholds( + bucket.clone(), + &thresholds, + &std::collections::HashSet::new(), + ) + }) .collect(); // Must be tabular (not newspaper) layout From d95584d7f29b1fcd4632c3e2ec2f08cae1815806 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 09:44:15 -0700 Subject: [PATCH 36/46] fix(tables): reject row-stripe tables with oversized cell text (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newsletter-style PDFs have decorative background rects (sidebar, header, section bands) that pass row-stripe detection as false tables. Reject when any cell exceeds 500 chars — real alternating-row data tables have short cell content; layout backgrounds produce paragraph- length "cells". Co-authored-by: Claude Opus 4.6 --- src/tables/detect_rects.rs | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index d308c29..adf4c99 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -1276,6 +1276,23 @@ fn detect_row_stripe_table( return None; } + // Reject if any cell has excessive text — layout background rects (sidebar, + // header, section bands) produce "cells" that contain paragraphs of body text. + // Real alternating-row-stripe data tables have short cell content. + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + if max_cell_len > 500 { + debug!( + " row-stripe rejected: max cell length {} > 500 (layout background)", + max_cell_len + ); + return None; + } + // No empty columns for col in 0..num_cols { let col_has_content = cells @@ -1444,6 +1461,22 @@ fn detect_merged_cluster_table( return None; } + // Reject if any cell has excessive text — layout background rects produce + // "cells" containing paragraphs, not short data-table values. + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + if max_cell_len > 500 { + debug!( + " merged-cluster rejected: max cell length {} > 500 (layout background)", + max_cell_len + ); + return None; + } + // No empty columns for col in 0..num_cols { let col_has_content = cells @@ -1852,6 +1885,28 @@ mod tests { assert!(!is_row_stripe_pattern(&rects)); } + #[test] + fn test_row_stripe_rejects_layout_background_long_cells() { + // Simulate a newsletter page with wide background rects (sidebar, header, body) + // that look like row stripes but contain paragraphs of body text. + let rects = vec![ + (10.0, 700.0, 550.0, 50.0), // header band + (10.0, 640.0, 550.0, 50.0), // nav band + (10.0, 200.0, 550.0, 430.0), // body background + ]; + let items = vec![ + make_item("General News", 20.0, 650.0, 10.0), + make_item("People News", 20.0, 710.0, 10.0), + // Simulate a long body text (>500 chars) in the main content area + make_item(&"A".repeat(600), 200.0, 650.0, 10.0), + ]; + let result = detect_row_stripe_table(&items, &rects, 1); + assert!( + result.is_none(), + "layout background rects should not be detected as a table" + ); + } + // --- propagate_merged_cells --- #[test] From 61a4c7a08881a4157229e16e05df1f05411933b4 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 11:47:57 -0700 Subject: [PATCH 37/46] fix(tables): exclude page-bg rects from clustering to prevent bridging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Origin-anchored full-page rects (x<5, y<5, h>20× median) are clipping paths or page fills that bridge separate table regions into one cluster, corrupting row-stripe detection. Exclude them from union-find adjacency while keeping them available for hint generation and fallback paths. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/tables/detect_rects.rs | 108 ++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index adf4c99..83330c6 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -327,7 +327,42 @@ pub fn detect_tables_from_rects( // Full grid detection requires ≥ 6 rects if page_rects.len() >= 6 { - let clusters = cluster_rects(&page_rects, 3.0, 6); + // Identify origin-anchored page-background rects (clipping paths or + // page fills) that would bridge separate table regions if included in + // clustering. Exclude them from adjacency but add them back to each + // cluster they overlap, so grid detection still has their edges. + let is_page_bg = { + let mut heights: Vec = page_rects.iter().map(|&(_, _, _, h)| h).collect(); + heights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_height = heights[heights.len() / 2]; + let height_threshold = median_height * 20.0; + let flags: Vec = page_rects + .iter() + .map(|&(x, y, _, h)| x < 5.0 && y < 5.0 && h > height_threshold) + .collect(); + if flags.iter().any(|&b| b) { + debug!( + "page {}: {} origin-anchored page-bg rects excluded from clustering", + page, + flags.iter().filter(|&&b| b).count(), + ); + } + flags + }; + + // Build filtered rect list for clustering (excluding page backgrounds) + let non_bg_indices: Vec = + (0..page_rects.len()).filter(|&i| !is_page_bg[i]).collect(); + let non_bg_rects: Vec<(f32, f32, f32, f32)> = + non_bg_indices.iter().map(|&i| page_rects[i]).collect(); + let raw_clusters = cluster_rects(&non_bg_rects, 3.0, 6); + + // Map cluster indices back to page_rects indices + let clusters: Vec> = raw_clusters + .iter() + .map(|cluster| cluster.iter().map(|&i| non_bg_indices[i]).collect()) + .collect(); + debug!("page {}: {} clusters with >= 6 rects", page, clusters.len()); for cluster_indices in &clusters { let group_rects: Vec<(f32, f32, f32, f32)> = @@ -2579,4 +2614,75 @@ mod tests { "narrow header band (20pt) should not produce hint" ); } + + // --- page-bg clustering exclusion --- + + #[test] + fn page_bg_rects_do_not_bridge_separate_clusters() { + // Simulate page 27 scenario: two groups of row stripes at different Y + // ranges, connected by full-page background rects at (0,0). + // Without exclusion, all rects cluster into one group. + // With exclusion, two separate clusters form. + let mut rects = Vec::new(); + let page = 1; + + // Group 1: 7 row stripes at Y=444..537 (Reference Group table) + for i in 0..7 { + let y = 444.0 + i as f32 * 15.5; + rects.push(PdfRect { + x: 44.0, + y, + width: 505.0, + height: 15.5, + page, + }); + } + + // Group 2: 4 row stripes at Y=176..238 (smaller table) + for i in 0..4 { + let y = 176.0 + i as f32 * 15.5; + rects.push(PdfRect { + x: 44.0, + y, + width: 505.0, + height: 15.5, + page, + }); + } + + // 3 full-page background rects at origin + for _ in 0..3 { + rects.push(PdfRect { + x: 0.0, + y: 0.0, + width: 594.0, + height: 774.0, + page, + }); + } + + // Items in group 1 region for row-stripe detection + let mut items = Vec::new(); + for i in 0..7 { + let y = 449.0 + i as f32 * 15.5; + items.push(make_item("Company Name", 50.0, y, 9.0)); + items.push(make_item("P", 320.0, y, 9.0)); + items.push(make_item("P", 450.0, y, 9.0)); + } + + let (tables, _hints) = detect_tables_from_rects(&items, &rects, page); + // Should detect the group 1 table (7 row stripes) without being + // confused by group 2 stripes bridged via page-bg rects. + assert!( + !tables.is_empty(), + "should detect table from row stripes when page-bg rects are excluded from clustering" + ); + // The table should have rows from group 1 only, not spanning to group 2 + let table = &tables[0]; + assert!( + table.rows.len() <= 8, + "table should have at most ~7 rows from group 1, got {}", + table.rows.len() + ); + } } From b6764e7ca90c8ceb5c812114b292c09f604cc9dc Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 12:33:39 -0700 Subject: [PATCH 38/46] feat(tables): extract tables from tagged PDF structure tree (#17) When a PDF has a well-formed structure tree with /Table > /TR > /TD|TH elements linked to MCIDs, build tables directly from the semantic hierarchy. Runs as highest-priority detection (step 0) before rect-based, line-based, and heuristic strategies. - Add StructTree::extract_tables() to walk the tree and collect table descriptors with row/cell/MCID info - Add detect_tables_from_struct_tree() to match MCIDs to TextItems - Reject tables with <30% MCID cell coverage (stale structure trees) - Update 2013-app2 snapshot (struct-tree gives valid but different column ordering) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lib.rs | 32 ++-- src/markdown/mod.rs | 36 +++- src/structure_tree.rs | 105 +++++++++++ src/tables/detect_struct.rs | 354 +++++++++++++++++++++++++++++++++++ src/tables/mod.rs | 2 + tests/snapshots/2013-app2.md | 297 +++++++++++++++++++++++++++++ 6 files changed, 809 insertions(+), 17 deletions(-) create mode 100644 src/tables/detect_struct.rs diff --git a/src/lib.rs b/src/lib.rs index 56368a2..2f4611b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -386,20 +386,23 @@ fn process_document( }; // Parse structure tree for tagged PDFs (reuses the loaded document) - let struct_roles = structure_tree::StructTree::from_doc(&doc).and_then(|tree| { - let page_ids = doc.get_pages(); - let roles = tree.mcid_to_roles(&page_ids); - if roles.is_empty() { - None - } else { - log::debug!( - "structure tree: {} pages with MCID roles, {} total MCIDs", - roles.len(), - tree.mcid_count() - ); - Some(roles) - } - }); + let (struct_roles, struct_tables) = structure_tree::StructTree::from_doc(&doc) + .map(|tree| { + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + let tables = tree.extract_tables(&page_ids); + if !roles.is_empty() { + log::debug!( + "structure tree: {} pages with MCID roles, {} total MCIDs, {} tagged tables", + roles.len(), + tree.mcid_count(), + tables.len() + ); + } + let roles = if roles.is_empty() { None } else { Some(roles) }; + (roles, tables) + }) + .unwrap_or((None, Vec::new())); let (markdown, layout, has_encoding_issues, gid_pages) = match extracted { Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { @@ -463,6 +466,7 @@ fn process_document( &lines, &page_thresholds, struct_roles.as_ref(), + &struct_tables, )) }; diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 0a1ab1c..e1e3d3e 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -497,7 +497,15 @@ pub fn to_markdown_from_items_with_rects( options: MarkdownOptions, rects: &[crate::types::PdfRect], ) -> String { - to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new(), None) + to_markdown_from_items_with_rects_and_lines( + items, + options, + rects, + &[], + &HashMap::new(), + None, + &[], + ) } /// Convert positioned text items to markdown, using rectangles and line segments for table detection. @@ -511,10 +519,11 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( pdf_lines: &[crate::types::PdfLine], page_thresholds: &HashMap, struct_roles: Option<&HashMap>>, + struct_tables: &[crate::structure_tree::StructTable], ) -> String { use crate::tables::{ - detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown, - try_build_rect_guided_table, + detect_tables, detect_tables_from_lines, detect_tables_from_rects, + detect_tables_from_struct_tree, table_to_markdown, try_build_rect_guided_table, }; use crate::types::ItemType; @@ -648,6 +657,27 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // Track which band-local indices are claimed by structural detection let mut rect_claimed: HashSet = HashSet::new(); + // 0. Structure-tree detection (highest priority — semantic PDF tagging) + if !struct_tables.is_empty() { + let st_tables = detect_tables_from_struct_tree(band_items, struct_tables, page); + for table in &st_tables { + for &idx in &table.item_indices { + rect_claimed.insert(idx); + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } + // 1. Rect-based detection first (well-tested, high precision) let (rect_tables, hint_regions) = detect_tables_from_rects(band_items, band_rects, page); diff --git a/src/structure_tree.rs b/src/structure_tree.rs index d7e440a..fa40110 100644 --- a/src/structure_tree.rs +++ b/src/structure_tree.rs @@ -273,6 +273,111 @@ impl StructTree { flatten_recursive(&self.children, &mut out, 0); out } + + /// Extract table structures from the tagged PDF tree. + /// + /// Walks the tree to find `/Table` elements with `/TR` > `/TD|TH` children, + /// collecting MCIDs at each cell. Returns structured descriptors that can + /// be matched against extracted [`TextItem`]s to build tables without + /// relying on geometry-based detection. + pub fn extract_tables( + &self, + page_ids: &std::collections::BTreeMap, + ) -> Vec { + let obj_to_page: HashMap = + page_ids.iter().map(|(&num, &id)| (id, num)).collect(); + let mut tables = Vec::new(); + collect_tables(&self.children, &obj_to_page, &mut tables); + tables + } +} + +// ─── Tagged table structures ──────────────────────────────────────── + +/// A table cell extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTableCell { + /// Whether this cell is a header cell (`/TH`). + pub is_header: bool, + /// MCIDs with their resolved page numbers. + pub mcids: Vec<(i64, u32)>, +} + +/// A table row extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTableRow { + pub cells: Vec, +} + +/// A complete table extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTable { + pub rows: Vec, +} + +fn collect_tables( + elements: &[StructElement], + obj_to_page: &HashMap, + tables: &mut Vec, +) { + for elem in elements { + if elem.role == StructRole::Table { + let mut rows = Vec::new(); + collect_rows(&elem.children, obj_to_page, &mut rows); + if rows.len() >= 2 && rows.iter().any(|r| !r.cells.is_empty()) { + tables.push(StructTable { rows }); + } + } else { + collect_tables(&elem.children, obj_to_page, tables); + } + } +} + +/// Collect rows from Table children, transparently descending through +/// THead/TBody/TFoot grouping elements. +fn collect_rows( + elements: &[StructElement], + obj_to_page: &HashMap, + rows: &mut Vec, +) { + for elem in elements { + match elem.role { + StructRole::TR => { + let mut cells = Vec::new(); + for child in &elem.children { + if child.role == StructRole::TD || child.role == StructRole::TH { + let is_header = child.role == StructRole::TH; + let mut mcids = Vec::new(); + collect_mcids_recursive(child, obj_to_page, &mut mcids); + cells.push(StructTableCell { is_header, mcids }); + } + } + rows.push(StructTableRow { cells }); + } + StructRole::THead | StructRole::TBody | StructRole::TFoot => { + collect_rows(&elem.children, obj_to_page, rows); + } + _ => {} + } + } +} + +/// Recursively collect all MCIDs from an element and its descendants. +fn collect_mcids_recursive( + elem: &StructElement, + obj_to_page: &HashMap, + mcids: &mut Vec<(i64, u32)>, +) { + for mcref in &elem.content_refs { + if let Some(page_id) = mcref.page_id { + if let Some(&page_num) = obj_to_page.get(&page_id) { + mcids.push((mcref.mcid, page_num)); + } + } + } + for child in &elem.children { + collect_mcids_recursive(child, obj_to_page, mcids); + } } /// A flattened view of a structure element for linear traversal. diff --git a/src/tables/detect_struct.rs b/src/tables/detect_struct.rs new file mode 100644 index 0000000..d573257 --- /dev/null +++ b/src/tables/detect_struct.rs @@ -0,0 +1,354 @@ +//! Structure-tree-based table detection. +//! +//! When a PDF has a well-formed structure tree with `/Table` > `/TR` > `/TD|TH` +//! elements linked to MCIDs, this module builds `Table` structs directly from +//! the semantic hierarchy — no geometry heuristics needed. + +use std::collections::HashMap; + +use crate::structure_tree::StructTable; +use crate::types::TextItem; + +use super::Table; + +/// Build tables from structure-tree table descriptors by matching MCIDs to TextItems. +/// +/// Returns tables for the given page. Tables where fewer than 50% of cells +/// resolve to TextItems are rejected (stale or broken structure tree). +pub fn detect_tables_from_struct_tree( + items: &[TextItem], + struct_tables: &[StructTable], + page: u32, +) -> Vec

    { + if struct_tables.is_empty() { + return Vec::new(); + } + + // Build MCID → item indices for this page + let mut mcid_to_items: HashMap> = HashMap::new(); + for (idx, item) in items.iter().enumerate() { + if item.page == page { + if let Some(mcid) = item.mcid { + mcid_to_items.entry(mcid).or_default().push(idx); + } + } + } + + let mut tables = Vec::new(); + + for st in struct_tables { + // Filter rows to this page + let page_rows: Vec<_> = st + .rows + .iter() + .filter(|row| { + row.cells + .iter() + .any(|cell| cell.mcids.iter().any(|&(_, p)| p == page)) + }) + .collect(); + + if page_rows.len() < 2 { + continue; + } + + // Determine column count from max cells per row + let num_cols = page_rows.iter().map(|r| r.cells.len()).max().unwrap_or(0); + if num_cols < 2 { + continue; + } + + // Build cell text and collect item indices + let mut cells: Vec> = Vec::new(); + let mut all_item_indices: Vec = Vec::new(); + let mut total_cells = 0u32; + let mut matched_cells = 0u32; + + for row in &page_rows { + let mut row_cells = Vec::with_capacity(num_cols); + for (col_idx, cell) in row.cells.iter().enumerate() { + if col_idx >= num_cols { + break; + } + total_cells += 1; + + // Collect all items for this cell's MCIDs + let mut cell_items: Vec<(usize, &TextItem)> = Vec::new(); + for &(mcid, p) in &cell.mcids { + if p == page { + if let Some(indices) = mcid_to_items.get(&mcid) { + for &idx in indices { + cell_items.push((idx, &items[idx])); + } + } + } + } + + if !cell_items.is_empty() { + matched_cells += 1; + } + + // Sort by Y (descending = top-to-bottom) then X + cell_items.sort_by(|a, b| { + b.1.y + .partial_cmp(&a.1.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + + let text: String = cell_items + .iter() + .map(|(_, item)| item.text.as_str()) + .collect::>() + .join(" "); + + for (idx, _) in &cell_items { + all_item_indices.push(*idx); + } + + row_cells.push(text); + } + + // Pad to num_cols + while row_cells.len() < num_cols { + row_cells.push(String::new()); + } + cells.push(row_cells); + } + + // Reject if too few cells matched (stale structure tree) + if total_cells == 0 || (matched_cells as f32 / total_cells as f32) < 0.3 { + continue; + } + + // Derive row/column positions from item geometry + let mut row_positions: Vec = Vec::new(); + for row in &page_rows { + let y = row + .cells + .iter() + .flat_map(|c| c.mcids.iter()) + .filter(|(_, p)| *p == page) + .filter_map(|(mcid, _)| mcid_to_items.get(mcid)) + .flatten() + .map(|&idx| items[idx].y) + .reduce(f32::max) + .unwrap_or(0.0); + row_positions.push(y); + } + + // Column positions: use X positions of first non-empty cell in each column + let mut col_positions: Vec = vec![0.0; num_cols]; + for (col, col_pos) in col_positions.iter_mut().enumerate() { + for row in &page_rows { + if col < row.cells.len() { + if let Some(x) = row.cells[col] + .mcids + .iter() + .filter(|(_, p)| *p == page) + .filter_map(|(mcid, _)| mcid_to_items.get(mcid)) + .flatten() + .map(|&idx| items[idx].x) + .reduce(f32::min) + { + *col_pos = x; + break; + } + } + } + } + + all_item_indices.sort_unstable(); + all_item_indices.dedup(); + + tables.push(Table { + columns: col_positions, + rows: row_positions, + cells, + item_indices: all_item_indices, + }); + } + + tables +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::structure_tree::{StructTableCell, StructTableRow}; + use crate::types::ItemType; + + fn make_item(text: &str, x: f32, y: f32, page: u32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * 5.0, + height: 10.0, + font: "Test".to_string(), + font_size: 10.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid, + } + } + + #[test] + fn basic_struct_table() { + let items = vec![ + make_item("Name", 50.0, 700.0, 1, Some(10)), + make_item("Age", 200.0, 700.0, 1, Some(11)), + make_item("Alice", 50.0, 680.0, 1, Some(20)), + make_item("30", 200.0, 680.0, 1, Some(21)), + make_item("Bob", 50.0, 660.0, 1, Some(30)), + make_item("25", 200.0, 660.0, 1, Some(31)), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: true, + mcids: vec![(10, 1)], + }, + StructTableCell { + is_header: true, + mcids: vec![(11, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(30, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(31, 1)], + }, + ], + }, + ], + }]; + + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert_eq!(tables.len(), 1); + let table = &tables[0]; + assert_eq!(table.cells.len(), 3); + assert_eq!(table.cells[0], vec!["Name", "Age"]); + assert_eq!(table.cells[1], vec!["Alice", "30"]); + assert_eq!(table.cells[2], vec!["Bob", "25"]); + assert_eq!(table.item_indices.len(), 6); + } + + #[test] + fn rejects_low_mcid_coverage() { + // Items have no MCIDs matching the struct table + let items = vec![ + make_item("Orphan", 50.0, 700.0, 1, Some(999)), + make_item("Text", 200.0, 700.0, 1, None), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(10, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(11, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 1)], + }, + ], + }, + ], + }]; + + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert!( + tables.is_empty(), + "should reject table with no MCID matches" + ); + } + + #[test] + fn filters_by_page() { + let items = vec![ + make_item("A", 50.0, 700.0, 2, Some(10)), + make_item("B", 200.0, 700.0, 2, Some(11)), + make_item("C", 50.0, 680.0, 2, Some(20)), + make_item("D", 200.0, 680.0, 2, Some(21)), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(10, 2)], + }, + StructTableCell { + is_header: false, + mcids: vec![(11, 2)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 2)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 2)], + }, + ], + }, + ], + }]; + + // Page 1 should find nothing + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert!(tables.is_empty()); + + // Page 2 should find the table + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 2); + assert_eq!(tables.len(), 1); + } +} diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 4b7c79f..ece5e81 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -5,6 +5,7 @@ mod detect_heuristic; mod detect_lines; mod detect_rects; +mod detect_struct; mod financial; mod format; mod grid; @@ -13,6 +14,7 @@ pub use detect_heuristic::detect_tables; pub use detect_lines::detect_tables_from_lines; pub(crate) use detect_rects::cluster_rects; pub use detect_rects::{detect_tables_from_rects, RectHintRegion}; +pub use detect_struct::detect_tables_from_struct_tree; pub use format::table_to_markdown; use crate::types::TextItem; diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index ace09ee..e3850c7 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -1,3 +1,50 @@ +|Date|Procurement Title|PE|Bidder|Amount|| +|---|---|---|---|---|---| +|JAN|||||| +|1|8/1|Procurement of Criticals Spare Parts for Engine Maintenance on Mahe|PUC|Wartsila Eastern Africa Ltd|Euro97,922.30| +|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| +|3|15/1|Manufacturing and Deliveries of 900 Students Desks|MOE|SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00|| +|4|15/1|Renovation Works at Mont Fleuri Secondary School|MOE|Sai-Fu Enterprise|SR2,815,771.00| +|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| +|6|15/1|Storm Water Channel Project at Au Cap-Additional Works|DOE|United Concrete Products (Sey)Ltd|SR184,300.00| +|7|22/1|Procurement of Security at ex-Maritime Training Centre|SFA|Elite Surveillance Security Agency|SR30,000.00| +|8|22/1|Installation of Sewerage Treatment Plant at Anse Gaulette|MLUH|Green Island Construction Compnay|SR4,524,884.85| +|9|29/1|Procurement of Cylinder Liner|PUC|Wartsila Eastern Africa Ltd|Euro28,186.75| +|10|29/1|Procurement of Service Pack for Coupling|PUC|Wartsila Eastern Africa Ltd|Euro11,151.00| +|11|29/1|Construction of Drainage for Roads A & B Eve Island Praslin|MLUH|Ascent Projects Sey|SR2,931,174.00| +|12|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| +|FEB|||||| +|13|5/2|Procurement of DI Pipes and Fittings for Le Rocher Refurbishment|PUC|Legend General Supply|USD43,807.50| +|14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| +|15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| +|16|5/2|constraction of 6*2 bedroom Houses- Mont Buxton|MLUH|O-NIVO Construction|SR3,688,844.00| +|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| +|18|5/2|Procurement of Vehcile x1|MOE|Abhaye Valabhji Pty Ltd|SR 650,000.00| +|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| +|20|12/2|Procurement of Turbo charger Rotor for Engine on Praslin|PUC|Marine Power International FZC|Euro 835,669.00| +|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| +|22|12/2|Procurement of Air Cooler Cartridge for Wartsila Engine|PUC|Marine Power International FZC|Euro 36,508.93| +|23|12/2|Procurement of Spares for Genset 8P on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro 288,202.50| +|24|19/2|Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka|SLTA|Esparon's Enterprise|SR 1,105,069.00| +|25|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41|PUC|Marine Power International|Euro 144,726.26| +|26|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31|PUC|Marine Power International FZC|Euro 143,746.75| +|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| +|28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| +|29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| +|30|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| +|MAR|||||| +|31|5/3|Procurement of Spare Parts for the Asphalt Plant- Petite Paris|SLTA|Astec Factory (USA)|USD 101,337.81| +|32|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines|PUC|Marine Power International FZC|Euro 44,050.00| +|33|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines B11|PUC|Marine Power International FZC|Euro 98,743.00| +|34|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines 8B|PUC|Marine Power International FZC|Euro 103,215.00| +|35|5/3|Procurement of Critical Spare Parts for Wartsila Engines 8B|PUC|Wartsila Global Logistic|Euro 97,979.10| +|36|5/3|Procurement Spare Parts for the 8MW Engines|PUC|Wartsila Global Logistic|Euro 41,444.00| +|37|5/3|Procurement of Voltage Districbution Boxes and Fuses-Variations|PUC|Indian Ocean Export Company Pty Ltd|GBP1,300.00| +|38|5/3|33Kv South Mahe Project-Variations|PUC|United Concrete Products (Sey)Ltd|SR459,684.00| +|39|5/3|construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations|SFRS|Furui Construction Pty Ltd|SR184,836.03| +|40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| + ||Date|Procurement Title|PE|Bidder|Amount| |---|---|---|---|---|---| ||JAN||||| @@ -45,6 +92,57 @@ |40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| |41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| ||||||1| + +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 2 Roy & Sons Import|SR 359,200.00| +|---|---|---|---|---|---| +|41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 4 HIS & PJ Enterprise|SR 1,753,614.70| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 5 Roy & Sons Import|SR 1,080.000.00| +|42|19/3|Procurement of Medium Voltage Cables|PUC|Nexans France Lens|Euro 52,354.45| +|43|19/3|Procurement of Brass/Bronze Fitting|PUC|Jainsons Industries|Euro 97,718.48| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre|SFA|Neil Marine (Sri Lanka)|USD 284,521.15| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre-Equipment|SFA|Neil Marine (Sri Lanka)|USD 97,861.50| +|45|19/3|Procurement of Stationery Items-Lot 1|MOE|Print V Care|SR 349,450.00| +|45|19/3|Procurement of Stationery Items-Lot 2|MOE|Print V Care|SR 392,500.00| +|45|19/3|Procurement of Stationery Items-Lot 3|MOE|JD's Stationery & Educational Centre|SR 19,150.00| +|45|19/3|Procurement of Stationery Items-Lot 5|MOE|Vitoria Computer Services Pty LTd|SR 94,500.00| +|45|19/3|Procurement of Stationery Items-Lot 6|MOE|Roy & Sons Imports|SR 114,750.00| +|45|19/3|Procurement of Stationery Items-Lot 7|MOE|Trade Supplies Pty Ltd|SR 35,224.00| +|46|26/3|Procurement of Microsoft Windows License|STB|Victoria Computer Service|SR 977,150.40| +|47|26/3|Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations|PUC|Ras Tek Pvt Ltd|Euro 58,000.00| +|APR|||||| +|48|2/4|Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin|PUC|Vijay Construction|SR 623,025.00| +|49|2/4|Supply of Fish to the Prison Department|Prison Service|Mr. Danny Loizeau|SR 27.50 per KG| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 1|PUC|Magilyn Ltee|USD 28,885.00| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 2|PUC|Magilyn Ltee|USD 35,910.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations|PUC|Bluezone Mauritius|Euro 30,761.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3|PUC|Bluezone Mauritius|Euro 24,254.00| +|51|2/4|Procurement of Grundfos Pumps Sewerage Pump Station|PUC|Bluezone Mauritius|Euro 65,203.00| +|52|2/4|Procurement of Virtual Studio Equipment|SBC|New Tek Europe|Euro 27,240.00| +|53|2/4|Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works|PUC|Tracetebel Engineering|Euro 244,810.00| +|54|2/4|Security in MOH's Institution-Lot 1|MOH|Alliance Security|SR 58,968.00| +|55|2/4|Procurement of Bearings for TurboCharger Wartsila Engines|PUC|ABB France|Euro 57,191.50| +|56|2/4|Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine|PUC|Marine Power International|Euro 33,769.00| +|57|2/4|Procurement of Lighting Equipments|PUC|Thorn Europhane|Euro 38,982.30| +|58|9/4|Construction of Fooothpath at Olivier Maradan Street|SLTA|Benoiton Construction|SR 1,548.520.00| +|59|9/4|Procurement of Utrasonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 31,740.00| +|60|9/4|Implementation of Nwe Financial System|SCAA|Blanche Birger|Euro 65,779.50| +|61|9/4|Construction of Fuel station and Admin Block|SPTC|Onivo Construction|SR2,196.748.00| +|62|9/4|Procurement of Plastic Chairs|MOE|J Galt International|ZAR 498,560.00| +|63|9/4|Security Service in All Education Institution -Lot 2|MOE|Xtreme Security Service|SR 28,000.00| +|64|16/4|Procurement of Critical Spare Parts for Wartsila Engines B41|PUC|Wartsila Global Logistic Service|Euro 37,460.92| +|65|16/4|Security Service STC Premises-Supermarket|STC|Isles Security Agency Ltd|SR 908,107.20| +|65|16/4|Security Service STC Premises-Meat & Veg|STC|Allaince Security|SR 456,183.36| +|65|16/4|Security Service STC Premises-Warehouse, Duty free, BDR Complex|STC|Isles Security Agency Ltd|SR 490,752.00| +|65|16/4|Security Service STC Premises-Head Office|STC|Isles Security Agency Ltd|SR 503,712.00| +|65|16/4|Security Service STC Premises-Praslin ( Amitie Store /Duty-Free|STC|Alliance Security|SR 327,598.56| +|66|16/4|Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract|MLUH|Allied Builders|SR 1,732,989.80| +|67|16/4|Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials|MLUH|Ascent Projects (Sey) Pty Ltd|SR 3,888,632.42| +|68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| +|69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| + +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 2 Roy & Sons Import|SR 359,200.00| +|---|---|---|---|---|---| |41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| |41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 4 HIS & PJ Enterprise|SR 1,753,614.70| |41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 5 Roy & Sons Import|SR 1,080.000.00| @@ -91,6 +189,57 @@ |68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| |69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| ||||||2| + +|70|23/4|Procurement of ADOBE CSE Editing Software for Audio and Video|SBC|Nuclei|Euro 91,187.70| +|---|---|---|---|---|---| +|71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| +|72|30/4|Procurement of Vehicle|PA|Abhaye Valabhji Pty Ltd|SR575,000.00| +|73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| +|74|30/4|Procurement of Electrical Meters|PUC|ISKRAEMECO|Euro 55,733.60| +|75|30/4|Procurement of Grunfos Pump for Le Rocher Pump Station|PUC|Bluezone Mauritius Ltd|Euro 37,910.60| +|76|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| +|MAY|||||| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power|PUC|Technoglass|US$ 156, 920.00| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite|PUC|Technoglass|US$ 83,980.00| +|78|7/5|Procurement of Connecting Rod (Variations)|PUC|Wartsila Eastern Africa Ltd|Euro 1,200.00| +|79|7/5|Project Management Consultancy-Extension of contract|SIBA|Philippe Adrienne Consultancy Engineers|SR 225.000.00| +|80|7/5|Procurement of Liquid Chlorine|PUC|Al Afaq LLC|USD 77,088.00| +|81|21/5|Procurement of Agriculture Inputs|SAA|Rodley Mathieu|SR1,241,225.00| +|82|21/5|Procurement of Metelogical Equipment|MEE|Vaisala|Euro 33,815.00| +|83|21/5|Procurement of Turbocharger for Rotor Shaft for Wartsila Engine|PUC|Marine Power International|Euro 39,945.00| +|84|21/5|Provision of Security Srevices for MOH's Institutions|MOH|Alliance Security|SR 19,656.00| +|85|21/5|Procurement of Low Voltage ABC Cables|PUC|Nextans|Euro 50,118.69| +|86|28/5|Procurement of HDPE pipes-Pipe for water applications|PUC|STR Marketing-|USD 68,030.02| +|86|28/5|Procurement of HDPE pipes-Pipe for sewerage applications|PUC|STR Marketing|USD 12,509.55| +|87|28/5|Procurement of Incenerator for Baie Ste Anne Praslin Hospital|MOH|Incinco Limited|GBP 101,992.00| +|88|28/5|Geotechnical Survey on Ile Soleil|2020 Development Ltd|Geoconsul Ltee|SR 741,520.00| +|89|28/5|Extra Works at Palais De Justice|The Judiciary|Quingjian Group Co|SR 1,614,226.00| +|90|28/5|Fire Fighting and rescue training Course|SFRS|Emergency Training Solution Pty Ltd|ZAR 824,453.80| +|91|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| +|JUN|||||| +|92|4/6|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale|ZAR 438,081.83| +|93|4/6|Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC||Wartsila Global Service|Euro 74,768.40| +|94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| +|95|4/6|Procurement of Digital Microwave Equipment|SBC|MOCHINO|Euro 125,845.00| +|96|4/6|Procurement of Vehicle 1|LWMA|Abhaye Valabhji Pty Ltd-Jeep|SR 475,000.00| +|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| +|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| +|98|11/6|Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka|SLTA|Bazil Construction|Sr 1,081,530.00| +|99|11/6|Procurement of Seychelles Paswsports|DIA|Groupe Impimerie Nationale|Euro 102,400.00| +|100|19/6|Anse Boileau Footpath and Drainage Construction Phase II|SLTA|TCH Building Contractor|SR 960,107.50| +|101|21/6|Procurement of Vehicle x 1|PSD|Abhaye Valabhji-Bus x1|SR 475,000.00| +|101|21/6|Procurement of Vehicle x 1|PSD|PMC Auto- Car x1|SR 267,850.00| +|102|21/6|Renovation Works on Block A- Beau Vollon Secondary School|MOE|Sai-Fu Enterprise Company Ltd|SR 2,253,300.00| +|103|21/6|Procurement of 35 VMS Terminal Accessories|SFA|Communication Specialist Ltd|Euro 63,770.00| +|104|21/6|Renewal of SFA's Themis FMC Services|SFA|CLS- France|Euro 36,000.00| +|105|21/6|Procurement of Electrical Spares for Wartsila Engine|PUC|Wartsila Eastern Africa|Euro 1,757.00| +|106|21/6|Procurement of Class D water Meters|PUC|Elster Metering Limited (Pty) Ltd|USD 109,7051.00| +|107|21/6|Procurement of Sludge Incinerator|PUC|Atlas Incinerator|€ 94,470.00| +|108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| +|109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| + +|70|23/4|Procurement of ADOBE CSE Editing Software for Audio and Video|SBC|Nuclei|Euro 91,187.70| +|---|---|---|---|---|---| |71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| |72|30/4|Procurement of Vehicle|PA|Abhaye Valabhji Pty Ltd|SR575,000.00| |73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| @@ -137,6 +286,57 @@ |108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| |109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| ||||||3| + +|110|21/6|Refurbishment of Turbo Charger for Rotor Shaft|PUC|Marine Power International FZC|Euro 57,326.00| +|---|---|---|---|---|---| +|111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| +|112|21/6|Refurbishment of Pharmaceutical Production Unit-Contract Extension|MOH|Mahe Design|SR808,375.00| +|113|21/6|Technical Service for Repair on Generator-1B|PUC|Goltens|USD 75,600.00| +|114|21/6|Procurement of Gate Valves|PUC|AVK Valves Southern africa (Pty) Ltd|ZAR 461,608.44| +|115|25/6|Procurement of Hot-Dipped Galvanised Materials|PUC|HDSA Shipping (Pty) Ltd|Zar 435,614.00| +|116|25/6|Procurement of Critical Spares for Wartsila Engine B51- Lot 1|PUC|Wartsila Global Services|Euro 92,472.70| +|117|25/6|Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin|PUC|RUYSCH|Euro 83,262.64| +|118|25/6|Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013|PUC|Tornado Group|USD 266,820.82| +|119|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| +|JUL|||||| +|120|2/7|Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road)|SLTA|Esparon's Enterprise|SR 183,410.00| +|121|2/7|Bridge Renovation at Cascade|SLTA|Benioton Construction|SR 1,361,100.00| +|122|2/7|Procurement of Bitumen|SLTA|Termcotank S.A|USD 519,418.20| +|123|2/7|Procurement of Vehicle x 2|STB|PMC Auto Pty Ltd|SR 960,126.00| +|124|2/7|Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning|MFA|John Nevill|SR 180,000.00| +|125|9/7|Procurement of Stationery Items Lot 4|MOE|JD's Stationey EDU Centre|SR 629,950.00| +|126|9/7|Procurement of Atomic Spectrometer|SBS|SMM Instrument (Pty) Ltd|USD 181,147.00| +|127|9/7|Operation and Maintenance of Containerised Desalination Units by Tornado-2012|PUC|Tornado Group|USD 45,174.62| +|128|9/7|Procurement of Sensors and Transmitters for Wartsila Engines|PUC|Wartsila Global Logistic Services|Euro 50,360.00| +|129|9/7|Procurement of Bulk Water Meters Strainers|PUC|Elster Metering Limited (Pty) Ltd|ZAR 931,966.00| +|130|9/7|Procurement of Gudgeon Pins for engine 8P|PUC|Wartsila Global Logistic Services|Euro 23,460.00| +|131|9/7|Procurement of Piston for Wartsila Engine A21|PUC|Wartsila Global Logistic Services|Euro 219,622.00| +|132|9/7|Procurement of Piston for Wartsila Engine B11|PUC|Marine Power International FZC|Euro 243,799.98| +|133|9/7|Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance|MLUH|Sai-Fu Enterprise Company Ltd|SR 17,376,392.65| +|134|9/7|La Louise Non- Performance Pipieline Replacement|PUC|Ascent Projects Sey Pty Ltd|SR 1,289,375.00| +|135|9/7|Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR 1,462,875.00| +|136|9/7|Consultancy Services for Quality management System (QMS)|DE|Mr. John Horack|USD 22,325.00| +|137|9/7|Construction of New Road at Cascade Primary School|MLUH|Esparon's Enterprise|SR 2,443,814.00| +|138|9/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale Pty Ltd|ZAR 518,694.02| +|139|9/7|Procurement of Non-Critical Spares- Specialized Tools|PUC|Marine Power International FZC|Euro 10,866.55| +|140|9/7|Upgarding of Roche Caiman Road and Roundabout|SLTA|Bazil Construction|SR 1,592,002.00| +|141|9/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|142|16/7|Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR1,462,875.00| +|143|16/7|Consultancy Service for Quality Management System (QMS)|DE|Mr. John Horack|CND$22,325.00| +|144|16/7|Construction of New Road at Cascade Primary School|SLTA|Esparon's Enterprise|SR 2,443,814.00| +|145|16/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brit Pale Pty Ltd|ZAR518,694.02| +|146|16/7|Procurement od Non- critcal spares-specialized tools|PUC|Marine International FZC|Euro10,866.55| +|147|16/7|Upgrading of Roche Caiman Road and roundabout|SLTA|Bazil Construction|SR1,592,002.00| +|148|16/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|149|23/7|Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area.|MLUH|LC International|SR2,814,108.00| +|150|23/7|Procurement of CT Scan Tube|MOH|Ireland Blyth Ltd from Mauritius|Euro 122,000.00| +|151|23/7|Procurement of Ultraviolet disinfection System for Sewerage Treatment|PUC|Orica Wtercare|SR851,104.72| +|152|23/7|Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps|PUC|Hidrostal Sewage Sa Pty Ltd|Euro97,498.32| +|153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| +|154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| + +|110|21/6|Refurbishment of Turbo Charger for Rotor Shaft|PUC|Marine Power International FZC|Euro 57,326.00| +|---|---|---|---|---|---| |111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| |112|21/6|Refurbishment of Pharmaceutical Production Unit-Contract Extension|MOH|Mahe Design|SR808,375.00| |113|21/6|Technical Service for Repair on Generator-1B|PUC|Goltens|USD 75,600.00| @@ -183,6 +383,57 @@ |153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| |154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| ||||||4| + +|155|23/7|Renovation work at the schools section MOE headquater|MOE|Prime Builders|SR1,768,620.00| +|---|---|---|---|---|---| +|156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| +|157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00| +|158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00| +|159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00| +|160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00| +|161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00| +|162|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| +|AUG|||||| +|163|6/8|Procurement of Vehichels X 4|MOH|Kim-Koom & Co Pty Ltd|SR1,100,000.00| +|164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe|WMF|Mr. Donal Ernesta|| +|164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe|WMF|Mr. Kali Deenudayali|| +|165|13/8|Procurement of exercise books|MOE|JD's Stationey EDU Centre|SR,1,700,000.00| +|166|13/8|Procurement of High Pressure pump spares -BZM00003022|PUC|Bluezone Mauritius Ltd|Euro27,712.73| +|166|13/8|Procurement of High Pressure pump spares -BZM00003023|PUC|Bluezone Mauritius Ltd|Euro12,639.55| +|167|13/8|Procurement of CR64 pump spares|PUC|Bluezone Mauritius Ltd|Euro31,822.00| +|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| +|169|13/8|Renovation work to one classroom block at Pionte Larue Secondary School|MOE|Belverdere Builders|SR1,114,575.00| +|170|13/8|Completion of Amitie Housing Project 12 x 3 Bedrooms|MLUH|Allied Builders Sey Ltd|6,006,410.37| +|171|13/8|Copolia Road widening-Phase 2|SLTA|Belverdere Builders|SR1,037,235.00| +|172|20/8|Procurement of ABB Turbocharger Cartridge|PUC|ABB France|Euro106,266.66| +|173|20/8|Procurement of services to carry out the full refit and overhaul of tug Alouette|SPA|SECREN (Madagascar)|Euro189,618.42| +|174|20/8|Awarding of cranshatf and Block replacement solution for A11 engine|PUC|Wartsila|Euro800,000.00| +|175|29/8|Servicing of geartrain for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro 21,141.90| +|176|29/8|Spare parts for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro38,440.00| +|177|29/8|Procurement for sience equipment and chemical for 2013|MOE|Findel Education|GBP37,831.26| +|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| +|179|29/8|Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA||Furui Construction Pty Ltd|SR277,521.45| +|180|29/8|Propsed road widening and drainage improvement at Quincy Vilage|SLTA|TCH Contractor|SR1,015,806.24| +|181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| +|182|3/9|Procurement of Bitumen in drums for the asphalt production Praslin|SLTA|Benzene International Pte Ltd|Euro87,220.00| +|183|3/9|Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S|PUC|Wartsila Eastern Africa|Euro9,861.00| +|184|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| +|SEP|||||| +|185|10/9|Cleaning and maintenance of wetlands and rivers on Praslin|ED|"W" Cleaning Service|SR769,590.00| +|186|10/9|Re-construction and partition of Independence House|MLUH|Green Island Construction Co Pty|SR868,022.94| +|187|10/9|Procurement of critical spare fro major overhaul|PUC|Wartsila Eastern Afirca|Euro33,623.00| +|188|10/9|Procurement of control panel for six pumps station|PUC|CA Engineering Consultancy Pte Ltd|SGD37,400.00| +|189|17/9|Security service for Wellness Centre|MOH|Alliance Security|SR78,624.00| +|190|17/9|Procurement of reinforce plastic (FRP) handrails and grating|PUC|Webforge Group|SR839,795.02| +|191|17/9|Procurement of helital fitting|PUC|Cu A1 Engineering (Pty) Ltd|ZAR343,000.00| +|192|17/9|Procurement of Flygt pumps|PUC|Aqualia DPI LTD|Euro56,080.00| +|193|17/9|Procurement of black-up and emergency pumps|PUC|M.A.H.Y Khoory & Co|UAE186,650.00| +|194|17/9|Procurement of additional requirement of polo t-shirts and jeans|PUC|Magilyn Ltee|USD42,127.50| +|195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| +|196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| + +|155|23/7|Renovation work at the schools section MOE headquater|MOE|Prime Builders|SR1,768,620.00| +|---|---|---|---|---|---| |156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| |157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00| |158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00| @@ -229,6 +480,33 @@ |195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| |196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| ||||||5| + +|197|17/9|Supply of pipes and fittings for Network diversion in Victoria|PUC|Ascent Projects (Sey) Pty Ltd|USD181,661.00| +|---|---|---|---|---|---| +|198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| +|199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| +|200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| +|201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00| +|OCT|||||| +|202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| +|203|1/10|Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations|SLTA|G & S Enterprise|SR200,448.00| +|204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| +|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| +|206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| +|207|1/10|Procurement of Viking Johnson fittings|PUC|Viking Johnson (UK)|GBP92,286.50| +|208|1/10|Procurement of HDPE Pipes and Fittings|PUC|STR Marketing Ltee|USD196,672.57| +|209|1/10|Procurement of Piston and Gudgeon Pins|PUC|Marine Power International FZC|Euro277,098.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 1|PUC|Ms. Suzanne Edmond|SR500.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 2|PUC|Ms. Suzanne Edmond|SR200.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 3|PUC|Ms. Suzanne Edmond|SR475.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 4|PUC|Sey Sytle|SR450.00| +|211|8/10|Procurement of vehicle x 2|FIU|PMC Auto Pty Ltd|SR526,864.00| +|212|15/10|Renovation Works to Le Chantier Mall-Variation Works|SSF|Allied Builders Sey Ltd|SR1,673,752.51| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| + +|197|17/9|Supply of pipes and fittings for Network diversion in Victoria|PUC|Ascent Projects (Sey) Pty Ltd|USD181,661.00| +|---|---|---|---|---|---| |198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| |199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| |200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| @@ -276,6 +554,25 @@ |232|29/10|Renovation and Partitioning of Independence House-Variation|MLUH|Green Island Construction Co Pty|SR1,847,477.43| ||||||6| +|298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| +|---|---|---|---|---|---| +|299|17/12|Supply of New Equipment for Kitchen|Prison Service|K. K. Chua|SR943,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Sun Motors|SR770,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Abhaye Valabhji|SR435,000.00| +|301|23/12|Procurement of Spares for LT Water Circulating Pump|PUC|Wartsila Global Logistcs Services|Euro 19,389.00| +|302|23/12|Procurement of Filter Cartridge-Desalination Plants|PUC|Trans Crescent Technical Equipment Company|Euro22,030.00| +|303|23/12|Procurement of LED Streetlights|SLTA|Lighting Orient (China)|USD104,282.00| +|304|23/12|Proposed New Road at Cascade Primary School-Variations|SLTA|Esparon's Enterprise|SR861,400.00| +|305|23/12|General renovation at School Section at MOE Headquarters-Variations|MOE|Prime Builders|SR2,761,935.75| +|306|23/12|Remedial Works at Mont Fleuri Primary School and Creche-Variations|MOE|Sai-Fu Enterprise Company Ltd|SR1,006,778.50| +|307|23/12|General Renovation of Toilet at La Digue School|MOE|Furui Construction|SR338,785.40| +|308|26/12|Procurement of 2500 Ream of A4 Paper|MOE|Island Motors|SR1,200,000.00| +|309|26/12|Procurement of Additional of 400 Desktop Computers|MOE|Orion Computers|SR3,100,000.00| +|310|26/12|Procuremet of Vehcile x 5|MOE|PMC Auto Pty Ltd|SR1,374,897.00| +|311|26/12|Procurement of Canon Ink / Riso Meter|MOE|Paradise Computer Services|SR473,000.00| +|312|26/12|Supply of Metal Fencing|MOE|BBT (UK)|GBP105,278.00| +|313|26/12|Procurement of Mobile Dental Clinic / Surgeries x 2|MOH|Quayle Dental (UK)|GBP233,842.00| + |298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| |---|---|---|---|---|---| |299|17/12|Supply of New Equipment for Kitchen||Prison Service K. K. Chua|SR943,000.00| From fd13f2acb16b7c1e01327f3301132311c9cc78ee Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 12:50:02 -0700 Subject: [PATCH 39/46] fix(tables): prevent struct-tree and rect detection from emitting duplicate tables Skip rect-detected tables that overlap with items already claimed by struct-tree detection. Previously both strategies emitted separate tables for the same content, doubling the output on tagged PDFs. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/markdown/mod.rs | 12 +- src/tables/detect_struct.rs | 25 ++- tests/snapshots/2013-app2.md | 325 ++--------------------------------- 3 files changed, 48 insertions(+), 314 deletions(-) diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index e1e3d3e..6070c2d 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -678,10 +678,20 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } - // 1. Rect-based detection first (well-tested, high precision) + // 1. Rect-based detection (always runs; struct-tree claims prevent + // double-counting via table_items but rect may find additional tables) let (rect_tables, hint_regions) = detect_tables_from_rects(band_items, band_rects, page); for table in &rect_tables { + // Skip rect tables that overlap with struct-tree claimed items + if !rect_claimed.is_empty() + && table + .item_indices + .iter() + .any(|idx| rect_claimed.contains(idx)) + { + continue; + } for &idx in &table.item_indices { rect_claimed.insert(idx); if let Some(&page_idx) = band_index_map.get(idx) { diff --git a/src/tables/detect_struct.rs b/src/tables/detect_struct.rs index d573257..182c543 100644 --- a/src/tables/detect_struct.rs +++ b/src/tables/detect_struct.rs @@ -6,6 +6,8 @@ use std::collections::HashMap; +use log::debug; + use crate::structure_tree::StructTable; use crate::types::TextItem; @@ -48,6 +50,13 @@ pub fn detect_tables_from_struct_tree( }) .collect(); + debug!( + "page {}: struct table has {} rows on this page (from {} total)", + page, + page_rows.len(), + st.rows.len() + ); + if page_rows.len() < 2 { continue; } @@ -121,7 +130,21 @@ pub fn detect_tables_from_struct_tree( } // Reject if too few cells matched (stale structure tree) - if total_cells == 0 || (matched_cells as f32 / total_cells as f32) < 0.3 { + let coverage = if total_cells > 0 { + matched_cells as f32 / total_cells as f32 + } else { + 0.0 + }; + debug!( + "page {}: struct table {}x{}, {}/{} cells matched ({:.0}%)", + page, + page_rows.len(), + num_cols, + matched_cells, + total_cells, + coverage * 100.0 + ); + if total_cells == 0 || coverage < 0.3 { continue; } diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index e3850c7..711d202 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -45,54 +45,6 @@ |40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| |41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| -||Date|Procurement Title|PE|Bidder|Amount| -|---|---|---|---|---|---| -||JAN||||| -|1|8/1|Procurement of Criticals Spare Parts for Engine Maintenance on Mahe|PUC|Wartsila Eastern Africa Ltd|Euro97,922.30| -|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| -|3|15/1|Manufacturing and Deliveries of 900 Students Desks|MOE||SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00| -|4|15/1|Renovation Works at Mont Fleuri Secondary School|MOE|Sai-Fu Enterprise|SR2,815,771.00| -|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| -|6|15/1|Storm Water Channel Project at Au Cap-Additional Works|DOE|United Concrete Products (Sey)Ltd|SR184,300.00| -|7|22/1|Procurement of Security at ex-Maritime Training Centre|SFA|Elite Surveillance Security Agency|SR30,000.00| -|8|22/1|Installation of Sewerage Treatment Plant at Anse Gaulette|MLUH|Green Island Construction Compnay|SR4,524,884.85| -|9|29/1|Procurement of Cylinder Liner|PUC|Wartsila Eastern Africa Ltd|Euro28,186.75| -|10|29/1|Procurement of Service Pack for Coupling|PUC|Wartsila Eastern Africa Ltd|Euro11,151.00| -|11|29/1|Construction of Drainage for Roads A & B Eve Island Praslin|MLUH|Ascent Projects Sey|SR2,931,174.00| -|12|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| -||FEB||||| -|13|5/2|Procurement of DI Pipes and Fittings for Le Rocher Refurbishment|PUC|Legend General Supply|USD43,807.50| -|14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| -|15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| -|16|5/2|constraction of 6*2 bedroom Houses- Mont Buxton|MLUH|O-NIVO Construction|SR3,688,844.00| -|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| -|18|5/2|Procurement of Vehcile x1|MOE|Abhaye Valabhji Pty Ltd|SR 650,000.00| -|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| -|20|12/2|Procurement of Turbo charger Rotor for Engine on Praslin|PUC|Marine Power International FZC|Euro 835,669.00| -|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| -|22|12/2|Procurement of Air Cooler Cartridge for Wartsila Engine|PUC|Marine Power International FZC|Euro 36,508.93| -|23|12/2|Procurement of Spares for Genset 8P on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro 288,202.50| -|24|19/2|Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka|SLTA|Esparon's Enterprise|SR 1,105,069.00| -|25|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41|PUC|Marine Power International|Euro 144,726.26| -|26|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31|PUC|Marine Power International FZC|Euro 143,746.75| -|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| -|28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| -|29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| -|30|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| -||MAR||||| -|31|5/3|Procurement of Spare Parts for the Asphalt Plant- Petite Paris|SLTA|Astec Factory (USA)|USD 101,337.81| -|32|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines|PUC|Marine Power International FZC|Euro 44,050.00| -|33|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines B11|PUC|Marine Power International FZC|Euro 98,743.00| -|34|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines 8B|PUC|Marine Power International FZC|Euro 103,215.00| -|35|5/3|Procurement of Critical Spare Parts for Wartsila Engines 8B|PUC|Wartsila Global Logistic|Euro 97,979.10| -|36|5/3|Procurement Spare Parts for the 8MW Engines|PUC|Wartsila Global Logistic|Euro 41,444.00| -|37|5/3|Procurement of Voltage Districbution Boxes and Fuses-Variations|PUC|Indian Ocean Export Company Pty Ltd|GBP1,300.00| -|38|5/3|33Kv South Mahe Project-Variations|PUC|United Concrete Products (Sey)Ltd|SR459,684.00| -|39|5/3|construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations|SFRS|Furui Construction Pty Ltd|SR184,836.03| -|40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| -|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| -||||||1| - |41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 2 Roy & Sons Import|SR 359,200.00| |---|---|---|---|---|---| |41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| @@ -141,55 +93,6 @@ |68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| |69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| -|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 2 Roy & Sons Import|SR 359,200.00| -|---|---|---|---|---|---| -|41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| -|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 4 HIS & PJ Enterprise|SR 1,753,614.70| -|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 5 Roy & Sons Import|SR 1,080.000.00| -|42|19/3|Procurement of Medium Voltage Cables|PUC|Nexans France Lens|Euro 52,354.45| -|43|19/3|Procurement of Brass/Bronze Fitting|PUC|Jainsons Industries|Euro 97,718.48| -|44|19/3|Procurement of Training Vessel for Maritime Training Centre|SFA|Neil Marine (Sri Lanka)|USD 284,521.15| -|44|19/3|Procurement of Training Vessel for Maritime Training Centre-Equipment|SFA|Neil Marine (Sri Lanka)|USD 97,861.50| -|45|19/3|Procurement of Stationery Items-Lot 1|MOE|Print V Care|SR 349,450.00| -|45|19/3|Procurement of Stationery Items-Lot 2|MOE|Print V Care|SR 392,500.00| -|45|19/3|Procurement of Stationery Items-Lot 3|MOE|JD's Stationery & Educational Centre|SR 19,150.00| -|45|19/3|Procurement of Stationery Items-Lot 5|MOE|Vitoria Computer Services Pty LTd|SR 94,500.00| -|45|19/3|Procurement of Stationery Items-Lot 6|MOE|Roy & Sons Imports|SR 114,750.00| -|45|19/3|Procurement of Stationery Items-Lot 7|MOE|Trade Supplies Pty Ltd|SR 35,224.00| -|46|26/3|Procurement of Microsoft Windows License|STB|Victoria Computer Service|SR 977,150.40| -|47|26/3||Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations PUC|Ras Tek Pvt Ltd|Euro 58,000.00| -||APR||||| -|48|2/4|Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin|PUC|Vijay Construction|SR 623,025.00| -|49|2/4|Supply of Fish to the Prison Department||Prison Service Mr. Danny Loizeau|SR 27.50 per KG| -|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 1|PUC|Magilyn Ltee|USD 28,885.00| -|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 2|PUC|Magilyn Ltee|USD 35,910.00| -|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations|PUC|Bluezone Mauritius|Euro 30,761.00| -|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3|PUC|Bluezone Mauritius|Euro 24,254.00| -|51|2/4|Procurement of Grundfos Pumps Sewerage Pump Station|PUC|Bluezone Mauritius|Euro 65,203.00| -|52|2/4|Procurement of Virtual Studio Equipment|SBC|New Tek Europe|Euro 27,240.00| -|53|2/4|Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works|PUC|Tracetebel Engineering|Euro 244,810.00| -|54|2/4|Security in MOH's Institution-Lot 1|MOH|Alliance Security|SR 58,968.00| -|55|2/4|Procurement of Bearings for TurboCharger Wartsila Engines|PUC|ABB France|Euro 57,191.50| -|56|2/4|Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine|PUC|Marine Power International|Euro 33,769.00| -|57|2/4|Procurement of Lighting Equipments|PUC|Thorn Europhane|Euro 38,982.30| -|58|9/4|Construction of Fooothpath at Olivier Maradan Street|SLTA|Benoiton Construction|SR 1,548.520.00| -|59|9/4|Procurement of Utrasonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 31,740.00| -|60|9/4|Implementation of Nwe Financial System|SCAA|Blanche Birger|Euro 65,779.50| -|61|9/4|Construction of Fuel station and Admin Block|SPTC|Onivo Construction|SR2,196.748.00| -|62|9/4|Procurement of Plastic Chairs|MOE|J Galt International|ZAR 498,560.00| -|63|9/4|Security Service in All Education Institution -Lot 2|MOE|Xtreme Security Service|SR 28,000.00| -|64|16/4|Procurement of Critical Spare Parts for Wartsila Engines B41|PUC|Wartsila Global Logistic Service|Euro 37,460.92| -|65|16/4|Security Service STC Premises-Supermarket|STC|Isles Security Agency Ltd|SR 908,107.20| -|65|16/4|Security Service STC Premises-Meat & Veg|STC|Allaince Security|SR 456,183.36| -|65|16/4|Security Service STC Premises-Warehouse, Duty free, BDR Complex|STC|Isles Security Agency Ltd|SR 490,752.00| -|65|16/4|Security Service STC Premises-Head Office|STC|Isles Security Agency Ltd|SR 503,712.00| -|65|16/4|Security Service STC Premises-Praslin ( Amitie Store /Duty-Free|STC|Alliance Security|SR 327,598.56| -|66|16/4|Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract|MLUH|Allied Builders|SR 1,732,989.80| -|67|16/4|Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials|MLUH|Ascent Projects (Sey) Pty Ltd|SR 3,888,632.42| -|68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| -|69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| -||||||2| - |70|23/4|Procurement of ADOBE CSE Editing Software for Audio and Video|SBC|Nuclei|Euro 91,187.70| |---|---|---|---|---|---| |71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| @@ -238,55 +141,6 @@ |108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| |109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| -|70|23/4|Procurement of ADOBE CSE Editing Software for Audio and Video|SBC|Nuclei|Euro 91,187.70| -|---|---|---|---|---|---| -|71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| -|72|30/4|Procurement of Vehicle|PA|Abhaye Valabhji Pty Ltd|SR575,000.00| -|73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| -|74|30/4|Procurement of Electrical Meters|PUC|ISKRAEMECO|Euro 55,733.60| -|75|30/4|Procurement of Grunfos Pump for Le Rocher Pump Station|PUC|Bluezone Mauritius Ltd|Euro 37,910.60| -|76|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| -||MAY||||| -|77|7/5|Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power|PUC|Technoglass|US$ 156, 920.00| -|77|7/5|Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite|PUC|Technoglass|US$ 83,980.00| -|78|7/5|Procurement of Connecting Rod (Variations)|PUC|Wartsila Eastern Africa Ltd|Euro 1,200.00| -|79|7/5|Project Management Consultancy-Extension of contract|SIBA|Philippe Adrienne Consultancy Engineers|SR 225.000.00| -|80|7/5|Procurement of Liquid Chlorine|PUC|Al Afaq LLC|USD 77,088.00| -|81|21/5|Procurement of Agriculture Inputs|SAA|Rodley Mathieu|SR1,241,225.00| -|82|21/5|Procurement of Metelogical Equipment|MEE|Vaisala|Euro 33,815.00| -|83|21/5|Procurement of Turbocharger for Rotor Shaft for Wartsila Engine|PUC|Marine Power International|Euro 39,945.00| -|84|21/5|Provision of Security Srevices for MOH's Institutions|MOH|Alliance Security|SR 19,656.00| -|85|21/5|Procurement of Low Voltage ABC Cables|PUC|Nextans|Euro 50,118.69| -|86|28/5|Procurement of HDPE pipes-Pipe for water applications|PUC|STR Marketing-|USD 68,030.02| -|86|28/5|Procurement of HDPE pipes-Pipe for sewerage applications|PUC|STR Marketing|USD 12,509.55| -|87|28/5|Procurement of Incenerator for Baie Ste Anne Praslin Hospital|MOH|Incinco Limited|GBP 101,992.00| -|88|28/5|Geotechnical Survey on Ile Soleil||2020 Development Ltd Geoconsul Ltee|SR 741,520.00| -|89|28/5|Extra Works at Palais De Justice||The Judiciary Quingjian Group Co|SR 1,614,226.00| -|90|28/5|Fire Fighting and rescue training Course|SFRS|Emergency Training Solution Pty Ltd|ZAR 824,453.80| -|91|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| -||JUN||||| -|92|4/6|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale|ZAR 438,081.83| -|93|4/6||Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC|Wartsila Global Service|Euro 74,768.40| -|94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| -|95|4/6|Procurement of Digital Microwave Equipment|SBC|MOCHINO|Euro 125,845.00| -|96|4/6|Procurement of Vehicle 1|LWMA|Abhaye Valabhji Pty Ltd-Jeep|SR 475,000.00| -|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| -|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| -|98|11/6|Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka|SLTA|Bazil Construction|Sr 1,081,530.00| -|99|11/6|Procurement of Seychelles Paswsports|DIA|Groupe Impimerie Nationale|Euro 102,400.00| -|100|19/6|Anse Boileau Footpath and Drainage Construction Phase II|SLTA|TCH Building Contractor|SR 960,107.50| -|101|21/6|Procurement of Vehicle x 1|PSD|Abhaye Valabhji-Bus x1|SR 475,000.00| -|101|21/6|Procurement of Vehicle x 1|PSD|PMC Auto- Car x1|SR 267,850.00| -|102|21/6|Renovation Works on Block A- Beau Vollon Secondary School|MOE|Sai-Fu Enterprise Company Ltd|SR 2,253,300.00| -|103|21/6|Procurement of 35 VMS Terminal Accessories|SFA|Communication Specialist Ltd|Euro 63,770.00| -|104|21/6|Renewal of SFA's Themis FMC Services|SFA|CLS- France|Euro 36,000.00| -|105|21/6|Procurement of Electrical Spares for Wartsila Engine|PUC|Wartsila Eastern Africa|Euro 1,757.00| -|106|21/6|Procurement of Class D water Meters|PUC|Elster Metering Limited (Pty) Ltd|USD 109,7051.00| -|107|21/6|Procurement of Sludge Incinerator|PUC|Atlas Incinerator|€ 94,470.00| -|108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| -|109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| -||||||3| - |110|21/6|Refurbishment of Turbo Charger for Rotor Shaft|PUC|Marine Power International FZC|Euro 57,326.00| |---|---|---|---|---|---| |111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| @@ -335,55 +189,6 @@ |153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| |154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| -|110|21/6|Refurbishment of Turbo Charger for Rotor Shaft|PUC|Marine Power International FZC|Euro 57,326.00| -|---|---|---|---|---|---| -|111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| -|112|21/6|Refurbishment of Pharmaceutical Production Unit-Contract Extension|MOH|Mahe Design|SR808,375.00| -|113|21/6|Technical Service for Repair on Generator-1B|PUC|Goltens|USD 75,600.00| -|114|21/6|Procurement of Gate Valves|PUC|AVK Valves Southern africa (Pty) Ltd|ZAR 461,608.44| -|115|25/6|Procurement of Hot-Dipped Galvanised Materials|PUC|HDSA Shipping (Pty) Ltd|Zar 435,614.00| -|116|25/6|Procurement of Critical Spares for Wartsila Engine B51- Lot 1|PUC|Wartsila Global Services|Euro 92,472.70| -|117|25/6|Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin|PUC|RUYSCH|Euro 83,262.64| -|118|25/6||Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013 PUC|Tornado Group|USD 266,820.82| -|119|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| -||JUL||||| -|120|2/7|Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road)|SLTA|Esparon's Enterprise|SR 183,410.00| -|121|2/7|Bridge Renovation at Cascade|SLTA|Benioton Construction|SR 1,361,100.00| -|122|2/7|Procurement of Bitumen|SLTA|Termcotank S.A|USD 519,418.20| -|123|2/7|Procurement of Vehicle x 2|STB|PMC Auto Pty Ltd|SR 960,126.00| -|124|2/7||Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning MFA|John Nevill|SR 180,000.00| -|125|9/7|Procurement of Stationery Items Lot 4|MOE|JD's Stationey EDU Centre|SR 629,950.00| -|126|9/7|Procurement of Atomic Spectrometer|SBS|SMM Instrument (Pty) Ltd|USD 181,147.00| -|127|9/7|Operation and Maintenance of Containerised Desalination Units by Tornado-2012|PUC|Tornado Group|USD 45,174.62| -|128|9/7|Procurement of Sensors and Transmitters for Wartsila Engines|PUC|Wartsila Global Logistic Services|Euro 50,360.00| -|129|9/7|Procurement of Bulk Water Meters Strainers|PUC|Elster Metering Limited (Pty) Ltd|ZAR 931,966.00| -|130|9/7|Procurement of Gudgeon Pins for engine 8P|PUC|Wartsila Global Logistic Services|Euro 23,460.00| -|131|9/7|Procurement of Piston for Wartsila Engine A21|PUC|Wartsila Global Logistic Services|Euro 219,622.00| -|132|9/7|Procurement of Piston for Wartsila Engine B11|PUC|Marine Power International FZC|Euro 243,799.98| -|133|9/7|Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance|MLUH|Sai-Fu Enterprise Company Ltd|SR 17,376,392.65| -|134|9/7|La Louise Non- Performance Pipieline Replacement|PUC|Ascent Projects Sey Pty Ltd|SR 1,289,375.00| -|135|9/7|Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR 1,462,875.00| -|136|9/7|Consultancy Services for Quality management System (QMS)|DE|Mr. John Horack|USD 22,325.00| -|137|9/7|Construction of New Road at Cascade Primary School|MLUH|Esparon's Enterprise|SR 2,443,814.00| -|138|9/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale Pty Ltd|ZAR 518,694.02| -|139|9/7|Procurement of Non-Critical Spares- Specialized Tools|PUC|Marine Power International FZC|Euro 10,866.55| -|140|9/7|Upgarding of Roche Caiman Road and Roundabout|SLTA|Bazil Construction|SR 1,592,002.00| -|141|9/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| -|142|16/7|Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR1,462,875.00| -|143|16/7|Consultancy Service for Quality Management System (QMS)|DE|Mr. John Horack|CND$22,325.00| -|144|16/7|Construction of New Road at Cascade Primary School|SLTA|Esparon's Enterprise|SR 2,443,814.00| -|145|16/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brit Pale Pty Ltd|ZAR518,694.02| -|146|16/7|Procurement od Non- critcal spares-specialized tools|PUC|Marine International FZC|Euro10,866.55| -|147|16/7|Upgrading of Roche Caiman Road and roundabout|SLTA|Bazil Construction|SR1,592,002.00| -|148|16/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| -|149|23/7|Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area.|MLUH|LC International|SR2,814,108.00| -|150|23/7|Procurement of CT Scan Tube|MOH|Ireland Blyth Ltd from Mauritius|Euro 122,000.00| -|151|23/7|Procurement of Ultraviolet disinfection System for Sewerage Treatment|PUC|Orica Wtercare|SR851,104.72| -|152|23/7||Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps PUC|Hidrostal Sewage Sa Pty Ltd|Euro97,498.32| -|153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| -|154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| -||||||4| - |155|23/7|Renovation work at the schools section MOE headquater|MOE|Prime Builders|SR1,768,620.00| |---|---|---|---|---|---| |156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| @@ -432,55 +237,6 @@ |195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| |196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| -|155|23/7|Renovation work at the schools section MOE headquater|MOE|Prime Builders|SR1,768,620.00| -|---|---|---|---|---|---| -|156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| -|157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00| -|158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00| -|159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00| -|160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00| -|161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00| -|162|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| -||AUG||||| -|163|6/8|Procurement of Vehichels X 4|MOH|Kim-Koom & Co Pty Ltd|SR1,100,000.00| -|164|6/8||Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe WMF|Mr. Donal Ernesta|| -|164|6/8||Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe WMF|Mr. Kali Deenudayali|| -|165|13/8|Procurement of exercise books|MOE|JD's Stationey EDU Centre|SR,1,700,000.00| -|166|13/8|Procurement of High Pressure pump spares -BZM00003022|PUC|Bluezone Mauritius Ltd|Euro27,712.73| -|166|13/8|Procurement of High Pressure pump spares -BZM00003023|PUC|Bluezone Mauritius Ltd|Euro12,639.55| -|167|13/8|Procurement of CR64 pump spares|PUC|Bluezone Mauritius Ltd|Euro31,822.00| -|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| -|169|13/8|Renovation work to one classroom block at Pionte Larue Secondary School|MOE|Belverdere Builders|SR1,114,575.00| -|170|13/8|Completion of Amitie Housing Project 12 x 3 Bedrooms|MLUH|Allied Builders Sey Ltd|6,006,410.37| -|171|13/8|Copolia Road widening-Phase 2|SLTA|Belverdere Builders|SR1,037,235.00| -|172|20/8|Procurement of ABB Turbocharger Cartridge|PUC|ABB France|Euro106,266.66| -|173|20/8|Procurement of services to carry out the full refit and overhaul of tug Alouette|SPA|SECREN (Madagascar)|Euro189,618.42| -|174|20/8|Awarding of cranshatf and Block replacement solution for A11 engine|PUC|Wartsila|Euro800,000.00| -|175|29/8|Servicing of geartrain for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro 21,141.90| -|176|29/8|Spare parts for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro38,440.00| -|177|29/8|Procurement for sience equipment and chemical for 2013|MOE|Findel Education|GBP37,831.26| -|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| -|179|29/8||Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA|Furui Construction Pty Ltd|SR277,521.45| -|180|29/8|Propsed road widening and drainage improvement at Quincy Vilage|SLTA|TCH Contractor|SR1,015,806.24| -|181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| -|182|3/9|Procurement of Bitumen in drums for the asphalt production Praslin|SLTA|Benzene International Pte Ltd|Euro87,220.00| -|183|3/9|Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S|PUC|Wartsila Eastern Africa|Euro9,861.00| -|184|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| -||SEP||||| -|185|10/9|Cleaning and maintenance of wetlands and rivers on Praslin|ED|"W" Cleaning Service|SR769,590.00| -|186|10/9|Re-construction and partition of Independence House|MLUH|Green Island Construction Co Pty|SR868,022.94| -|187|10/9|Procurement of critical spare fro major overhaul|PUC|Wartsila Eastern Afirca|Euro33,623.00| -|188|10/9|Procurement of control panel for six pumps station|PUC|CA Engineering Consultancy Pte Ltd|SGD37,400.00| -|189|17/9|Security service for Wellness Centre|MOH|Alliance Security|SR78,624.00| -|190|17/9|Procurement of reinforce plastic (FRP) handrails and grating|PUC|Webforge Group|SR839,795.02| -|191|17/9|Procurement of helital fitting|PUC|Cu A1 Engineering (Pty) Ltd|ZAR343,000.00| -|192|17/9|Procurement of Flygt pumps|PUC|Aqualia DPI LTD|Euro56,080.00| -|193|17/9|Procurement of black-up and emergency pumps|PUC|M.A.H.Y Khoory & Co|UAE186,650.00| -|194|17/9|Procurement of additional requirement of polo t-shirts and jeans|PUC|Magilyn Ltee|USD42,127.50| -|195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| -|196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| -||||||5| - |197|17/9|Supply of pipes and fittings for Network diversion in Victoria|PUC|Ascent Projects (Sey) Pty Ltd|USD181,661.00| |---|---|---|---|---|---| |198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| @@ -505,54 +261,19 @@ |213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| |213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| -|197|17/9|Supply of pipes and fittings for Network diversion in Victoria|PUC|Ascent Projects (Sey) Pty Ltd|USD181,661.00| -|---|---|---|---|---|---| -|198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| -|199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| -|200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| -|201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00| -||OCT||||| -|202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| -|203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00| -|204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| -|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| -|206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| -|207|1/10|Procurement of Viking Johnson fittings|PUC|Viking Johnson (UK)|GBP92,286.50| -|208|1/10|Procurement of HDPE Pipes and Fittings|PUC|STR Marketing Ltee|USD196,672.57| -|209|1/10|Procurement of Piston and Gudgeon Pins|PUC|Marine Power International FZC|Euro277,098.00| -|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 1|PUC|Ms. Suzanne Edmond|SR500.00| -|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 2|PUC|Ms. Suzanne Edmond|SR200.00| -|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 3|PUC|Ms. Suzanne Edmond|SR475.00| -|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 4|PUC|Sey Sytle|SR450.00| -|211|8/10|Procurement of vehicle x 2|FIU|PMC Auto Pty Ltd|SR526,864.00| -|212|15/10|Renovation Works to Le Chantier Mall-Variation Works|SSF|Allied Builders Sey Ltd|SR1,673,752.51| -|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| -|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| -|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 4|MSACDS|Security Protection Services|SR71,000.00| -|214|15/10|Anse Boileau Footpath and Drainage Construction Phase 2 - Variations|SLTA|TCH Contractor|SR326,780.00| -|215|15/10|Construction of Fooothpath at Olivier Maradan Street-Variations||Benoiton Construction Pty Ltd|SR368,642.50| -|216|15/10|Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014|PUC|Tornado Group (UAE)|USD1,214,400.00| -|217|15/10|Procurement of Non Critical Spare Parts for Wartsila Engine 5B||MAN Diesel & Turbo (UK)|GDP88,360.88| -|218|15/10|Procurement of Meter Boxes|PUC|CAHORS (France)|Euro35,221.20| -|219|15/10|Construction of Services Road to Commercial Zone West of Inter Island Quay|MLUH|Allied Builders Sey Ltd|SR6,234,343.36| -|220|15/10|Proposed New traffic lane from Roche Caiman to Eden Island|SLTA|Franky's Constrcution|SR3,017,155.00| -|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Seytex|SR695,810.58| -|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|VCS Pty Ltd|SR36,430.00| -|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|KIS Distribution Company|SR10,500.00| -|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|MNM General Supply|SR16,216.20| -|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Roy & Sons Import|SR107,625.00| -|222|22/10|Road Widening and Drainage Improvement at Quincy Village-Variations|SLTA|TCH Building Contractor|SR328,213.75| -|223|22/10|Procurement of Grunfos Pumps for Mare Aux Cochons|PUC|Bluezone Mauritius|Euro44,914.00| -|224|22/10|Procurement of Vehicle x 3|MLUH|Sun Motors|SR2,310,000.00| -|225|22/10|Procurement of Agriculture Inputs|SAA|Launch Export (SA)|ZAR720,100.00| -|226|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09||Allied Builders Sey Ltd|SR14,964,125.91| -|227|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08||Allied Builders Sey Ltd|SR35,927,978.92| -|228|29/10|Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri|MLUH|Allied Builders Sey Ltd|SR10,372,824.02| -|229|29/10|Construction of One Block of Condominium Flats at Pointe Larue|MLUH|Franky's Constrcution|SR18,063,876.00| -|230|29/10|Proposed Construction of Micro-Enterprise Building at Providence|MLUH|O Nivo Construction|SR28,462,412.00| -|231|29/10|Renovation of English River Secondary Schools|MLUH|Divy Constrution|SR2,133,560.00| -|232|29/10|Renovation and Partitioning of Independence House-Variation|MLUH|Green Island Construction Co Pty|SR1,847,477.43| -||||||6| +213 15/10 Security Services for District's Administration Offices and Community Centres-Lot 4 214 15/10 Anse Boileau Footpath and Drainage Construction Phase 2 - Variations 215 15/10 Construction of Fooothpath at Olivier Maradan Street-Variations 216 15/10 Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014 217 15/10 Procurement of Non Critical Spare Parts for Wartsila Engine 5B 218 15/10 Procurement of Meter Boxes 219 15/10 Construction of Services Road to Commercial Zone West of Inter Island Quay 220 15/10 Proposed New traffic lane from Roche Caiman to Eden Island 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 222 22/10 Road Widening and Drainage Improvement at Quincy Village-Variations 223 22/10 Procurement of Grunfos Pumps for Mare Aux Cochons 224 22/10 Procurement of Vehicle x 3 225 22/10 Procurement of Agriculture Inputs 226 22/10 Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09 22/10 Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08 29/10 Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri 29/10 Construction of One Block of Condominium Flats at Pointe Larue 29/10 Proposed Construction of Micro-Enterprise Building at Providence 29/10 Renovation of English River Secondary Schools 29/10 Renovation and Partitioning of Independence House-Variation + +MSACDS SLTA + +PUC + +PUC MLUH SLTA MOE MOE MOE MOE MOE SLTA PUC MLUH SAA + +MLUH MLUH MLUH MLUH MLUH + +Security Protection Services TCH Contractor Benoiton Construction Pty Ltd Tornado Group (UAE) MAN Diesel & Turbo (UK) CAHORS (France) Allied Builders Sey Ltd Franky's Constrcution Seytex VCS Pty Ltd KIS Distribution Company MNM General Supply Roy & Sons Import TCH Building Contractor Bluezone Mauritius Sun Motors Launch Export (SA) Allied Builders Sey Ltd Allied Builders Sey Ltd Allied Builders Sey Ltd Franky's Constrcution O Nivo Construction Divy Constrution Green Island Construction Co Pty + +SR71,000.00 SR326,780.00 SR368,642.50 USD1,214,400.00 GDP88,360.88 Euro35,221.20 SR6,234,343.36 SR3,017,155.00 SR695,810.58 SR36,430.00 SR10,500.00 SR16,216.20 SR107,625.00 SR328,213.75 Euro44,914.00 SR2,310,000.00 ZAR720,100.00 SR14,964,125.91 SR35,927,978.92 SR10,372,824.02 SR18,063,876.00 SR28,462,412.00 SR2,133,560.00 SR1,847,477.43 |298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| |---|---|---|---|---|---| @@ -573,23 +294,3 @@ |312|26/12|Supply of Metal Fencing|MOE|BBT (UK)|GBP105,278.00| |313|26/12|Procurement of Mobile Dental Clinic / Surgeries x 2|MOH|Quayle Dental (UK)|GBP233,842.00| -|298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| -|---|---|---|---|---|---| -|299|17/12|Supply of New Equipment for Kitchen||Prison Service K. K. Chua|SR943,000.00| -|300|23/12|Procurement of Vehicle x 1|MLUH|Sun Motors|SR770,000.00| -|300|23/12|Procurement of Vehicle x 1|MLUH|Abhaye Valabhji|SR435,000.00| -|301|23/12|Procurement of Spares for LT Water Circulating Pump|PUC|Wartsila Global Logistcs Services|Euro 19,389.00| -|302|23/12|Procurement of Filter Cartridge-Desalination Plants|PUC|Trans Crescent Technical Equipment Company|Euro22,030.00| -|303|23/12|Procurement of LED Streetlights|SLTA|Lighting Orient (China)|USD104,282.00| -|304|23/12|Proposed New Road at Cascade Primary School-Variations|SLTA|Esparon's Enterprise|SR861,400.00| -|305|23/12|General renovation at School Section at MOE Headquarters-Variations|MOE|Prime Builders|SR2,761,935.75| -|306|23/12|Remedial Works at Mont Fleuri Primary School and Creche-Variations|MOE|Sai-Fu Enterprise Company Ltd|SR1,006,778.50| -|307|23/12|General Renovation of Toilet at La Digue School|MOE|Furui Construction|SR338,785.40| -|308|26/12|Procurement of 2500 Ream of A4 Paper|MOE|Island Motors|SR1,200,000.00| -|309|26/12|Procurement of Additional of 400 Desktop Computers|MOE|Orion Computers|SR3,100,000.00| -|310|26/12|Procuremet of Vehcile x 5|MOE|PMC Auto Pty Ltd|SR1,374,897.00| -|311|26/12|Procurement of Canon Ink / Riso Meter|MOE|Paradise Computer Services|SR473,000.00| -|312|26/12|Supply of Metal Fencing|MOE|BBT (UK)|GBP105,278.00| -|313|26/12|Procurement of Mobile Dental Clinic / Surgeries x 2|MOH|Quayle Dental (UK)|GBP233,842.00| -||||||9| - From 14e5dbbec79d94586d568db23da947ca4193dcdb Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 13:02:35 -0700 Subject: [PATCH 40/46] fix(tables): require 50% band coverage for struct-tree tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Struct-tree tables with incomplete page tagging (e.g., only 22 of 50+ rows tagged on a page) would claim items and block rect detection, leaving unclaimed items as loose text. Now require struct-tree tables to capture ≥50% of band items before using them; incomplete trees fall through to geometry-based detection. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/markdown/mod.rs | 11 ++++++--- tests/snapshots/2013-app2.md | 43 ++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 6070c2d..23ab03e 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -658,9 +658,16 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( let mut rect_claimed: HashSet = HashSet::new(); // 0. Structure-tree detection (highest priority — semantic PDF tagging) + // Only use struct-tree tables when they capture a majority (≥50%) of + // band items. Incomplete struct trees (partial tagging) should fall + // through to geometry detection which sees all items. if !struct_tables.is_empty() { let st_tables = detect_tables_from_struct_tree(band_items, struct_tables, page); for table in &st_tables { + let coverage = table.item_indices.len() as f32 / band_items.len().max(1) as f32; + if coverage < 0.5 { + continue; + } for &idx in &table.item_indices { rect_claimed.insert(idx); if let Some(&page_idx) = band_index_map.get(idx) { @@ -678,12 +685,10 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } - // 1. Rect-based detection (always runs; struct-tree claims prevent - // double-counting via table_items but rect may find additional tables) + // 1. Rect-based detection (skips tables overlapping struct-tree claims) let (rect_tables, hint_regions) = detect_tables_from_rects(band_items, band_rects, page); for table in &rect_tables { - // Skip rect tables that overlap with struct-tree claimed items if !rect_claimed.is_empty() && table .item_indices diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index 711d202..e5d4030 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -243,9 +243,9 @@ |199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| |200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| |201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00| -|OCT|||||| +||OCT||||| |202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| -|203|1/10|Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations|SLTA|G & S Enterprise|SR200,448.00| +|203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00| |204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| |205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| |206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| @@ -260,20 +260,31 @@ |212|15/10|Renovation Works to Le Chantier Mall-Variation Works|SSF|Allied Builders Sey Ltd|SR1,673,752.51| |213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| |213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| - -213 15/10 Security Services for District's Administration Offices and Community Centres-Lot 4 214 15/10 Anse Boileau Footpath and Drainage Construction Phase 2 - Variations 215 15/10 Construction of Fooothpath at Olivier Maradan Street-Variations 216 15/10 Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014 217 15/10 Procurement of Non Critical Spare Parts for Wartsila Engine 5B 218 15/10 Procurement of Meter Boxes 219 15/10 Construction of Services Road to Commercial Zone West of Inter Island Quay 220 15/10 Proposed New traffic lane from Roche Caiman to Eden Island 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 221 22/10 Procurement of Textbooks for Primary and Secondary Schools 222 22/10 Road Widening and Drainage Improvement at Quincy Village-Variations 223 22/10 Procurement of Grunfos Pumps for Mare Aux Cochons 224 22/10 Procurement of Vehicle x 3 225 22/10 Procurement of Agriculture Inputs 226 22/10 Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09 22/10 Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08 29/10 Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri 29/10 Construction of One Block of Condominium Flats at Pointe Larue 29/10 Proposed Construction of Micro-Enterprise Building at Providence 29/10 Renovation of English River Secondary Schools 29/10 Renovation and Partitioning of Independence House-Variation - -MSACDS SLTA - -PUC - -PUC MLUH SLTA MOE MOE MOE MOE MOE SLTA PUC MLUH SAA - -MLUH MLUH MLUH MLUH MLUH - -Security Protection Services TCH Contractor Benoiton Construction Pty Ltd Tornado Group (UAE) MAN Diesel & Turbo (UK) CAHORS (France) Allied Builders Sey Ltd Franky's Constrcution Seytex VCS Pty Ltd KIS Distribution Company MNM General Supply Roy & Sons Import TCH Building Contractor Bluezone Mauritius Sun Motors Launch Export (SA) Allied Builders Sey Ltd Allied Builders Sey Ltd Allied Builders Sey Ltd Franky's Constrcution O Nivo Construction Divy Constrution Green Island Construction Co Pty - -SR71,000.00 SR326,780.00 SR368,642.50 USD1,214,400.00 GDP88,360.88 Euro35,221.20 SR6,234,343.36 SR3,017,155.00 SR695,810.58 SR36,430.00 SR10,500.00 SR16,216.20 SR107,625.00 SR328,213.75 Euro44,914.00 SR2,310,000.00 ZAR720,100.00 SR14,964,125.91 SR35,927,978.92 SR10,372,824.02 SR18,063,876.00 SR28,462,412.00 SR2,133,560.00 SR1,847,477.43 +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 4|MSACDS|Security Protection Services|SR71,000.00| +|214|15/10|Anse Boileau Footpath and Drainage Construction Phase 2 - Variations|SLTA|TCH Contractor|SR326,780.00| +|215|15/10|Construction of Fooothpath at Olivier Maradan Street-Variations||Benoiton Construction Pty Ltd|SR368,642.50| +|216|15/10|Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014|PUC|Tornado Group (UAE)|USD1,214,400.00| +|217|15/10|Procurement of Non Critical Spare Parts for Wartsila Engine 5B||MAN Diesel & Turbo (UK)|GDP88,360.88| +|218|15/10|Procurement of Meter Boxes|PUC|CAHORS (France)|Euro35,221.20| +|219|15/10|Construction of Services Road to Commercial Zone West of Inter Island Quay|MLUH|Allied Builders Sey Ltd|SR6,234,343.36| +|220|15/10|Proposed New traffic lane from Roche Caiman to Eden Island|SLTA|Franky's Constrcution|SR3,017,155.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Seytex|SR695,810.58| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|VCS Pty Ltd|SR36,430.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|KIS Distribution Company|SR10,500.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|MNM General Supply|SR16,216.20| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Roy & Sons Import|SR107,625.00| +|222|22/10|Road Widening and Drainage Improvement at Quincy Village-Variations|SLTA|TCH Building Contractor|SR328,213.75| +|223|22/10|Procurement of Grunfos Pumps for Mare Aux Cochons|PUC|Bluezone Mauritius|Euro44,914.00| +|224|22/10|Procurement of Vehicle x 3|MLUH|Sun Motors|SR2,310,000.00| +|225|22/10|Procurement of Agriculture Inputs|SAA|Launch Export (SA)|ZAR720,100.00| +|226|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09||Allied Builders Sey Ltd|SR14,964,125.91| +|227|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08||Allied Builders Sey Ltd|SR35,927,978.92| +|228|29/10|Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri|MLUH|Allied Builders Sey Ltd|SR10,372,824.02| +|229|29/10|Construction of One Block of Condominium Flats at Pointe Larue|MLUH|Franky's Constrcution|SR18,063,876.00| +|230|29/10|Proposed Construction of Micro-Enterprise Building at Providence|MLUH|O Nivo Construction|SR28,462,412.00| +|231|29/10|Renovation of English River Secondary Schools|MLUH|Divy Constrution|SR2,133,560.00| +|232|29/10|Renovation and Partitioning of Independence House-Variation|MLUH|Green Island Construction Co Pty|SR1,847,477.43| +||||||6| |298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| |---|---|---|---|---|---| From 44092bcc9e2976827b22e863f02b6f632ebca641 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 16:57:51 -0700 Subject: [PATCH 41/46] feat(tables): borderless table detection improvements (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(tables): improve heuristic detection for borderless wrapped-cell tables Three changes to the body-font heuristic detector: 1. Adaptive Y-gap in find_table_regions_strict: use median qualifying-row spacing × 3 instead of fixed 25pt. Tables with wrapped cells have larger gaps between qualifying rows (those with 3+ X-clusters). 2. Y-only region filtering: use full X range when collecting region items. The strict X bounds from qualifying rows excluded continuation lines in wrapped cells, starving find_column_boundaries of items. 3. Merged-band retry: when split_side_by_side splits a page into bands but no band produces a table, retry heuristic detection on all items merged as a single band. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): gap-histogram column detection for small tables + lower avg_cells Two changes to fix PDF 045 (borderless table with narrow "No." column): 1. Extend gap-histogram column threshold to small tables: when the gap between within-column jitter and between-column spacing is >10pt (unambiguous bimodal signal), use the detected threshold even with fewer than 500 items. Previously only triggered for dense tables. 2. Lower BodyFont avg_cells_per_row minimum from 2.5 to 2.0 to handle tables with wrapped multi-line cells where continuation lines have only 1 filled cell. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(tables): trim empty outer columns + relax partial H-line validation - Rect detection: trim empty first/last columns instead of rejecting the whole table. Rect edges often extend beyond text boundaries. - Line detection: accept tables with 6+ partial horizontal lines (>15% width) when <3 full-spanning lines exist. Handles tables with column-level separators instead of full-width rules. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): cell-rect fallback for tables with variable-width backgrounds When rect clustering produces a grid that fails validation (empty interior columns from variable-width cell backgrounds), fall through to a new strategy: use rect Y-edges for row boundaries and text X-position clustering for columns. This handles tables like the opendataloader-bench 088-090 comparison tables where each cell has its own background rect at different widths. Also widen failed-cluster hint width cap for large clusters (≥30 rects) to allow page-spanning table regions. TEDS score on opendataloader-bench: 0.300 → 0.353. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(tables): relax vertical line spanning validation for partial borders Accept tables with 4+ partial vertical lines (>10% table height) when fewer than 2 span >30%. Handles tables like opendataloader-bench 053 with column-level vertical separators that don't extend the full height. TEDS: 0.353 → 0.377 on opendataloader-bench. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(tables): lower cell-rect density threshold + add validation logging - Lower cell-rect density minimum from 25% to 15% to accept sparser tables with decorative backgrounds (fixes 147). - Add debug logging to all heuristic validation paths for diagnosability. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): relax body-font validations for text-only and 2-column tables Four fixes closing 71% of the TEDS gap vs opendataloader: 1. Validation 7 (table-like content): bypass numeric content requirement for tables with 3+ columns that passed all structural checks. Text-only tables (category lists, program descriptions) are legitimate. 2. Qualifying row threshold: lower from 3+ to 2+ X-clusters per row. Enables 2-column body-font table detection (fixes 166). 3. Row-stripe max cell length: raise from 500 to 2000 for 3+ column tables. Tables with paragraph descriptions in one column are valid (fixes 121). 4. Row-stripe empty-column trimming: apply the same outer-column trim as grid detection (fixes 121 column-0 rejection). TEDS: 0.377 → 0.438 on opendataloader-bench (gap: -0.056 vs odl). TEDS=0 docs: 14 → 9. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): enable 2-column body-font tables + lower all minimums - Lower BodyFont minimum columns from 3 to 2 in detect_table_in_region - Lower BodyFont minimum rows from 3 to 2 - Lower avg_cells_per_row minimum from 2.0 to 1.5 (handles wrapped cells in 2-column tables) - Apply empty-outer-column trimming to row-stripe detection (not just grid) TEDS: 0.438 → 0.468 on opendataloader-bench (gap: -0.027 vs odl). TEDS=0 docs: 9 → 8. 86% of original gap closed. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): text-based row fallback + fix 120 flow-chart and 188 leaderboard Three changes that push TEDS past opendataloader: 1. Cell-rect Y-edge fallback: when rects have too few Y-edges for row structure, derive rows from text Y-position clustering within the rect bounding box. Fixes flow-chart tables (120) and column-header- only rects (188). 2. Lower cell-rect minimum from 20 to 6 rects to catch smaller tables. 3. Relax validation 1 (first-column presence) from 50% to 25% of rows. Tables with wrapped model names have continuation lines without first column content. TEDS: 0.468 → 0.508 on opendataloader-bench. Now BEATS opendataloader (0.508 vs 0.494, gap=+0.014). TEDS=0 docs: 8 → 5. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(tables): wrapped-cell continuation row merging Merge rows that have fewer filled cells than the header row into the previous row. Handles wrapped multi-line cells where text overflow creates extra rows (e.g., "Direct" + "communications" → "Direct communications"). Conditions: fewer filled cells than header, more than previous row had, not a data row (numeric), not a short subheader label. TEDS: 0.508 → 0.522 on opendataloader-bench (now +0.028 vs odl). Co-Authored-By: Claude Opus 4.6 (1M context) * fix(tables): tighten cell-rect validation + fix continuation-row merging Cell-rect false positives: - Raise density threshold back to 25% (from 15%) - Add max cell length check (500 chars) to reject paragraph content - Reject disproportionate grids (>20 rows, <4 cols) Continuation-row merging: - Wide tables (5+ cols): only merge rows with ≤50% header cells - Narrow tables (2-4 cols): merge rows with fewer cells than header - Prevents merging normal data rows in large tables (6_KE_Chart) while keeping wrapped-cell merging for narrow tables (178) TEDS: 0.498 on opendataloader-bench (still +0.004 vs odl). pdf-evals: 191/192 passed, 0 regressions. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- src/markdown/mod.rs | 57 +++++ src/tables/detect_heuristic.rs | 134 +++++++++-- src/tables/detect_lines.rs | 53 ++++- src/tables/detect_rects.rs | 360 +++++++++++++++++++++++++++++- src/tables/format.rs | 36 ++- src/tables/grid.rs | 21 +- tests/snapshots/2013-app2.md | 24 +- tests/snapshots/nexo-price-en.md | 3 +- tests/snapshots/td9264.md | 209 ++++++++--------- tests/snapshots/thermo-freon12.md | 10 +- 10 files changed, 718 insertions(+), 189 deletions(-) diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 23ab03e..84a1471 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -649,6 +649,33 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .collect() }; + // When the page is split into bands but no band produces a table, + // retry with all items merged as a single band. This handles + // borderless tables whose column alignment is misclassified as + // page-layout columns by split_side_by_side. + let was_split = band_specs.len() > 1; + log::debug!( + "page {}: {} bands (was_split={})", + page, + band_specs.len(), + was_split + ); + let merged_band: BandSpec = if was_split { + let identity: Vec = (0..page_items.len()).collect(); + ( + page_items.clone(), + identity, + rects.iter().filter(|r| r.page == page).cloned().collect(), + pdf_lines + .iter() + .filter(|l| l.page == page) + .cloned() + .collect(), + ) + } else { + (Vec::new(), Vec::new(), Vec::new(), Vec::new()) + }; + for (band_items, band_index_map, band_rects, band_lines) in &band_specs { if band_items.is_empty() { continue; @@ -875,6 +902,36 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } } + + // Merged-band retry: if we split into bands but found no tables in + // any band, retry heuristic detection with all items as a single band. + // This catches borderless tables whose text-column alignment was + // misclassified as page-layout columns. + if was_split && !page_tables.contains_key(&page) && !merged_band.0.is_empty() { + let (ref band_items, ref band_index_map, _, _) = merged_band; + log::debug!( + "page {}: merged-band retry ({} items, was_split={})", + page, + band_items.len(), + was_split + ); + let heuristic_tables = detect_tables(band_items, base_size, false); + for table in &heuristic_tables { + for &idx in &table.item_indices { + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } } // Check structure tree coverage on ALL text items (before table filtering) diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 80ecb07..d2579f2 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -212,18 +212,35 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo }) .collect(); + log::debug!( + "body-font pass: {} candidates (base={:.1}, range={:.1}..{:.1})", + body_candidates.len(), + base_font_size, + body_font_low, + body_font_high, + ); if body_candidates.len() >= 9 { let regions = find_table_regions_strict(&body_candidates); + log::debug!("body-font: {} strict regions found", regions.len()); - for (y_min, y_max, x_min, x_max) in regions { + for (y_min, y_max, _x_min, _x_max) in ®ions { + // Use full X range for region items — the strict X bounds from + // qualifying rows can exclude continuation lines in wrapped cells. + // Y bounds from the region are sufficient to scope the table area. let region_items: Vec<(usize, &TextItem)> = body_candidates .iter() - .filter(|(_, item)| { - item.y >= y_min && item.y <= y_max && item.x >= x_min && item.x <= x_max - }) + .filter(|(_, item)| item.y >= *y_min && item.y <= *y_max) .cloned() .collect(); + log::debug!( + " region y={:.0}..{:.0}: {} items of {} candidates", + y_min, + y_max, + region_items.len(), + body_candidates.len() + ); + if region_items.len() < 9 { continue; } @@ -249,6 +266,12 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo .collect(); table.item_indices = original_indices.into_iter().collect(); table.item_indices.sort_unstable(); + log::debug!( + " heuristic table: {}x{}, {} item indices", + table.rows.len(), + table.columns.len(), + table.item_indices.len() + ); } tables @@ -339,24 +362,43 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 } } - if cluster_starts.len() >= 3 { + if cluster_starts.len() >= 2 { qualifying_rows.push((*y, cluster_starts)); } } + log::debug!( + "find_table_regions_strict: {} row groups, {} qualifying (2+ X-clusters)", + row_groups.len(), + qualifying_rows.len() + ); if qualifying_rows.len() < 3 { return vec![]; } - // Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap) + // Step 3: Find contiguous runs of qualifying rows. + // Use adaptive gap: median spacing × 3 (handles wrapped cells where + // qualifying rows are spaced further apart), with a floor of 25pt. qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + let max_gap = if qualifying_rows.len() >= 3 { + let mut gaps: Vec = qualifying_rows + .windows(2) + .map(|w| (w[1].0 - w[0].0).abs()) + .collect(); + gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_gap = gaps[gaps.len() / 2]; + (median_gap * 3.0).max(25.0) + } else { + 25.0 + }; + let mut candidate_regions: Vec)>> = Vec::new(); let mut current_region: Vec<&(f32, Vec)> = vec![&qualifying_rows[0]]; for row in qualifying_rows.iter().skip(1) { let prev_y = current_region.last().unwrap().0; - if row.0 - prev_y > 25.0 { + if row.0 - prev_y > max_gap { if current_region.len() >= 3 { candidate_regions.push(current_region); } @@ -406,6 +448,11 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 } else { 0.0 }; + log::debug!( + " candidate region: {} rows, avg alignment score={:.2}", + num_rows, + avg_score + ); if avg_score >= 0.5 { let y_min = region_rows.first().unwrap().0; let y_max = region_rows.last().unwrap().0; @@ -431,24 +478,35 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Option
    { // Find column boundaries let columns = find_column_boundaries(items, mode); - let min_cols = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; + let min_cols = 2; if columns.len() < min_cols || columns.len() > 25 { + log::debug!( + " detect_table_in_region: rejected {} cols (need {}..25)", + columns.len(), + min_cols + ); return None; } // Find row boundaries let rows = find_row_boundaries(items); - let min_rows = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; + let min_rows = 2; if rows.len() < min_rows { + log::debug!( + " detect_table_in_region: rejected {} rows (need {}+)", + rows.len(), + min_rows + ); return None; } + log::debug!( + " detect_table_in_region: {} cols, {} rows, {} items", + columns.len(), + rows.len(), + items.len() + ); + // Verify this looks like a table: multiple items should align to columns let col_alignment = check_column_alignment(items, &columns, mode); let min_alignment = match mode { @@ -456,6 +514,13 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode TableDetectionMode::BodyFont => 0.7, }; if col_alignment < min_alignment { + log::debug!( + " detect_table_in_region: rejected alignment {:.2} < {:.2} ({} cols, {} rows)", + col_alignment, + min_alignment, + columns.len(), + rows.len() + ); return None; } @@ -515,9 +580,16 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode cells.push(row_cells); } - // Validation 1: most rows should have content in first column + // Validation 1: some rows should have content in first column. + // Use a lower threshold (25%) for tables with wrapped cells where + // continuation lines leave the first column empty. let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count(); - if rows_with_first_col < rows.len() / 2 { + if rows_with_first_col < rows.len() / 4 { + log::debug!( + " validation 1 fail: {}/{} rows have first col", + rows_with_first_col, + rows.len() + ); return None; } @@ -531,6 +603,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode TableDetectionMode::BodyFont => (rows.len() / 2).max(1), // 50% }; if rows_with_multi_cols < multi_col_threshold { + log::debug!( + " validation 2 fail: {}/{} rows multi-col (need {})", + rows_with_multi_cols, + rows.len(), + multi_col_threshold + ); return None; } @@ -549,36 +627,43 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode .map(|row| row.iter().filter(|c| !c.is_empty()).count()) .sum(); let avg_cells_per_row = total_filled as f32 / rows.len() as f32; - let min_avg_cells = match mode { - TableDetectionMode::SmallFont => 1.5, - TableDetectionMode::BodyFont => 2.5, - }; + let min_avg_cells = 1.5; if avg_cells_per_row < min_avg_cells { + log::debug!( + " validation 4 fail: avg_cells={:.1} < {:.1}", + avg_cells_per_row, + min_avg_cells + ); return None; } // Validation 5: Check for key-value pair layout (NOT a table) if is_key_value_layout(&cells) { + log::debug!(" validation 5 fail: key-value layout"); return None; } // Validation 6: Check column count consistency if !has_consistent_columns(&cells) { + log::debug!(" validation 6 fail: inconsistent columns"); return None; } // Validation 7: Tables should have some numeric/data content if !has_table_like_content(&cells, mode) { + log::debug!(" validation 7 fail: no table-like content"); return None; } // Validation 8: Check for Table of Contents pattern if is_table_of_contents(&cells) { + log::debug!(" validation 8 fail: table of contents"); return None; } // Validation 9: Reject paragraph-like content falsely detected as tables if is_paragraph_content(&cells) { + log::debug!(" validation 9 fail: paragraph content"); return None; } @@ -715,9 +800,10 @@ fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bo TableDetectionMode::BodyFont => 0.3, }; - // For SmallFont, bypass content check for wide tables (5+ columns may have text headers). - // For BodyFont, always require data-like content to prevent paragraph false positives. - pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5) + // Bypass content check for wide tables (3+ columns) — text-only tables + // (category lists, program descriptions) are legitimate if they passed + // all structural validations (alignment, consistency, not key-value). + pct_data > min_pct || num_cols >= 3 } /// Check if a cell value looks like table data diff --git a/src/tables/detect_lines.rs b/src/tables/detect_lines.rs index 79f8882..8dfea17 100644 --- a/src/tables/detect_lines.rs +++ b/src/tables/detect_lines.rs @@ -72,6 +72,13 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 let v_xs: Vec = verticals.iter().map(|(x, _, _)| *x).collect(); let col_edges = snap_edges(&v_xs, 3.0); + log::debug!( + "detect_lines p{}: {} row edges, {} col edges after snap", + page, + row_edges.len(), + col_edges.len() + ); + // Require at least 2 columns (3 col edges) and 2 rows (3 row edges). // A single column of horizontal lines is just separator rules, not a table. if row_edges.len() < 3 || col_edges.len() < 3 { @@ -80,6 +87,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 // Cap grid size: >20 columns is almost certainly a diagram, not a table if col_edges.len() > 21 || row_edges.len() > 80 { + log::debug!( + "detect_lines p{}: rejected — too many edges ({}x{})", + page, + row_edges.len(), + col_edges.len() + ); return Vec::new(); } @@ -101,26 +114,54 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 // dimension in both axes, it's a border frame, not a table. // Standard pages are ~595×842 (A4) or ~612×792 (Letter). if table_width > 500.0 && table_height > 700.0 { + log::debug!( + "detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})", + page, + table_width, + table_height + ); return Vec::new(); } - // Validate horizontal lines: at least 3 should span >50% of table width. - // This filters out short segments that accidentally align. + // Validate horizontal lines: at least 3 should span a meaningful width. + // Full-width spanning (>50%) is ideal, but tables with partial horizontal + // rules (column-level separators) are also valid if there are enough. let spanning_h = horizontals .iter() .filter(|(_, x_min, x_max)| (x_max - x_min) > table_width * 0.5) .count(); - if spanning_h < 3 { + let partial_h = horizontals + .iter() + .filter(|(_, x_min, x_max)| (x_max - x_min) > table_width * 0.15) + .count(); + if spanning_h < 3 && partial_h < 6 { + log::debug!( + "detect_lines p{}: rejected — {} spanning + {} partial H lines", + page, + spanning_h, + partial_h + ); return Vec::new(); } - // Validate vertical lines: at least 2 should span >30% of table height. - // Real table columns extend most of the table height. + // Validate vertical lines: at least 2 should span a meaningful height. + // Full spanning (>30%) is ideal, but accept many shorter lines (>10%) + // for tables with partial column separators. let spanning_v = verticals .iter() .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3) .count(); - if spanning_v < 2 { + let partial_v = verticals + .iter() + .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10) + .count(); + if spanning_v < 2 && partial_v < 4 { + log::debug!( + "detect_lines p{}: rejected — {} spanning + {} partial V lines", + page, + spanning_v, + partial_v + ); return Vec::new(); } diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 83330c6..cfdac1e 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -430,6 +430,27 @@ pub fn detect_tables_from_rects( } } + // Cell-rect fallback: when per-cluster attempts all fail, try using + // rect Y-edges for rows + text X-positions for columns on each failed + // cluster. Handles tables with cell-background rects that don't form + // a clean grid (variable column widths, decoration fills). + if tables.is_empty() { + debug!( + "page {}: cell-rect fallback: {} failed clusters", + page, + failed_clusters.len() + ); + for fc_rects in &failed_clusters { + if fc_rects.len() >= 6 { + if let Some(table) = + detect_row_stripe_table_from_cell_rects(items, fc_rects, page) + { + tables.push(table); + } + } + } + } + // Row-stripe fallback: when clustering produces no large clusters // (row stripes don't overlap so each is its own cluster of 1), // try all page rects directly as a row-stripe table. @@ -526,8 +547,11 @@ pub fn detect_tables_from_rects( .count(); let w = x_right - x_left; // Require reasonable dimensions: height ≥100pt (≈5+ rows), - // width ≤500pt (not page-spanning), height ≤600pt (not full page) - if (100.0..=600.0).contains(&h) && w <= 500.0 && items_inside >= 6 { + // height ≤600pt (not full page). + // Width check: ≤500pt normally, but allow wider for large + // clusters (≥30 rects) that are clearly structured. + let max_w = if fc_rects.len() >= 30 { 800.0 } else { 500.0 }; + if (100.0..=600.0).contains(&h) && w <= max_w && items_inside >= 6 { debug!( "page {}: failed-cluster hint from {} rects ({} items): x={:.1}..{:.1} y={:.1}..{:.1} ({:.0}×{:.0})", page, fc_rects.len(), items_inside, x_left, x_right, y_bottom, y_top, @@ -966,16 +990,52 @@ fn try_build_grid( } } - // Reject tables with any completely empty column — indicates a bad grid. - for col in 0..num_cols { + // Trim empty outer columns (rect edges beyond text), reject if any + // interior column is empty — that indicates a bad grid. + let first_non_empty = (0..num_cols).find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let last_non_empty = (0..num_cols).rev().find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let (first_col, last_col) = match (first_non_empty, last_non_empty) { + (Some(f), Some(l)) if l > f => (f, l), + _ => { + debug!(" rejected: no content columns"); + return GridResult::Failed; + } + }; + // Check interior columns + for col in first_col..=last_col { let col_has_content = cells .iter() .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); if !col_has_content { - debug!(" rejected: column {} is completely empty", col); + debug!(" rejected: interior column {} is completely empty", col); return GridResult::Failed; } } + // Trim outer empty columns + let (columns, cells) = if first_col > 0 || last_col < num_cols - 1 { + let trimmed_cols: Vec = columns[first_col..=last_col].to_vec(); + let trimmed_cells: Vec> = cells + .iter() + .map(|row| row[first_col..=last_col].to_vec()) + .collect(); + debug!( + " trimmed {} empty outer columns ({}..={})", + (num_cols - 1 - last_col + first_col), + first_col, + last_col + ); + (trimmed_cols, trimmed_cells) + } else { + (columns, cells) + }; GridResult::Ok(Table { columns, @@ -1320,24 +1380,53 @@ fn detect_row_stripe_table( .map(|c| c.len()) .max() .unwrap_or(0); - if max_cell_len > 500 { + // Allow longer cells for multi-column tables (descriptions in one column + // are common). Single-column or 2-column "tables" with giant cells are + // almost always layout backgrounds. + let max_allowed = if num_cols >= 3 { 2000 } else { 500 }; + if max_cell_len > max_allowed { debug!( - " row-stripe rejected: max cell length {} > 500 (layout background)", - max_cell_len + " row-stripe rejected: max cell length {} > {} (layout background)", + max_cell_len, max_allowed ); return None; } - // No empty columns - for col in 0..num_cols { + // Trim empty outer columns, reject if interior columns are empty + let first_col = (0..num_cols).find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let last_col = (0..num_cols).rev().find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let (first_col, last_col) = match (first_col, last_col) { + (Some(f), Some(l)) if l > f => (f, l), + _ => return None, + }; + for col in first_col..=last_col { let col_has_content = cells .iter() .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); if !col_has_content { - debug!(" row-stripe rejected: column {} is empty", col); + debug!(" row-stripe rejected: interior column {} is empty", col); return None; } } + let (col_edges, cells) = if first_col > 0 || last_col < num_cols - 1 { + let new_edges: Vec = col_edges[first_col..=last_col + 1].to_vec(); + let new_cells: Vec> = cells + .iter() + .map(|row| row[first_col..=last_col].to_vec()) + .collect(); + (new_edges, new_cells) + } else { + (col_edges, cells) + }; + let num_cols = col_edges.len() - 1; let column_centers: Vec = (0..num_cols) .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) @@ -1361,6 +1450,255 @@ fn detect_row_stripe_table( }) } +/// Detect a table from cell-background rects that failed grid detection. +/// +/// Uses rect Y-edges for row boundaries and text X-position clustering for +/// columns. Handles tables with cell backgrounds that don't form a clean +/// X-edge grid (variable column widths, decorative fills). +fn detect_row_stripe_table_from_cell_rects( + items: &[TextItem], + group_rects: &[(f32, f32, f32, f32)], + page: u32, +) -> Option
    { + if group_rects.len() < 6 { + return None; + } + + // Extract Y-edges from rects + let mut y_edges: Vec = Vec::new(); + for &(_, y, _, h) in group_rects { + y_edges.push(y); + y_edges.push(y + h); + } + let y_edges = snap_edges(&y_edges, 6.0); + + // If rect Y-edges are insufficient for row structure, use the rect + // bounding box to scope items and derive rows from text Y-positions. + let row_edges = if y_edges.len() >= 4 { + let mut edges = y_edges; + edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + edges + } else { + // Fall back: gather items in the rect region and cluster by Y + let y_min = y_edges.first().copied().unwrap_or(0.0); + let y_max = y_edges.last().copied().unwrap_or(0.0); + let x_min = group_rects + .iter() + .map(|r| r.0) + .reduce(f32::min) + .unwrap_or(0.0); + let x_max = group_rects + .iter() + .map(|r| r.0 + r.2) + .reduce(f32::max) + .unwrap_or(0.0); + let region_items: Vec<&TextItem> = items + .iter() + .filter(|i| { + i.page == page + && i.y >= y_min - 5.0 + && i.y <= y_max + 5.0 + && i.x >= x_min - 5.0 + && i.x <= x_max + 5.0 + }) + .collect(); + if region_items.len() < 4 { + return None; + } + // Cluster Y positions using median font height as threshold + let median_h = { + let mut hs: Vec = region_items.iter().map(|i| i.height).collect(); + hs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + hs[hs.len() / 2] + }; + let mut ys: Vec = region_items.iter().map(|i| i.y).collect(); + ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + let mut edges = Vec::new(); + let threshold = median_h * 0.8; + let mut cluster_start = ys[0]; + let mut cluster_sum = ys[0]; + let mut cluster_count = 1.0f32; + for &y in &ys[1..] { + if (cluster_sum / cluster_count - y).abs() > threshold { + let center = cluster_sum / cluster_count; + edges.push(center + median_h * 0.5); + edges.push(center - median_h * 0.5); + cluster_start = y; + cluster_sum = y; + cluster_count = 1.0; + } else { + cluster_sum += y; + cluster_count += 1.0; + } + } + let center = cluster_sum / cluster_count; + edges.push(center + median_h * 0.5); + edges.push(center - median_h * 0.5); + let _ = cluster_start; // suppress unused warning + edges = snap_edges(&edges, 3.0); + edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + if edges.len() < 4 { + return None; + } + edges + }; + + // Compute bounding box from non-full-page rects + let median_h = { + let mut heights: Vec = group_rects.iter().map(|&(_, _, _, h)| h).collect(); + heights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + heights[heights.len() / 2] + }; + let content_rects: Vec<_> = group_rects + .iter() + .filter(|&&(_, _, _, h)| h < median_h * 10.0) + .collect(); + if content_rects.is_empty() { + return None; + } + + let x_left = content_rects + .iter() + .map(|&&(x, _, _, _)| x) + .reduce(f32::min)?; + let x_right = content_rects + .iter() + .map(|&&(x, _, w, _)| x + w) + .reduce(f32::max)?; + let y_top = row_edges[0]; + let y_bottom = *row_edges.last()?; + + // Gather items within the rect region + let page_items: Vec<(usize, &TextItem)> = items + .iter() + .enumerate() + .filter(|(_, item)| { + item.page == page + && item.y >= y_bottom - 2.0 + && item.y <= y_top + 2.0 + && item.x >= x_left - 5.0 + && item.x + item.width <= x_right + 5.0 + }) + .collect(); + + if page_items.is_empty() { + return None; + } + + // Derive columns from text X-position clustering + let columns = cluster_x_positions(&page_items, 15.0); + if columns.len() < 2 { + return None; + } + + // Build column edges + let mut col_edges: Vec = Vec::with_capacity(columns.len() + 1); + let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?; + col_edges.push(min_x - 5.0); + for pair in columns.windows(2) { + col_edges.push((pair[0] + pair[1]) / 2.0); + } + let max_x_right = page_items + .iter() + .map(|(_, i)| i.x + i.width) + .reduce(f32::max)?; + col_edges.push(max_x_right + 5.0); + + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + debug!( + " cell-rect table: {}x{} from {} rects, {} items", + num_rows, + num_cols, + group_rects.len(), + page_items.len() + ); + + let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + + if item_indices.is_empty() { + return None; + } + + // Validate: >=2 non-empty rows, >=25% density + let non_empty_rows = cells + .iter() + .filter(|row| row.iter().any(|c| !c.trim().is_empty())) + .count(); + if non_empty_rows < 2 { + debug!( + " cell-rect rejected: only {} non-empty rows", + non_empty_rows + ); + return None; + } + + let total_cells = (num_cols * num_rows) as f32; + let non_empty_cells = cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let density = if total_cells > 0.0 { + non_empty_cells as f32 / total_cells + } else { + 0.0 + }; + if density < 0.25 { + debug!( + " cell-rect rejected: density {:.0}% < 25%", + density * 100.0 + ); + return None; + } + + // Reject tables with paragraph-length cells (layout backgrounds, not tables) + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + if max_cell_len > 500 { + debug!( + " cell-rect rejected: max cell length {} > 500", + max_cell_len + ); + return None; + } + + // Reject wildly disproportionate grids (e.g. 68x6 from decorative rects) + if num_rows > 20 && num_cols < 4 { + debug!( + " cell-rect rejected: disproportionate grid {}x{}", + num_rows, num_cols + ); + return None; + } + + let column_centers: Vec = (0..num_cols) + .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) + .collect(); + let row_centers: Vec = (0..num_rows) + .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) + .collect(); + + debug!( + " cell-rect table accepted: {}x{}, {:.0}% density", + num_rows, + num_cols, + non_empty_cells as f32 / total_cells * 100.0 + ); + + Some(Table { + columns: column_centers, + rows: row_centers, + cells, + item_indices, + }) +} + /// Detect a table by merging all cluster rects into one group. /// /// This handles clip-path PDFs where each column's cell rects form a separate diff --git a/src/tables/format.rs b/src/tables/format.rs index 9d442a4..14e5aa3 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -107,11 +107,43 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { let looks_like_data_row = non_first_cells.len() >= 2 && avg_cell_len <= 10.0 && numeric_cells > non_first_cells.len() / 2; - let is_continuation = first_cell.is_empty() + // Classic continuation: first cell empty, content in other cells + let is_classic_continuation = first_cell.is_empty() && !non_first_cells.is_empty() && !is_short_subheader && !looks_like_data_row - && cleaned.len() > 1; // Don't merge into the first row (header) + && cleaned.len() > 1; + + // Wrapped-cell continuation: row has fewer filled cells than the header + // row, suggesting it's overflow text from the previous row's cells. + // Only trigger when the previous row has significantly more filled cells. + let num_cols = row.len(); + let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count(); + let prev_filled = cleaned + .last() + .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) + .unwrap_or(0); + let header_filled = cleaned + .first() + .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) + .unwrap_or(num_cols); + // Merge when the row has significantly fewer filled cells than header. + // For wide tables (5+ cols), require ≤50% of header cells. + // For narrow tables (2-4 cols), require fewer than header cells. + // This prevents merging normal data rows in wide tables (6_KE_Chart) + // while allowing continuation merging in narrow tables (178). + let max_filled_for_merge = if header_filled >= 5 { + header_filled / 2 + } else { + header_filled.saturating_sub(1) + }; + let is_wrapped_continuation = cleaned.len() > 1 + && filled_cells <= max_filled_for_merge + && prev_filled > filled_cells + && !looks_like_data_row + && !is_short_subheader; + + let is_continuation = is_classic_continuation || is_wrapped_continuation; if is_continuation { // Merge with previous row diff --git a/src/tables/grid.rs b/src/tables/grid.rs index e3361ef..950dde4 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -66,13 +66,19 @@ pub(crate) fn find_column_boundaries( let threshold = (consec_gaps[best_split] + consec_gaps[(best_split + 1).min(consec_gaps.len() - 1)]) / 2.0; - // Only override for genuinely dense tables: many items packed into - // a wide range (e.g. 1200+ items for a 24-column train schedule). - // Smaller item counts (< 500) with a bimodal gap pattern are usually - // normal tables where center-based clustering works correctly. + // Override for tables with a clear bimodal gap pattern: + // - Dense tables (500+ items, e.g. 24-column train schedule): use + // edge-based clustering with the detected threshold. + // - Smaller tables with a strong bimodal signal (jump > 10pt): + // lower the threshold but keep center-based clustering to avoid + // over-splitting. if threshold < 15.0 && best_jump > 2.0 && x_positions.len() > 500 { cluster_threshold = threshold.clamp(8.0, 25.0); use_edge_clustering = true; + } else if best_jump > 10.0 && threshold < cluster_threshold { + // Strong bimodal signal even with fewer items — the gap between + // within-column jitter and between-column spacing is unambiguous. + cluster_threshold = threshold.max(8.0); } } @@ -116,6 +122,13 @@ pub(crate) fn find_column_boundaries( }) .collect(); + log::debug!( + " find_column_boundaries: {} columns before filter, threshold={:.1}, {} items", + columns.len(), + cluster_threshold, + items.len() + ); + // Anti-paragraph safeguard for BodyFont mode: // Paragraphs concentrate items at the left margin; tables distribute evenly. // Reject if any single column has >60% of all items. diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index e5d4030..dfaf416 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -12,8 +12,7 @@ |9|29/1|Procurement of Cylinder Liner|PUC|Wartsila Eastern Africa Ltd|Euro28,186.75| |10|29/1|Procurement of Service Pack for Coupling|PUC|Wartsila Eastern Africa Ltd|Euro11,151.00| |11|29/1|Construction of Drainage for Roads A & B Eve Island Praslin|MLUH|Ascent Projects Sey|SR2,931,174.00| -|12|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| -|FEB|||||| +|12 FEB|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| |13|5/2|Procurement of DI Pipes and Fittings for Le Rocher Refurbishment|PUC|Legend General Supply|USD43,807.50| |14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| |15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| @@ -31,8 +30,7 @@ |27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| |28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| |29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| -|30|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| -|MAR|||||| +|30 MAR|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| |31|5/3|Procurement of Spare Parts for the Asphalt Plant- Petite Paris|SLTA|Astec Factory (USA)|USD 101,337.81| |32|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines|PUC|Marine Power International FZC|Euro 44,050.00| |33|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines B11|PUC|Marine Power International FZC|Euro 98,743.00| @@ -61,8 +59,7 @@ |45|19/3|Procurement of Stationery Items-Lot 6|MOE|Roy & Sons Imports|SR 114,750.00| |45|19/3|Procurement of Stationery Items-Lot 7|MOE|Trade Supplies Pty Ltd|SR 35,224.00| |46|26/3|Procurement of Microsoft Windows License|STB|Victoria Computer Service|SR 977,150.40| -|47|26/3|Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations|PUC|Ras Tek Pvt Ltd|Euro 58,000.00| -|APR|||||| +|47 APR|26/3|Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations|PUC|Ras Tek Pvt Ltd|Euro 58,000.00| |48|2/4|Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin|PUC|Vijay Construction|SR 623,025.00| |49|2/4|Supply of Fish to the Prison Department|Prison Service|Mr. Danny Loizeau|SR 27.50 per KG| |50|2/4|Supply of Polo T- Shirt and Jeans -Lot 1|PUC|Magilyn Ltee|USD 28,885.00| @@ -100,8 +97,7 @@ |73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| |74|30/4|Procurement of Electrical Meters|PUC|ISKRAEMECO|Euro 55,733.60| |75|30/4|Procurement of Grunfos Pump for Le Rocher Pump Station|PUC|Bluezone Mauritius Ltd|Euro 37,910.60| -|76|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| -|MAY|||||| +|76 MAY|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| |77|7/5|Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power|PUC|Technoglass|US$ 156, 920.00| |77|7/5|Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite|PUC|Technoglass|US$ 83,980.00| |78|7/5|Procurement of Connecting Rod (Variations)|PUC|Wartsila Eastern Africa Ltd|Euro 1,200.00| @@ -118,8 +114,7 @@ |88|28/5|Geotechnical Survey on Ile Soleil|2020 Development Ltd|Geoconsul Ltee|SR 741,520.00| |89|28/5|Extra Works at Palais De Justice|The Judiciary|Quingjian Group Co|SR 1,614,226.00| |90|28/5|Fire Fighting and rescue training Course|SFRS|Emergency Training Solution Pty Ltd|ZAR 824,453.80| -|91|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| -|JUN|||||| +|91 JUN|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| |92|4/6|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale|ZAR 438,081.83| |93|4/6|Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC||Wartsila Global Service|Euro 74,768.40| |94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| @@ -151,8 +146,7 @@ |116|25/6|Procurement of Critical Spares for Wartsila Engine B51- Lot 1|PUC|Wartsila Global Services|Euro 92,472.70| |117|25/6|Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin|PUC|RUYSCH|Euro 83,262.64| |118|25/6|Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013|PUC|Tornado Group|USD 266,820.82| -|119|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| -|JUL|||||| +|119 JUL|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| |120|2/7|Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road)|SLTA|Esparon's Enterprise|SR 183,410.00| |121|2/7|Bridge Renovation at Cascade|SLTA|Benioton Construction|SR 1,361,100.00| |122|2/7|Procurement of Bitumen|SLTA|Termcotank S.A|USD 519,418.20| @@ -197,8 +191,7 @@ |159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00| |160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00| |161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00| -|162|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| -|AUG|||||| +|162 AUG|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| |163|6/8|Procurement of Vehichels X 4|MOH|Kim-Koom & Co Pty Ltd|SR1,100,000.00| |164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe|WMF|Mr. Donal Ernesta|| |164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe|WMF|Mr. Kali Deenudayali|| @@ -222,8 +215,7 @@ |181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| |182|3/9|Procurement of Bitumen in drums for the asphalt production Praslin|SLTA|Benzene International Pte Ltd|Euro87,220.00| |183|3/9|Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S|PUC|Wartsila Eastern Africa|Euro9,861.00| -|184|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| -|SEP|||||| +|184 SEP|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| |185|10/9|Cleaning and maintenance of wetlands and rivers on Praslin|ED|"W" Cleaning Service|SR769,590.00| |186|10/9|Re-construction and partition of Independence House|MLUH|Green Island Construction Co Pty|SR868,022.94| |187|10/9|Procurement of critical spare fro major overhaul|PUC|Wartsila Eastern Afirca|Euro33,623.00| diff --git a/tests/snapshots/nexo-price-en.md b/tests/snapshots/nexo-price-en.md index f261102..5bc4ae2 100644 --- a/tests/snapshots/nexo-price-en.md +++ b/tests/snapshots/nexo-price-en.md @@ -2,8 +2,7 @@ |Classification Exclusive Exclusive|Selling price before tax benefit Supply value(surtax) 80,509,000 73,190,000(7,319,000)|Selling price after tax benefit 76,435,000|Standard equipment • Powertrain/Performance: Fuel cell system(150kW drive motor, lithium-ion battery, and reducer), Regenerative braking system, Column-Type Shift By Wire(vibration warning), Drive mode select • Safety: 9 airbag system(1st-row advanced/center side airbags, 1st/2nd-row side airbags, and rollover-resistant curtain airbags), Multi-Collision Brake System, Active hood system(for pedestrian protection), Safety unlock function, Artificial engine sound(for pedestrian protection), Child seat fastening system (2 in 2nd-row), Fire extinguisher for vehicles, Pedal Misapplication Safety Assist • Smart Safety Technology: Forward Collision-avoidance Assist(vehicles/ pedestrians/two-wheeled vehicles/junction turning/front oncoming), Smart Cruise Control with Stop & Go, Lane Keeping Assist, Lane Following Assist 2, Blind-spot Collision Warning(driving), Blind-spot Collision-avoidance Assist(forward exit), Rear Cross-traffic Collision-avoidance Assist, Safety Exit Assist, Driver Attention Warning, High Beam Assist, Advanced Rear Occupant Alert, Intelligent Speed Limit Assist, Hands-On Detection, Highway Driving Assist, Navigation-based Smart Cruise Control(safety speed zone/curve control), Vibration warning steering wheel • Exterior: Full LED headlamps(projection type), LED turn signal lamps (front and rear), LED Daytime Running Lights, LED positioning lights, LED rear combination lamps, LED third brake lights, 18-inch alloy wheels & tires, Solar glass(windshield), Double-glazed soundproof glass(windshield, and 1st/ 2nd-row doors), Outside mirror(heating, power-folding, power adjustment,|Options (before tax benefit) ▶ Hi-pass(e hi-pass) [200,000]| |---|---|---|---|---| -||with 3.5% individual consumption tax applied 79,287,000|with 3.5% individual consumption tax applied 76,435,000|and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather-|| -|Special|83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|79,275,000 with 3.5% individual consumption tax applied 79,275,000|upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| +|Special|with 3.5% individual consumption tax applied 79,287,000 83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|with 3.5% individual consumption tax applied 76,435,000 79,275,000 with 3.5% individual consumption tax applied 79,275,000|and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather- upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| |Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(front and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| **Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index 43201d8..8161541 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -13,13 +13,17 @@ IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER (v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in -termination of membership in the controlled group in which such corporation has been included. +|termination of membership in the controlled group in which such corporation has been included. (B) Election not filed.|In the event no election is filed in accordance with the| +|---|---| +|provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service|| +|will determine the group in which such corporation is to be included. Such|| +|determination will be binding for all subsequent years unless the corporation files a|| +|valid election with respect to any such subsequent year or until a change in the|| +|stock ownership of the corporation results in termination of membership in the|| +|controlled group in which such corporation has been included.|| +|(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3).|| +|(e) Effective date-- (1) Applicability date.|This section applies to any original| -(B) Election not filed. In the event no election is filed in accordance with the -provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service will determine the group in which such corporation is to be included. Such determination will be binding for all subsequent years unless the corporation files a valid election with respect to any such subsequent year or until a change in the stock ownership of the corporation results in termination of membership in the controlled group in which such corporation has been included. - -(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3). -(e) Effective date-- (1) Applicability date. This section applies to any original Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30, 2006. @@ -48,14 +52,9 @@ company other than a life insurance company shall make a return on Form 1120PC. annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return. -(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and -(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. -(4) Exception for insurance companies filing their Federal income tax returns -electronically. If an insurance company described in paragraph (c)(1), (c)(2), or - -(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). -(5) Definition. For purposes of this section, the term annual statement means -the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of +||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and| +|---|---| +||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of| Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. @@ -65,146 +64,118 @@ Federal income tax return (including any amended return filed on or before the d 2006. (2) Expiration date. The applicability of this section will expire on May 26, -2009. Par. 53. For each entry in the “Location” column of the following table, remove the language in the “Remove” column and add the language in the “Add” column in its place: Location Remove Add The last sentence of the The following rules shall The rules described in introductory text to be applicable in paragraph (a) of §1.302- §1.302-4 determining whether the 4T and in paragraphs (b) -specific requirements of through (g) of this section section 302(c)(2) are apply in determining met: whether the specific requirements of section 302(c)(2) are met. §1.338(h)(10)-1(f) §1.331-1(d), and §1.332-§1.331-1T(d) and §1.332- 6 6T +2009. -|The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| +|||Par. 53. For each entry in the “Location” column of the following table,| |---|---|---| -|§1.382-2T(h)(4)(vi)(B)|section|11T| -|The first sentence of|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +||remove the language in the “Remove” column and add the language in the “Add”|| +|column in its place:||| +|Location|Remove|Add| +|The last sentence of the|The following rules shall|The rules described in| +|introductory text to|be applicable in|paragraph (a) of §1.302-| +|§1.302-4|determining whether the specific requirements of section 302(c)(2) are met:|4T and in paragraphs (b) through (g) of this section apply in determining whether the specific| -§1.382-6(b)(2)(i) The second sentence of paragraph (c) of this paragraphs (c)(1), (c)(3), §1.382-8(a) section (c)(4) and (c)(5) of this section and paragraph +requirements of section 302(c)(2) are met. + +|§1.338(h)(10)-1(f)|§1.331-1(d), and §1.332-|§1.331-1T(d) and §1.332-| +|---|---|---| +||6|6T| +|The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| +|§1.382-2T(h)(4)(vi)(B)|section|11T| +|The first sentence of §1.382-6(b)(2)(i)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The second sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|§1.382-8(a)|section|(c)(4) and (c)(5) of this| + +section and paragraph (c)(2) of §1.382-8T -The third sentence of §1.382-8(a) +|The third sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|---|---|---| +|§1.382-8(a)|section|(c)(4) and (c)(5) of this section and paragraph (c)(2) of §1.382-8T| +|§1.382-8(c)(3)|paragraph (c)(2) of this section|paragraph (c)(2) of §1.382-8T| +|The first sentence of|paragraphs (c)(1), (2),|paragraphs (c)(1) and| +|§1.382-8(c)(4)|and (3) of this section|(c)(3) of this section and paragraph (c)(2) of §1.382-8T| +|§1.382-8(c)(5)|this paragraph (c)|paragraphs (c)(1), (c)(3),| -§1.382-8(c)(3) The first sentence of §1.382-8(c)(4) - -§1.382-8(c)(5) - -The fifth sentence of §1.382-8(f) - -§1.382-8(g), Example - -(1)(b)(2) The second sentence of §1.382-8(g), Example -(1)(c) -paragraph (c) of this section - -paragraph (c)(2) of this section paragraphs (c)(1), (2), and (3) of this section - -this paragraph (c) - -paragraphs (c)(1), (c)(3), - -(c)(4) and (c)(5) of this section and paragraph -(c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T paragraphs (c)(1) and -(c)(3) of this section and paragraph (c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), (c)(4), and (c)(5) of this section, and paragraph (c)(2) of §1.382-8T +|The fifth sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|---|---|---| +|§1.382-8(f)|section|(c)(4), and (c)(5) of this section, and paragraph (c)(2) of §1.382-8T| +|§1.382-8(g), Example|paragraph (c) of this|paragraphs (c)(1), (c)(3), section, and paragraph (c)(2) of §1.382-8T| +|The second sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|§1.382-8(g), Example|section|(c)(4), and (c)(5) of this| + +(1)(b)(2) section (c)(4), and (c)(5) of this +(1)(c) section, and paragraph + +(c)(2) of §1.382-8T + |§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| |---|---|---| |The first sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| |§1.382-8(g), Example|section|§1.382-8T| - -(2)(c) section §1.382-8T - -(2)(e) - |§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| -|---|---|---| |§1.382-8(g), Example|paragraphs (c)(1) and (2)|paragraph (c)(1) of this| +(2)(c) section §1.382-8T +(2)(e) (3)(b) section §1.382-8T (3)(c)(1)(B) of this section section and paragraph -The second sentence of §1.382-8(g), Example +(c)(2) of §1.382-8T -(4)(c) The second sentence of §1.382-8(g), Example -(5)(c) -paragraph (c) of this section - -paragraph (c) of this section - -paragraph (c) of this section - -paragraph (c)(2) of this section paragraph (c)(2) of this section - -(c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T - -|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| +|The second sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| |---|---|---| +|§1.382-8(g), Example|section|§1.382-8T| +|The second sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| +|§1.382-8(g), Example|section|§1.382-8T| +|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| |§1.1502-32(b)(4)(v)(A)|this section|§1.1502-32T| |The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| |§1.1502-32(b)(4)(v)(B)|this section|§1.1502-32T| -§1.1502-35(c)(4)(ii)(B) §1.1502-76(b)(2)(ii)(D) §1.1502-76(b)(2)(ii)(A)(2) paragraph (b)(2)(ii)(D) of this section §1.1502-92(e)(1) §1.382-2T(a)(2)(ii) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-92(e)(2) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) The second sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) +(4)(c) +(5)(c) -|The last sentence of|paragraph (f) of this|paragraph (f) of §1.1502-| +|§1.1502-35(c)(4)(ii)(B)|§1.1502-76(b)(2)(ii)(D)|§1.1502-76T(b)(2)(ii)(D)| |---|---|---| +|§1.1502-76(b)(2)(ii)(A)(2)|paragraph (b)(2)(ii)(D) of this section|paragraph (b)(2)(ii)(D) of §1.1502-76T| +|§1.1502-92(e)(1)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The first sentence of §1.1502-92(e)(2)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The first sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The second sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The last sentence of|paragraph (f) of this|paragraph (f) of §1.1502-| |§1.1502-95(b)(3)|section|95T| |The last sentence of|subdivision (ii) of this|paragraph (c)(2)(i) of| -|§1.1563-1(c)(2)(iv),|subparagraph|§1.1563-1T| +|§1.1563-1(c)(2)(iv), Example (1)|subparagraph|§1.1563-1T| +|The last sentence of|the district director with|the Internal Revenue| +|§1.1563-1(c)(2)(iv), Example (1)|audit jurisdiction of N’s return|Service| +|The third sentence of|subdivision (iii) of this|paragraph (c)(2)(ii) of| +|§1.1563-1(c)(2)(iv), Example (2)|subparagraph|§1.1563-1T| +|The third sentence of|the district director with|the Internal Revenue| +|§1.1563-1(c)(2)(iv), Example (2)|audit jurisdiction of the return of the corporation whose taxable year ends on the earliest date|Service| +|The last sentence of|district director|Internal Revenue Service| -Example (1) The last sentence of §1.1563-1(c)(2)(iv), Example (1) The third sentence of §1.1563-1(c)(2)(iv), Example (2) The third sentence of §1.1563-1(c)(2)(iv), Example (2) +§1.1563-1(c)(2)(iv), Example (2) -The last sentence of §1.1563-1(c)(2)(iv), Example (2) The second sentence of §1.1563-3(d)(2)(i) - -the district director with audit jurisdiction of N’s return subdivision (iii) of this subparagraph the district director with audit jurisdiction of the return of the corporation whose taxable year ends on the earliest date district director - -subdivisions (ii), (iii), and (iv) of this subparagraph - -§1.1502-76T(b)(2)(ii)(D) paragraph (b)(2)(ii)(D) of §1.1502-76T §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) - -|The first sentence of|§1.332-6(b), 1.368-3(a),|§1.332-6T(a), §1.368-| +|The second sentence of|subdivisions (ii), (iii), and|paragraphs (d)(2)(ii) and| |---|---|---| +|§1.1563-3(d)(2)(i)|(iv) of this subparagraph|(iii) of this section, and paragraph (d)(2)(iv) of §1.1563-3T| +|The first sentence of|§1.332-6(b), 1.368-3(a),|§1.332-6T(a), §1.368-| |§1.6043-2(a)|or 1.1081-11|3T(a), or §1.1081-11T| -|The first sentence of|§1.6012-2|paragraphs (a), (b) and| +|The first sentence of §301.6011-5T(a) (twice)|§1.6012-2|paragraphs (a), (b) and (d) through (j) of §1.6012- 2, and paragraph (c) of §1.6012-2T| -§301.6011-5T(a) (twice) - -the Internal Revenue Service paragraph (c)(2)(ii) of §1.1563-1T the Internal Revenue Service - -Internal Revenue Service - -paragraphs (d)(2)(ii) and (iii) of this section, and paragraph (d)(2)(iv) of §1.1563-3T - -(d) through (j) of §1.6012- 2, and paragraph (c) of §1.6012-2T - -PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The authority citation for part 602 continues to read as follows: Authority: 26 U.S.C. 7805. Par. 55. In §602.101, paragraph (b) is amended to read as follows: - -1. The following entries to the table are removed: -§602.101 OMB Control numbers. - -* * * * * -(b) * * * -CFR part or section where Current OMB identified or described control No. - -* * * * * -1.332-6…………………………………………………………………. 1545-2019 -1.382-11……………………………………………………………….. 1545-2019 -1.351-3…………………………………………………………………. 1545-2019 -1.355-5…………………………………………………………………. 1545-2019 -1.368-3…………………………………………………………………. 1545-2019 -1.1081-11………………………………………………………………. 1545-2019 -* * * * * **______________________________________________________________** -2. The following entries are added in numerical order to the table: -§602.101 OMB Control numbers. - -* * * * * -(b) * * * -CFR part or section where Current OMB identified or described control No. - -* * * * * -1.302-2T………………………………………………………………… 1545-2019 -1.302-4T………………………………………………………………… 1545-2019 +|||PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK|| +|---|---|---|---| +||REDUCTION ACT Authority: 26 U.S.C. 7805. 1. The following entries to the table are removed: §602.101 OMB Control numbers.|Par. 54. The authority citation for part 602 continues to read as follows: Par. 55. In §602.101, paragraph (b) is amended to read as follows:|| +|* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| +|* * * * *|1.332-6………………………………………………………………….|1.382-11……………………………………………………………….. 1545-2019 1.351-3…………………………………………………………………. 1545-2019 1.355-5…………………………………………………………………. 1545-2019 1.368-3…………………………………………………………………. 1545-2019 1.1081-11………………………………………………………………. 1545-2019|1545-2019| +|* * * * *|§602.101 OMB Control numbers.|______________________________________________________________ 2. The following entries are added in numerical order to the table:|| +|* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| +|* * * * *|1.302-2T………………………………………………………………… 1545 1.302-4T………………………………………………………………… 1545||-2019 -2019| |1.331-1T………………………………………………………………… 1545|-2019| |---|---| diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 789ed1a..25ea607 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -46,10 +46,12 @@ l **Freon** **®** **12 Saturation Properties-Temperature Table** -|Temp|Pressure|Volume||Density|||Enthalpy||Entropy||Temp| -|---|---|---|---|---|---|---|---|---|---|---|---| -|°C|[kPa]|[m3/kg] Liquid v|Vapour v|[kg/m3] Liquid d|Vapour d|Liquid H|[kJ/kg] Latent H|Vapour H|[kJ/K-kg] Liquid S|Vapour S|°C| +|Temp|Pressure||Volume|||Density||Enthalpy|||Entropy|Temp| +|---|---|---|---|---|---|---|---|---|---|---|---|---| +|°C|[kPa]|[m3 Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m3] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C| + |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| +|---|---|---|---|---|---|---|---|---|---|---|---| |-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99| |-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98| |-97|1.6|0.0006|7.7250|1671.0|0.129|115.8|191.6|307.4|0.6218|1.7100|-97| @@ -104,5 +106,3 @@ l |-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48| |-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47| -**f g f g f** fg **g f g** - From 4d52d7af5276279656d931da6d6d02d57af0b49a Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Wed, 25 Mar 2026 22:58:56 -0700 Subject: [PATCH 42/46] fix(extractor): content stream comment parsing + CJK mojibake detection (#18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(extractor): strip PDF comments that break lopdf content stream parsing Some PDF generators (notably PD4ML used by school districts) embed comments (% to end of line) in content streams. lopdf's Content::decode parser fails to parse operators that follow comments, silently dropping ET (end text) and Q (restore graphics state) operators. This caused entire pages to produce 0 text items despite having valid text. Fix: pre-process content streams to strip comments before parsing. Comments inside string literals (parentheses) and hex strings are preserved. The comment is replaced with a space to maintain token separation. Impact: fixes 13+ school district PDFs and similar PD4ML-generated documents that were producing near-empty output (454 → 31,955 chars for a 22-page school improvement plan). Co-Authored-By: Claude Opus 4.6 (1M context) * fix(detect): flag sparse-extraction pages as needing OCR When a TEXT-BASED PDF produces <50 chars/page average with <500 total chars, flag all pages as needing OCR. This catches PDFs where the extractable text is minimal (form templates, image-heavy layouts) and the bulk of content requires OCR to access. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(detect): improve CID mojibake detection for Japanese/CJK PDFs Extend is_cid_garbage to detect CID-as-Latin-1 mojibake: when ≥40% of characters are high Latin-1 (U+00A0-00FF) and <33% are ASCII letters, the text is likely CID values misinterpreted as Latin-1 characters (common in Japanese/CJK PDFs with broken ToUnicode CMaps). Also add sparse-extraction OCR flagging: TEXT-BASED PDFs with <50 chars/page and <500 total chars get all pages flagged for OCR. Impact: Softbank Japanese PDFs now produce empty output with pages_needing_ocr=all instead of mojibake garbage. Korean PDFs with valid extraction (nexo-price-en) remain unaffected. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(detect): sparse extraction check only when markdown is generated The sparse-extraction OCR check was triggering in Analyze mode where markdown is not generated (md_len=0), causing false OCR flags on every PDF processed via detect-pdf --analyze. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- src/extractor/content_stream.rs | 91 +++++++++++++++++++++++++++++++++ src/lib.rs | 40 ++++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 0829440..7696563 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -20,6 +20,60 @@ use super::fonts::{ use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; use super::{get_number, multiply_matrices}; +/// Strip PDF comments (% to end of line) from content stream bytes. +/// +/// Some PDF generators (e.g. PD4ML) embed comments in content streams that +/// confuse lopdf's `Content::decode` parser. Comments inside string literals +/// (parentheses) are NOT stripped — only top-level comments. +fn strip_pdf_comments(data: &[u8]) -> Vec { + // Quick check: if no '%' present, return as-is (common case) + if !data.contains(&b'%') { + return data.to_vec(); + } + + let mut result = Vec::with_capacity(data.len()); + let mut i = 0; + let mut in_string = 0i32; // parenthesis nesting depth + let mut in_hex_string = false; + + while i < data.len() { + let b = data[i]; + match b { + b'(' if !in_hex_string => { + in_string += 1; + result.push(b); + } + b')' if !in_hex_string && in_string > 0 => { + in_string -= 1; + result.push(b); + } + b'<' if in_string == 0 && !in_hex_string => { + in_hex_string = true; + result.push(b); + } + b'>' if in_hex_string => { + in_hex_string = false; + result.push(b); + } + b'%' if in_string == 0 && !in_hex_string => { + // Skip until end of line + while i < data.len() && data[i] != b'\n' && data[i] != b'\r' { + i += 1; + } + // Replace comment with a space to preserve token separation + result.push(b' '); + continue; // Don't increment i again + } + _ => { + result.push(b); + } + } + i += 1; + } + + result +} + /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates /// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( @@ -113,6 +167,11 @@ pub(crate) fn extract_page_text_items( .get_page_content(page_id) .map_err(|e| PdfError::Parse(e.to_string()))?; + // Strip PDF comments (% to end of line) from the content stream. + // Some PDF generators (e.g. PD4ML) embed comments that confuse lopdf's + // Content::decode parser, causing it to skip operators like ET and Q. + let content_data = strip_pdf_comments(&content_data); + let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; const MAX_OPERATIONS: usize = 1_000_000; @@ -1174,4 +1233,36 @@ mod tests { assert!(rects.is_empty()); assert!(lines.is_empty()); } + + #[test] + fn test_strip_pdf_comments() { + // Basic comment stripping + let input = b"BT\n% comment\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"BT\n \nTj\nET\n"); + + // No comments = unchanged + let input = b"BT\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, input.to_vec()); + + // Don't strip inside string literals + let input = b"(text with % not a comment)\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"(text with % not a comment)\n \n"); + + // Don't strip inside hex strings + let input = b"<0033% not a comment>\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"<0033% not a comment>\n \n"); + + // PD4ML style: comment between Tj and ET + let input = b"<0033> Tj\n\t% Mission Statement\n\tET\n"; + let output = strip_pdf_comments(input); + let output_str = String::from_utf8_lossy(&output); + assert!( + output_str.contains("ET"), + "ET should be preserved after comment stripping" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 2f4611b..64eb001 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -520,6 +520,27 @@ fn process_document( pages_needing_ocr.sort_unstable(); } + // Detect sparse extraction: when a TEXT-BASED PDF produces very few + // characters per page, the text is likely embedded in images/forms + // that need OCR. Flag all pages for OCR in this case. + // Only check when markdown was actually generated (not in Analyze mode). + if pdf_type == PdfType::TextBased + && page_count > 0 + && pages_needing_ocr.is_empty() + && markdown.is_some() + { + let md_len = markdown.as_ref().map_or(0, |m| m.len()); + let chars_per_page = md_len as f32 / page_count as f32; + if chars_per_page < 50.0 && md_len < 500 { + log::debug!( + "sparse extraction: {:.0} chars/page — recommending OCR for all {} pages", + chars_per_page, + page_count + ); + pages_needing_ocr = (1..=page_count).collect(); + } + } + let markdown = if all_gid { log::debug!( "all {} pages have gid-encoded fonts — suppressing markdown output", @@ -622,6 +643,7 @@ fn is_cid_garbage(text: &str) -> bool { } let mut total = 0usize; let mut c1_control = 0usize; + let mut high_latin = 0usize; for ch in text.chars() { if ch.is_whitespace() { continue; @@ -631,9 +653,25 @@ fn is_cid_garbage(text: &str) -> bool { if ('\u{0080}'..='\u{009F}').contains(&ch) { c1_control += 1; } + // High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text + // but when combined with ASCII in CID passthrough, indicates mojibake + // from CID values being misinterpreted as Latin-1 characters. + if ('\u{00A0}'..='\u{00FF}').contains(&ch) { + high_latin += 1; + } + } + if total < 5 { + return false; } // If ≥5% of non-whitespace chars are C1 controls, it's garbage - total >= 20 && c1_control * 20 >= total + if c1_control * 20 >= total { + return true; + } + // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few + // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs + // where CID values 0x80-0xFF become accented Latin characters). + let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count(); + high_latin * 5 >= total * 2 && ascii_letters * 3 < total } /// Analyse extracted items and rects for layout complexity. From 263ed0aa420f98b5fe8bed98c4703558b193199d Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 2 Apr 2026 02:35:05 -0700 Subject: [PATCH 43/46] add region-based text extraction for hybrid OCR pipelines Add `extract_text_in_regions_mem` that takes layout-detected bounding boxes (top-left origin, PDF points) and returns text within each region in reading order. Designed for pipelines where a layout model detects regions and text-based pages can skip GPU OCR by extracting text from the PDF structure directly. Also add `classify_pdf_mem` for lightweight PDF type classification returning 0-indexed pages_needing_ocr. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/extractor/mod.rs | 2 +- src/lib.rs | 226 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 226 insertions(+), 2 deletions(-) diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 9a2395c..93b465e 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -2,7 +2,7 @@ //! //! This module extracts text with position information for structure detection. -mod content_stream; +pub(crate) mod content_stream; mod fonts; mod layout; mod links; diff --git a/src/lib.rs b/src/lib.rs index 64eb001..68d6b2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,7 @@ pub use process_mode::ProcessMode; pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem}; use lopdf::Document; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::Path; use tounicode::FontCMaps; @@ -264,6 +264,230 @@ pub fn process_pdf_mem_with_config( ) } +// ========================================================================= +// Region-based text extraction (for hybrid OCR pipelines) +// ========================================================================= + +/// Lightweight classification result for routing decisions. +#[derive(Debug)] +pub struct PdfClassification { + /// The detected PDF type. + pub pdf_type: PdfType, + /// Total page count. + pub page_count: u32, + /// 0-indexed page numbers that need OCR (scanned/image pages). + pub pages_needing_ocr: Vec, + /// Detection confidence score (0.0–1.0). + pub confidence: f32, +} + +/// Classify a PDF from a memory buffer without extracting text. +/// Returns the PDF type and which pages need OCR (~10-50ms). +pub fn classify_pdf_mem(buffer: &[u8]) -> Result { + validate_pdf_bytes(buffer)?; + let (doc, page_count) = load_document_from_mem(buffer)?; + let detection = detector::detect_from_document(&doc, page_count, &DetectionConfig::default())?; + Ok(PdfClassification { + pdf_type: detection.pdf_type, + page_count, + // Convert from 1-indexed to 0-indexed for caller convenience + pages_needing_ocr: detection.pages_needing_ocr.iter().map(|&p| p - 1).collect(), + confidence: detection.confidence, + }) +} + +/// Extract text within bounding-box regions from a PDF in memory. +/// +/// This is designed for hybrid OCR pipelines: a layout model detects regions +/// in a rendered page image, and this function extracts the PDF text that +/// falls within each region — avoiding GPU OCR for text-based pages. +/// +/// # Arguments +/// +/// * `buffer` — PDF file bytes +/// * `page_regions` — list of `(page_number_0indexed, Vec<[x1, y1, x2, y2]>)`. +/// Coordinates are in **PDF points** with **top-left origin** (matching typical +/// layout model output after coordinate conversion). +/// +/// # Returns +/// +/// A `Vec` parallel to `page_regions`, each containing a `Vec` parallel +/// to that page's region list. Each string is the extracted text within that +/// region, in reading order (top→bottom, left→right), with lines joined by `\n`. +pub fn extract_text_in_regions_mem( + buffer: &[u8], + page_regions: &[(u32, Vec<[f32; 4]>)], +) -> Result>, PdfError> { + validate_pdf_bytes(buffer)?; + let (doc, _page_count) = load_document_from_mem(buffer)?; + let font_cmaps = FontCMaps::from_doc(&doc); + let pages = doc.get_pages(); + + // Build a set of pages we need to extract + let needed_pages: HashSet = page_regions.iter().map(|(p, _)| p + 1).collect(); // to 1-indexed + + // Extract text items for needed pages only + let mut items_by_page: HashMap> = HashMap::new(); + let mut page_heights: HashMap = HashMap::new(); + + for (page_num, &page_id) in pages.iter() { + if !needed_pages.contains(page_num) { + continue; + } + + // Get page height from MediaBox for coordinate flip + let height = get_page_height(&doc, page_id).unwrap_or(792.0); + page_heights.insert(*page_num, height); + + // Extract text items for this page + let ((mut items, _rects, _lines), _has_gid) = + extractor::content_stream::extract_page_text_items( + &doc, + page_id, + *page_num, + &font_cmaps, + false, + )?; + text_utils::fix_letterspaced_items(&mut items); + items_by_page.insert(*page_num, items); + } + + // For each page's regions, filter and assemble text + let mut results = Vec::with_capacity(page_regions.len()); + + for (page_0idx, regions) in page_regions { + let page_1idx = page_0idx + 1; + let items = items_by_page.get(&page_1idx); + let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0); + + let mut page_results = Vec::with_capacity(regions.len()); + + for rect in regions { + let [rx1, ry1, rx2, ry2] = *rect; + + let text = match items { + Some(items) => collect_text_in_region(items, rx1, ry1, rx2, ry2, page_h), + None => String::new(), + }; + + page_results.push(text); + } + + results.push(page_results); + } + + Ok(results) +} + +/// Get page height in points from MediaBox. +fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option { + let page_dict = doc.get_dictionary(page_id).ok()?; + // Try MediaBox directly, then follow reference + let media_box = page_dict.get(b"MediaBox").ok()?; + let arr = match media_box { + lopdf::Object::Array(a) => a, + lopdf::Object::Reference(r) => { + if let Ok(lopdf::Object::Array(a)) = doc.get_object(*r) { + a + } else { + return None; + } + } + _ => return None, + }; + if arr.len() >= 4 { + let y1 = obj_to_f32(&arr[1])?; + let y2 = obj_to_f32(&arr[3])?; + Some((y2 - y1).abs()) + } else { + None + } +} + +fn obj_to_f32(obj: &lopdf::Object) -> Option { + match obj { + lopdf::Object::Integer(i) => Some(*i as f32), + lopdf::Object::Real(f) => Some(*f as f32), + _ => None, + } +} + +/// Collect text items that fall within a region bbox (top-left origin, PDF points) +/// and return them as a single string in reading order. +fn collect_text_in_region( + items: &[TextItem], + rx1: f32, + ry1: f32, + rx2: f32, + ry2: f32, + page_height: f32, +) -> String { + // Convert region from top-left to bottom-left origin + let by1 = page_height - ry2; // top-left y2 → bottom-left y1 + let by2 = page_height - ry1; // top-left y1 → bottom-left y2 + + // Collect items whose center falls within the region + let mut matched: Vec<&TextItem> = items + .iter() + .filter(|item| { + let cx = item.x + item.width / 2.0; + let cy = item.y + item.height / 2.0; + cx >= rx1 && cx <= rx2 && cy >= by1 && cy <= by2 + }) + .collect(); + + if matched.is_empty() { + return String::new(); + } + + // Sort top→bottom (descending Y in bottom-left coords), then left→right + matched.sort_by(|a, b| { + let line_threshold = a.font_size.max(b.font_size) * 0.5; + let y_diff = b.y - a.y; // descending Y = top to bottom + if y_diff.abs() < line_threshold { + a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal) + } else { + y_diff + .partial_cmp(&0.0_f32) + .unwrap_or(std::cmp::Ordering::Equal) + } + }); + + // Group into lines and join + let mut lines: Vec = Vec::new(); + let mut current_line = String::new(); + let mut last_y = f32::NAN; + let mut last_x_end = 0.0_f32; + + for item in &matched { + let line_threshold = item.font_size * 0.5; + let same_line = (item.y - last_y).abs() < line_threshold; + + if !same_line && !current_line.is_empty() { + lines.push(current_line.clone()); + current_line.clear(); + } + + if !current_line.is_empty() { + // Insert space if there's a gap between items on the same line + let gap = item.x - last_x_end; + if gap > item.font_size * 0.15 { + current_line.push(' '); + } + } + + current_line.push_str(&item.text); + last_y = item.y; + last_x_end = item.x + item.width; + } + + if !current_line.is_empty() { + lines.push(current_line); + } + + lines.join("\n") +} + // ========================================================================= // Internal: single-load document pipeline // ========================================================================= From af6ecebaa6ac9d3335a799e19a60b48ea40753b6 Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 2 Apr 2026 10:46:38 -0700 Subject: [PATCH 44/46] add per-region quality checks to extract_text_in_regions_mem Return RegionText with a needs_ocr flag per region, set when: - extracted text is empty (image region with no PDF text) - page uses GID-encoded fonts (unreliable CID mapping) - text fails garbage detection (mostly non-alphanumeric) - text has encoding issues (U+FFFD, dollar-as-space patterns) This lets callers skip GPU OCR only when text quality is reliable, falling back for any region where extraction is suspect. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lib.rs | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 68d6b2c..97dec4a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -296,12 +296,35 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result { }) } +/// Result for a single region's text extraction. +#[derive(Debug)] +pub struct RegionText { + /// Extracted text (may be empty if region has no text items). + pub text: String, + /// `true` when the text should not be trusted and OCR should be used instead. + /// Set when: the region is empty, the page uses GID-encoded fonts, or the + /// extracted text fails garbage/encoding checks. + pub needs_ocr: bool, +} + +/// Result for a page's region extractions. +#[derive(Debug)] +pub struct PageRegionResult { + /// 0-indexed page number. + pub page: u32, + /// Per-region results, parallel to the input regions. + pub regions: Vec, +} + /// Extract text within bounding-box regions from a PDF in memory. /// /// This is designed for hybrid OCR pipelines: a layout model detects regions /// in a rendered page image, and this function extracts the PDF text that /// falls within each region — avoiding GPU OCR for text-based pages. /// +/// Each region result includes a `needs_ocr` flag that is set when extraction +/// quality is suspect (empty text, GID-encoded fonts, garbage/encoding issues). +/// /// # Arguments /// /// * `buffer` — PDF file bytes @@ -311,13 +334,11 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result { /// /// # Returns /// -/// A `Vec` parallel to `page_regions`, each containing a `Vec` parallel -/// to that page's region list. Each string is the extracted text within that -/// region, in reading order (top→bottom, left→right), with lines joined by `\n`. +/// A `Vec` parallel to `page_regions`. pub fn extract_text_in_regions_mem( buffer: &[u8], page_regions: &[(u32, Vec<[f32; 4]>)], -) -> Result>, PdfError> { +) -> Result, PdfError> { validate_pdf_bytes(buffer)?; let (doc, _page_count) = load_document_from_mem(buffer)?; let font_cmaps = FontCMaps::from_doc(&doc); @@ -329,6 +350,7 @@ pub fn extract_text_in_regions_mem( // Extract text items for needed pages only let mut items_by_page: HashMap> = HashMap::new(); let mut page_heights: HashMap = HashMap::new(); + let mut gid_pages: HashSet = HashSet::new(); for (page_num, &page_id) in pages.iter() { if !needed_pages.contains(page_num) { @@ -340,7 +362,7 @@ pub fn extract_text_in_regions_mem( page_heights.insert(*page_num, height); // Extract text items for this page - let ((mut items, _rects, _lines), _has_gid) = + let ((mut items, _rects, _lines), has_gid) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -349,6 +371,9 @@ pub fn extract_text_in_regions_mem( false, )?; text_utils::fix_letterspaced_items(&mut items); + if has_gid { + gid_pages.insert(*page_num); + } items_by_page.insert(*page_num, items); } @@ -359,6 +384,7 @@ pub fn extract_text_in_regions_mem( let page_1idx = page_0idx + 1; let items = items_by_page.get(&page_1idx); let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0); + let page_has_gid = gid_pages.contains(&page_1idx); let mut page_results = Vec::with_capacity(regions.len()); @@ -370,10 +396,18 @@ pub fn extract_text_in_regions_mem( None => String::new(), }; - page_results.push(text); + let needs_ocr = text.trim().is_empty() + || page_has_gid + || is_garbage_text(&text) + || detect_encoding_issues(&text); + + page_results.push(RegionText { text, needs_ocr }); } - results.push(page_results); + results.push(PageRegionResult { + page: *page_0idx, + regions: page_results, + }); } Ok(results) From 7d0e0b295b3380ed97f89cb625512ae70ff5745e Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 2 Apr 2026 10:52:29 -0700 Subject: [PATCH 45/46] fix clippy: remove unnecessary f32 cast in obj_to_f32 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 97dec4a..426eaa1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -441,7 +441,7 @@ fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option { fn obj_to_f32(obj: &lopdf::Object) -> Option { match obj { lopdf::Object::Integer(i) => Some(*i as f32), - lopdf::Object::Real(f) => Some(*f as f32), + lopdf::Object::Real(f) => Some(*f), _ => None, } } From 87d2bae0d9a3a51ca389cc31c50781a807c0ca0b Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Thu, 2 Apr 2026 11:29:57 -0700 Subject: [PATCH 46/46] add napi binding package for GitHub Packages publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the napi bridge from fire-pdf into pdf-inspector as `napi/`. Package name: @firecrawl/pdf-inspector-js, published to GitHub Packages (npm.pkg.github.com) as a public package on v* tags. Exposes two functions: - classifyPdf(buffer) → type, page count, pages needing OCR - extractTextInRegions(buffer, pageRegions) → per-region text with needsOcr quality flag (GID fonts, garbage, encoding issues) Includes publish workflow that builds linux-x64-gnu + darwin-arm64 binaries and publishes main + platform-specific packages. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/publish.yml | 99 +++ napi/.gitignore | 6 + napi/Cargo.lock | 1509 +++++++++++++++++++++++++++++++++ napi/Cargo.toml | 19 + napi/build.rs | 3 + napi/bun.lock | 231 +++++ napi/index.d.ts | 49 ++ napi/index.js | 580 +++++++++++++ napi/package.json | 34 + napi/src/lib.rs | 116 +++ 10 files changed, 2646 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 napi/.gitignore create mode 100644 napi/Cargo.lock create mode 100644 napi/Cargo.toml create mode 100644 napi/build.rs create mode 100644 napi/bun.lock create mode 100644 napi/index.d.ts create mode 100644 napi/index.js create mode 100644 napi/package.json create mode 100644 napi/src/lib.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..efab531 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,99 @@ +name: Publish npm package + +on: + push: + tags: ['v*'] + +permissions: + contents: read + packages: write + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: macos-14 + target: aarch64-apple-darwin + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + napi/target/ + key: ${{ runner.os }}-cargo-napi-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-napi- + + - name: Install dependencies + working-directory: napi + run: bun install + + - name: Build native addon + working-directory: napi + run: bunx napi build --platform --release + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.target }} + path: napi/*.node + if-no-files-found: error + + publish: + name: Publish to GitHub Packages + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + working-directory: napi + run: bun install + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: napi/artifacts + + - name: Move artifacts + run: bunx napi artifacts --dir napi/artifacts --napi-dir napi + working-directory: napi + + - name: List packages + run: ls -R npm/ + working-directory: napi + + - name: Publish + working-directory: napi + run: | + echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > .npmrc + echo "@firecrawl:registry=https://npm.pkg.github.com" >> .npmrc + bunx napi prepublish -t npm + npm publish --access public + for dir in npm/*/; do + if [ -f "$dir/package.json" ]; then + echo "Publishing $dir" + (cd "$dir" && npm publish --access public) + fi + done diff --git a/napi/.gitignore b/napi/.gitignore new file mode 100644 index 0000000..01254fc --- /dev/null +++ b/napi/.gitignore @@ -0,0 +1,6 @@ +target/ +node_modules/ +*.node + +# Override root .gitignore — lock file needed for reproducible napi builds +!Cargo.lock diff --git a/napi/Cargo.lock b/napi/Cargo.lock new file mode 100644 index 0000000..8a11ccd --- /dev/null +++ b/napi/Cargo.lock @@ -0,0 +1,1509 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "352d39c2f7bef1d6ad73db6f5160efcaed66d94ef8c6c573a8410c00bf909a98" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dtor" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1057d6c64987086ff8ed0fd3fbf377a6b7d205cc7715868cd401705f715cbe4" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lopdf" +version = "0.40.0" +source = "git+https://github.com/J-F-Liu/lopdf?rev=052674053814a9f4897af94f0b8e46a545c9b329#052674053814a9f4897af94f0b8e46a545c9b329" +dependencies = [ + "aes", + "bitflags", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "nom_locate", + "rand", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "napi" +version = "3.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb7848c221fb7bb789e02f01875287ebb1e078b92a6566a34de01ef8806e7c2b" +dependencies = [ + "bitflags", + "ctor", + "futures", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", + "serde", + "serde_json", +] + +[[package]] +name = "napi-build" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1" + +[[package]] +name = "napi-derive" +version = "3.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60867ff9a6f76e82350e0c3420cb0736f5866091b61d7d8a024baa54b0ec17dd" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "5.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0864cf6a82e2cfb69067374b64c9253d7e910e5b34db833ed7495dda56ccb18" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb602b84d7c1edae45e50bbf1374696548f36ae179dfa667f577e384bb90c2b" +dependencies = [ + "libloading", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pdf-inspector" +version = "0.1.0" +dependencies = [ + "env_logger", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + +[[package]] +name = "pdf-inspector-napi" +version = "0.2.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "pdf-inspector", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/napi/Cargo.toml b/napi/Cargo.toml new file mode 100644 index 0000000..55a8730 --- /dev/null +++ b/napi/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pdf-inspector-napi" +version = "0.2.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +pdf-inspector = { path = ".." } +napi = { version = "3.0.0", features = ["serde-json"] } +napi-derive = "3.0.0" + +[build-dependencies] +napi-build = "2" + +[profile.release] +lto = true +strip = "debuginfo" diff --git a/napi/build.rs b/napi/build.rs new file mode 100644 index 0000000..bbfc9e4 --- /dev/null +++ b/napi/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/napi/bun.lock b/napi/bun.lock new file mode 100644 index 0000000..cd7f371 --- /dev/null +++ b/napi/bun.lock @@ -0,0 +1,231 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@firecrawl/pdf-inspector-js", + "devDependencies": { + "@napi-rs/cli": "^3.4.1", + }, + }, + }, + "packages": { + "@emnapi/core": ["@emnapi/core@1.9.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.9.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw=="], + + "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], + + "@inquirer/ansi": ["@inquirer/ansi@2.0.4", "", {}, "sha512-DpcZrQObd7S0R/U3bFdkcT5ebRwbTTC4D3tCc1vsJizmgPLxNJBo+AAFmrZwe8zk30P2QzgzGWZ3Q9uJwWuhIg=="], + + "@inquirer/checkbox": ["@inquirer/checkbox@5.1.2", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-PubpMPO2nJgMufkoB3P2wwxNXEMUXnBIKi/ACzDUYfaoPuM7gSTmuxJeMscoLVEsR4qqrCMf5p0SiYGWnVJ8kw=="], + + "@inquirer/confirm": ["@inquirer/confirm@6.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-tiNyA73pgpQ0FQ7axqtoLUe4GDYjNCDcVsbgcA5anvwg2z6i+suEngLKKJrWKJolT//GFPZHwN30binDIHgSgQ=="], + + "@inquirer/core": ["@inquirer/core@11.1.7", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4", "cli-width": "^4.1.0", "fast-wrap-ansi": "^0.2.0", "mute-stream": "^3.0.0", "signal-exit": "^4.1.0" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-1BiBNDk9btIwYIzNZpkikIHXWeNzNncJePPqwDyVMhXhD1ebqbpn1mKGctpoqAbzywZfdG0O4tvmsGIcOevAPQ=="], + + "@inquirer/editor": ["@inquirer/editor@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/external-editor": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-VJx4XyaKea7t8hEApTw5dxeIyMtWXre2OiyJcICCRZI4hkoHsMoCnl/KbUnJJExLbH9csLLHMVR144ZhFE1CwA=="], + + "@inquirer/expand": ["@inquirer/expand@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-fC0UHJPXsTRvY2fObiwuQYaAnHrp3aDqfwKUJSdfpgv18QUG054ezGbaRNStk/BKD5IPijeMKWej8VV8O5Q/eQ=="], + + "@inquirer/external-editor": ["@inquirer/external-editor@2.0.4", "", { "dependencies": { "chardet": "^2.1.1", "iconv-lite": "^0.7.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-Prenuv9C1PHj2Itx0BcAOVBTonz02Hc2Nd2DbU67PdGUaqn0nPCnV34oDyyoaZHnmfRxkpuhh/u51ThkrO+RdA=="], + + "@inquirer/figures": ["@inquirer/figures@2.0.4", "", {}, "sha512-eLBsjlS7rPS3WEhmOmh1znQ5IsQrxWzxWDxO51e4urv+iVrSnIHbq4zqJIOiyNdYLa+BVjwOtdetcQx1lWPpiQ=="], + + "@inquirer/input": ["@inquirer/input@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-nvZ6qEVeX/zVtZ1dY2hTGDQpVGD3R7MYPLODPgKO8Y+RAqxkrP3i/3NwF3fZpLdaMiNuK0z2NaYIx9tPwiSegQ=="], + + "@inquirer/number": ["@inquirer/number@4.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-Ht8OQstxiS3APMGjHV0aYAjRAysidWdwurWEo2i8yI5xbhOBWqizT0+MU1S2GCcuhIBg+3SgWVjEoXgfhY+XaA=="], + + "@inquirer/password": ["@inquirer/password@5.0.10", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-QbNyvIE8q2GTqKLYSsA8ATG+eETo+m31DSR0+AU7x3d2FhaTWzqQek80dj3JGTo743kQc6mhBR0erMjYw5jQ0A=="], + + "@inquirer/prompts": ["@inquirer/prompts@8.3.2", "", { "dependencies": { "@inquirer/checkbox": "^5.1.2", "@inquirer/confirm": "^6.0.10", "@inquirer/editor": "^5.0.10", "@inquirer/expand": "^5.0.10", "@inquirer/input": "^5.0.10", "@inquirer/number": "^4.0.10", "@inquirer/password": "^5.0.10", "@inquirer/rawlist": "^5.2.6", "@inquirer/search": "^4.1.6", "@inquirer/select": "^5.1.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-yFroiSj2iiBFlm59amdTvAcQFvWS6ph5oKESls/uqPBect7rTU2GbjyZO2DqxMGuIwVA8z0P4K6ViPcd/cp+0w=="], + + "@inquirer/rawlist": ["@inquirer/rawlist@5.2.6", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-jfw0MLJ5TilNsa9zlJ6nmRM0ZFVZhhTICt4/6CU2Dv1ndY7l3sqqo1gIYZyMMDw0LvE1u1nzJNisfHEhJIxq5w=="], + + "@inquirer/search": ["@inquirer/search@4.1.6", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-3/6kTRae98hhDevENScy7cdFEuURnSpM3JbBNg8yfXLw88HgTOl+neUuy/l9W0No5NzGsLVydhBzTIxZP7yChQ=="], + + "@inquirer/select": ["@inquirer/select@5.1.2", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-kTK8YIkHV+f02y7bWCh7E0u2/11lul5WepVTclr3UMBtBr05PgcZNWfMa7FY57ihpQFQH/spLMHTcr0rXy50tA=="], + + "@inquirer/type": ["@inquirer/type@4.0.4", "", { "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-PamArxO3cFJZoOzspzo6cxVlLeIftyBsZw/S9bKY5DzxqJVZgjoj1oP8d0rskKtp7sZxBycsoer1g6UeJV1BBA=="], + + "@napi-rs/cli": ["@napi-rs/cli@3.6.0", "", { "dependencies": { "@inquirer/prompts": "^8.0.0", "@napi-rs/cross-toolchain": "^1.0.3", "@napi-rs/wasm-tools": "^1.0.1", "@octokit/rest": "^22.0.1", "clipanion": "^4.0.0-rc.4", "colorette": "^2.0.20", "emnapi": "^1.9.1", "es-toolkit": "^1.41.0", "js-yaml": "^4.1.0", "obug": "^2.0.0", "semver": "^7.7.3", "typanion": "^3.14.0" }, "peerDependencies": { "@emnapi/runtime": "^1.7.1" }, "optionalPeers": ["@emnapi/runtime"], "bin": { "napi": "dist/cli.js", "napi-raw": "cli.mjs" } }, "sha512-aA8m4+9XxnK1+0sr4GplZP0Ze90gkzO8sMKaplOK0zXbLnzsLl6O2BQQt6rTCcTRzIN24wrrByakr/imM+CxhA=="], + + "@napi-rs/cross-toolchain": ["@napi-rs/cross-toolchain@1.0.3", "", { "dependencies": { "@napi-rs/lzma": "^1.4.5", "@napi-rs/tar": "^1.1.0", "debug": "^4.4.1" }, "peerDependencies": { "@napi-rs/cross-toolchain-arm64-target-aarch64": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-armv7": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-ppc64le": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-s390x": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-x86_64": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-aarch64": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-armv7": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-ppc64le": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-s390x": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-x86_64": "^1.0.3" }, "optionalPeers": ["@napi-rs/cross-toolchain-arm64-target-aarch64", "@napi-rs/cross-toolchain-arm64-target-armv7", "@napi-rs/cross-toolchain-arm64-target-ppc64le", "@napi-rs/cross-toolchain-arm64-target-s390x", "@napi-rs/cross-toolchain-arm64-target-x86_64", "@napi-rs/cross-toolchain-x64-target-aarch64", "@napi-rs/cross-toolchain-x64-target-armv7", "@napi-rs/cross-toolchain-x64-target-ppc64le", "@napi-rs/cross-toolchain-x64-target-s390x", "@napi-rs/cross-toolchain-x64-target-x86_64"] }, "sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg=="], + + "@napi-rs/lzma": ["@napi-rs/lzma@1.4.5", "", { "optionalDependencies": { "@napi-rs/lzma-android-arm-eabi": "1.4.5", "@napi-rs/lzma-android-arm64": "1.4.5", "@napi-rs/lzma-darwin-arm64": "1.4.5", "@napi-rs/lzma-darwin-x64": "1.4.5", "@napi-rs/lzma-freebsd-x64": "1.4.5", "@napi-rs/lzma-linux-arm-gnueabihf": "1.4.5", "@napi-rs/lzma-linux-arm64-gnu": "1.4.5", "@napi-rs/lzma-linux-arm64-musl": "1.4.5", "@napi-rs/lzma-linux-ppc64-gnu": "1.4.5", "@napi-rs/lzma-linux-riscv64-gnu": "1.4.5", "@napi-rs/lzma-linux-s390x-gnu": "1.4.5", "@napi-rs/lzma-linux-x64-gnu": "1.4.5", "@napi-rs/lzma-linux-x64-musl": "1.4.5", "@napi-rs/lzma-wasm32-wasi": "1.4.5", "@napi-rs/lzma-win32-arm64-msvc": "1.4.5", "@napi-rs/lzma-win32-ia32-msvc": "1.4.5", "@napi-rs/lzma-win32-x64-msvc": "1.4.5" } }, "sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg=="], + + "@napi-rs/lzma-android-arm-eabi": ["@napi-rs/lzma-android-arm-eabi@1.4.5", "", { "os": "android", "cpu": "arm" }, "sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q=="], + + "@napi-rs/lzma-android-arm64": ["@napi-rs/lzma-android-arm64@1.4.5", "", { "os": "android", "cpu": "arm64" }, "sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ=="], + + "@napi-rs/lzma-darwin-arm64": ["@napi-rs/lzma-darwin-arm64@1.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA=="], + + "@napi-rs/lzma-darwin-x64": ["@napi-rs/lzma-darwin-x64@1.4.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw=="], + + "@napi-rs/lzma-freebsd-x64": ["@napi-rs/lzma-freebsd-x64@1.4.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw=="], + + "@napi-rs/lzma-linux-arm-gnueabihf": ["@napi-rs/lzma-linux-arm-gnueabihf@1.4.5", "", { "os": "linux", "cpu": "arm" }, "sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw=="], + + "@napi-rs/lzma-linux-arm64-gnu": ["@napi-rs/lzma-linux-arm64-gnu@1.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg=="], + + "@napi-rs/lzma-linux-arm64-musl": ["@napi-rs/lzma-linux-arm64-musl@1.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w=="], + + "@napi-rs/lzma-linux-ppc64-gnu": ["@napi-rs/lzma-linux-ppc64-gnu@1.4.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ=="], + + "@napi-rs/lzma-linux-riscv64-gnu": ["@napi-rs/lzma-linux-riscv64-gnu@1.4.5", "", { "os": "linux", "cpu": "none" }, "sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA=="], + + "@napi-rs/lzma-linux-s390x-gnu": ["@napi-rs/lzma-linux-s390x-gnu@1.4.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA=="], + + "@napi-rs/lzma-linux-x64-gnu": ["@napi-rs/lzma-linux-x64-gnu@1.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw=="], + + "@napi-rs/lzma-linux-x64-musl": ["@napi-rs/lzma-linux-x64-musl@1.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ=="], + + "@napi-rs/lzma-wasm32-wasi": ["@napi-rs/lzma-wasm32-wasi@1.4.5", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA=="], + + "@napi-rs/lzma-win32-arm64-msvc": ["@napi-rs/lzma-win32-arm64-msvc@1.4.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg=="], + + "@napi-rs/lzma-win32-ia32-msvc": ["@napi-rs/lzma-win32-ia32-msvc@1.4.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg=="], + + "@napi-rs/lzma-win32-x64-msvc": ["@napi-rs/lzma-win32-x64-msvc@1.4.5", "", { "os": "win32", "cpu": "x64" }, "sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q=="], + + "@napi-rs/tar": ["@napi-rs/tar@1.1.0", "", { "optionalDependencies": { "@napi-rs/tar-android-arm-eabi": "1.1.0", "@napi-rs/tar-android-arm64": "1.1.0", "@napi-rs/tar-darwin-arm64": "1.1.0", "@napi-rs/tar-darwin-x64": "1.1.0", "@napi-rs/tar-freebsd-x64": "1.1.0", "@napi-rs/tar-linux-arm-gnueabihf": "1.1.0", "@napi-rs/tar-linux-arm64-gnu": "1.1.0", "@napi-rs/tar-linux-arm64-musl": "1.1.0", "@napi-rs/tar-linux-ppc64-gnu": "1.1.0", "@napi-rs/tar-linux-s390x-gnu": "1.1.0", "@napi-rs/tar-linux-x64-gnu": "1.1.0", "@napi-rs/tar-linux-x64-musl": "1.1.0", "@napi-rs/tar-wasm32-wasi": "1.1.0", "@napi-rs/tar-win32-arm64-msvc": "1.1.0", "@napi-rs/tar-win32-ia32-msvc": "1.1.0", "@napi-rs/tar-win32-x64-msvc": "1.1.0" } }, "sha512-7cmzIu+Vbupriudo7UudoMRH2OA3cTw67vva8MxeoAe5S7vPFI7z0vp0pMXiA25S8IUJefImQ90FeJjl8fjEaQ=="], + + "@napi-rs/tar-android-arm-eabi": ["@napi-rs/tar-android-arm-eabi@1.1.0", "", { "os": "android", "cpu": "arm" }, "sha512-h2Ryndraj/YiKgMV/r5by1cDusluYIRT0CaE0/PekQ4u+Wpy2iUVqvzVU98ZPnhXaNeYxEvVJHNGafpOfaD0TA=="], + + "@napi-rs/tar-android-arm64": ["@napi-rs/tar-android-arm64@1.1.0", "", { "os": "android", "cpu": "arm64" }, "sha512-DJFyQHr1ZxNZorm/gzc1qBNLF/FcKzcH0V0Vwan5P+o0aE2keQIGEjJ09FudkF9v6uOuJjHCVDdK6S6uHtShAw=="], + + "@napi-rs/tar-darwin-arm64": ["@napi-rs/tar-darwin-arm64@1.1.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Zz2sXRzjIX4e532zD6xm2SjXEym6MkvfCvL2RMpG2+UwNVDVscHNcz3d47Pf3sysP2e2af7fBB3TIoK2f6trPw=="], + + "@napi-rs/tar-darwin-x64": ["@napi-rs/tar-darwin-x64@1.1.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-EI+CptIMNweT0ms9S3mkP/q+J6FNZ1Q6pvpJOEcWglRfyfQpLqjlC0O+dptruTPE8VamKYuqdjxfqD8hifZDOA=="], + + "@napi-rs/tar-freebsd-x64": ["@napi-rs/tar-freebsd-x64@1.1.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J0PIqX+pl6lBIAckL/c87gpodLbjZB1OtIK+RDscKC9NLdpVv6VGOxzUV/fYev/hctcE8EfkLbgFOfpmVQPg2g=="], + + "@napi-rs/tar-linux-arm-gnueabihf": ["@napi-rs/tar-linux-arm-gnueabihf@1.1.0", "", { "os": "linux", "cpu": "arm" }, "sha512-SLgIQo3f3EjkZ82ZwvrEgFvMdDAhsxCYjyoSuWfHCz0U16qx3SuGCp8+FYOPYCECHN3ZlGjXnoAIt9ERd0dEUg=="], + + "@napi-rs/tar-linux-arm64-gnu": ["@napi-rs/tar-linux-arm64-gnu@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-d014cdle52EGaH6GpYTQOP9Py7glMO1zz/+ynJPjjzYFSxvdYx0byrjumZk2UQdIyGZiJO2MEFpCkEEKFSgPYA=="], + + "@napi-rs/tar-linux-arm64-musl": ["@napi-rs/tar-linux-arm64-musl@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ=="], + + "@napi-rs/tar-linux-ppc64-gnu": ["@napi-rs/tar-linux-ppc64-gnu@1.1.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ=="], + + "@napi-rs/tar-linux-s390x-gnu": ["@napi-rs/tar-linux-s390x-gnu@1.1.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ=="], + + "@napi-rs/tar-linux-x64-gnu": ["@napi-rs/tar-linux-x64-gnu@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww=="], + + "@napi-rs/tar-linux-x64-musl": ["@napi-rs/tar-linux-x64-musl@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ=="], + + "@napi-rs/tar-wasm32-wasi": ["@napi-rs/tar-wasm32-wasi@1.1.0", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw=="], + + "@napi-rs/tar-win32-arm64-msvc": ["@napi-rs/tar-win32-arm64-msvc@1.1.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-vfpG71OB0ijtjemp3WTdmBKJm9R70KM8vsSExMsIQtV0lVzP07oM1CW6JbNRPXNLhRoue9ofYLiUDk8bE0Hckg=="], + + "@napi-rs/tar-win32-ia32-msvc": ["@napi-rs/tar-win32-ia32-msvc@1.1.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-hGPyPW60YSpOSgzfy68DLBHgi6HxkAM+L59ZZZPMQ0TOXjQg+p2EW87+TjZfJOkSpbYiEkULwa/f4a2hcVjsqQ=="], + + "@napi-rs/tar-win32-x64-msvc": ["@napi-rs/tar-win32-x64-msvc@1.1.0", "", { "os": "win32", "cpu": "x64" }, "sha512-L6Ed1DxXK9YSCMyvpR8MiNAyKNkQLjsHsHK9E0qnHa8NzLFqzDKhvs5LfnWxM2kJ+F7m/e5n9zPm24kHb3LsVw=="], + + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.2", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-sNXv5oLJ7ob93xkZ1XnxisYhGYXfaG9f65/ZgYuAu3qt7b3NadcOEhLvx28hv31PgX8SZJRYrAIPQilQmFpLVw=="], + + "@napi-rs/wasm-tools": ["@napi-rs/wasm-tools@1.0.1", "", { "optionalDependencies": { "@napi-rs/wasm-tools-android-arm-eabi": "1.0.1", "@napi-rs/wasm-tools-android-arm64": "1.0.1", "@napi-rs/wasm-tools-darwin-arm64": "1.0.1", "@napi-rs/wasm-tools-darwin-x64": "1.0.1", "@napi-rs/wasm-tools-freebsd-x64": "1.0.1", "@napi-rs/wasm-tools-linux-arm64-gnu": "1.0.1", "@napi-rs/wasm-tools-linux-arm64-musl": "1.0.1", "@napi-rs/wasm-tools-linux-x64-gnu": "1.0.1", "@napi-rs/wasm-tools-linux-x64-musl": "1.0.1", "@napi-rs/wasm-tools-wasm32-wasi": "1.0.1", "@napi-rs/wasm-tools-win32-arm64-msvc": "1.0.1", "@napi-rs/wasm-tools-win32-ia32-msvc": "1.0.1", "@napi-rs/wasm-tools-win32-x64-msvc": "1.0.1" } }, "sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ=="], + + "@napi-rs/wasm-tools-android-arm-eabi": ["@napi-rs/wasm-tools-android-arm-eabi@1.0.1", "", { "os": "android", "cpu": "arm" }, "sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw=="], + + "@napi-rs/wasm-tools-android-arm64": ["@napi-rs/wasm-tools-android-arm64@1.0.1", "", { "os": "android", "cpu": "arm64" }, "sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg=="], + + "@napi-rs/wasm-tools-darwin-arm64": ["@napi-rs/wasm-tools-darwin-arm64@1.0.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g=="], + + "@napi-rs/wasm-tools-darwin-x64": ["@napi-rs/wasm-tools-darwin-x64@1.0.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A=="], + + "@napi-rs/wasm-tools-freebsd-x64": ["@napi-rs/wasm-tools-freebsd-x64@1.0.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg=="], + + "@napi-rs/wasm-tools-linux-arm64-gnu": ["@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q=="], + + "@napi-rs/wasm-tools-linux-arm64-musl": ["@napi-rs/wasm-tools-linux-arm64-musl@1.0.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ=="], + + "@napi-rs/wasm-tools-linux-x64-gnu": ["@napi-rs/wasm-tools-linux-x64-gnu@1.0.1", "", { "os": "linux", "cpu": "x64" }, "sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw=="], + + "@napi-rs/wasm-tools-linux-x64-musl": ["@napi-rs/wasm-tools-linux-x64-musl@1.0.1", "", { "os": "linux", "cpu": "x64" }, "sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ=="], + + "@napi-rs/wasm-tools-wasm32-wasi": ["@napi-rs/wasm-tools-wasm32-wasi@1.0.1", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA=="], + + "@napi-rs/wasm-tools-win32-arm64-msvc": ["@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ=="], + + "@napi-rs/wasm-tools-win32-ia32-msvc": ["@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw=="], + + "@napi-rs/wasm-tools-win32-x64-msvc": ["@napi-rs/wasm-tools-win32-x64-msvc@1.0.1", "", { "os": "win32", "cpu": "x64" }, "sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ=="], + + "@octokit/auth-token": ["@octokit/auth-token@6.0.0", "", {}, "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w=="], + + "@octokit/core": ["@octokit/core@7.0.6", "", { "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "before-after-hook": "^4.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q=="], + + "@octokit/endpoint": ["@octokit/endpoint@11.0.3", "", { "dependencies": { "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag=="], + + "@octokit/graphql": ["@octokit/graphql@9.0.3", "", { "dependencies": { "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA=="], + + "@octokit/openapi-types": ["@octokit/openapi-types@27.0.0", "", {}, "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA=="], + + "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@14.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw=="], + + "@octokit/plugin-request-log": ["@octokit/plugin-request-log@6.0.0", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q=="], + + "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@17.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw=="], + + "@octokit/request": ["@octokit/request@10.0.8", "", { "dependencies": { "@octokit/endpoint": "^11.0.3", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "fast-content-type-parse": "^3.0.0", "json-with-bigint": "^3.5.3", "universal-user-agent": "^7.0.2" } }, "sha512-SJZNwY9pur9Agf7l87ywFi14W+Hd9Jg6Ifivsd33+/bGUQIjNujdFiXII2/qSlN2ybqUHfp5xpekMEjIBTjlSw=="], + + "@octokit/request-error": ["@octokit/request-error@7.1.0", "", { "dependencies": { "@octokit/types": "^16.0.0" } }, "sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw=="], + + "@octokit/rest": ["@octokit/rest@22.0.1", "", { "dependencies": { "@octokit/core": "^7.0.6", "@octokit/plugin-paginate-rest": "^14.0.0", "@octokit/plugin-request-log": "^6.0.0", "@octokit/plugin-rest-endpoint-methods": "^17.0.0" } }, "sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw=="], + + "@octokit/types": ["@octokit/types@16.0.0", "", { "dependencies": { "@octokit/openapi-types": "^27.0.0" } }, "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg=="], + + "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + + "before-after-hook": ["before-after-hook@4.0.0", "", {}, "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ=="], + + "chardet": ["chardet@2.1.1", "", {}, "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ=="], + + "cli-width": ["cli-width@4.1.0", "", {}, "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ=="], + + "clipanion": ["clipanion@4.0.0-rc.4", "", { "dependencies": { "typanion": "^3.8.0" } }, "sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q=="], + + "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "emnapi": ["emnapi@1.9.2", "", { "peerDependencies": { "node-addon-api": ">= 6.1.0" }, "optionalPeers": ["node-addon-api"] }, "sha512-OdUoQe/8so7FvubnE/DNV9sNNSFwDYQiK4ZCAz4agMnD1s6faLuDn2gzxfJrmMoKfxZhhsckqGNwqPnS5K140A=="], + + "es-toolkit": ["es-toolkit@1.45.1", "", {}, "sha512-/jhoOj/Fx+A+IIyDNOvO3TItGmlMKhtX8ISAHKE90c4b/k1tqaqEZ+uUqfpU8DMnW5cgNJv606zS55jGvza0Xw=="], + + "fast-content-type-parse": ["fast-content-type-parse@3.0.0", "", {}, "sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg=="], + + "fast-string-truncated-width": ["fast-string-truncated-width@3.0.3", "", {}, "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g=="], + + "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], + + "fast-wrap-ansi": ["fast-wrap-ansi@0.2.0", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w=="], + + "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + + "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + + "json-with-bigint": ["json-with-bigint@3.5.8", "", {}, "sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "mute-stream": ["mute-stream@3.0.0", "", {}, "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw=="], + + "obug": ["obug@2.1.1", "", {}, "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "typanion": ["typanion@3.14.0", "", {}, "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug=="], + + "universal-user-agent": ["universal-user-agent@7.0.3", "", {}, "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A=="], + } +} diff --git a/napi/index.d.ts b/napi/index.d.ts new file mode 100644 index 0000000..8254452 --- /dev/null +++ b/napi/index.d.ts @@ -0,0 +1,49 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ +/** + * Classify a PDF: detect type (TextBased/Scanned/Mixed/ImageBased), + * page count, and which pages need OCR. Takes PDF bytes as Buffer. + */ +export declare function classifyPdf(buffer: Buffer): PdfClassification + +/** + * Extract text within bounding-box regions from a PDF. + * + * For hybrid OCR: layout model detects regions in rendered images, + * this extracts PDF text within those regions — skipping GPU OCR + * for text-based pages. + * + * Each region result includes `needs_ocr` — set when the extracted text + * is unreliable (empty, GID-encoded fonts, garbage, encoding issues). + * + * Coordinates are PDF points with top-left origin. + */ +export declare function extractTextInRegions(buffer: Buffer, pageRegions: Array): Array + +/** A page's regions for text extraction: (page_index_0based, bboxes). */ +export interface PageRegions { + page: number + /** Each bbox is [x1, y1, x2, y2] in PDF points, top-left origin. */ + regions: Array> +} + +/** Extracted text for one page's regions. */ +export interface PageRegionTexts { + page: number + regions: Array +} + +/** Lightweight PDF classification result. */ +export interface PdfClassification { + pdfType: string + pageCount: number + pagesNeedingOcr: Array + confidence: number +} + +/** Extracted text for a single region. */ +export interface RegionText { + text: string + /** `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues). */ + needsOcr: boolean +} diff --git a/napi/index.js b/napi/index.js new file mode 100644 index 0000000..23f7a6e --- /dev/null +++ b/napi/index.js @@ -0,0 +1,580 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* auto-generated by NAPI-RS */ + +const { readFileSync } = require('node:fs') +let nativeBinding = null +const loadErrors = [] + +const isMusl = () => { + let musl = false + if (process.platform === 'linux') { + musl = isMuslFromFilesystem() + if (musl === null) { + musl = isMuslFromReport() + } + if (musl === null) { + musl = isMuslFromChildProcess() + } + } + return musl +} + +const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-') + +const isMuslFromFilesystem = () => { + try { + return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl') + } catch { + return null + } +} + +const isMuslFromReport = () => { + let report = null + if (typeof process.report?.getReport === 'function') { + process.report.excludeNetwork = true + report = process.report.getReport() + } + if (!report) { + return null + } + if (report.header && report.header.glibcVersionRuntime) { + return false + } + if (Array.isArray(report.sharedObjects)) { + if (report.sharedObjects.some(isFileMusl)) { + return true + } + } + return false +} + +const isMuslFromChildProcess = () => { + try { + return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl') + } catch (e) { + // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false + return false + } +} + +function requireNative() { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + try { + return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH); + } catch (err) { + loadErrors.push(err) + } + } else if (process.platform === 'android') { + if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.android-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-android-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-android-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./pdf-inspector.android-arm-eabi.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-android-arm-eabi') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-android-arm-eabi/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`)) + } + } else if (process.platform === 'win32') { + if (process.arch === 'x64') { + if (process.config?.variables?.shlib_suffix === 'dll.a' || process.config?.variables?.node_target_type === 'shared_library') { + try { + return require('./pdf-inspector.win32-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-x64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.win32-x64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-x64-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-x64-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ia32') { + try { + return require('./pdf-inspector.win32-ia32-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-ia32-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-ia32-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.win32-arm64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-arm64-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-arm64-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`)) + } + } else if (process.platform === 'darwin') { + try { + return require('./pdf-inspector.darwin-universal.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-universal') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-universal/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + if (process.arch === 'x64') { + try { + return require('./pdf-inspector.darwin-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.darwin-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`)) + } + } else if (process.platform === 'freebsd') { + if (process.arch === 'x64') { + try { + return require('./pdf-inspector.freebsd-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-freebsd-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-freebsd-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.freebsd-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-freebsd-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-freebsd-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`)) + } + } else if (process.platform === 'linux') { + if (process.arch === 'x64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-x64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-x64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-x64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-x64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-arm64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-arm64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-arm-musleabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm-musleabihf') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm-musleabihf/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-arm-gnueabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm-gnueabihf') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm-gnueabihf/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'loong64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-loong64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-loong64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-loong64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-loong64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-loong64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-loong64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'riscv64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-riscv64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-riscv64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-riscv64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-riscv64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-riscv64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-riscv64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ppc64') { + try { + return require('./pdf-inspector.linux-ppc64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-ppc64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-ppc64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 's390x') { + try { + return require('./pdf-inspector.linux-s390x-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-s390x-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-s390x-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`)) + } + } else if (process.platform === 'openharmony') { + if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.openharmony-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'x64') { + try { + return require('./pdf-inspector.openharmony-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./pdf-inspector.openharmony-arm.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-arm') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-arm/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`)) + } + } else { + loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`)) + } +} + +nativeBinding = requireNative() + +if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { + let wasiBinding = null + let wasiBindingError = null + try { + wasiBinding = require('./pdf-inspector.wasi.cjs') + nativeBinding = wasiBinding + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + wasiBindingError = err + } + } + if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { + try { + wasiBinding = require('@firecrawl/pdf-inspector-js-wasm32-wasi') + nativeBinding = wasiBinding + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + if (!wasiBindingError) { + wasiBindingError = err + } else { + wasiBindingError.cause = err + } + loadErrors.push(err) + } + } + } + if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) { + const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error') + error.cause = wasiBindingError + throw error + } +} + +if (!nativeBinding) { + if (loadErrors.length > 0) { + throw new Error( + `Cannot find native binding. ` + + `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` + + 'Please try `npm i` again after removing both package-lock.json and node_modules directory.', + { + cause: loadErrors.reduce((err, cur) => { + cur.cause = err + return cur + }), + }, + ) + } + throw new Error(`Failed to load native binding`) +} + +module.exports = nativeBinding +module.exports.classifyPdf = nativeBinding.classifyPdf +module.exports.extractTextInRegions = nativeBinding.extractTextInRegions diff --git a/napi/package.json b/napi/package.json new file mode 100644 index 0000000..86b28eb --- /dev/null +++ b/napi/package.json @@ -0,0 +1,34 @@ +{ + "name": "@firecrawl/pdf-inspector-js", + "version": "0.2.0", + "main": "index.js", + "types": "index.d.ts", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/firecrawl/pdf-inspector" + }, + "publishConfig": { + "registry": "https://npm.pkg.github.com", + "access": "public" + }, + "napi": { + "binaryName": "pdf-inspector", + "targets": [ + "x86_64-unknown-linux-gnu", + "aarch64-apple-darwin" + ], + "package": { + "name": "@firecrawl/pdf-inspector-js" + } + }, + "scripts": { + "build": "napi build --platform --release", + "build:debug": "napi build --platform", + "prepublishOnly": "napi prepublish -t npm", + "artifacts": "napi artifacts" + }, + "devDependencies": { + "@napi-rs/cli": "^3.4.1" + } +} diff --git a/napi/src/lib.rs b/napi/src/lib.rs new file mode 100644 index 0000000..c13ffdf --- /dev/null +++ b/napi/src/lib.rs @@ -0,0 +1,116 @@ +#![deny(clippy::all)] + +use napi::bindgen_prelude::*; +use napi_derive::napi; + +/// Lightweight PDF classification result. +#[napi(object)] +pub struct PdfClassification { + pub pdf_type: String, + pub page_count: u32, + pub pages_needing_ocr: Vec, + pub confidence: f64, +} + +/// A page's regions for text extraction: (page_index_0based, bboxes). +#[napi(object)] +pub struct PageRegions { + pub page: u32, + /// Each bbox is [x1, y1, x2, y2] in PDF points, top-left origin. + pub regions: Vec>, +} + +/// Extracted text for a single region. +#[napi(object)] +pub struct RegionText { + pub text: String, + /// `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues). + pub needs_ocr: bool, +} + +/// Extracted text for one page's regions. +#[napi(object)] +pub struct PageRegionTexts { + pub page: u32, + pub regions: Vec, +} + +/// Classify a PDF: detect type (TextBased/Scanned/Mixed/ImageBased), +/// page count, and which pages need OCR. Takes PDF bytes as Buffer. +#[napi] +pub fn classify_pdf(buffer: Buffer) -> Result { + let result = pdf_inspector::classify_pdf_mem(&buffer).map_err(|e| { + Error::new(Status::GenericFailure, format!("classify_pdf failed: {e}")) + })?; + + Ok(PdfClassification { + pdf_type: match result.pdf_type { + pdf_inspector::PdfType::TextBased => "TextBased".to_string(), + pdf_inspector::PdfType::Scanned => "Scanned".to_string(), + pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(), + pdf_inspector::PdfType::Mixed => "Mixed".to_string(), + }, + page_count: result.page_count, + pages_needing_ocr: result.pages_needing_ocr, + confidence: result.confidence as f64, + }) +} + +/// Extract text within bounding-box regions from a PDF. +/// +/// For hybrid OCR: layout model detects regions in rendered images, +/// this extracts PDF text within those regions — skipping GPU OCR +/// for text-based pages. +/// +/// Each region result includes `needs_ocr` — set when the extracted text +/// is unreliable (empty, GID-encoded fonts, garbage, encoding issues). +/// +/// Coordinates are PDF points with top-left origin. +#[napi] +pub fn extract_text_in_regions( + buffer: Buffer, + page_regions: Vec, +) -> Result> { + // Convert from napi types to the Rust API's expected format + let regions: Vec<(u32, Vec<[f32; 4]>)> = page_regions + .iter() + .map(|pr| { + let bboxes: Vec<[f32; 4]> = pr + .regions + .iter() + .map(|r| { + if r.len() != 4 { + [0.0, 0.0, 0.0, 0.0] + } else { + [r[0] as f32, r[1] as f32, r[2] as f32, r[3] as f32] + } + }) + .collect(); + (pr.page, bboxes) + }) + .collect(); + + let results = pdf_inspector::extract_text_in_regions_mem(&buffer, ®ions).map_err(|e| { + Error::new( + Status::GenericFailure, + format!("extract_text_in_regions failed: {e}"), + ) + })?; + + Ok( + results + .into_iter() + .map(|page_result| PageRegionTexts { + page: page_result.page, + regions: page_result + .regions + .into_iter() + .map(|r| RegionText { + text: r.text, + needs_ocr: r.needs_ocr, + }) + .collect(), + }) + .collect(), + ) +}