From d9c2143c320c21e48d76337b06ed998dce94fd4a Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Mar 2026 20:50:24 -0700 Subject: [PATCH] feat: tagged PDF structure tree support (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: tagged PDF structure tree support for semantic markdown generation Parse /StructTreeRoot from tagged PDFs and use semantic roles (H1-H6, P, LI, BlockQuote, Code, Caption) to improve markdown output. Structure tree headings add to font-size heuristics without suppressing them. Coverage threshold (≥50%) ensures only properly tagged PDFs activate this path. Phase 1: Parse structure tree with role maps, MCID collection, flattening Phase 2: Capture MCIDs from BMC/BDC operators, tag TextItems Phase 3: Structure-aware markdown generation in convert loop Co-Authored-By: Claude Opus 4.6 * fix: accumulate consecutive code lines into single fenced block Per-line code fencing produced broken markdown for multi-line code blocks (separate ``` open/close per line). Unify struct-tree Code role and font-based monospace detection into a single is_code_line check with in_code_block state for proper accumulation. Co-Authored-By: Claude Opus 4.6 * test: add tagged PDF fixture with Firecrawl docs content Synthetic 7-page PDF with rich structure tree exercising H1, H2, H3, P, Code, LI, Caption, TH, TD roles. Generated via fpdf2 script. Integration test verifies struct tree parsing and code fence output. Co-Authored-By: Claude Opus 4.6 * chore: remove python PDF generator script from repo Keep the generated fixture PDF but don't track the generator script. Co-Authored-By: Claude Opus 4.6 * fix: handle malformed bare-name struct types in tagged PDFs Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code in structure elements. lopdf silently drops these objects since bare tokens are invalid PDF syntax. Add a pre-processor that scans for known bare struct type names and prepends / before loading. Unifies path and memory loading through the same fix pipeline. Co-Authored-By: Claude Opus 4.6 * chore: update lopdf dependency to main branch The firecrawl/zlib-checksum-encrypted branch was merged and deleted. Point to main which includes all previously merged fixes. Co-Authored-By: Claude Opus 4.6 * chore: switch lopdf to upstream repo pinned at 845cd3d Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- Cargo.toml | 2 +- src/extractor/content_stream.rs | 92 +- src/extractor/layout.rs | 1 + src/extractor/links.rs | 2 + src/extractor/mod.rs | 23 + src/extractor/xobjects.rs | 2 + src/lib.rs | 37 +- src/markdown/convert.rs | 435 ++++++++- src/markdown/mod.rs | 39 +- src/structure_tree.rs | 1035 ++++++++++++++++++++++ src/tables/detect_heuristic.rs | 1 + src/tables/detect_lines.rs | 1 + src/tables/detect_rects.rs | 1 + src/tables/financial.rs | 1 + src/tables/grid.rs | 3 + src/tables/mod.rs | 4 + src/text_utils.rs | 3 + src/types.rs | 3 + tests/fixtures/bare_name_struct.pdf | Bin 0 -> 1865 bytes tests/fixtures/firecrawl_docs_tagged.pdf | Bin 0 -> 64026 bytes tests/integration_tests.rs | 38 + 21 files changed, 1650 insertions(+), 73 deletions(-) create mode 100644 src/structure_tree.rs create mode 100644 tests/fixtures/bare_name_struct.pdf create mode 100644 tests/fixtures/firecrawl_docs_tagged.pdf diff --git a/Cargo.toml b/Cargo.toml index 22ae989..eea0ced 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/firecrawl/pdf-inspector" [dependencies] # PDF parsing -lopdf = { git = "https://github.com/firecrawl/lopdf", branch = "firecrawl/zlib-checksum-encrypted", features = ["rayon"] } +lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "845cd3d4648d9cefb7b5def5fb387df61ba3a0e5", features = ["rayon"] } # Error handling thiserror = "2.0" diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 2ce06e6..ff4a3c1 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -124,10 +124,18 @@ pub(crate) fn extract_page_text_items( let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; - // Marked content (ActualText) tracking - let mut marked_content_stack: Vec> = Vec::new(); + // Marked content tracking: (ActualText, MCID) per nesting level + struct MarkedContentEntry { + actual_text: Option, + mcid: Option, + } + let mut marked_content_stack: Vec = Vec::new(); let mut suppress_glyph_extraction = false; let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry + /// Get the innermost MCID from the marked content stack. + fn current_mcid(stack: &[MarkedContentEntry]) -> Option { + stack.iter().rev().find_map(|e| e.mcid) + } for op in &content.operations { trace!("{} {:?}", op.operator, op.operands); @@ -292,6 +300,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -440,6 +449,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -496,6 +506,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -531,11 +542,15 @@ pub(crate) fn extract_page_text_items( } "BMC" => { // Begin Marked Content (no properties) - marked_content_stack.push(None); + marked_content_stack.push(MarkedContentEntry { + actual_text: None, + mcid: None, + }); } "BDC" => { - // Begin Marked Content with properties — extract ActualText + // Begin Marked Content with properties — extract ActualText and MCID let mut actual_text: Option = None; + let mut mcid: Option = None; if op.operands.len() >= 2 { let dict = match &op.operands[1] { Object::Dictionary(d) => Some(d.clone()), @@ -549,47 +564,56 @@ pub(crate) fn extract_page_text_items( _ => None, }; } + if let Ok(Object::Integer(id)) = d.get(b"MCID") { + mcid = Some(*id); + } } } if actual_text.is_some() { suppress_glyph_extraction = true; actual_text_start_tm = Some(text_matrix); } - marked_content_stack.push(actual_text); + marked_content_stack.push(MarkedContentEntry { actual_text, mcid }); } "EMC" => { // End Marked Content — emit ActualText item with correct width - if let Some(Some(at)) = marked_content_stack.pop() { - // Compute width from text matrix advancement during BDC..EMC - if let Some(start_tm) = actual_text_start_tm.take() { - let combined = multiply_matrices(&start_tm, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - // Width in device space from text matrix delta - let delta_ts = text_matrix[4] - start_tm[4]; - let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; - let width = (delta_ts * scale_x).abs(); - if !at.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&at), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); + if let Some(entry) = marked_content_stack.pop() { + if let Some(at) = entry.actual_text { + // Compute width from text matrix advancement during BDC..EMC + if let Some(start_tm) = actual_text_start_tm.take() { + let combined = multiply_matrices(&start_tm, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + // Width in device space from text matrix delta + let delta_ts = text_matrix[4] - start_tm[4]; + let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; + let width = (delta_ts * scale_x).abs(); + if !at.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&at), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + mcid: entry + .mcid + .or_else(|| current_mcid(&marked_content_stack)), + }); + } } + suppress_glyph_extraction = + marked_content_stack.iter().any(|e| e.actual_text.is_some()); } - suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); } } "re" => { diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index 471ec81..312c6ca 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -796,6 +796,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } diff --git a/src/extractor/links.rs b/src/extractor/links.rs index f7e9090..74a1fae 100644 --- a/src/extractor/links.rs +++ b/src/extractor/links.rs @@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V is_bold: false, is_italic: false, item_type: ItemType::Link(url), + mcid: None, }); } } @@ -316,5 +317,6 @@ pub(crate) fn walk_form_fields( is_bold: false, is_italic: false, item_type: ItemType::FormField, + mcid: None, }); } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 767506f..2b85eed 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -319,6 +319,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { is_bold: first.is_bold, is_italic: first.is_italic, item_type: first.item_type.clone(), + mcid: first.mcid, }); i = j; @@ -359,6 +360,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -372,6 +374,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Next line".into(), @@ -385,6 +388,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -438,6 +442,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Prague".into(), @@ -451,6 +456,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Rules".into(), @@ -464,6 +470,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -488,6 +495,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "A".into(), @@ -501,6 +509,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "V".into(), @@ -514,6 +523,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -540,6 +550,7 @@ mod tests { is_bold: true, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -573,6 +584,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -607,6 +619,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "履行義務".into(), @@ -620,6 +633,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "を識別す".into(), @@ -633,6 +647,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -654,6 +669,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -765,6 +781,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "\u{05D1}".into(), // bet at x=200 (rightmost) @@ -778,6 +795,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -801,6 +819,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -814,6 +833,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -853,6 +873,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -896,6 +917,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -939,6 +961,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index c9e5da1..e011b88 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -408,6 +408,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } @@ -549,6 +550,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } diff --git a/src/lib.rs b/src/lib.rs index 7011967..4acabe5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ pub mod extractor; pub mod glyph_names; pub mod markdown; pub mod process_mode; +pub mod structure_tree; pub mod tables; pub mod text_utils; pub mod tounicode; @@ -273,20 +274,21 @@ pub fn process_pdf_mem_with_config( /// are combined here, but lopdf loads the full doc in `load()` so we extract /// page count from it directly to avoid the metadata-only round-trip. fn load_document_from_path>(path: P) -> Result<(Document, u32), PdfError> { - let doc = match Document::load(&path) { - Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_with_password(&path, "")?, - Err(e) => return Err(e.into()), - }; - let page_count = doc.get_pages().len() as u32; - Ok((doc, page_count)) + let buffer = std::fs::read(&path)?; + load_document_from_mem(&buffer) } /// Load a PDF from a memory buffer. fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> { - let doc = match Document::load_mem(buffer) { + // Fix malformed struct element names before parsing. Some PDF generators + // write bare names (/S Code) instead of proper PDF names (/S /Code), which + // causes lopdf to silently drop the entire object. + let fixed = structure_tree::fix_bare_struct_names(buffer); + let buf = fixed.as_ref(); + + let doc = match Document::load_mem(buf) { Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buffer, "")?, + Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buf, "")?, Err(e) => return Err(e.into()), }; let page_count = doc.get_pages().len() as u32; @@ -350,6 +352,22 @@ fn process_document( Some(extracted?) }; + // Parse structure tree for tagged PDFs (reuses the loaded document) + let struct_roles = structure_tree::StructTree::from_doc(&doc).and_then(|tree| { + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + if roles.is_empty() { + None + } else { + log::debug!( + "structure tree: {} pages with MCID roles, {} total MCIDs", + roles.len(), + tree.mcid_count() + ); + Some(roles) + } + }); + let (markdown, layout, has_encoding_issues) = match extracted { Some(((items, rects, lines), page_thresholds)) => { let layout = compute_layout_complexity(&items, &rects, &lines); @@ -363,6 +381,7 @@ fn process_document( &rects, &lines, &page_thresholds, + struct_roles.as_ref(), )) }; diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 42fb107..1fef66f 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; +use crate::structure_tree::StructRole; use crate::types::TextLine; use super::analysis::{ @@ -13,6 +14,50 @@ use super::postprocess::clean_markdown; use super::preprocess::{merge_drop_caps, merge_heading_lines}; use super::MarkdownOptions; +/// Resolve the dominant structure role for a text line by looking up its items' MCIDs. +/// +/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span). +/// These wrapper roles don't carry useful semantic info for markdown generation. +fn resolve_line_struct_role( + line: &TextLine, + struct_roles: &std::collections::HashMap>, +) -> Option { + let page_roles = struct_roles.get(&line.page)?; + for item in &line.items { + if let Some(mcid) = item.mcid { + if let Some(role) = page_roles.get(&mcid) { + match role { + // Skip container/wrapper roles — not useful for line classification + StructRole::Document + | StructRole::Part + | StructRole::Art + | StructRole::Sect + | StructRole::Div + | StructRole::NonStruct + | StructRole::Span + | StructRole::Private => continue, + _ => return Some(role.clone()), + } + } + } + } + None +} + +/// Map a StructRole heading variant to a markdown heading level (1–6). +fn struct_role_heading_level(role: &StructRole) -> Option { + match role { + StructRole::H => Some(1), // Generic heading → H1 + StructRole::H1 => Some(1), + StructRole::H2 => Some(2), + StructRole::H3 => Some(3), + StructRole::H4 => Some(4), + StructRole::H5 => Some(5), + StructRole::H6 => Some(6), + _ => None, + } +} + /// Merge continuation tables that span across page breaks. /// /// When consecutive pages each have exactly one table with the same number of columns @@ -182,6 +227,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( page_tables: std::collections::HashMap>, page_images: std::collections::HashMap>, band_split_pages: &HashSet, + struct_roles: Option< + &std::collections::HashMap>, + >, ) -> String { if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { return String::new(); @@ -215,6 +263,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let mut in_list = false; let mut in_paragraph = false; let mut last_list_x: Option = None; + let mut in_code_block = false; let mut prev_had_dot_leaders = false; let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); @@ -233,6 +282,10 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( if line.page != current_page { // Flush current page's remaining tables and images if current_page > 0 { + if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } flush_page_tables_and_images( current_page, &page_tables, @@ -352,7 +405,25 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Detect figure/table captions and source citations // These should be on their own line followed by a paragraph break - if is_caption_line(plain_trimmed) { + let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(&line, roles)); + + // Determine if this line is code (struct-tree or font-based) for block accumulation + let is_code_line = struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Code)) + || (options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))); + + // Close code block when transitioning to non-code + if in_code_block && !is_code_line { + output.push_str("```\n"); + in_code_block = false; + } + + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Caption)) + || is_caption_line(plain_trimmed) + { if in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -362,27 +433,48 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( continue; } - // Detect headers by font size - // Note: Headers typically shouldn't have bold markers since they're already emphasized - // Skip very short text (drop caps/labels) and very long text (body paragraphs) - if options.detect_headers + // Detect headers: structure-tree headings win, then font-size heuristics. + // Structure roles ADD headings (e.g. same-size text tagged H2) but do NOT + // suppress headings that the font heuristic would detect (some tagged PDFs + // mark obvious headings as P or Span). + let struct_heading = struct_role.as_ref().and_then(struct_role_heading_level); + let heuristic_heading = if options.detect_headers && plain_trimmed.len() > 3 && plain_trimmed.split_whitespace().count() <= 15 { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers) - { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let prefix = "#".repeat(header_level); - // Use plain text for headers to avoid redundant formatting - output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); - in_list = false; - continue; + detect_header_level(line_font_size, base_size, &heading_tiers) + } else { + None + }; + + if let Some(level) = struct_heading.or(heuristic_heading) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + let prefix = "#".repeat(level); + // Use plain text for headers to avoid redundant formatting + output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); + in_list = false; + continue; + } + + // Structure-tree list item (LI only — LBody is a continuation, not a new item) + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::LI)) + && !is_list_item(plain_trimmed) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(&format!("- {}", trimmed)); + output.push('\n'); + in_list = true; + last_list_x = line.items.first().map(|i| i.x); + continue; } // Detect list items @@ -428,18 +520,32 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( } } - // Detect code blocks by font - if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; + // Structure-tree block quote + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::BlockQuote)) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + output.push_str(&format!("> {}\n", trimmed)); + continue; + } + + // Code block accumulation (struct-tree Code role or monospace font) + if is_code_line { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + if !in_code_block { + output.push_str("```\n"); + in_code_block = true; + } + output.push_str(plain_trimmed); + output.push('\n'); + continue; } // Regular text - join lines within same paragraph with space @@ -456,6 +562,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( prev_had_dot_leaders = cur_dot_leaders; } + // Close any trailing code block + if in_code_block { + output.push_str("```\n"); + } + // Flush current page and any remaining pages with tables/images // (handles table-only pages after the last text line, and trailing image-only pages) flush_page_tables_and_images( @@ -680,3 +791,271 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Clean up and post-process clean_markdown(output, &options) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::structure_tree::StructRole; + use crate::types::TextItem; + use std::collections::HashMap; + + fn make_item(text: &str, page: u32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x: 72.0, + y: 700.0, + width: 100.0, + height: 12.0, + font: "Helvetica".to_string(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid, + } + } + + fn make_line(items: Vec) -> TextLine { + let y = items.first().map(|i| i.y).unwrap_or(0.0); + let page = items.first().map(|i| i.page).unwrap_or(1); + TextLine { + items, + y, + page, + adaptive_threshold: 0.10, + } + } + + #[test] + fn test_struct_role_heading() { + let lines = vec![ + make_line(vec![make_item("Introduction", 1, Some(0))]), + make_line(vec![{ + let mut item = make_item("Body text here.", 1, Some(1)); + item.y = 680.0; + item + }]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::H1); + page_roles.insert(1i64, StructRole::P); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("# Introduction"), + "Should have H1 heading: {md}" + ); + assert!( + md.contains("Body text here."), + "Should have body text: {md}" + ); + } + + #[test] + fn test_struct_role_list_item() { + let lines = vec![make_line(vec![make_item("First item", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::LI); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("- First item"), + "Should format as list item: {md}" + ); + } + + #[test] + fn test_struct_role_blockquote() { + let lines = vec![make_line(vec![make_item("Quoted text", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::BlockQuote); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("> Quoted text"), + "Should format as blockquote: {md}" + ); + } + + #[test] + fn test_struct_role_heading_levels() { + let mcids = vec![ + (StructRole::H1, "Title"), + (StructRole::H2, "Section"), + (StructRole::H3, "Subsection"), + ]; + + let mut lines = Vec::new(); + let mut page_roles = HashMap::new(); + for (i, (role, text)) in mcids.iter().enumerate() { + let mut item = make_item(text, 1, Some(i as i64)); + item.y = 700.0 - (i as f32 * 30.0); + lines.push(make_line(vec![item])); + page_roles.insert(i as i64, role.clone()); + } + + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!(md.contains("# Title"), "H1 → #: {md}"); + assert!(md.contains("## Section"), "H2 → ##: {md}"); + assert!(md.contains("### Subsection"), "H3 → ###: {md}"); + } + + #[test] + fn test_no_struct_roles_falls_back_to_heuristics() { + let mut item = make_item("Big Title", 1, None); + item.font_size = 24.0; + item.height = 24.0; + + let lines = vec![ + make_line(vec![item]), + make_line(vec![{ + let mut body = make_item("Normal body text.", 1, None); + body.y = 660.0; + body + }]), + ]; + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + None, + ); + + assert!( + md.contains("# Big Title"), + "Font heuristic should detect heading: {md}" + ); + } + + #[test] + fn test_resolve_line_struct_role_skips_containers() { + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Div); + page_roles.insert(1i64, StructRole::H2); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let line = make_line(vec![ + make_item("Part ", 1, Some(0)), + make_item("Title", 1, Some(1)), + ]); + + let role = resolve_line_struct_role(&line, &roles); + assert_eq!(role, Some(StructRole::H2)); + } + + #[test] + fn test_struct_role_code() { + let lines = vec![make_line(vec![make_item("fn main() {}", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("```\nfn main() {}\n```"), + "Should format as code block: {md}" + ); + } + + #[test] + fn test_struct_role_code_multiline_accumulation() { + let mut line1 = make_item("fn main() {", 1, Some(0)); + line1.y = 700.0; + let mut line2 = make_item(" println!(\"hello\");", 1, Some(1)); + line2.y = 688.0; + let mut line3 = make_item("}", 1, Some(2)); + line3.y = 676.0; + + let lines = vec![ + make_line(vec![line1]), + make_line(vec![line2]), + make_line(vec![line3]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + page_roles.insert(1i64, StructRole::Code); + page_roles.insert(2i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + // Should produce a single fenced block, not three separate ones + assert!( + md.contains("```\nfn main() {\nprintln!(\"hello\");\n}\n```"), + "Should accumulate consecutive code lines into one block: {md}" + ); + // Should NOT have adjacent fences + assert!( + !md.contains("```\n```"), + "Should not have adjacent close/open fences: {md}" + ); + } +} diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 8a8e241..ed0cbd5 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -452,7 +452,7 @@ pub fn to_markdown_from_items_with_rects( options: MarkdownOptions, rects: &[crate::types::PdfRect], ) -> String { - to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new()) + to_markdown_from_items_with_rects_and_lines(items, options, rects, &[], &HashMap::new(), None) } /// Convert positioned text items to markdown, using rectangles and line segments for table detection. @@ -465,6 +465,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( rects: &[crate::types::PdfRect], pdf_lines: &[crate::types::PdfLine], page_thresholds: &HashMap, + struct_roles: Option<&HashMap>>, ) -> String { use crate::tables::{ detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown, @@ -757,6 +758,40 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } + // Check structure tree coverage on ALL text items (before table filtering) + // to decide whether to use structure-aware markdown generation. + let struct_roles_coverage_ok = struct_roles.is_some_and(|roles| { + let total = text_items.len(); + if total == 0 { + return false; + } + let tagged = text_items + .iter() + .filter(|item| { + item.mcid + .and_then(|mcid| { + roles + .get(&item.page) + .and_then(|page_roles| page_roles.get(&mcid)) + }) + .is_some() + }) + .count(); + let coverage = tagged as f32 / total as f32; + log::debug!( + "structure tree coverage: {}/{} items ({:.0}%)", + tagged, + total, + coverage * 100.0 + ); + coverage >= 0.5 + }); + let effective_struct_roles = if struct_roles_coverage_ok { + struct_roles + } else { + None + }; + // Filter out table items and process the rest let non_table_items: Vec = text_items .into_iter() @@ -841,6 +876,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( page_tables, page_images, &band_split_page_set, + effective_struct_roles, ) } @@ -926,6 +962,7 @@ mod tests { is_bold: false, is_italic: false, item_type: crate::types::ItemType::Text, + mcid: None, } } diff --git a/src/structure_tree.rs b/src/structure_tree.rs new file mode 100644 index 0000000..d7e440a --- /dev/null +++ b/src/structure_tree.rs @@ -0,0 +1,1035 @@ +//! Tagged PDF structure tree parser. +//! +//! Reads the `/StructTreeRoot` from the document catalog and builds an +//! in-memory tree of [`StructElement`] nodes. Each leaf maps back to +//! content-stream marked content via MCID (Marked Content ID), which lets +//! downstream code attach semantic roles (heading, paragraph, table cell, +//! list item, …) to extracted [`TextItem`]s. + +use log::debug; +use lopdf::{Document, Object, ObjectId}; +use std::borrow::Cow; +use std::collections::HashMap; + +// ─── Standard structure types ──────────────────────────────────────── + +/// Standard PDF structure element types (ISO 32000-1, Table 333–340). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum StructRole { + Document, + Part, + Art, + Sect, + Div, + BlockQuote, + Caption, + TOC, + TOCI, + Index, + NonStruct, + Private, + // Heading & paragraph + H, + H1, + H2, + H3, + H4, + H5, + H6, + P, + // List + L, + LI, + Lbl, + LBody, + // Table + Table, + TR, + TH, + TD, + THead, + TBody, + TFoot, + // Inline + Span, + Quote, + Note, + Reference, + BibEntry, + Code, + Link, + Annot, + // Illustration + Figure, + Formula, + Form, + // Ruby / Warichu (CJK) + Ruby, + RB, + RT, + RP, + Warichu, + WT, + WP, + // Fallback + Other(String), +} + +impl StructRole { + fn from_name(name: &str) -> Self { + match name { + "Document" => Self::Document, + "Part" => Self::Part, + "Art" => Self::Art, + "Sect" => Self::Sect, + "Div" => Self::Div, + "BlockQuote" => Self::BlockQuote, + "Caption" => Self::Caption, + "TOC" => Self::TOC, + "TOCI" => Self::TOCI, + "Index" => Self::Index, + "NonStruct" => Self::NonStruct, + "Private" => Self::Private, + "H" => Self::H, + "H1" => Self::H1, + "H2" => Self::H2, + "H3" => Self::H3, + "H4" => Self::H4, + "H5" => Self::H5, + "H6" => Self::H6, + "P" => Self::P, + "L" => Self::L, + "LI" => Self::LI, + "Lbl" => Self::Lbl, + "LBody" => Self::LBody, + "Table" => Self::Table, + "TR" => Self::TR, + "TH" => Self::TH, + "TD" => Self::TD, + "THead" => Self::THead, + "TBody" => Self::TBody, + "TFoot" => Self::TFoot, + "Span" => Self::Span, + "Quote" => Self::Quote, + "Note" => Self::Note, + "Reference" => Self::Reference, + "BibEntry" => Self::BibEntry, + "Code" => Self::Code, + "Link" => Self::Link, + "Annot" => Self::Annot, + "Figure" => Self::Figure, + "Formula" => Self::Formula, + "Form" => Self::Form, + "Ruby" => Self::Ruby, + "RB" => Self::RB, + "RT" => Self::RT, + "RP" => Self::RP, + "Warichu" => Self::Warichu, + "WT" => Self::WT, + "WP" => Self::WP, + other => Self::Other(other.to_string()), + } + } + + /// Resolve a possibly-custom tag name through a role map. + fn from_name_with_role_map(name: &str, role_map: &HashMap) -> Self { + // Follow role map chain (max 8 hops to avoid cycles) + let mut current = name.to_string(); + for _ in 0..8 { + let role = Self::from_name(¤t); + if !matches!(role, Self::Other(_)) { + return role; + } + if let Some(mapped) = role_map.get(current.as_str()) { + current = mapped.clone(); + } else { + return role; + } + } + Self::Other(name.to_string()) + } +} + +// ─── Marked content reference ──────────────────────────────────────── + +/// A leaf reference linking a structure element to content-stream content. +#[derive(Debug, Clone)] +pub struct MarkedContentRef { + /// The Marked Content ID used in the content stream's `BDC`/`BMC`. + pub mcid: i64, + /// Page ObjectId this content belongs to (from `/Pg` key). + pub page_id: Option, +} + +// ─── Structure element ─────────────────────────────────────────────── + +/// A node in the PDF structure tree. +#[derive(Debug, Clone)] +pub struct StructElement { + /// Semantic role (H1, P, Table, TD, …). + pub role: StructRole, + /// Alternative text for figures / illustrations. + pub alt_text: Option, + /// Actual text override (e.g. for ligatures). + pub actual_text: Option, + /// Language override (e.g. "en-US"). + pub lang: Option, + /// Direct marked-content references (leaf content). + pub content_refs: Vec, + /// Child structure elements. + pub children: Vec, +} + +// ─── Structure tree (top level) ────────────────────────────────────── + +/// Parsed PDF structure tree. +/// +/// Built from `/StructTreeRoot` in the document catalog. Use +/// [`StructTree::from_doc`] to parse, then [`StructTree::mcid_to_roles`] +/// to get per-page MCID → role lookup tables. +#[derive(Debug, Clone)] +pub struct StructTree { + /// Root children (the top-level structure elements). + pub children: Vec, +} + +impl StructTree { + /// Attempt to parse the structure tree from a PDF document. + /// + /// Returns `None` if the PDF is not tagged (no `/StructTreeRoot`). + pub fn from_doc(doc: &Document) -> Option { + let catalog = doc.catalog().ok()?; + let struct_root_obj = catalog.get(b"StructTreeRoot").ok()?; + let struct_root = resolve_dict(doc, struct_root_obj)?; + + // Parse role map: custom tag → standard tag + let role_map = parse_role_map(doc, struct_root); + debug!("structure tree: {} role map entries", role_map.len()); + + // Parse child elements from /K + let children = parse_kids(doc, struct_root, &role_map, None, 0); + debug!("structure tree: {} top-level elements", children.len()); + + if children.is_empty() { + return None; + } + + Some(StructTree { children }) + } + + /// Build per-page MCID → StructRole lookup. + /// + /// Returns a map: page_number (1-indexed) → (MCID → StructRole). + /// The `page_ids` map should come from `doc.get_pages()`. + pub fn mcid_to_roles( + &self, + page_ids: &std::collections::BTreeMap, + ) -> HashMap> { + // Invert: ObjectId → page number + let obj_to_page: HashMap = + page_ids.iter().map(|(&num, &id)| (id, num)).collect(); + + let mut result: HashMap> = HashMap::new(); + self.collect_mcid_roles(&self.children, &obj_to_page, &mut result); + result + } + + fn collect_mcid_roles( + &self, + elements: &[StructElement], + obj_to_page: &HashMap, + result: &mut HashMap>, + ) { + for elem in elements { + for mcref in &elem.content_refs { + if let Some(page_id) = mcref.page_id { + if let Some(&page_num) = obj_to_page.get(&page_id) { + result + .entry(page_num) + .or_default() + .insert(mcref.mcid, elem.role.clone()); + } + } + } + self.collect_mcid_roles(&elem.children, obj_to_page, result); + } + } + + /// Count total marked-content references across the tree. + pub fn mcid_count(&self) -> usize { + fn count(elements: &[StructElement]) -> usize { + elements + .iter() + .map(|e| e.content_refs.len() + count(&e.children)) + .sum() + } + count(&self.children) + } + + /// Build a flat list of structure elements with their roles and MCIDs, + /// preserving document order. Useful for structure-aware markdown generation. + pub fn flatten(&self) -> Vec { + let mut out = Vec::new(); + flatten_recursive(&self.children, &mut out, 0); + out + } +} + +/// A flattened view of a structure element for linear traversal. +#[derive(Debug, Clone)] +pub struct FlatStructElement { + /// Semantic role. + pub role: StructRole, + /// Nesting depth (0 = top-level). + pub depth: usize, + /// Alt text (figures). + pub alt_text: Option, + /// Direct MCIDs with page ObjectIds. + pub content_refs: Vec, + /// Number of child elements (in the original tree). + pub child_count: usize, +} + +fn flatten_recursive(elements: &[StructElement], out: &mut Vec, depth: usize) { + for elem in elements { + out.push(FlatStructElement { + role: elem.role.clone(), + depth, + alt_text: elem.alt_text.clone(), + content_refs: elem.content_refs.clone(), + child_count: elem.children.len(), + }); + flatten_recursive(&elem.children, out, depth + 1); + } +} + +// ─── Parsing helpers ───────────────────────────────────────────────── + +/// Parse the `/RoleMap` dictionary (custom tag → standard tag). +fn parse_role_map(doc: &Document, struct_root: &lopdf::Dictionary) -> HashMap { + let mut map = HashMap::new(); + let Ok(rm_obj) = struct_root.get(b"RoleMap") else { + return map; + }; + let Some(rm_dict) = resolve_dict(doc, rm_obj) else { + return map; + }; + for (key, val) in rm_dict.iter() { + let key_str = String::from_utf8_lossy(key).to_string(); + if let Ok(name) = val.as_name() { + let val_str = String::from_utf8_lossy(name).to_string(); + map.insert(key_str, val_str); + } + } + map +} + +/// Max recursion depth for structure tree parsing (prevents stack overflow on +/// malformed PDFs). +const MAX_DEPTH: usize = 64; + +/// Parse child elements from a `/K` entry. +fn parse_kids( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, +) -> Vec { + if depth >= MAX_DEPTH { + return Vec::new(); + } + + let Ok(k_obj) = dict.get(b"K") else { + return Vec::new(); + }; + + // /Pg on this element (inherited by children) + let page_id = get_page_ref(doc, dict).or(inherited_page); + + match k_obj { + Object::Array(arr) => { + let mut children = Vec::new(); + for item in arr { + let resolved = resolve_obj(doc, item); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + } + children + } + other => { + let resolved = resolve_obj(doc, other); + let mut children = Vec::new(); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + children + } + } +} + +/// Parse a single child (either a struct element dict or an MCID integer). +fn parse_kid( + doc: &Document, + obj: &Object, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + match obj { + // Direct MCID integer — create a leaf wrapper + Object::Integer(mcid) => { + // This is a bare MCID at the struct-element level. + // We attach it to the parent element, so we create a wrapper struct element. + // Actually, bare MCIDs inside /K are content refs for the parent, + // not separate child elements. We handle this at the caller level. + // For now, create a minimal Span wrapper. + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id: inherited_page, + }], + children: Vec::new(), + }); + } + Object::Dictionary(d) => { + parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out); + } + Object::Stream(s) => { + // Some PDFs wrap struct elements in streams (rare) + parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out); + } + _ => {} + } +} + +/// Parse a dictionary that could be either a struct element or a marked-content +/// reference (MCR) dictionary. +fn parse_struct_element_dict( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + if depth >= MAX_DEPTH { + return; + } + // Check if this is a marked-content reference dict (has /Type /MCR) + if is_mcr_dict(dict) { + if let Ok(Object::Integer(mcid)) = dict.get(b"MCID") { + let page_id = get_page_ref(doc, dict).or(inherited_page); + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id, + }], + children: Vec::new(), + }); + } + return; + } + + // Check if this is an object reference dict (has /Type /OBJR) — skip these + if is_objr_dict(dict) { + return; + } + + // It's a struct element — parse its /S (structure type) + let role_name = match dict.get(b"S") { + Ok(s_obj) => { + let resolved = resolve_obj(doc, s_obj); + match resolved.as_name() { + Ok(name) => String::from_utf8_lossy(name).to_string(), + Err(_) => return, + } + } + Err(_) => return, + }; + + let role = StructRole::from_name_with_role_map(&role_name, role_map); + let page_id = get_page_ref(doc, dict).or(inherited_page); + + // Extract optional attributes + let alt_text = get_text_string(dict, b"Alt"); + let actual_text = get_text_string(dict, b"ActualText"); + let lang = get_text_string(dict, b"Lang"); + + // Parse children from /K + let mut content_refs = Vec::new(); + let mut children = Vec::new(); + + if let Ok(k_obj) = dict.get(b"K") { + let k_resolved = resolve_obj(doc, k_obj); + match k_resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Array(arr) => { + for item in arr { + let resolved = resolve_obj(doc, item); + match resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else if is_objr_dict(d) { + // Skip object references + } else { + parse_struct_element_dict( + doc, + d, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + } + Object::Stream(s) => { + parse_struct_element_dict( + doc, + &s.dict, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + _ => {} + } + } + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else { + parse_struct_element_dict(doc, d, role_map, page_id, depth + 1, &mut children); + } + } + _ => {} + } + } + + out.push(StructElement { + role, + alt_text, + actual_text, + lang, + content_refs, + children, + }); +} + +/// Check if dict has `/Type /MCR`. +fn is_mcr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"MCR") +} + +/// Check if dict has `/Type /OBJR`. +fn is_objr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"OBJR") +} + +/// Get the `/Pg` page reference from a dictionary. +fn get_page_ref(doc: &Document, dict: &lopdf::Dictionary) -> Option { + let pg = dict.get(b"Pg").ok()?; + match pg { + Object::Reference(id) => Some(*id), + _ => { + let resolved = resolve_obj(doc, pg); + if let Object::Reference(id) = resolved { + Some(*id) + } else { + None + } + } + } +} + +/// Extract a text string from a dictionary key (handles PDF text encoding). +fn get_text_string(dict: &lopdf::Dictionary, key: &[u8]) -> Option { + let obj = dict.get(key).ok()?; + match obj { + Object::String(bytes, _) => Some(crate::text_utils::decode_text_string(bytes)), + _ => None, + } +} + +/// Resolve an Object reference, returning the target object. +fn resolve_obj<'a>(doc: &'a Document, obj: &'a Object) -> &'a Object { + match obj { + Object::Reference(id) => doc.get_object(*id).unwrap_or(obj), + _ => obj, + } +} + +/// Resolve an Object to a dictionary (handling references). +fn resolve_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a lopdf::Dictionary> { + match obj { + Object::Dictionary(d) => Some(d), + Object::Reference(id) => doc.get_dictionary(*id).ok(), + _ => None, + } +} + +// ─── PDF byte pre-processing ──────────────────────────────────────── + +/// Fix malformed structure element `/S` entries in raw PDF bytes. +/// +/// Some PDF generators (notably fpdf2) write bare names like `/S Code` +/// instead of the correct `/S /Code`. lopdf cannot parse dictionaries +/// containing bare tokens, so the entire object is silently dropped. +/// +/// This function scans for the pattern `/S ` inside struct +/// element dictionaries and prepends `/` to make them valid PDF names. +/// Returns `Cow::Borrowed` if no fixes were needed. +pub fn fix_bare_struct_names(buf: &[u8]) -> Cow<'_, [u8]> { + // Quick check: if no StructTreeRoot, nothing to fix + if !contains_bytes(buf, b"/StructTreeRoot") { + return Cow::Borrowed(buf); + } + + // Known struct type names that may appear as bare tokens. + // We only fix names that are valid PDF structure types to avoid + // false positives on arbitrary dictionary values. + const KNOWN_NAMES: &[&[u8]] = &[ + b"Document", + b"Part", + b"Art", + b"Sect", + b"Div", + b"BlockQuote", + b"Caption", + b"TOC", + b"TOCI", + b"Index", + b"NonStruct", + b"Private", + b"H", + b"H1", + b"H2", + b"H3", + b"H4", + b"H5", + b"H6", + b"P", + b"L", + b"LI", + b"Lbl", + b"LBody", + b"Table", + b"TR", + b"TH", + b"TD", + b"THead", + b"TBody", + b"TFoot", + b"Span", + b"Quote", + b"Note", + b"Reference", + b"BibEntry", + b"Code", + b"Link", + b"Annot", + b"Figure", + b"Formula", + b"Form", + b"Ruby", + b"RB", + b"RT", + b"RP", + b"Warichu", + b"WT", + b"WP", + ]; + + let pattern = b"/S "; + let mut result: Option> = None; + let mut pos = 0; + + while pos + pattern.len() < buf.len() { + let Some(idx) = find_bytes(&buf[pos..], pattern).map(|i| i + pos) else { + break; + }; + + let after = idx + pattern.len(); + // Check if the next char is already '/' (correct name) or not + if after < buf.len() && buf[after] == b'/' { + pos = after; + continue; + } + + // Try to match a known bare struct name at this position + let mut matched = false; + for name in KNOWN_NAMES { + let end = after + name.len(); + if end <= buf.len() + && &buf[after..end] == *name + // Must be followed by a delimiter (whitespace, newline, /, >) + && (end >= buf.len() || matches!(buf[end], b'\n' | b'\r' | b' ' | b'/' | b'>')) + { + // Found a bare name — lazily allocate output buffer + let out = result.get_or_insert_with(|| buf[..after].to_vec()); + // Append everything from last position up to the bare name + if out.len() < after { + out.extend_from_slice(&buf[out.len()..after]); + } + out.push(b'/'); + out.extend_from_slice(name); + pos = end; + matched = true; + debug!( + "fix_bare_struct_names: patched /S {} → /S /{}", + String::from_utf8_lossy(name), + String::from_utf8_lossy(name) + ); + break; + } + } + + if !matched { + pos = after; + } + } + + match result { + Some(mut out) => { + // Append remaining bytes + if out.len() < buf.len() { + out.extend_from_slice(&buf[out.len()..]); + } + Cow::Owned(out) + } + None => Cow::Borrowed(buf), + } +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + find_bytes(haystack, needle).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_struct_role_from_name() { + assert_eq!(StructRole::from_name("H1"), StructRole::H1); + assert_eq!(StructRole::from_name("P"), StructRole::P); + assert_eq!(StructRole::from_name("Table"), StructRole::Table); + assert_eq!(StructRole::from_name("TD"), StructRole::TD); + assert_eq!( + StructRole::from_name("CustomTag"), + StructRole::Other("CustomTag".to_string()) + ); + } + + #[test] + fn test_struct_role_with_role_map() { + let mut role_map = HashMap::new(); + role_map.insert("Heading1".to_string(), "H1".to_string()); + role_map.insert("Body".to_string(), "P".to_string()); + // Chain: MyTag → Heading1 → H1 + role_map.insert("MyTag".to_string(), "Heading1".to_string()); + + assert_eq!( + StructRole::from_name_with_role_map("Heading1", &role_map), + StructRole::H1 + ); + assert_eq!( + StructRole::from_name_with_role_map("Body", &role_map), + StructRole::P + ); + assert_eq!( + StructRole::from_name_with_role_map("MyTag", &role_map), + StructRole::H1 + ); + // Standard names bypass the map + assert_eq!( + StructRole::from_name_with_role_map("H2", &role_map), + StructRole::H2 + ); + } + + #[test] + fn test_struct_role_role_map_cycle() { + // A→B→A cycle should not infinite-loop + let mut role_map = HashMap::new(); + role_map.insert("A".to_string(), "B".to_string()); + role_map.insert("B".to_string(), "A".to_string()); + + let role = StructRole::from_name_with_role_map("A", &role_map); + // Should terminate (as Other) rather than loop forever + assert!(matches!(role, StructRole::Other(_))); + } + + #[test] + fn test_flat_struct_element() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + let flat = tree.flatten(); + assert_eq!(flat.len(), 3); + assert_eq!(flat[0].role, StructRole::Document); + assert_eq!(flat[0].depth, 0); + assert_eq!(flat[1].role, StructRole::H1); + assert_eq!(flat[1].depth, 1); + assert_eq!(flat[2].role, StructRole::P); + assert_eq!(flat[2].depth, 1); + } + + #[test] + fn test_mcid_count() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![ + MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }, + MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }, + ], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 2, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + assert_eq!(tree.mcid_count(), 3); + } + + #[test] + fn test_mcid_to_roles() { + use std::collections::BTreeMap; + + let page_id: ObjectId = (5, 0); + let mut page_ids = BTreeMap::new(); + page_ids.insert(1u32, page_id); + + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + ], + }], + }; + + let roles = tree.mcid_to_roles(&page_ids); + let page1 = roles.get(&1).unwrap(); + assert_eq!(page1.get(&0), Some(&StructRole::H1)); + assert_eq!(page1.get(&1), Some(&StructRole::P)); + } + + #[test] + fn test_fix_bare_struct_names() { + // Verify the byte-level pre-processor fixes bare names. + // All inputs include /StructTreeRoot to pass the early-return guard. + let input = b"/StructTreeRoot /S Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!( + fixed.windows(b"/S /Code".len()).any(|w| w == b"/S /Code"), + "Should fix bare Code: {:?}", + String::from_utf8_lossy(&fixed) + ); + + // Already correct — should return borrowed + let input = b"/StructTreeRoot /S /Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + + // Multiple bare names + let input = b"/StructTreeRoot /S H1\n/foo\n/S P\n/bar"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S /H1"), "Should fix H1: {s}"); + assert!(s.contains("/S /P"), "Should fix P: {s}"); + + // Unknown name should not be touched + let input = b"/StructTreeRoot /S FooBar\n"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S FooBar"), "Should not fix unknown: {s}"); + + // No StructTreeRoot — skip entirely + let input = b"/S Code\nno struct tree"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + } + + #[test] + fn test_bare_name_struct_types() { + // Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code. + // lopdf silently drops objects with invalid tokens. Our pre-processor + // fixes these before loading. + let raw = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap(); + let fixed = fix_bare_struct_names(&raw); + let doc = Document::load_mem(fixed.as_ref()).unwrap(); + + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "Should parse bare-name struct tree"); + let tree = tree.unwrap(); + + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + + assert!( + roles.iter().any(|r| matches!(r, StructRole::H1)), + "Should find H1 from bare name: {:?}", + roles + ); + assert!( + roles.iter().any(|r| matches!(r, StructRole::Code)), + "Should find Code from bare name: {:?}", + roles + ); + } + + #[test] + fn test_parse_real_tagged_pdf() { + let doc = Document::load("tests/fixtures/2013-app2.pdf").unwrap(); + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "2013-app2.pdf should have a structure tree"); + let tree = tree.unwrap(); + + // Should have a non-trivial structure + assert!(!tree.children.is_empty()); + assert!( + tree.mcid_count() > 0, + "Should have marked content references" + ); + + // Flatten and verify we get heading/paragraph/table elements + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + assert!( + roles.iter().any(|r| matches!(r, StructRole::P)), + "Should contain paragraph elements" + ); + + // Verify mcid_to_roles produces a populated map + let page_ids = doc.get_pages(); + let role_map = tree.mcid_to_roles(&page_ids); + assert!(!role_map.is_empty(), "Should have MCID→role mappings"); + } +} diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 873caa0..4576d9a 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -105,6 +105,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec Option> is_bold: item.is_bold, is_italic: item.is_italic, item_type: item.item_type.clone(), + mcid: item.mcid, }); } Some(sub_items) diff --git a/src/tables/grid.rs b/src/tables/grid.rs index ceb9e69..e3361ef 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -381,6 +381,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -727,6 +728,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); @@ -762,6 +764,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 043dacb..1e9e0b8 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -231,6 +231,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec Vec, } /// A line of text (grouped text items) diff --git a/tests/fixtures/bare_name_struct.pdf b/tests/fixtures/bare_name_struct.pdf new file mode 100644 index 0000000000000000000000000000000000000000..bb7af199f6f613cea3bc2ca6bc70dbbfa944353f GIT binary patch literal 1865 zcmbtVO^6&-5LOg4Pl&kSMIjcFRio^__j;yhd%|SHOwV?9*{rk7EGilH(CK+Qw)XTJ zx?eB5LA)e*5kW*ENDxgfIjGgm;{Gs9=gWcs%oyI&Cw4CLuoNUcVwEt60ICv@5G#n3I`r}jFbE28JS4Y>tf zh8DAG&Kee2WjM|y0d;(}2w@Tgh^mz0wZQwa5+Qp&NKLH1?!ffv_&kF$P+QlWri%l>`)9WU$IYvPjwJ7<&D+SN-_*Ped=n}=^1n}7a%?&}}7kKey?>xuL9&Zn2D zIONFV?>^ps<5*oQV3gfR3uca5!;OU_v08Ld1e-$no;%Z!L66Xd5PAAxALxsISPx@= zY`_*Lr&X#l-58W-)*lU%D`4fWQBcy6FQWesubVmPk0yXRaX4shC_={o9lxE*A(CE4 zDCkn0G!&26Lbgi-lyYNqw@3G-^<4g1xZ?oJ(5-5?bOf}xw7h}q@xx#l{k^)+0cp=1 z5YKeU^+dJ_9}o5Fcv7^QrI!pa%3kP&Kn!xQY4kDf^T_}g1cUBf(et#@17kPj#WW{K z+2KPU(tNPRusCUK#;h1Vzfll}{}Ya#PD{Q=$|Z|c>#SnW Y|NDtl%dv7Jm3EDnIVaQ83oVEI1%RjtG5`Po literal 0 HcmV?d00001 diff --git a/tests/fixtures/firecrawl_docs_tagged.pdf b/tests/fixtures/firecrawl_docs_tagged.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e4b3a78b1193e31027208e40e741dd12fa71fe67 GIT binary patch literal 64026 zcmeFa2|QI@`~ROI6^#;!ijpYP*?XTUq%zAa${2Exc`T6zG%7@?D9uSEB_R|_6Vjk4 zL#PlHr3|I|?{m)X?zGo=>B~zu)iw`~RQ&^?L5^XSL5~uf6v5Uh7)dwbt76%ysmX z2`Y&2ytmySZwM1))MWf!y@WM1gjKcuL;XTz$ik}2*SZDE*u&^wSqc*w|0Xm3O=bL> zK>v-UFoF5^1jYr9!m1`Tx3wXtrun(i?+T$;{<^z~OOUsrpS!<|hK4HrziDnVAwi)u^kP-B0Gi*@H6d&L{jgfe zsOqf^F_7^K_3=Tq>8i#q8~j5ud0=d>WUy2 zzu*9uAe!Hr4Kk|QXo(JO-P$!YOZ{c^r!c4gO~|hbS<-_2Lxa}Pf@PpTs>r-5SVrwv z<>*hxQZsrbe$`BGtxpIo2yF}>mk^o`ZH>PhO<2{K=I0UODFdlN6#CDCLxN~7zQSQg zj@mx=dv#kha$u#{@SI6%UV*ERj3%#%Xt6yKczcH2(v5+S&e}m93W;kIokU;1v{JT| zO3fC}o~@)U|Mf_=m38F#x250a#tzA9%ra5&9sKd_-qYtr!8x0Tr!*fqHnZe#`RfZ4 zOiPtS&IT78n)Q9$;2CdNJxXoes?M?yA@N*gk8kRiH4;Q98&Zw-8T>d@W>(kJQf!cN z=B~F>y6z9*v#SHCgKs+9y54yPm2ADb{>03j3of2o&KC8{iUyy2TN02ubam2`JC5Rc zuQw1?Utf;W4_tXB@0?6^H-DmW?=W%Wa2PQ;swU0lw)ey@B}yq3ngQn?Z``-*sfuVC zZG7|7fSNwr?IOh;lZNH@t+kC78!|T7*#Bi+MA7LI0fSTGyF(OX(>)S&r7qN!_AQ$E zavD)dH|p-%ysX-Ta6++o!kNk|dJgA=!&GL-9lj}j({bs(n~q8D@E5b;ctNLR+mtLD zT5m>1@MH3yV5_mZOU_-JfFmC`&@;3{1<=Uvs%t_XYn;v zuhWs#lCSlNtHmvIB&^EbTDlzEJCi^@uH7kAwjk!ow<}S$2SXCXCM~l$mr<&9kS4F@ z^KHnM81!H&-}`kdN^5T_D63X@Q><$8vQ*R5_9f0Re}B1j*uuz7_xkj>V>H)|pAJ16 z2nhbcm$F+m#!?{rSa-y{Z#5OhITw%C-aA+```e>yA*9xz2wimt>e8KGJ}4|E1@-z= zk-aP04$7NcFEgp8eLp#;d1-}Y*!jV`$~P)AXNY9%Rr}!Ze#3#ZGh)HA)@^PYtC0n> zyt>!M4OQoUR^`j|ypU#nGIDB7`uBtasp~&B8Tj}gy13VH&6LP2>ov1)yI%6hOkbz< zy6x`L!pPoN3D0E*{ca6UzfID+KX6H_bl5X!dy3z5p`!7QJI9Rsx=(+qpu!W;=vtxB z>!#g-LyfW3v(}xfJ2-n$m570ySB0l@MXBr7<=K>j6+z2qJf_*+4Yxlc=BzQQQ&(xv z&mQIaXQhet0&5m6sJCg|q1@3h)#>cdo3Dpk!_xL_{`vV$-}Kzj;nuc+R}CxWiZw^| z3|6f(2va;SpSh3klk?N_w5Bxamvy1GM@{A1c4fxh$@zG=WlVUse^J#50lT&nD_^Ev zQL4YJJ_b2=^!}BmXrm{wIr>9_>*{0%w!WD0IciH`KOfOEVu#p?j-skljT4tRsgQLC zXKRGI535W~@}>4~==U59f2ZklhR<>6bz)g*qG?rmg2Uz2TYb01ZO=!(#<{6Ibkk6G zBQ%NLoj2y%9*@xlV+-9Zk3)_}+uH*3ZGPUq_+(O8V(Gb+7v>Ij^iL}(+baJ0_N?H} zO^+@=Pkukwth=Ob<^-{xyQLM9osNDnGmlQZ+GZ)U%tlGbt02C>>Q-E&`II83tOdO^ z;rsRDt#i7+p8r}mO#v%+{0mq zG$=_n=~Eh;6L%K~>~@|joRKCp2w#tRWH#Z&#KS-Gpzicuy#PfyPf0K5mbc@NP+TuX zj*8BmbMcWze0t3pO4H1l@~boFke5wDkom%gbz`?Ygr|Rx++!zwth`0GzowHW^-gfi zkn@-wm#*gaZw&ArY&{7NsG7}hUFu$6=ALi*!v)^+e6o7etvSxA_Nhb5rYTHR4b3~M zbIEy~eBILmuUSSe_kxkyY3Ev@3bT!20q%iHQ#+Dk<_}~NSHkB?ZB;LdnH^BMo8{l$ zFwRQKX`P*6i21@xHxmk1eBCI%nZ&Q&O+2`97o3)bH276adAIqv?=fT5e22D3m5BGJ z-y5&nl6F^H;?j!!#j-lGr9blJJUu`5vq<{eA@LVY zs;^}WBmH+)R>w6u6`c6A?4GmMT!XNbTETWZFWA#M^YQ$ENE1J=i$@#X9d?E8)NkLj zWFk^A?sZ<9yW3M8>xf(J_5Bu6<DMC_f?WPuZU#Kp<@NDz& z!6k9Tyo{Oay;Ar6q&R)6Si|3xi)wsEH9jBoBM>(hsgfUj5h@H^1s#{z(7&DR=tr#9B zzG-#ve8kJ|FeQZCY<2zkW95gR<#Wj4Qf1Y&58F~&YP0L2r1lwB86>5aHs1BUrBPJ4 z)yQp5hDd1S>X5p=6>Sxf_uk)?EA&kb+nFvPHxPPL&-vm!a&+pvfzSrM_t%md_ZYi& zrR^K(7Rr*}!JZ9B; z-`ef!dDW%jd>IWy{vhS6lAl(Fe$2~iF>M)AmT5f`?r+fhP($iNT=JrId*|C;6Vuf` z(sbSL`Ph2XwQ#nrXH$+vgYP!dtF~zI6QReYi2Qj^UhE04ICgx6|L4n5HA^a_x6NPm z`MsTwR7{-4)NQoDgYG)(MyKe{U7~)t`kdYlanFf+UwCI;mR$EBclV1-_lIL2n!3TA z2ZgO2btPw6emv(S_qk{6keAHMPxTSYyGsHhq;93g&Q`q=)qKaR*Xp8t>EicJ&;2jf zxUHYR;(EaRD-n~XWi>S37@%#OLiet)VIT9e=swynr{N!WhV;M4ouNB*sB6XaxPG}a zg#X~q&@1t)W}Ms^0tr$3)16u6Yt~>e`NxBc)8Bm1wkx|`)V)$?VV?B4&o35^$vEE? zbY9#;+d=wbP4|*h=YM{8e_za3xk!1ptJ=rKHB}E5_Zd$wZt63bTsG55RCQPDk2BYr zBQ#^m6i0VWt%+_GG~FKYldt*p{)f|D`#i5ss2N{7=G@rF!S~RW8?AGy}O&$R{8oGhy6>^*dKRb3J%^ zyZG3e`NirPV=YsqQ*unN@3K8yDWCfx;pCk#m-3y*-bIm<7hSk~`N4|Zq>D@Pb7=E@ ze75e@6kX>b>Jn_>P%}Z$^?vz;{j0W=AG+luA#LVUn#z|}XPJh^#1++z z4{LLs$H*G4yMG~DSF_H2sh~$%ht6eB!_F}=LPrg>j&$37p8NVpZ}z*wm3d?J?|QDR z*0i%iw&mMCduikQbKKL3n%7KYXK!eYj@`PbI=iD@tW$Qp1TA^Dbk2QU#Ysj1&sU$x0~ctx8GYQ;jA2V(8=-Y!}YeB+bXBMn{Imj=>Y?i;YkWd{HMn>jGvx- z>gd*Z{;TiCNN=9ykRZ@6G6`Nc=6n3~qAmFcXa6YQZ~Vr1%lVd#1aaM?XVh+*JUC$^ zV;_`?>@72xZM=XeV4*Z4Qbz5F`;1;L16G2d(n zYpeLqdd$|@_S$^Ejc~EQft0xY=f_uHT^Zl=+WMw>(zili6P;52xmF9OmPwMoxDlox zFDi5e_wT#wI5;6A=)fBvoAc+l*Db3YkZ*mErQF{C7GxIX1_ zTWq#t-pR`xt-56v*GLoqO@_Uq%pp= z614s|dMZwz<=>WdOq#7a(dp>1nZp}C6_Uf-WAcwg%z{NN=WF!mZ*ZC`@OtTsG;)`E zpi=r#{brQ|?zvXcXWmUxc;)9^p`210J-wskgO%I@!V}fTq`kU=1+%AgJ$U(|D6i>` zvFFdv$<;Xn7u98Al>%l>en!ktp0ZNGac-*a4C66AYkl$80_RY}bQ^8{S&n&Y(xBO_0}wa}VX`9$g} zpVeU_!o}tr1)QFR^3bNd@d|OD`l^fUdlfIuhcYk5wz>zz%z$aIfI*7=tocdFf~EL<9WOI>96h(UUr@P%_}t0kig zkz4i0_UP+2#3spa&7EVb|fQ-K*>QYAMkj6I069*g!! z##wEw-1o{paG^?S+J-IS&Pw^dmkbteb3cE3@S~ZJS@n$9Z{OZYS2xi@daKuOGplQq z3LR6uDfQh(%cpxcZ5jID6Mb)8L%@TbHAfGthb07E@dzxaj6T?xX}kSx)YWCz&u6X+ zD7sN$@`8{z74{%$3!l54Q4p0omVD-(9kf{`@xz$9S#Ftr#97X`Pm5pUd5h@EvG&F;luA z-`XA5Yjq+-YyG26b@yw~YMqvPJDH~q#!E6%4sV#f_+IAn&rdJ=$vYfzO*d*8r; z!fB3WtG?PB?>acVOs%P=Wa`sAhanT?gD$P@ZrjeSY;av+Z9QfsrEu@cpAo~`?p*E4 zdhzyW!5qm1a44lmLVgWc{|u#AG~s_dGQbS6;D%D@KMbYxO8lxB$Dx#p;35Nd-~1a$ z65mTUSLl*tCO%XXH7prc9{oa2cR(Te%~qE~S|Pi2=i9j%9&n@RCj5+Cd2vwo>*I5q z&+!+vQ;R!3^(KtlvATIx)ko)D@k?GvlV(?ZPn|>#`1DA~9;%E=m2_w<9ugirn{)Vm zw11$yT%*q5god`v&5_I7@?An~Vk!kyZq7N9X*BK4l*ZxMHKnutRTdw6dfcYvX1l;r zGrz@UW?Dx#F8($oowdHF<>rvBncS+5vPgO9-i^lsMSNrrcb5kCX$g;>ZG&jO^t*mF zseNIY$);|NcX4-C<~Tlx+7grJxMQe%Peh;c$|BjJg>sWL+Ks}`CRd#H*sifMH(=k2 zB8!TJ=f#c$hM%;gjF+NKstvNcwPBjaThgbYg_j);rg)#}=bw3?o0y^bE{kNjJs2y632@xFS_AbOjI5xztP^e zj+F7J7sk~(dBa(@-qeRFDSqjBlJ!@6bxRis6ppKR@`HVp*IzsI zGK2rJ;0NTzx!|tf)LQ_+I+GIhF@qzASNx5GWhh>l6Todv|R9d2C7lnB7)S zK8TU$CLJnU;>VwRuwl=JZPnjnpq0}T9$$89a{RL1s`c5`PRm0&64|?UhQH7cS-i(f zFiiEDWRudO+43Z>7e_nY-9&>A)s3xk3W3|rKWBdxoK~p3$Wm?R1I?-Xa+(dYCDvPz zCcWDKIPy@_*kPyj@YW)wAQmzzV!$SY<-|Kyv%=_p0 zuUmKiLv+jSnMv=i`XhtikcG9O4jCH`O>Mb3sCRteWNPc)ld&!vPn~Vri8}y>q82f09evKFw=!d{TmemFx6ZV}(oF z-6y(N?;ymbM5Gr=7Az3o|N5qw{_+0AR15WcsjlAIjkfjD^(%E6zX)7w)Hr@)_j{pF zUrY_clZ(E1W^0xg>F~|gj?g^)SfNHu`zEs7}W2Ndh-nsDT-uz1^Uth3#vBB(( z>$?Mdfm$E04=0vIU8+Odng`qQOq^C zVDPQ`bVc@}lh@=twuGg5=#IA5cc4#kJYUS#6SFNTsL^m7~?i;K;^>tmiTI#HyU%tJ6`te)kvG~m; zgWb+SI_1F=ihBsD*|OBr568cy8rQ#0&{`5)__+CO>5bci;cbnuEW`xCU-?pYs$jkZTytst1KT6x)FV8X& zTp2CH-~J>|ZpUXt+cjKRBU&Rb6j|@IZGnU#G2rQh$*TpTKKMO3V7vI1=W%j@@1A>) zh$1g9Jxt5UwDWYLO|yx;>>c$sI%V-ylioAmoguG^6QS!7l8k^;&u;l)=^DGIZisq; zU+?YSM*iIXkDtU*j49vqFg^{UY zO2kWBu9;*x)+)8lXyN6I1(lKFDW?cy?-8aIe4ceE@6p)a=~teux$H#$%SFlhtg`;oU`^?Jxav zvu__Ph<;!qakhI@@(Ee#XK?|8;&$GLq|4r_hshI&ez&^68NOK8@ZshY%JR$K?sXpX z_9=AGT{9S%uKZ)of^+gBfg25y7WzZf%Ev}$p6)j6h%1P=ZAFxH=?JJ?cF}FmVTBDT zVYl9w)@I0yd_=^{l^gjwQQgCJFq%8Nr*!1&b)2BIv)Y;oFIhd)nu`{wo%gOe64CsE)Q!u{< z`G4FG0zKn`PPzZb{SPeT@#joLeQxmwkyse{uLxE^y&Bf`1i?gic zt{Ck4I`*o_6~o=`{zcVI1(PDWb(bs;$O!@1l1IOiY>cbFsUvmWoT{h;6X=zGYI@E`_+0dve8Yo!7YwP1H@yx;L@A|J9f0 zoh1`){8;ogEONcTGyfZ94XNFB%Y|#-K7AZay(DE<>squqvNFS<=~=nr>v-GRiP`+a z0r{g(`<9+ktqJxiSGq89zqG<#-?pOk#-Yg%-Yj!4)s;Q9uwY*<<$z4fzz(y`ODw*d zmK1&qmFs=8EvIyg=%Ln?l@8kjWo7LTR9bJ)(0g{Pe&Ss{{_JFNbO2N6{_XoVu@ z#`c9RXr-wbt@2nHR<=aORCd~-D|5E$oHkqdQ7&HJc|d20X1TBOq;Cd-?dmTtrf=BO zcrmx)$|kAm*sxud)F1w~x~ocJ(ygUi%aa=RsHZhPFueC(Xpi7l`ND=F{(Sv_!|_iA z9ctz;zEcocE>Ljgu!+JLrBS)LkBIG$l4tl^<;r$?Y_C-5B~~3*(X2E%@0xf2lW6lC ztGw|^0&DxH7c{MW7o`7)*jAl$+M4DmFzDB}ZQ{djzL4pcGQ`aXbZ&I)DzK9-adVlo zdl~J8=<b}dD-8n9$o{0qtH)I~Zw3BO&ejRoz z?`O|}Wf3;&`P7eJpj~f7oqB8r+!T^OzRGpDF{`S)^2XxGAAO&yzc;fBGKtl8~-Gnwn}QEs$8QV(#o^AnG3bD6{OP zgk5P`VySZX{^6VbCQ7^Ov_j%NG^dq)jBG2oeJ8W#c(d-9mG6V}&u?%4=)Lbu{Fk2j zH}4gajGkE$rE7?^*U5S%k#qMh&5d+TxBFNIMMZo)*=0M%PDu01t-zUWZ%H3!d{6H$ z)ckU@wSIEv=x6Eb-HOxVq9x8oo_oxn6zT5~{^h)rpw5>8q1v>|tvOrNN^~lvmk+s7 zq=wB0b$nN7sV)(pxAkl4x~89NS9NBkKcLkApnlDEt!&+7uefc*a&VKh3t zw<5x-D-J1bm?4tMUoCG}D12@0qV&L%N~?W!$2RT`nE2eXJ+DqA_sS~e{kj+4E{mM{ z^S7|^X~mP$5}7mEqT6a`Z`I0*E?JCD6;uP|=RN0>3!ksH)7rB-$i`De>5Uy=sI%!+}xus#-ogUw6 zYj1K7$=K>X^C_`@Z`b@SPxi^W70oSCX}9}o6{UBn8x58`*xI7xEMFGsSAPA&;i@Z3 zhfdTks@+@bzUox%hZD6gJf$+K3kFP_w8bP``b}I9KT7DGIBmc;sjZ%x;W0v!Q#T}E?(aJi8dJBeX{`V45|ibJPsJ`s*`#G--*CcWYUO~MD%~O(EZ+h9s zctX&scv{5N$jtmt>n)E4gnWJbY~QGkZ|3?B1kdfzT}=)xSt)xxW5&T({Ru_$pKEs= zH>`W1ZKRZHJ^z4zSFq-Oa*dO4R>-CBzR)q-Zks)TjML61S#JtFLwbI{&-cg48H-xK z-W|75<+YFXY@d10t&Wr{mFYV<^V4cBWjV}KIVEz}Zp|Z-cf+ml=@+anEwSGkZ@Ey^ zMJFcaT669|SiXvgm}**)rp1Y_El8)P#q+-Jcj8tkyt!33NvHp8T%O{?Ni#RpJltn@ zXjQwcV^0-9v|?k*39_W;j-Xhfe<;>ap!oQ``xD>dPX0jW9C2Uoxy!S4I+pC~xvst1#B%%RvzcRN zFRS_)Km1*j!8Jdgg z9SW?iY#AeW`sM1Rgq!geBAVB(kN)a+HcCLO@vg&|ZwiOnb@!3IRvoYv7%-f^ec)Tf zxq(SzPiU==Gb>U*>UTi#bC-3;m2W?kg;(Fu)Aui$w0Eaf+~^Ivri5l6xOCED+l;RB zB=Y99`?GdT4Ytp;_x0QR$!>!81}j0ohMg5xW7-rpZS~EyBkm=%88sd5xpT%QPyd+R z=U0>G|9r(~(ZffGo#HB~`SbZs_f-q6Nl8Xy22@P^9=#^WH&A~(&byMMRDFxIwC4)t zveI%%aX-~B18G?s%1H}vXHJtCGMe1_EN1WH$@=@WU*~>>GR`mjtbeA@V)6b(7w@<4 z>bkSn^OO1hWnW+Do)Q$fCPoX;+yGxbR%`yIPWY*jxbIQ3X$xl0*y0xL9+cIhec|C8 z3G3~T?$-;wd6RP}I>OBKWk^HEeE)Mh&6DDnXLKa!d#-reY422%R}{zN}@3Mnsld*2xN!E#mh6F_n&p&rcu<@pHZ}G{ycDl9;J-XOJa#$IB;p>Itxi9$l$7VcETHrlV zH7eetqon2WtGo4cRF+jW#Z}dgEx%SJOkAZnVE?^DAl=f{B1$Lzki;vgJ*B%Y%#kwN zU$vsC<^U1_>_Ilt-n-SZB2rq@?NPU-?wg) z;}_mGKcM=IpT|R4#Zz8k+m?qu@u%h~nNN|ud8;QdSkqKI>%Od#Uw?<+7Nu3KiXV0R zzS?bgrtDHSKHjRzQTI#HE#W%_hYikIn97RK!o>WiUjDAR^I?P1j_`~Uf{Tl=1pl44 zv4@u_8@C>5t?fCv(XHzTbZh(55TU1=a+>eRPZ=OT>bw{3)hgm4Huc(5TfVrC&|Ogj zZ}OBry;C1IDnvuhlqTX&nLzkH?xNjJmvFHUeoz&yzIT+wPsIA|yB6g`&TC`l-e{=J zTR=3Btcaya^}NeIyiHW8m9j2>R;Iaa#?pC?r4p$V2?2V7xuc>g#L_p-R~co|F;AyW zfikA-@coGo^D9i(S9gDSQ=w9;RrsTEN$BkJHx_@t9>n)p6ne(P!C`iE*2 z#pHJ0krKb9mgd?u@MErj@dcwV(F;S)(D>|E_rG5}W|hLNRqsUnIugZ&2@3*VZ??=a z9nPJU@D|NkYK2^Se^w>MQo3mOyM47c$JyyvJH;KFnP0!IPkh^n&5rVEhx3#9_MGio z)-id(*z#Iq3j_W)_XAHn%UQL@^kH+|Q=-^r&8=zv8i%XDZ1s&gY2n%REgb#B?fywx zl&k$-b1}WJu58J^Yo4RX5)XC=h##?--dd(K z%@IgF`N3yi%xJ-JgrBy(NBIQnPm+nt!kV;NP5NGoW z#>tBGss(QYwT5?-7S4-#l#}WfwAHq0UqXU_N!x__EMNJlxT zLXMvAI`2gp>WkL(Jy={?eQ;0Gcl%F+tzJ5dBgJo}C+q43KbDFdJXrct@Kn{@v^VCm z_dJDGJBz>k(QPYw-PqJ1(BxQ@+&l@l6DL0!o_zde&Z$b74QUmsj|N}qLtn?FE&tJX zVtvE+-VCW4v7zq+X8BD&h?jm0MmspJUr%scZJ8=bSZO$&_9|&Zhwo9P=9+u9(4D?z`fvj;nafhwJo>A zXXQl(wMT1(^&*h}SjBa9>+c?Qh_$$*cP8>?vF_E9ijNz+0;BUo9)Afr|M*I-6b6FYpCAc16f1ls_KzNK)`H{~i17k``L&M%A@h(aRRiFt~GKJbBuZ2P?#5VmCS- z?fICZFJQ(|$lLGv4c(3d+3M4e zr}OPUV$@jGQ%w<19rN{D;P_3M!AtLa%9uNOk6*FF?J0^k7kM1{puMd9N~uJ@e}cZ& z-s?VAyYp&jsx$5vT#ihAA?~uQeM+FsM*i|Hugco~mT2+&A!lc_*S{$x3As;9zaGB~ zx^p&7ZB4tOP}yrK;}1o(`vxIpgVShlWOwF?oCD28TK1uDAd{q;2nO#%h|c zEnd}~a54XGkXByzt$yj52U_Fmvv0Tr$$a$YA1_PGuhpm2cBL8`e3*OObonNq)&s-t z4=tpVXp>9ISN?R&M3zqx_N$yQSvXHh+GXQ`c^6;Lt3uawsYS+h1jpPY>tk^lzO{Y@N~t*=%CDbnpZpR!u!sjnq4;M$UcuR zySL;fca9y@3z-`>U}$peo|3__S+3`DCi53(6udetw*HXq$JYFsW8a!K_4pxgCx+L3 ze>AyO#_sZ5Ij`r{N~bHPm_JXPX}@7!k>qC;`6oKns6e)bSF5;%k&#tpO)B^19&lU4}Q!OobXa9iR z{Waa$`^nMkq7`<-gRW|kbDT%VY<-zH%f)=Tzk6@KQTfoqIOVM5&Sb+7#Xa8~eqJ5; z(lB-J z`zF^=TgJOt`Q9`3y17~M$+vUlf}-NXpM|0?3J8`s-k0so>vZ4PaC!2>b&5MK6pA*D zU$o-hqyzi4$Dw~)A={bv+Wqt8-6HLh=3(Q?&%SOt?g?E*ikt2`&6@62@6%vmR2MWO ze@;&Hj%{;36;E&6FL2m(nRb+Ca=iQbS*Hps-y!*J1s`mdUFvq78M1b3>LB4rVsnji zq1`!~h|HN&ljBq!(;9B135~0)?3j}hn9$h0Wm0vtoSe^o71`FHY4dHToa%J*OqI+x zJE-zd|JY))wt0tl>`IX@t2`-$wg{i@>lr$&=G)DT`=_j3oTYd~m!rJIg0X= z)9oQmld5;H=nOpv%MT~_Jk~r@rwOmzYWb#QqAn|n|ve>)VWvU=w>NZ3Qxp9BcTX5zzQ~RW~ zf&(Ut3nSjxPr5Z`{*CW83wO`)g7!MAgvRH;YtM00PqE2;5@j%5GckX-BEm?KD0upU zA))+oeBjHyvzN-Jzg?E`bWm;wag>+nZ4UBQE}| z@ae4Q^*1+?a);koo>AO?qenj2J8|0j6FF)#!+b6Vm>RhS1uk9mjp#9k93^iZSX4h4 zKia>mdVJ+mHwBdwCkaQR->rT^;lJo~>Pysxdp6R-20z0p<(D2hIXzCE7^rfNw#e{& z+Jw<|*3x^-7Uh?jS-u&1e?oL!)?4ZI+4o8>%sfl2lXm*4E98|eI?W`tvQg1^=B5=c z_hsfq9D3Dw!`;k0yyINHu+xJd@oOaX-vvK1Zj)5~Bp)27jt4c1qE@y0k57?Qg^0 zg;ediG-0p6`vy~s03(~FqDoI}1m8UlvR@{n}}OlX`25&8Np_KTYJ5wWxG(o8;z5H`td3K@juc6LQ;Db;h z?8w|0gNrMW{*w##x2qkgFfKuo?Camg3@hlBkR8YEIeKfuxvQDpvQL8=oFWpMGaefq zP$eukeeE(cz3F>Dd7PudWH&Es_%P;VM3=UG_cDmevD& zvrdRbQ3sdys83fRPz$Q$8u{D9Vp7%4{X8L8HA>r9?~sT}-mbMvC%0~r-?UEr%LLOm zlP>6-yOEacW&2rH!bq;7ZqjU-!AW^77t=PwT@#`S8;2>yk|wv8XhlXWZ8<8^KXwZUU$}36&`59cIOU1KIm8`r7u`DfeqGr9>CdDnXD=%p{F-y^QlQHL z5wDr|w!Na>xiWQM-G)oY&3kf#?OWT4dFzvnR44ybnb!*!e9Mq1_~|n| zT6CAz+?Zp_=RGn1aC{@FPORTsSg6YDc8*3{%EIw_d#lNIHrmFcUqsHFf2o-`9Dey( zkdwFM7DL}!NZK{xg7_rTj8GZ!F7Nf@iY+ap;?6A64!zP5p(x@wK8d27_f9}RZSrz* zb{Z|iNqUpS`p7U+i{~uYVu9hX7Gse!3w@fm`1Y?|A8?WH$PB?JLlb*nZ5^AYw@lZ= z894zPz2vXyTwL-#Jf@_hIArXRZLW$RvugK5ph3A~}!MRx~Wbk<9EcL~;WICaK$@U2zW zWn^E*_8Z%FX9`_?HM4oPlFjH6r(V*f6q~EN4tzK%rP4ZU{PcCA4>D~u?~qb;gS36F z%abM{whg6U*C>~cx=|*UJ9|ya`l7GNN1i6o+$YzpZF2Z=7e3SK1%1(f61zEjw(1A@ z_|?NZ`jR$K!U{@`P3$cjHT%ew-F-rVyJMd&Xufu)SyJ`m=VXs|f%+p~xF3#)fEk$PrOqVDb8 z95MSyYW+gbOlNJ8c57OG(Y5)MFVSyrzUu7%VJ6lzrMRK-vuAonrAe{pl?O6`JX9N)e0esTET#WtB^B@zzI5YQ=U zdU|oQc4Vwoy-lv1;*(WBV~pR2luLNMn*JrjSf_Y+-nVb0M_1SW6c}GEcYsYd#JWyw znM*KD58Xv3qiR6&Sw{<5yT(O%nZJ*lu&S;fx@l{zpNEX<%C&w={esv2DnQpyhq{I^ zHqFtunf;G;j zGgqIrfuZQewqH&2Z`+Eo&+gZq7_S4}!>31(Az*#D9wftPPBf9O2g?wcn;lj45OgaY zJ-yD@!WZPf#)=jqW3P(tqC_|R(ZbMoYUqpZ&|8N6sg3<<=*aSxpg-ctbVK;xc4yXr z#Q)xB{v+J*zxA0}J%GP_=8?SIKL+*xtq;oD$Nc4kqPG1%VbHAI&T<)hhye>@D=vMH zD}9@-42&&GVeXP;#;AVnzhz~EwSPgSVCHUEn3)KInJZvnW&{mpW^rJqGY2!dg_(Au&}*OcfGSg~U`LF;z%R6%td0#8e?MRY*(~GE;@jR3S4}$V?S7Q-#b_ zAv0CTOcgRyh0Ii;FjXi_6$(>@!c?I!RVYjq3R8u`RG~0cC`=V9Q-#V@p)ysdOcg3q zh00W+GF7Nd6)IDO%2Xk!F=Ys9OdWz6Q;49(R3fM`r3h+FErJ?TjG)F;Bd9Uu2x^!f zw*HCmYiBwE)BCkW9Sgt`n0wWQ3D|6afc5+YtlKAGeLjKN;R_S6o!JCN|Bdc=hp_rV zSp6WZeh^ka2&*5qyPUwp0^KQ(Juhb41kAVzm~|5{^Cn>SO~4GCfLS;JGjRfD;{?pe z37C}=Ff%7$c22+yoq$<70W)<1X6ppZ*a?`m6EJf}vzZJe2$;bWFpDQ(CQra@o`4xW z0ke7nX7&Wk?g^OT6EMpsV5U#NY#-gr#k>z@{RGVX37Gv8FaQV`0R#*I0>%IVgMffh zK)^5{U>p!I5C|9v1PlcN#sUF@fq>CKz;Ga7JP6E zA27xTjPU_u;|GlK0b_i?7#}dk2aNFnV|>6EA27xTjPU_ue83nVFvbUr@d0Cez!)De z#s`e?0b_i?7#}dk2aNFnV|>6EA27xT{L44`$2nq-NANHA=$C)=%R&0(A^mcZFg{?6 z4;bSE#`u6SK46Rw7~=!R_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6S zK46Rw7~=!R_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6SK46Rw7~=!R z_<%7!V2lqK;{(R{fH6K`j1L&&1IGA(F+O074;bSE#`u6SK46Rw7~=!R_<%7!V2lqK z;{(R{fH6K0j1L6k1Ht$}Fg_5B4+P@_!T3NhJ`ju#1mgq2_&_i|5R4B5;{(C?KrlWK zj1L6k1Ht$}Fg_5B4+P@_!T3NhJ`ju#1mgq2_&_i|5R4B5;{(C?KrlWKj1L4GKM;%$ z1mgq2_&_i|5R4Dx*Bs*4B;waB;@33dpLqn+e?flDBz{dLe$6F*O(rlt5R4Cm$p?L! z^v?lA^h1LD*M$0_v!-7`W9C#z*N5iIGEHJU;;-O^X{axHK9kyiCrjx3?f-L=B}ep3 zF8lvZnCNqJ!vC2GlOuXAnf-q!P1q#+zmq2BBhMmwt>9=w_7~iS;w0<*Z^Hte>luF+%RF zVk5DCM%!E*aj<_bBO4=sz$(ST`Z*76j0^;;5(m@gViSzHDytL+!(U;* z(V1Z6L0F_<4rb3y3fJkw9E_fu6t2sMIT$@RDO`sSb1?Z8I-J`I=jFg044#`5uDgdh zm^(KqTxSn+Fm`TII6nvGVC>waaBdFF!P>b6}P z-mAe!lz0-u`89xL@Fa%wYyh+1NsN9mOPc|GAx~mBUmoZUc@o2U^FV*dlNipQ2V;6X zso`9D!iX4PEc{}51z=QdL#9$D)47kwImv6_ulkOK-_IyC}g zMI5UZ2NF0(B$%5PSfxh90b^+%tJ;V_U@Y)sRT~irjAeo>YG8Da<&APn2QW8tu&RxS z1jdp@R<#ig2aJV}tZE}7fw6p&RgDt~T-%MY2$NNf0|s12WC9~w&#J}&guhUu8vv`? zh*)4OD`iz15etkpI;?6VVu7*5l|>B<`SHR6BiqEPHX;`OSc^*EvNs|Y@X1>+_Q%VU zGM4_bKIw>9U@RbJRT~itjAh8IY9nHSv6z`vZA2_EmPE6vjfe%t!fIAE4lHnq?K3yO zv1ozez~A=4WhiB=?`D$67BA!vR+<{N#WG09>_j za|48f`E%96O$`tZ#?MuY-n6Xk#liHszDL}|0GNN$eSP#u0CfJzC5ge@Udd7|z`A&l z!c7W*9+n3w+>8LQt-qJT=W+qlVv*t;0%vYDWoawGvUrffSv+7^JV@aT9lemNtm^?m< z3$7mFpE;x<+d%MSHb-|@0JZRzidvFTLgBH&2!3pHwYvB{S;OY?` zwD2+i5!gLGwF|Bv`CYHSN#HP#x?nL_a23gKL2fQIKEn$xBl)exJ;sAi^1>X<|1Z>V z4i7kc@M&LgBFR(TaLx}n zmE=hc=l&oZhJn9$Qk>h5a2N>wLJjBnBO_t~p9Kb&n()+a^ovMKJdTI;Nk_s0^H3jFH4ekVU(^l90+EA5^cQM47KkHZfq6_3>ywUz z1?FK#tZE!s;4)%Ci~}!Nw7_L3JdnUVc8OJOBqT5oZ(>y&2?@+2pjg#LL;^k^Od1gh z_SYR9x#QIJ- zu)rnL#5|^sMT^3L1kRZz=79|?Qh*ZOL1AVUxQ$i#OfUe{Zz(Q$6Xtn)EVTk?`g
    5dFAe~3+4{&l&X-jSm{+jWi*w?Qc_c8a z76;?!S}&aGgZo!`s29%g!2|>kTDTzq%sud+g&P3C+yf6J<~E%Mb^&wncDVQy_#82qSzu{1ZhI7%2hX!M7zbyEyNd|y z1?J%I{Dm5h05G?})05&DAdSfF$8QQIaqxP$KPf&(3< z#icXC?-K@iXOZHH0rP7VtcE)x4wxUcV3ivY2+VI|u*!{y1m-6;Smj290`p59ta8A} z$J?vGu>c0Xc$34y07kxelf%&fhQ4@{!wn5!?29)!`mn_MDn>>HK2;1xzj&%0<6@TD zf#EOS5a8-t-sEspfFUs6zks2G{2DRy!OO zV04T(IUE&WfQ&ae92H=Uj75%%>%{z?6iep`2FX~&IQve_Z&I;pjck~}=aIoU8B5i; zr|D#kg>%>1w>Yny>V z!{2X{OPZPabxu|>P7rVvqu-ZRj1vP~#c%|$iIDK{m^c6koCx4rGu$x1COX1H&2X*} zo7e~sVmJc8l|j7N8IAxpago2T89v?2CMv>%7;YF~6BFS<3^xj}iHPtZM!%GW05j12u!)B7AciA=O)P{5F&qJGA|X78;Rs+82jM{sM*y2B2oGX70>I75Jm7>+JhO>_ z{C&;v2T!w!fAAoNBY;ixg9kDCr7Q%niGA=Oh9iJY?BloC-(>5VUv*}!9;bnTn=6FR zJ+p~_uvCv*Vx0MfX_mUNiGT1UheLr){DUVs91LvYAH2xn^UrMJA3Vw7h6Fb851!<3 zNU({2@FYjSnDxzaa*Vi7s_^Gjvx$H3R6E?zz$X5|lN@evU=#n~Ne(wWu!(>0B!{Dd zP5gr=IUE&i;vYQ7F+cv!Vkm6lA3Vw7s9+QS;7JZg1-M_CWjy3+#>_9;vp(#Is6f~dW3N)3kl$cWnRQ^9Dw_kc@e`= zK;lFJmx(jNeYY|%HNz1AZdT?+3`YRCSD6F*Ryolil z0QV{LB1RvFSO@^ODf1$RBLLi`%!?R~0P;u(z@J(T?o#HZW;g;UoH7X9*u?Ktrf^Cj za23N5K;bwBaOoWJS!N2S90IqR;d9Jj3Wxr>u_I%-RSX9J*!(~E zhbyDf4;B8i3|6CeIk++c{cAC}GRAp%_%ixOOL1i|efb2g451%ZgDWG_BOmxO`o}79 zWhDBc-MBI`0z5BFp)ak*T}PpRIuTb!1-OLJz#4F0gz;^lKYxobPDjL-(GM}kmBC<} zA_)B~DBN{Kw5I~fKpR3xZ1l)%^eAlfC~WkopdL{T)I(npP(LC8)FVP{^k6o6jOg#5 zZ9qiW=n>iIk=W>w+0>8BMvuZqj|%FM(5m6z2T2XoBN0G75=u4jTgK+AKcAPxrjH;Y zZ1m7Z1@zDgf@N&#M`oi(VWUT7qelhx$ZYxuG6B>hGfu($^Lfb-FfKr3Hhlycp`VV0 z`z?_kYQ>iUV+uqjkpRxgWMB-1$Q1e);c$8sz!!kXRB#NSr~&=~x-ps_?f6rV!ls|3 zpdB=D9ZX*`h5Ie)NC0Iha9|nz>$14tlE8kALS|EE3Y$7pfpHL`P{BG=)xbJa8DBv9 zvo2JK9tXvjf$dC%=?CWHu4BUj>IwjTJC#H~B^CEu65y9WR5A&?2k2ixR4O=rP(hyy zB^vNS&^?iWuY!69fKP+&P6T`(v<#>-y6upNw|jJ$rhifEkLQKeNc6?M_%dJ&MZF<_ z540xqIXv#SRKS0R)zF5)Uq?`*f3V`u-=g1?1g`__046YYQsaILv(W=tfYHzLfcFJ` zYM4M`qX+s7s0$3Rg_Z$615BU-ei)2m4!jSD0Qg}rL;(CS7rYMvpPV_Yp8eB7*lI13oai`G%eb{PTT46#D8LTp5)J+ygaU;2tmm=(Az8QGt7) z6$I`9!}NI9pZA3ssUv(DFzx>WqUlc}tN8mXSGBDPn>+1o394(`N@%_(s zL8u^G=rjtr2ay2GabP0jtnxqgi0G({zX#gY0c}M@8wR)zy$M(b)|tqrKOmwZ3E;OB zHgyJlB$!AA>r7Gu>r5hmbta*m1pdAxh`v_n&vqff^pEf0${3+FTp1CV+oQdy8h8&9 z;D^B^G7-E7h5oHY+&!rD6Qc2DfDZzb)c_v^#SMsWprbz6PoiBDa9=X|Yy5rD*9Klk zWTQu9qelY!U39ttv^|*u%zx1?0`QUO-3CxUv^xdHRSE&@*U%|D&|gs?aO|P5X#)xZ zjujLXBz#*@P@@FP=*Q;#`7SABurH>t@ee3euc@r!)KA3Q5IWNZ`bk8M0QBvM z8U*;AXc^!uqr(iq78%Nf#*1)qZo%%cz#bil6YfZ+8|V*~5}eQ`kBqYn(& z20A?go&y~g@Z&y$`h>u9u!$$4SvFv9fLayc$D(Dx{1Uw_*w>*xAy7XeH;rRKs@LLivk3!JL1)hzJ z1`mPXQb5}vQ^EcU4Z#C8iDCpi2Zc==P#|zzpuiyKsA~qiUkU=+8ih^&M?ohb_-CV# zKn5vb934d;h;tznD%fwJQzqbjP|<)B{=QTK*q>6-0S>qh9W1~y&~~Zl#1^;?%q5`R z6`MMPc}WCyX~4QrKwG1t&M5A_Xl4iS8PLcLzz13e*an)40sL{I8X9Q;?tywCU>Wc( z|DQt6prv6Th{E^$ie7r|#yAln2x1fHNwCM_p@#|uX&~n0ulL)4VD~F{%#e`H%+Abv zZ?=tOeJD~ZzRGx2IT)mG>3eLdrzrKRr;Lx3GqeEJQ&_y}DK_=^xPkwu_Vx=@ql_Df zP?#DuQN58TrI`mIQhaaVKU$gQp-3L~Gx)FEr=WK66l~gmL5Wa2z<*^7A|$IF}!WD9?XF>oMWD;|_JJ&UNS_aacr_s#MP@vGhl^HbdbGS~-*H}W4s zh;pkj4~WM45Vrhxjm26Zlcfz3rtby9m3<)~2Z|-2TI&PI>b>{o0l~2y`91>^t*`xC zhvw;c{ya@p*HxSS`TMjSyxF{sBp&lPPC+U jNH4(%`f)q7PygTXo{#)|&Btk84*Wz$pw)WazYf(8fEH_C literal 0 HcmV?d00001 diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1d2cd85..bd599c9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -23,6 +23,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -47,6 +48,7 @@ fn make_text_item_with_font( is_bold: is_bold_font(font), is_italic: is_italic_font(font), item_type: ItemType::Text, + mcid: None, } } @@ -999,3 +1001,39 @@ startxref ); } } + +#[test] +fn test_firecrawl_tagged_pdf_struct_tree() { + use lopdf::Document; + use pdf_inspector::structure_tree::{StructRole, StructTree}; + + let doc = Document::load("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let tree = StructTree::from_doc(&doc).expect("Should have a structure tree"); + + // Verify structure tree contains expected roles + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + assert!(!roles.is_empty(), "Should have MCID roles across pages"); + + let flat = tree.flatten(); + let has_code = flat.iter().any(|e| matches!(e.role, StructRole::Code)); + let has_h1 = flat.iter().any(|e| matches!(e.role, StructRole::H1)); + let has_li = flat.iter().any(|e| matches!(e.role, StructRole::LI)); + let has_caption = flat.iter().any(|e| matches!(e.role, StructRole::Caption)); + assert!(has_code, "Should have Code elements"); + assert!(has_h1, "Should have H1 elements"); + assert!(has_li, "Should have LI elements"); + assert!(has_caption, "Should have Caption elements"); + + // Full conversion: code fences should be generated from Code struct elements + let buf = std::fs::read("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let result = pdf_inspector::process_pdf_mem(&buf).unwrap(); + let md = result.markdown.unwrap(); + let fence_count = md.matches("```").count(); + assert!( + fence_count > 0, + "Should produce code fences from tagged Code elements" + ); + // Fences come in open/close pairs + assert_eq!(fence_count % 2, 0, "Code fences should be balanced"); +}