feat: tagged PDF structure tree support (#4)
* feat: tagged PDF structure tree support for semantic markdown generation Parse /StructTreeRoot from tagged PDFs and use semantic roles (H1-H6, P, LI, BlockQuote, Code, Caption) to improve markdown output. Structure tree headings add to font-size heuristics without suppressing them. Coverage threshold (≥50%) ensures only properly tagged PDFs activate this path. Phase 1: Parse structure tree with role maps, MCID collection, flattening Phase 2: Capture MCIDs from BMC/BDC operators, tag TextItems Phase 3: Structure-aware markdown generation in convert loop Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: accumulate consecutive code lines into single fenced block Per-line code fencing produced broken markdown for multi-line code blocks (separate ``` open/close per line). Unify struct-tree Code role and font-based monospace detection into a single is_code_line check with in_code_block state for proper accumulation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * test: add tagged PDF fixture with Firecrawl docs content Synthetic 7-page PDF with rich structure tree exercising H1, H2, H3, P, Code, LI, Caption, TH, TD roles. Generated via fpdf2 script. Integration test verifies struct tree parsing and code fence output. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: remove python PDF generator script from repo Keep the generated fixture PDF but don't track the generator script. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: handle malformed bare-name struct types in tagged PDFs Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code in structure elements. lopdf silently drops these objects since bare tokens are invalid PDF syntax. Add a pre-processor that scans for known bare struct type names and prepends / before loading. Unifies path and memory loading through the same fix pipeline. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: update lopdf dependency to main branch The firecrawl/zlib-checksum-encrypted branch was merged and deleted. Point to main which includes all previously merged fixes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: switch lopdf to upstream repo pinned at 845cd3d Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
76ea52680b
commit
d9c2143c32
@@ -124,10 +124,18 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
|
||||
let mut in_text_block = false;
|
||||
|
||||
// Marked content (ActualText) tracking
|
||||
let mut marked_content_stack: Vec<Option<String>> = Vec::new();
|
||||
// Marked content tracking: (ActualText, MCID) per nesting level
|
||||
struct MarkedContentEntry {
|
||||
actual_text: Option<String>,
|
||||
mcid: Option<i64>,
|
||||
}
|
||||
let mut marked_content_stack: Vec<MarkedContentEntry> = Vec::new();
|
||||
let mut suppress_glyph_extraction = false;
|
||||
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
|
||||
/// Get the innermost MCID from the marked content stack.
|
||||
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
|
||||
stack.iter().rev().find_map(|e| e.mcid)
|
||||
}
|
||||
|
||||
for op in &content.operations {
|
||||
trace!("{} {:?}", op.operator, op.operands);
|
||||
@@ -292,6 +300,7 @@ pub(crate) fn extract_page_text_items(
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -440,6 +449,7 @@ pub(crate) fn extract_page_text_items(
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -496,6 +506,7 @@ pub(crate) fn extract_page_text_items(
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -531,11 +542,15 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
"BMC" => {
|
||||
// Begin Marked Content (no properties)
|
||||
marked_content_stack.push(None);
|
||||
marked_content_stack.push(MarkedContentEntry {
|
||||
actual_text: None,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
"BDC" => {
|
||||
// Begin Marked Content with properties — extract ActualText
|
||||
// Begin Marked Content with properties — extract ActualText and MCID
|
||||
let mut actual_text: Option<String> = None;
|
||||
let mut mcid: Option<i64> = None;
|
||||
if op.operands.len() >= 2 {
|
||||
let dict = match &op.operands[1] {
|
||||
Object::Dictionary(d) => Some(d.clone()),
|
||||
@@ -549,47 +564,56 @@ pub(crate) fn extract_page_text_items(
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
if let Ok(Object::Integer(id)) = d.get(b"MCID") {
|
||||
mcid = Some(*id);
|
||||
}
|
||||
}
|
||||
}
|
||||
if actual_text.is_some() {
|
||||
suppress_glyph_extraction = true;
|
||||
actual_text_start_tm = Some(text_matrix);
|
||||
}
|
||||
marked_content_stack.push(actual_text);
|
||||
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
|
||||
}
|
||||
"EMC" => {
|
||||
// End Marked Content — emit ActualText item with correct width
|
||||
if let Some(Some(at)) = marked_content_stack.pop() {
|
||||
// Compute width from text matrix advancement during BDC..EMC
|
||||
if let Some(start_tm) = actual_text_start_tm.take() {
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
// Width in device space from text matrix delta
|
||||
let delta_ts = text_matrix[4] - start_tm[4];
|
||||
let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2];
|
||||
let width = (delta_ts * scale_x).abs();
|
||||
if !at.trim().is_empty() {
|
||||
let base_font = font_base_names
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&at),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: rendered_size,
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
});
|
||||
if let Some(entry) = marked_content_stack.pop() {
|
||||
if let Some(at) = entry.actual_text {
|
||||
// Compute width from text matrix advancement during BDC..EMC
|
||||
if let Some(start_tm) = actual_text_start_tm.take() {
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
// Width in device space from text matrix delta
|
||||
let delta_ts = text_matrix[4] - start_tm[4];
|
||||
let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2];
|
||||
let width = (delta_ts * scale_x).abs();
|
||||
if !at.trim().is_empty() {
|
||||
let base_font = font_base_names
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&at),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height: rendered_size,
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: entry
|
||||
.mcid
|
||||
.or_else(|| current_mcid(&marked_content_stack)),
|
||||
});
|
||||
}
|
||||
}
|
||||
suppress_glyph_extraction =
|
||||
marked_content_stack.iter().any(|e| e.actual_text.is_some());
|
||||
}
|
||||
suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some());
|
||||
}
|
||||
}
|
||||
"re" => {
|
||||
|
||||
@@ -796,6 +796,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Link(url),
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -316,5 +317,6 @@ pub(crate) fn walk_form_fields(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::FormField,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -319,6 +319,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
is_bold: first.is_bold,
|
||||
is_italic: first.is_italic,
|
||||
item_type: first.item_type.clone(),
|
||||
mcid: first.mcid,
|
||||
});
|
||||
|
||||
i = j;
|
||||
@@ -359,6 +360,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "World".into(),
|
||||
@@ -372,6 +374,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "Next line".into(),
|
||||
@@ -385,6 +388,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -438,6 +442,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "Prague".into(),
|
||||
@@ -451,6 +456,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "Rules".into(),
|
||||
@@ -464,6 +470,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -488,6 +495,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "A".into(),
|
||||
@@ -501,6 +509,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "V".into(),
|
||||
@@ -514,6 +523,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -540,6 +550,7 @@ mod tests {
|
||||
is_bold: true,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -573,6 +584,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,6 +619,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "履行義務".into(),
|
||||
@@ -620,6 +633,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "を識別す".into(),
|
||||
@@ -633,6 +647,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -654,6 +669,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -765,6 +781,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "\u{05D1}".into(), // bet at x=200 (rightmost)
|
||||
@@ -778,6 +795,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
sort_line_items(&mut items);
|
||||
@@ -801,6 +819,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
TextItem {
|
||||
text: "World".into(),
|
||||
@@ -814,6 +833,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
];
|
||||
sort_line_items(&mut items);
|
||||
@@ -853,6 +873,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
};
|
||||
|
||||
@@ -896,6 +917,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
};
|
||||
|
||||
@@ -939,6 +961,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
};
|
||||
|
||||
|
||||
@@ -408,6 +408,7 @@ fn extract_form_xobject_text_inner(
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -549,6 +550,7 @@ fn extract_form_xobject_text_inner(
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user