diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 4ec4bc0..a2280df 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -83,6 +83,9 @@ pub struct TextItem { /// Underline detected geometrically (drawn rule/thin rect under the /// baseline) — PDFs carry no underline font flag. pub is_underline: bool, + /// Strikeout detected geometrically (rule crossing the glyphs at mid + /// x-height). + pub is_strikeout: bool, pub item_type: ItemType, /// URL for link items, `None` for other types. pub link_url: Option, @@ -294,6 +297,7 @@ pub fn extract_text_with_positions( is_bold: item.is_bold, is_italic: item.is_italic, is_underline: item.is_underline, + is_strikeout: item.is_strikeout, item_type, link_url, } diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index 21b1153..a28b996 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -39,6 +39,7 @@ class TextItem: is_bold: bool is_italic: bool is_underline: bool + is_strikeout: bool item_type: str class RegionText: diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index a87303f..774a8ae 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -74,7 +74,7 @@ fn format_items_json(items: &[TextItem]) -> String { _ => String::new(), }; format!( - r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#, + r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"is_strikeout":{},"item_type":"{}","mcid":{}{}}}"#, json_escape(&item.text), item.page, item.x, @@ -86,6 +86,7 @@ fn format_items_json(items: &[TextItem]) -> String { item.is_bold, item.is_italic, item.is_underline, + item.is_strikeout, item_type_label(&item.item_type), mcid, link_url, @@ -122,6 +123,7 @@ mod tests { is_bold: false, is_italic: true, is_underline: true, + is_strikeout: true, item_type: ItemType::Text, mcid: Some(7), }]; diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index abd3ae7..7c0852d 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -14,8 +14,9 @@ use lopdf::{Document, Encoding, Object, ObjectId}; use std::collections::HashMap; use super::fonts::{ - build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand, - get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, + build_font_encodings, build_font_widths, compute_string_width_ts, descriptor_style_flags, + extract_text_from_operand, get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, + FontStyleCache, }; use super::underline::UnderlineLine; use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; @@ -106,6 +107,25 @@ fn transformed_stroke_width( user_width * (ndx * ndx + ndy * ndy).sqrt() } +/// Text rise (Ts) displaces the glyph origin by (0, rise) in unscaled text +/// space — per the rendering-matrix definition it sits left of Tm, so the +/// offset maps through the text matrix's y column. Rise never contributes +/// to the advance, so callers apply it only to the rendering position and +/// keep advancing the unshifted text matrix. +fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] { + if rise == 0.0 { + return *tm; + } + [ + tm[0], + tm[1], + tm[2], + tm[3], + tm[4] + rise * tm[2], + tm[5] + rise * tm[3], + ] +} + /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates /// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( @@ -114,6 +134,7 @@ pub(crate) fn extract_page_text_items( page_num: u32, font_cmaps: &FontCMaps, include_invisible: bool, + style_cache: &mut FontStyleCache, ) -> Result<(PageExtraction, bool, bool), PdfError> { use lopdf::content::Content; @@ -153,6 +174,8 @@ pub(crate) fn extract_page_text_items( std::collections::HashMap::new(); let mut inline_cmaps: std::collections::HashMap = std::collections::HashMap::new(); + let mut font_style_flags: std::collections::HashMap = + std::collections::HashMap::new(); for (font_name, font_dict) in &fonts { let resource_name = String::from_utf8_lossy(font_name).to_string(); if let Ok(base_font) = font_dict.get(b"BaseFont") { @@ -161,6 +184,12 @@ pub(crate) fn extract_page_text_items( font_base_names.insert(resource_name.clone(), base_name); } } + // Descriptor style flags rescue subset fonts whose BaseFont names + // are opaque tags the name heuristics can't read. + let style = descriptor_style_flags(doc, font_dict, style_cache); + if style != (false, false) { + font_style_flags.insert(resource_name.clone(), style); + } // Track ToUnicode object reference, with FontFile2 fallback for Identity-H/V. // Also handle inline ToUnicode streams. match font_dict.get(b"ToUnicode") { @@ -235,6 +264,7 @@ pub(crate) fn extract_page_text_items( line_width: f32, char_spacing: f32, word_spacing: f32, + text_rise: f32, text_leading: f32, current_font: String, current_font_size: f32, @@ -247,6 +277,7 @@ pub(crate) fn extract_page_text_items( let mut text_leading: f32 = 0.0; // TL parameter (in text-space units) let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled) let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled) + let mut text_rise: f32 = 0.0; // Ts parameter (baseline shift for super/subscripts, unscaled) let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; @@ -268,6 +299,10 @@ pub(crate) fn extract_page_text_items( let mut suppress_glyph_extraction = false; let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC + // Text rise in effect at each captured matrix — the item must render at + // the rise of its GLYPHS, not whatever rise is set by EMC time. + let mut actual_text_start_rise: f32 = 0.0; + let mut actual_text_glyph_rise: Option = None; /// Get the innermost MCID from the marked content stack. fn current_mcid(stack: &[MarkedContentEntry]) -> Option { stack.iter().rev().find_map(|e| e.mcid) @@ -284,6 +319,7 @@ pub(crate) fn extract_page_text_items( line_width, char_spacing, word_spacing, + text_rise, text_leading, current_font: current_font.clone(), current_font_size, @@ -297,6 +333,7 @@ pub(crate) fn extract_page_text_items( line_width = saved.line_width; char_spacing = saved.char_spacing; word_spacing = saved.word_spacing; + text_rise = saved.text_rise; text_leading = saved.text_leading; current_font = saved.current_font; current_font_size = saved.current_font_size; @@ -369,6 +406,12 @@ pub(crate) fn extract_page_text_items( word_spacing = tw; } } + "Ts" => { + // Set text rise (baseline shift for superscripts/subscripts) + if let Some(ts) = op.operands.first().and_then(get_number) { + text_rise = ts; + } + } "Td" | "TD" => { // Move text position: TLM = T(tx,ty) × TLM; Tm = TLM // tx,ty are in text space — must be scaled by the text line matrix @@ -427,6 +470,7 @@ pub(crate) fn extract_page_text_items( if suppress_glyph_extraction { if actual_text_glyph_tm.is_none() { actual_text_glyph_tm = Some(text_matrix); + actual_text_glyph_rise = Some(text_rise); } if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; @@ -456,7 +500,8 @@ pub(crate) fn extract_page_text_items( &mut cmap_decisions, &font_widths, ) { - let combined = multiply_matrices(&text_matrix, &ctm); + let combined = + multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm); let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); if combined[0].abs() >= combined[1].abs() { @@ -478,6 +523,10 @@ pub(crate) fn extract_page_text_items( .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); items.push(TextItem { text: expand_ligatures(&text), x, @@ -487,9 +536,10 @@ pub(crate) fn extract_page_text_items( font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); @@ -507,6 +557,7 @@ pub(crate) fn extract_page_text_items( // Capture first-glyph position for ActualText if suppress_glyph_extraction && actual_text_glyph_tm.is_none() { actual_text_glyph_tm = Some(text_matrix); + actual_text_glyph_rise = Some(text_rise); } // Compute space threshold based on font metrics when available @@ -627,6 +678,10 @@ pub(crate) fn extract_page_text_items( .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]; for (text, start_w, end_w) in &sub_items { let offset_tm = [ @@ -637,7 +692,8 @@ pub(crate) fn extract_page_text_items( text_matrix[4] + start_w * text_matrix[0], text_matrix[5] + start_w * text_matrix[1], ]; - let combined = multiply_matrices(&offset_tm, &ctm); + let combined = + multiply_matrices(&rise_adjusted(&offset_tm, text_rise), &ctm); let (x, y) = (combined[4], combined[5]); let width = if font_info.is_some() { ((end_w - start_w) * scale_x).abs() @@ -653,9 +709,10 @@ pub(crate) fn extract_page_text_items( font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); @@ -679,6 +736,26 @@ pub(crate) fn extract_page_text_items( line_matrix[4] += (-tl) * line_matrix[2]; line_matrix[5] += (-tl) * line_matrix[3]; text_matrix = line_matrix; + // Capture first-glyph position for ActualText AFTER the + // line move — the BDC-entry matrix is on the previous line. + if suppress_glyph_extraction && actual_text_glyph_tm.is_none() { + actual_text_glyph_tm = Some(text_matrix); + actual_text_glyph_rise = Some(text_rise); + } + // Advance width, as for Tj — without it the item stays + // zero-width and geometric underline/strikeout detection + // rejects it (`is_underline_candidate` needs width > 0). + let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| { + op.operands.first().and_then(get_operand_bytes).map(|raw| { + compute_string_width_ts( + raw, + fi, + current_font_size, + char_spacing, + word_spacing, + ) + }) + }); if !((text_rendering_mode == 3 && !include_invisible) || suppress_glyph_extraction || op.operands.is_empty()) @@ -696,7 +773,8 @@ pub(crate) fn extract_page_text_items( &font_widths, ) { if !text.trim().is_empty() { - let combined = multiply_matrices(&text_matrix, &ctm); + let combined = + multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm); if combined[0].abs() >= combined[1].abs() { rotation_votes.horizontal += 1; } else { @@ -704,28 +782,45 @@ pub(crate) fn extract_page_text_items( } let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); + let width = w_ts_opt + .map(|w_ts| { + (w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2])) + .abs() + }) + .unwrap_or(0.0); let base_font = font_base_names .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); items.push(TextItem { text: expand_ligatures(&text), x, y, - width: 0.0, + width, height: rendered_size, font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); } } } + // Advance regardless of visibility so later show-text + // operators on the same line stay positioned (as for Tj). + if let Some(w_ts) = w_ts_opt { + text_matrix[4] += w_ts * text_matrix[0]; + text_matrix[5] += w_ts * text_matrix[1]; + } } "Do" => { // XObject invocation - could be an image or form @@ -757,6 +852,7 @@ pub(crate) fn extract_page_text_items( is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Image, mcid: current_mcid(&marked_content_stack), }); @@ -770,6 +866,7 @@ pub(crate) fn extract_page_text_items( font_cmaps, &ctm, &mut cmap_decisions, + style_cache, ); items.extend(form_items); } @@ -810,7 +907,9 @@ pub(crate) fn extract_page_text_items( if actual_text.is_some() { suppress_glyph_extraction = true; actual_text_start_tm = Some(text_matrix); + actual_text_start_rise = text_rise; actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ + actual_text_glyph_rise = None; } marked_content_stack.push(MarkedContentEntry { actual_text, mcid }); } @@ -823,9 +922,11 @@ pub(crate) fn extract_page_text_items( // Tj may have moved the text position to the correct line — // the BDC-entry position can be on the previous line. let glyph_tm = actual_text_glyph_tm.take(); + let glyph_rise = actual_text_glyph_rise.take(); let entry_tm = actual_text_start_tm.take(); if let Some(start_tm) = glyph_tm.or(entry_tm) { - let combined = multiply_matrices(&start_tm, &ctm); + let rise = glyph_rise.unwrap_or(actual_text_start_rise); + let combined = multiply_matrices(&rise_adjusted(&start_tm, rise), &ctm); if combined[0].abs() >= combined[1].abs() { rotation_votes.horizontal += 1; } else { @@ -842,6 +943,10 @@ pub(crate) fn extract_page_text_items( .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); items.push(TextItem { text: expand_ligatures(&at), x, @@ -851,9 +956,10 @@ pub(crate) fn extract_page_text_items( font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: entry .mcid @@ -1376,8 +1482,15 @@ mod tests { let (doc, page_id) = simple_doc_with_content(content); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _) = - extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let ((items, _, _), _, _) = extract_page_text_items( + &doc, + page_id, + 1, + &font_cmaps, + false, + &mut FontStyleCache::new(), + ) + .unwrap(); items } @@ -1462,6 +1575,108 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET assert!(!world.is_underline); } + #[test] + fn quote_operator_text_carries_advance_width() { + // `'` (move-to-next-line-and-show-text) must retain the string's + // advance width like Tj — zero-width items are invisible to + // geometric underline/strikeout detection. + let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (first) Tj (struck) ' ET +1 w +99 503 m 145 503 l S"; + + let items = extract_simple_items(content); + let struck = items.iter().find(|item| item.text == "struck").unwrap(); + + // 6 glyphs x 600/1000 x 12pt = 43.2pt, drawn one leading below Tm. + assert!((struck.width - 43.2).abs() < 0.1); + assert!((struck.y - 500.0).abs() < 0.1); + assert!(struck.is_strikeout); + assert!(!struck.is_underline); + } + + #[test] + fn quote_operator_advances_text_matrix() { + // Text shown after `'` on the same line must start past the shown + // string: "CD" lands at x=114.4 (2 glyphs x 600/1000 x 12pt after + // x=100), flush against "AB", so the merge pass joins them. Without + // the advance "CD" overlaps "AB" at x=100 and the items stay apart. + let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (AB) ' (CD) Tj ET"; + + let items = extract_simple_items(content); + let merged = items.iter().find(|item| item.text == "ABCD").unwrap(); + + assert!((merged.x - 100.0).abs() < 0.1); + assert!((merged.width - 28.8).abs() < 0.1); + assert!((merged.y - 500.0).abs() < 0.1); + } + + #[test] + fn text_rise_shifts_item_baseline() { + // Ts displaces the glyph origin vertically without touching the + // advance; the next run at rise 0 must return to the original + // baseline and follow the raised run horizontally. + let content = + b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (base) Tj 5 Ts (super) Tj 0 Ts (after) Tj ET"; + + let items = extract_simple_items(content); + let base = items.iter().find(|item| item.text == "base").unwrap(); + let raised = items.iter().find(|item| item.text == "super").unwrap(); + let after = items.iter().find(|item| item.text == "after").unwrap(); + + assert!((base.y - 500.0).abs() < 0.1); + assert!((raised.y - 505.0).abs() < 0.1); + assert!((after.y - 500.0).abs() < 0.1); + assert!(after.x > raised.x); + } + + #[test] + fn actual_text_item_uses_glyph_rise() { + // The ActualText replacement item must render at the rise in + // effect when its glyphs were drawn — not the unshifted BDC + // baseline, and not whatever rise is set by EMC time. + let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm \ +/Span <> BDC 5 Ts (sup) Tj 0 Ts EMC (after) Tj ET"; + + let items = extract_simple_items(content); + let sup = items.iter().find(|item| item.text == "super").unwrap(); + let after = items.iter().find(|item| item.text == "after").unwrap(); + + assert!((sup.y - 505.0).abs() < 0.1); + assert!((after.y - 500.0).abs() < 0.1); + } + + #[test] + fn actual_text_shown_with_quote_op_uses_moved_risen_baseline() { + // When the tagged span's show op is `'`, the glyph position is + // only known AFTER its line move — falling back to the BDC-entry + // matrix would place the item on the previous line, unrisen. + let content = b"BT /F1 12 Tf 14 TL 1 0 0 1 100 500 Tm \ +/Span <> BDC 3 Ts (raw) ' 0 Ts EMC ET"; + + let items = extract_simple_items(content); + let item = items.iter().find(|item| item.text == "replaced").unwrap(); + + // Line move: 500 - 14 = 486; rise: +3 -> 489. + assert!((item.y - 489.0).abs() < 0.1); + assert!(item.width > 0.0); + } + + #[test] + fn strikeout_detected_on_risen_text() { + // The rule crosses the glyphs at their risen position; without the + // rise in item.y the strike window sits 4pt too low and misses. + let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm 4 Ts (struck) Tj ET +1 w +99 507 m 145 507 l S"; + + let items = extract_simple_items(content); + let struck = items.iter().find(|item| item.text == "struck").unwrap(); + + assert!((struck.y - 504.0).abs() < 0.1); + assert!(struck.is_strikeout); + assert!(!struck.is_underline); + } + #[test] fn test_skip_excessive_operations() { use crate::tounicode::FontCMaps; @@ -1496,7 +1711,15 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET doc.add_object(catalog); let font_cmaps = FontCMaps::from_doc(&doc); - let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let result = extract_page_text_items( + &doc, + page_id, + 1, + &font_cmaps, + false, + &mut FontStyleCache::new(), + ) + .unwrap(); let ((items, rects, lines), _has_gid, _coords_rotated) = result; assert!(items.is_empty()); assert!(rects.is_empty()); @@ -1578,8 +1801,15 @@ BT 30 700 Tm <41> Tj ET"; doc.trailer.set("Root", Object::Reference(catalog_id)); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _) = - extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let ((items, _, _), _, _) = extract_page_text_items( + &doc, + page_id, + 1, + &font_cmaps, + false, + &mut FontStyleCache::new(), + ) + .unwrap(); let text = items .iter() .map(|item| item.text.as_str()) diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index e4247a4..b2d57e8 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -4,7 +4,7 @@ use crate::glyph_names::glyph_to_char; use crate::tounicode::FontCMaps; use crate::types::{FontEncodingMap, FontWidthInfo, PageFontEncodings, PageFontWidths}; use log::debug; -use lopdf::{Document, Encoding, Object}; +use lopdf::{Document, Encoding, Object, ObjectId}; use std::collections::HashMap; #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -739,6 +739,171 @@ pub(crate) fn get_font_file2_obj_num(doc: &Document, font_dict: &lopdf::Dictiona .map(|r| r.0) } +/// Document-scoped memo of embedded-font style flags, keyed by the +/// FontFile2/FontFile3 stream's object id. The same font program is +/// referenced from every page that uses the font, and decompressing + +/// parsing it dominates `descriptor_style_flags` — without the memo that +/// cost repeats per page whenever the descriptor leaves a flag unset +/// (the common case: regular fonts report neither italic nor bold). +#[derive(Debug, Default)] +pub(crate) struct FontStyleCache { + by_font_file: HashMap, +} + +impl FontStyleCache { + pub(crate) fn new() -> Self { + Self::default() + } +} + +/// Style flags from the FontDescriptor, which survive subset fonts whose +/// BaseFont names are opaque tags ("Tc1", "ABCDEF+F1") that defeat the +/// name-based bold/italic heuristics. +/// +/// Italic: `ItalicAngle` beyond a few degrees, or Flags bit 7 (Italic, +/// value 64). Bold: Flags bit 19 (ForceBold, value 1<<18). The small +/// ItalicAngle threshold skips fonts that declare a token slant. +pub(crate) fn descriptor_style_flags( + doc: &Document, + font_dict: &lopdf::Dictionary, + style_cache: &mut FontStyleCache, +) -> (bool, bool) { + let descriptor = font_dict + .get(b"FontDescriptor") + .ok() + .and_then(|obj| resolve_dict(doc, obj)) + .or_else(|| { + // Type0 fonts hang the descriptor off DescendantFonts[0]. + let desc_fonts = font_dict.get(b"DescendantFonts").ok()?; + let desc_fonts = resolve_array(doc, desc_fonts)?; + let cid_font_dict = resolve_dict(doc, desc_fonts.first()?)?; + resolve_dict(doc, cid_font_dict.get(b"FontDescriptor").ok()?) + }); + let Some(descriptor) = descriptor else { + return (false, false); + }; + + let italic_angle = descriptor + .get(b"ItalicAngle") + .ok() + .and_then(|obj| match obj { + Object::Integer(i) => Some(*i as f32), + Object::Real(r) => Some(*r), + _ => None, + }) + .unwrap_or(0.0); + let flags = descriptor + .get(b"Flags") + .ok() + .and_then(|obj| obj.as_i64().ok()) + .unwrap_or(0); + + let mut italic = italic_angle.abs() >= 4.0 || flags & (1 << 6) != 0; + let mut bold = flags & (1 << 18) != 0; + + // Descriptors lie: subset generators write ItalicAngle 0 for genuinely + // italic faces. The embedded font file keeps the truth — OS/2 + // fsSelection (via `Face::is_italic`) and the post table's italicAngle. + if !italic || !bold { + if let Some(ff_ref) = font_file_ref(descriptor) { + let (emb_italic, emb_bold) = *style_cache + .by_font_file + .entry(ff_ref) + .or_insert_with(|| embedded_style_flags(doc, ff_ref)); + italic = italic || emb_italic; + bold = bold || emb_bold; + } + } + (italic, bold) +} + +/// Style flags parsed from an embedded font program stream. +fn embedded_style_flags(doc: &Document, ff_ref: ObjectId) -> (bool, bool) { + let Some(data) = font_file_data(doc, ff_ref) else { + return (false, false); + }; + if let Ok(face) = ttf_parser::Face::parse(&data, 0) { + ( + face.is_italic() || face.italic_angle().abs() >= 4.0, + face.is_bold(), + ) + } else if let Some(name) = cff_font_name(&data) { + // FontFile3 is bare CFF (no sfnt container) — ttf_parser + // can't open it, but the CFF Name INDEX keeps the real + // PostScript name ("XXXXXX+Amplitude-LightItalic") even + // when the descriptor was rewritten to claim upright. + ( + crate::text_utils::is_italic_font(&name), + crate::text_utils::is_bold_font(&name), + ) + } else { + (false, false) + } +} + +/// First PostScript name from a bare CFF font's Name INDEX (CFF spec §7). +fn cff_font_name(data: &[u8]) -> Option { + // Header: major(1) minor(1) hdrSize(1) offSize(1); major must be 1. + if data.len() < 4 || data[0] != 1 { + return None; + } + let hdr_size = data[2] as usize; + // Name INDEX: count(u16) offSize(u8) offsets[count+1] data + let count = u16::from_be_bytes([*data.get(hdr_size)?, *data.get(hdr_size + 1)?]) as usize; + if count == 0 { + return None; + } + let off_size = *data.get(hdr_size + 2)? as usize; + if !(1..=4).contains(&off_size) { + return None; + } + let read_offset = |idx: usize| -> Option { + let at = hdr_size + 3 + idx * off_size; + let bytes = data.get(at..at + off_size)?; + let mut v = 0usize; + for b in bytes { + v = (v << 8) | *b as usize; + } + Some(v) + }; + let start = read_offset(0)?; + let end = read_offset(1)?; + if start == 0 || end < start { + return None; + } + // Offsets are 1-based from the byte before the object data. + let objects_base = hdr_size + 3 + (count + 1) * off_size - 1; + let name = data.get(objects_base + start..objects_base + end)?; + Some(String::from_utf8_lossy(name).to_string()) +} + +/// FontFile2/FontFile3 stream reference from a FontDescriptor. +fn font_file_ref(descriptor: &lopdf::Dictionary) -> Option { + descriptor + .get(b"FontFile2") + .ok() + .and_then(|o| o.as_reference().ok()) + .or_else(|| { + descriptor + .get(b"FontFile3") + .ok() + .and_then(|o| o.as_reference().ok()) + }) +} + +/// Decompressed embedded font program bytes. +fn font_file_data(doc: &Document, ff_ref: ObjectId) -> Option> { + let stream = doc + .get_object(ff_ref) + .and_then(lopdf::Object::as_stream) + .ok()?; + Some( + stream + .decompressed_content() + .unwrap_or_else(|_| stream.content.clone()), + ) +} + /// Decode text from a PDF string operand using font CMaps, encodings, and fallbacks. #[allow(clippy::too_many_arguments)] pub(crate) fn extract_text_from_operand( @@ -1262,6 +1427,172 @@ mod tests { } } + fn doc_with_descriptor(descriptor: lopdf::Dictionary) -> (Document, lopdf::Dictionary) { + let mut doc = Document::with_version("1.4"); + let desc_id = doc.add_object(descriptor); + let font_dict = dictionary! { + "Type" => "Font", + "Subtype" => "TrueType", + "BaseFont" => "Tc1", + "FontDescriptor" => desc_id, + }; + (doc, font_dict) + } + + #[test] + fn descriptor_italic_angle_sets_italic() { + // Subset font with an opaque BaseFont name ("Tc1") — the name + // heuristic sees nothing, the descriptor carries the truth. + let (doc, font_dict) = doc_with_descriptor(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "Tc1", + "ItalicAngle" => -12, + "Flags" => 32, + }); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (true, false) + ); + } + + #[test] + fn descriptor_italic_flag_bit_sets_italic() { + let (doc, font_dict) = doc_with_descriptor(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "Tc1", + "ItalicAngle" => 0, + "Flags" => 64, // bit 7: Italic + }); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (true, false) + ); + } + + #[test] + fn descriptor_force_bold_flag_sets_bold() { + let (doc, font_dict) = doc_with_descriptor(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "Tc1", + "ItalicAngle" => 0, + "Flags" => 1 << 18, // ForceBold + }); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (false, true) + ); + } + + #[test] + fn tiny_italic_angle_is_not_italic() { + // A token 1-degree slant is optical correction, not italic. + let (doc, font_dict) = doc_with_descriptor(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "Tc1", + "ItalicAngle" => lopdf::Object::Real(-1.0), + "Flags" => 32, + }); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (false, false) + ); + } + + #[test] + fn missing_descriptor_yields_no_flags() { + let doc = Document::with_version("1.4"); + let font_dict = dictionary! { "Type" => "Font", "BaseFont" => "Tc1" }; + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (false, false) + ); + } + + #[test] + fn type0_descendant_descriptor_is_resolved() { + let mut doc = Document::with_version("1.4"); + let desc_id = doc.add_object(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "ABCDEF+F1", + "ItalicAngle" => -15, + }); + let cid_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "CIDFontType2", + "FontDescriptor" => desc_id, + }); + let font_dict = dictionary! { + "Type" => "Font", + "Subtype" => "Type0", + "BaseFont" => "ABCDEF+F1", + "DescendantFonts" => vec![lopdf::Object::Reference(cid_id)], + }; + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (true, false) + ); + } + + /// Bare CFF: header + Name INDEX only — enough for `cff_font_name`. + fn bare_cff_with_name(name: &str) -> Vec { + let mut data = vec![1, 0, 4, 1]; // major, minor, hdrSize, offSize + data.extend_from_slice(&1u16.to_be_bytes()); // Name INDEX count + data.push(1); // offSize + data.push(1); // offset of first name + data.push(1 + name.len() as u8); // offset past last name + data.extend_from_slice(name.as_bytes()); + data + } + + #[test] + fn embedded_font_style_is_cached_by_font_file_object() { + use lopdf::{Object, Stream}; + + let mut doc = Document::with_version("1.4"); + let ff_id = doc.add_object(Object::Stream(Stream::new( + dictionary! {}, + bare_cff_with_name("ABCDEF+Test-BoldItalic"), + ))); + let desc_id = doc.add_object(dictionary! { + "Type" => "FontDescriptor", + "FontName" => "ABCDEF+Test-BoldItalic", + "ItalicAngle" => 0, + "Flags" => 32, + "FontFile3" => ff_id, + }); + let font_dict = dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Tc1", + "FontDescriptor" => desc_id, + }; + + let mut cache = FontStyleCache::new(); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut cache), + (true, true) + ); + assert_eq!(cache.by_font_file.len(), 1); + + // Replace the font program with garbage: a repeat call must serve + // the memo instead of re-reading the stream — repeated per-page + // decompression is exactly what the cache exists to avoid. + doc.objects.insert( + ff_id, + Object::Stream(Stream::new(dictionary! {}, vec![0u8; 4])), + ); + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut cache), + (true, true) + ); + // A cold cache parses the (now garbage) stream, proving the warm + // call above answered from the memo. + assert_eq!( + descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()), + (false, false) + ); + } + #[test] fn compute_string_width_ts_no_tc_tw() { // Without Tc/Tw (both 0), width = glyph widths only diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index 7c017e6..a8390ab 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -1495,6 +1495,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -1625,6 +1626,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }); diff --git a/src/extractor/links.rs b/src/extractor/links.rs index b08e97a..054a997 100644 --- a/src/extractor/links.rs +++ b/src/extractor/links.rs @@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Link(url), mcid: None, }); @@ -318,6 +319,7 @@ pub(crate) fn walk_form_fields( is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::FormField, mcid: None, }); diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 4bda278..4f2b9b0 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -24,6 +24,7 @@ use links::{extract_form_fields, extract_page_links}; // Re-export public types so existing `crate::extractor::X` paths keep working. pub use crate::text_utils::{is_bold_font, is_italic_font}; pub use crate::types::{ItemType, TextLine}; +pub(crate) use fonts::FontStyleCache; pub(crate) use layout::detect_columns; pub use layout::group_into_lines; pub(crate) use layout::group_into_lines_with_thresholds; @@ -161,6 +162,9 @@ fn extract_positioned_text_impl( let mut all_lines = Vec::new(); let mut page_thresholds: PageThresholds = HashMap::new(); let mut gid_encoded_pages: HashSet = HashSet::new(); + // Embedded-font style flags are document-scoped: the same font program + // is shared across pages, so parse it once, not once per page. + let mut style_cache = FontStyleCache::new(); // Build page ObjectId → page number map for form field extraction let page_id_to_num: HashMap = @@ -172,8 +176,14 @@ fn extract_positioned_text_impl( continue; } } - let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = - extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?; + let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items( + doc, + page_id, + *page_num, + font_cmaps, + include_invisible, + &mut style_cache, + )?; if has_gid_fonts { gid_encoded_pages.insert(*page_num); } @@ -234,7 +244,10 @@ fn suppress_table_underlines( lines: &[PdfLine], page: u32, ) { - if !items.iter().any(|item| item.is_underline) { + if !items + .iter() + .any(|item| item.is_underline || item.is_strikeout) + { return; } @@ -256,6 +269,7 @@ fn suppress_table_underlines( for index in table_item_indices { if let Some(item) = items.get_mut(index) { item.is_underline = false; + item.is_strikeout = false; } } } @@ -576,6 +590,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { if next.is_bold != first.is_bold || next.is_italic != first.is_italic || next.is_underline != first.is_underline + || next.is_strikeout != first.is_strikeout { break; } @@ -638,6 +653,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { is_bold: first.is_bold, is_italic: first.is_italic, is_underline: first.is_underline, + is_strikeout: first.is_strikeout, item_type: first.item_type.clone(), mcid: first.mcid, }); @@ -715,7 +731,9 @@ pub(crate) fn merge_subscript_items(items: Vec) -> Vec { .chars() .last() .is_some_and(|c| c.is_alphabetic()); - if parent.font_size >= sub_threshold && ends_with_letter { + let same_marks = parent.is_underline == item.is_underline + && parent.is_strikeout == item.is_strikeout; + if parent.font_size >= sub_threshold && ends_with_letter && same_marks { let parent_right = parent.x + parent.width; let gap = item.x - parent_right; // Subscripts must be tightly adjacent (within ~1pt) @@ -789,6 +807,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -991,6 +1010,7 @@ mod tests { items[3].y = 470.0; for item in &mut items { item.is_underline = true; + item.is_strikeout = true; } let lines = vec![ make_line(100.0, 500.0, 300.0, 500.0), @@ -1004,6 +1024,30 @@ mod tests { suppress_table_underlines(&mut items, &[], &lines, 1); assert!(items.iter().all(|item| !item.is_underline)); + assert!(items.iter().all(|item| !item.is_strikeout)); + } + + #[test] + fn subscript_digit_with_different_marks_is_not_absorbed() { + // A struck-out word followed by an unmarked footnote digit: merging + // would widen the parent's strikeout claim over the digit (and the + // reverse would drop the digit's own mark). Style boundaries break + // the merge, as in merge_text_items. + let mut word = make_merge_item("word", 100.0, 24.0); + word.font_size = 10.0; + word.is_strikeout = true; + let mut digit = make_merge_item("2", 124.5, 4.0); + digit.font_size = 6.0; + digit.y = word.y + 3.0; + + let merged = merge_subscript_items(vec![word.clone(), digit.clone()]); + assert_eq!(merged.len(), 2); + + // Same marks still merge (footnote ref inside the strike). + digit.is_strikeout = true; + let merged = merge_subscript_items(vec![word, digit]); + assert_eq!(merged.len(), 1); + assert!(merged[0].text.starts_with("word")); } #[test] @@ -1021,6 +1065,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1036,6 +1081,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1051,6 +1097,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1106,6 +1153,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1121,6 +1169,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1136,6 +1185,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1162,6 +1212,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1177,6 +1228,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1192,6 +1244,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1220,6 +1273,7 @@ mod tests { is_bold: true, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -1255,6 +1309,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -1291,6 +1346,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1306,6 +1362,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1321,6 +1378,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1344,6 +1402,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -1457,6 +1516,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1472,6 +1532,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1497,6 +1558,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1512,6 +1574,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }, @@ -1553,6 +1616,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }], @@ -1598,6 +1662,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }], @@ -1643,6 +1708,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }], @@ -1681,6 +1747,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } diff --git a/src/extractor/underline.rs b/src/extractor/underline.rs index b5ea643..0c76fb8 100644 --- a/src/extractor/underline.rs +++ b/src/extractor/underline.rs @@ -231,8 +231,27 @@ fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool { overlap >= min_overlap } +/// Strikeout window: a rule crossing the glyphs. Strikethroughs sit at +/// roughly 20-35% of the em above the baseline (about half the x-height); +/// accept a band well inside the glyph body so baseline underlines and +/// overlines never qualify. +fn rule_strikes_item(rule: &Rule, item: &TextItem) -> bool { + let y_min = item.y + item.font_size * 0.12; + let y_max = item.y + item.font_size * 0.55; + if rule.y < y_min || rule.y > y_max { + return false; + } + + let ix1 = item.x; + let ix2 = item.x + item.width; + let min_overlap = item.width * MIN_X_OVERLAP; + let overlap = rule.x2.min(ix2) - rule.x1.max(ix1); + overlap >= min_overlap +} + /// Mark `is_underline` on text items that have a horizontal rule just -/// below their baseline. `items`, `rects`, and `lines` are a single +/// below their baseline, and `is_strikeout` on items whose glyphs a rule +/// crosses at mid x-height. `items`, `rects`, and `lines` are a single /// page's extraction output (all in PDF coordinates, y-up, where /// `TextItem::y` is the text baseline). pub(crate) fn mark_underlined_items( @@ -258,6 +277,11 @@ pub(crate) fn mark_underlined_items( } if rule_matches_item(rule, item) { item.is_underline = true; + } + if rule_strikes_item(rule, item) { + item.is_strikeout = true; + } + if item.is_underline && item.is_strikeout { break; } } @@ -282,6 +306,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -358,6 +383,44 @@ mod tests { assert!(!items[0].is_underline); } + #[test] + fn mid_glyph_rule_marks_strikeout_not_underline() { + // Rule at ~30% of the em above the baseline crosses the glyphs. + let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(99.0, 161.0, 503.0)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(items[0].is_strikeout); + assert!(!items[0].is_underline); + } + + #[test] + fn baseline_rule_marks_underline_not_strikeout() { + let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(99.0, 161.0, 498.5)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(items[0].is_underline); + assert!(!items[0].is_strikeout); + } + + #[test] + fn overline_is_neither_underline_nor_strikeout() { + // Rule just above the cap height (overline / next line's rule). + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(99.0, 161.0, 507.0)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(!items[0].is_underline); + assert!(!items[0].is_strikeout); + } + + #[test] + fn thin_filled_rect_at_mid_glyph_marks_strikeout() { + let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)]; + let rects = vec![thin_rect(100.0, 502.6, 60.0)]; + mark_underlined_items(&mut items, &rects, &[], 1); + assert!(items[0].is_strikeout); + assert!(!items[0].is_underline); + } + #[test] fn line_above_baseline_is_not_an_underline() { // Strikethrough / overline geometry must not mark. diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 06eea8a..7b54979 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -1,5 +1,6 @@ //! Form XObject and image XObject extraction. +use super::fonts::descriptor_style_flags; use crate::text_utils::{effective_font_size, expand_ligatures, is_bold_font, is_italic_font}; use crate::tounicode::FontCMaps; use crate::types::{ItemType, TextItem}; @@ -8,7 +9,7 @@ use std::collections::HashMap; use super::fonts::{ build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand, - get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, + get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, FontStyleCache, }; use super::{get_number, image_bbox_from_ctm, multiply_matrices}; @@ -114,6 +115,7 @@ pub(crate) fn extract_form_xobject_text( font_cmaps: &FontCMaps, parent_ctm: &[f32; 6], cmap_decisions: &mut CMapDecisionCache, + style_cache: &mut FontStyleCache, ) -> Vec { extract_form_xobject_text_inner( doc, @@ -122,10 +124,12 @@ pub(crate) fn extract_form_xobject_text( font_cmaps, parent_ctm, cmap_decisions, + style_cache, 0, ) } +#[allow(clippy::too_many_arguments)] fn extract_form_xobject_text_inner( doc: &Document, form_id: ObjectId, @@ -133,6 +137,7 @@ fn extract_form_xobject_text_inner( font_cmaps: &FontCMaps, parent_ctm: &[f32; 6], cmap_decisions: &mut CMapDecisionCache, + style_cache: &mut FontStyleCache, depth: u8, ) -> Vec { use lopdf::content::Content; @@ -167,6 +172,7 @@ fn extract_form_xobject_text_inner( let mut font_tounicode_refs: HashMap = HashMap::new(); let mut inline_cmaps: HashMap = HashMap::new(); + let mut font_style_flags: HashMap = HashMap::new(); for (font_name, font_dict) in &form_fonts { let resource_name = String::from_utf8_lossy(font_name).to_string(); if let Ok(base_font) = font_dict.get(b"BaseFont") { @@ -175,6 +181,10 @@ fn extract_form_xobject_text_inner( font_base_names.insert(resource_name.clone(), base_name); } } + let style = descriptor_style_flags(doc, font_dict, style_cache); + if style != (false, false) { + font_style_flags.insert(resource_name.clone(), style); + } match font_dict.get(b"ToUnicode") { Ok(tounicode) => { if let Ok(obj_ref) = tounicode.as_reference() { @@ -272,6 +282,7 @@ fn extract_form_xobject_text_inner( font_cmaps, &ctm, cmap_decisions, + style_cache, depth + 1, ); items.extend(nested_items); @@ -295,6 +306,7 @@ fn extract_form_xobject_text_inner( is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Image, mcid: None, }); @@ -428,6 +440,10 @@ fn extract_form_xobject_text_inner( .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); items.push(TextItem { text: expand_ligatures(&text), x, @@ -437,9 +453,10 @@ fn extract_form_xobject_text_inner( font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }); @@ -560,6 +577,10 @@ fn extract_form_xobject_text_inner( .get(¤t_font) .map(|s| s.as_str()) .unwrap_or(¤t_font); + let (desc_italic, desc_bold) = font_style_flags + .get(¤t_font) + .copied() + .unwrap_or((false, false)); let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]; for (text, start_w, end_w) in &sub_items { let offset_tm = [ @@ -586,9 +607,10 @@ fn extract_form_xobject_text_inner( font: current_font.clone(), font_size: rendered_size, page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), + is_bold: is_bold_font(base_font) || desc_bold, + is_italic: is_italic_font(base_font) || desc_italic, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }); diff --git a/src/lib.rs b/src/lib.rs index 19fb321..8e93fba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -579,6 +579,7 @@ pub fn extract_text_in_regions_mem( let mut gid_pages: HashSet = HashSet::new(); let mut page_thresholds: HashMap = HashMap::new(); let mut rotated_pages: HashSet = HashSet::new(); + let mut style_cache = extractor::FontStyleCache::new(); for (page_num, &page_id) in pages.iter() { if !needed_pages.contains(page_num) { @@ -597,6 +598,7 @@ pub fn extract_text_in_regions_mem( *page_num, &font_cmaps, false, + &mut style_cache, )?; let threshold = text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { @@ -699,6 +701,7 @@ pub fn extract_tables_in_regions_mem( let mut gid_pages: HashSet = HashSet::new(); let mut page_thresholds: HashMap = HashMap::new(); let mut rotated_pages: HashSet = HashSet::new(); + let mut style_cache = extractor::FontStyleCache::new(); for (page_num, &page_id) in pages.iter() { if !needed_pages.contains(page_num) { @@ -714,6 +717,7 @@ pub fn extract_tables_in_regions_mem( *page_num, &font_cmaps, false, + &mut style_cache, )?; let threshold = text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { @@ -1019,6 +1023,7 @@ pub fn detect_vector_grid_in_region_mem( page_1idx, &font_cmaps, false, + &mut extractor::FontStyleCache::new(), )?; text_utils::fix_letterspaced_items(&mut items); @@ -1205,8 +1210,15 @@ mod vector_grid_tests { let &page_id = pages.get(&1).unwrap(); let needed: HashSet = HashSet::from([1]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated) = - extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap(); + let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items( + &doc, + page_id, + 1, + &cmaps, + false, + &mut crate::extractor::FontStyleCache::new(), + ) + .unwrap(); let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1); assert_eq!(rect_tables.len(), 1, "expected one rect-detected table"); @@ -1240,8 +1252,15 @@ mod vector_grid_tests { let &page_id = pages.get(&page_num).unwrap(); let needed: HashSet = HashSet::from([page_num]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated) = - extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap(); + let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items( + &doc, + page_id, + page_num, + &cmaps, + false, + &mut crate::extractor::FontStyleCache::new(), + ) + .unwrap(); let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num); rect_tables @@ -1966,6 +1985,7 @@ pub fn extract_tables_with_structure_cells_mem( let mut page_heights: HashMap = HashMap::new(); let mut page_thresholds: HashMap = HashMap::new(); let mut rotated_pages: HashSet = HashSet::new(); + let mut style_cache = extractor::FontStyleCache::new(); for (page_num, &page_id) in pages.iter() { if !needed_pages.contains(page_num) { @@ -1981,6 +2001,7 @@ pub fn extract_tables_with_structure_cells_mem( *page_num, &font_cmaps, false, + &mut style_cache, )?; let threshold = text_utils::fix_letterspaced_items(&mut items); if threshold > 0.10 { @@ -2782,6 +2803,7 @@ fn detect_tsr_quality_issue( page_1idx, &font_cmaps, false, + &mut extractor::FontStyleCache::new(), )?; let adaptive_threshold = text_utils::fix_letterspaced_items(&mut items); let coords = if coords_rotated { @@ -5056,6 +5078,7 @@ mod text_cluster_column_undercount_tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -5331,6 +5354,7 @@ mod table_candidate_selection_tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -6078,6 +6102,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index d538130..097ae58 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -1189,6 +1189,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: crate::types::ItemType::Text, mcid, } diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 76ab633..da1448e 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -1221,6 +1221,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: crate::types::ItemType::Text, mcid: None, } diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index fc89cee..ba13d89 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -543,6 +543,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid, } diff --git a/src/python.rs b/src/python.rs index cdd4d65..8269263 100644 --- a/src/python.rs +++ b/src/python.rs @@ -268,6 +268,8 @@ pub struct PyTextItem { #[pyo3(get)] pub is_underline: bool, #[pyo3(get)] + pub is_strikeout: bool, + #[pyo3(get)] pub item_type: String, } @@ -352,6 +354,7 @@ fn convert_text_items(items: Vec) -> Vec { is_bold: item.is_bold, is_italic: item.is_italic, is_underline: item.is_underline, + is_strikeout: item.is_strikeout, item_type: item_type_str(&item.item_type), }) .collect() diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 60eb0c7..b74ddc0 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -105,6 +105,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec Option> is_bold: item.is_bold, is_italic: item.is_italic, is_underline: item.is_underline, + is_strikeout: item.is_strikeout, item_type: item.item_type.clone(), mcid: item.mcid, }); diff --git a/src/tables/grid.rs b/src/tables/grid.rs index 2b5901b..66a5323 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -521,6 +521,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -886,6 +887,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, page: 1, @@ -923,6 +925,7 @@ mod tests { is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, page: 1, diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 270247f..2e3601a 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -236,6 +236,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec Vec Text is_bold: false, is_italic: false, is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, } @@ -131,6 +132,7 @@ fn make_text_item_with_font( is_bold: is_bold_font(font), is_italic: is_italic_font(font), is_underline: false, + is_strikeout: false, item_type: ItemType::Text, mcid: None, }