Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc6ebd8e70 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.10.0",
|
||||
"version": "1.9.11",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -83,9 +83,6 @@ pub struct TextItem {
|
||||
/// Underline detected geometrically (drawn rule/thin rect under the
|
||||
/// baseline) — PDFs carry no underline font flag.
|
||||
pub is_underline: bool,
|
||||
/// Strikeout detected geometrically (rule crossing the glyphs at mid
|
||||
/// x-height).
|
||||
pub is_strikeout: bool,
|
||||
pub item_type: ItemType,
|
||||
/// URL for link items, `None` for other types.
|
||||
pub link_url: Option<String>,
|
||||
@@ -297,7 +294,6 @@ pub fn extract_text_with_positions(
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type,
|
||||
link_url,
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ class TextItem:
|
||||
is_bold: bool
|
||||
is_italic: bool
|
||||
is_underline: bool
|
||||
is_strikeout: bool
|
||||
item_type: str
|
||||
|
||||
class RegionText:
|
||||
|
||||
+1
-3
@@ -74,7 +74,7 @@ fn format_items_json(items: &[TextItem]) -> String {
|
||||
_ => String::new(),
|
||||
};
|
||||
format!(
|
||||
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"is_strikeout":{},"item_type":"{}","mcid":{}{}}}"#,
|
||||
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
|
||||
json_escape(&item.text),
|
||||
item.page,
|
||||
item.x,
|
||||
@@ -86,7 +86,6 @@ fn format_items_json(items: &[TextItem]) -> String {
|
||||
item.is_bold,
|
||||
item.is_italic,
|
||||
item.is_underline,
|
||||
item.is_strikeout,
|
||||
item_type_label(&item.item_type),
|
||||
mcid,
|
||||
link_url,
|
||||
@@ -123,7 +122,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: true,
|
||||
is_underline: true,
|
||||
is_strikeout: true,
|
||||
item_type: ItemType::Text,
|
||||
mcid: Some(7),
|
||||
}];
|
||||
|
||||
+20
-250
@@ -14,9 +14,8 @@ use lopdf::{Document, Encoding, Object, ObjectId};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, descriptor_style_flags,
|
||||
extract_text_from_operand, get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
FontStyleCache,
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::underline::UnderlineLine;
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
@@ -107,25 +106,6 @@ fn transformed_stroke_width(
|
||||
user_width * (ndx * ndx + ndy * ndy).sqrt()
|
||||
}
|
||||
|
||||
/// Text rise (Ts) displaces the glyph origin by (0, rise) in unscaled text
|
||||
/// space — per the rendering-matrix definition it sits left of Tm, so the
|
||||
/// offset maps through the text matrix's y column. Rise never contributes
|
||||
/// to the advance, so callers apply it only to the rendering position and
|
||||
/// keep advancing the unshifted text matrix.
|
||||
fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] {
|
||||
if rise == 0.0 {
|
||||
return *tm;
|
||||
}
|
||||
[
|
||||
tm[0],
|
||||
tm[1],
|
||||
tm[2],
|
||||
tm[3],
|
||||
tm[4] + rise * tm[2],
|
||||
tm[5] + rise * tm[3],
|
||||
]
|
||||
}
|
||||
|
||||
/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
|
||||
/// the page uses fonts with unresolvable gid-encoded glyphs.
|
||||
pub(crate) fn extract_page_text_items(
|
||||
@@ -134,7 +114,6 @@ pub(crate) fn extract_page_text_items(
|
||||
page_num: u32,
|
||||
font_cmaps: &FontCMaps,
|
||||
include_invisible: bool,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> Result<(PageExtraction, bool, bool), PdfError> {
|
||||
use lopdf::content::Content;
|
||||
|
||||
@@ -174,8 +153,6 @@ pub(crate) fn extract_page_text_items(
|
||||
std::collections::HashMap::new();
|
||||
let mut inline_cmaps: std::collections::HashMap<String, crate::tounicode::CMapEntry> =
|
||||
std::collections::HashMap::new();
|
||||
let mut font_style_flags: std::collections::HashMap<String, (bool, bool)> =
|
||||
std::collections::HashMap::new();
|
||||
for (font_name, font_dict) in &fonts {
|
||||
let resource_name = String::from_utf8_lossy(font_name).to_string();
|
||||
if let Ok(base_font) = font_dict.get(b"BaseFont") {
|
||||
@@ -184,12 +161,6 @@ pub(crate) fn extract_page_text_items(
|
||||
font_base_names.insert(resource_name.clone(), base_name);
|
||||
}
|
||||
}
|
||||
// Descriptor style flags rescue subset fonts whose BaseFont names
|
||||
// are opaque tags the name heuristics can't read.
|
||||
let style = descriptor_style_flags(doc, font_dict, style_cache);
|
||||
if style != (false, false) {
|
||||
font_style_flags.insert(resource_name.clone(), style);
|
||||
}
|
||||
// Track ToUnicode object reference, with FontFile2 fallback for Identity-H/V.
|
||||
// Also handle inline ToUnicode streams.
|
||||
match font_dict.get(b"ToUnicode") {
|
||||
@@ -264,7 +235,6 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width: f32,
|
||||
char_spacing: f32,
|
||||
word_spacing: f32,
|
||||
text_rise: f32,
|
||||
text_leading: f32,
|
||||
current_font: String,
|
||||
current_font_size: f32,
|
||||
@@ -277,7 +247,6 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut text_leading: f32 = 0.0; // TL parameter (in text-space units)
|
||||
let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled)
|
||||
let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled)
|
||||
let mut text_rise: f32 = 0.0; // Ts parameter (baseline shift for super/subscripts, unscaled)
|
||||
let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
|
||||
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
|
||||
let mut in_text_block = false;
|
||||
@@ -299,10 +268,6 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut suppress_glyph_extraction = false;
|
||||
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
|
||||
let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
|
||||
// Text rise in effect at each captured matrix — the item must render at
|
||||
// the rise of its GLYPHS, not whatever rise is set by EMC time.
|
||||
let mut actual_text_start_rise: f32 = 0.0;
|
||||
let mut actual_text_glyph_rise: Option<f32> = None;
|
||||
/// Get the innermost MCID from the marked content stack.
|
||||
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
|
||||
stack.iter().rev().find_map(|e| e.mcid)
|
||||
@@ -319,7 +284,6 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
text_rise,
|
||||
text_leading,
|
||||
current_font: current_font.clone(),
|
||||
current_font_size,
|
||||
@@ -333,7 +297,6 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width = saved.line_width;
|
||||
char_spacing = saved.char_spacing;
|
||||
word_spacing = saved.word_spacing;
|
||||
text_rise = saved.text_rise;
|
||||
text_leading = saved.text_leading;
|
||||
current_font = saved.current_font;
|
||||
current_font_size = saved.current_font_size;
|
||||
@@ -406,12 +369,6 @@ pub(crate) fn extract_page_text_items(
|
||||
word_spacing = tw;
|
||||
}
|
||||
}
|
||||
"Ts" => {
|
||||
// Set text rise (baseline shift for superscripts/subscripts)
|
||||
if let Some(ts) = op.operands.first().and_then(get_number) {
|
||||
text_rise = ts;
|
||||
}
|
||||
}
|
||||
"Td" | "TD" => {
|
||||
// Move text position: TLM = T(tx,ty) × TLM; Tm = TLM
|
||||
// tx,ty are in text space — must be scaled by the text line matrix
|
||||
@@ -470,7 +427,6 @@ pub(crate) fn extract_page_text_items(
|
||||
if suppress_glyph_extraction {
|
||||
if actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
@@ -500,8 +456,7 @@ pub(crate) fn extract_page_text_items(
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
@@ -523,10 +478,6 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
@@ -536,10 +487,9 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -557,7 +507,6 @@ pub(crate) fn extract_page_text_items(
|
||||
// Capture first-glyph position for ActualText
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
|
||||
// Compute space threshold based on font metrics when available
|
||||
@@ -678,10 +627,6 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
|
||||
for (text, start_w, end_w) in &sub_items {
|
||||
let offset_tm = [
|
||||
@@ -692,8 +637,7 @@ pub(crate) fn extract_page_text_items(
|
||||
text_matrix[4] + start_w * text_matrix[0],
|
||||
text_matrix[5] + start_w * text_matrix[1],
|
||||
];
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&offset_tm, text_rise), &ctm);
|
||||
let combined = multiply_matrices(&offset_tm, &ctm);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
let width = if font_info.is_some() {
|
||||
((end_w - start_w) * scale_x).abs()
|
||||
@@ -709,10 +653,9 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -736,26 +679,6 @@ pub(crate) fn extract_page_text_items(
|
||||
line_matrix[4] += (-tl) * line_matrix[2];
|
||||
line_matrix[5] += (-tl) * line_matrix[3];
|
||||
text_matrix = line_matrix;
|
||||
// Capture first-glyph position for ActualText AFTER the
|
||||
// line move — the BDC-entry matrix is on the previous line.
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
// Advance width, as for Tj — without it the item stays
|
||||
// zero-width and geometric underline/strikeout detection
|
||||
// rejects it (`is_underline_candidate` needs width > 0).
|
||||
let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| {
|
||||
op.operands.first().and_then(get_operand_bytes).map(|raw| {
|
||||
compute_string_width_ts(
|
||||
raw,
|
||||
fi,
|
||||
current_font_size,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
)
|
||||
})
|
||||
});
|
||||
if !((text_rendering_mode == 3 && !include_invisible)
|
||||
|| suppress_glyph_extraction
|
||||
|| op.operands.is_empty())
|
||||
@@ -773,8 +696,7 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_widths,
|
||||
) {
|
||||
if !text.trim().is_empty() {
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
} else {
|
||||
@@ -782,45 +704,28 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
let width = w_ts_opt
|
||||
.map(|w_ts| {
|
||||
(w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]))
|
||||
.abs()
|
||||
})
|
||||
.unwrap_or(0.0);
|
||||
let base_font = font_base_names
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
width: 0.0,
|
||||
height: rendered_size,
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Advance regardless of visibility so later show-text
|
||||
// operators on the same line stay positioned (as for Tj).
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
text_matrix[5] += w_ts * text_matrix[1];
|
||||
}
|
||||
}
|
||||
"Do" => {
|
||||
// XObject invocation - could be an image or form
|
||||
@@ -852,7 +757,6 @@ pub(crate) fn extract_page_text_items(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -866,7 +770,6 @@ pub(crate) fn extract_page_text_items(
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
&mut cmap_decisions,
|
||||
style_cache,
|
||||
);
|
||||
items.extend(form_items);
|
||||
}
|
||||
@@ -907,9 +810,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if actual_text.is_some() {
|
||||
suppress_glyph_extraction = true;
|
||||
actual_text_start_tm = Some(text_matrix);
|
||||
actual_text_start_rise = text_rise;
|
||||
actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
|
||||
actual_text_glyph_rise = None;
|
||||
}
|
||||
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
|
||||
}
|
||||
@@ -922,11 +823,9 @@ pub(crate) fn extract_page_text_items(
|
||||
// Tj may have moved the text position to the correct line —
|
||||
// the BDC-entry position can be on the previous line.
|
||||
let glyph_tm = actual_text_glyph_tm.take();
|
||||
let glyph_rise = actual_text_glyph_rise.take();
|
||||
let entry_tm = actual_text_start_tm.take();
|
||||
if let Some(start_tm) = glyph_tm.or(entry_tm) {
|
||||
let rise = glyph_rise.unwrap_or(actual_text_start_rise);
|
||||
let combined = multiply_matrices(&rise_adjusted(&start_tm, rise), &ctm);
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
} else {
|
||||
@@ -943,10 +842,6 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&at),
|
||||
x,
|
||||
@@ -956,10 +851,9 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: entry
|
||||
.mcid
|
||||
@@ -1482,15 +1376,8 @@ mod tests {
|
||||
|
||||
let (doc, page_id) = simple_doc_with_content(content);
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
items
|
||||
}
|
||||
|
||||
@@ -1575,108 +1462,6 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
assert!(!world.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_operator_text_carries_advance_width() {
|
||||
// `'` (move-to-next-line-and-show-text) must retain the string's
|
||||
// advance width like Tj — zero-width items are invisible to
|
||||
// geometric underline/strikeout detection.
|
||||
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (first) Tj (struck) ' ET
|
||||
1 w
|
||||
99 503 m 145 503 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let struck = items.iter().find(|item| item.text == "struck").unwrap();
|
||||
|
||||
// 6 glyphs x 600/1000 x 12pt = 43.2pt, drawn one leading below Tm.
|
||||
assert!((struck.width - 43.2).abs() < 0.1);
|
||||
assert!((struck.y - 500.0).abs() < 0.1);
|
||||
assert!(struck.is_strikeout);
|
||||
assert!(!struck.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_operator_advances_text_matrix() {
|
||||
// Text shown after `'` on the same line must start past the shown
|
||||
// string: "CD" lands at x=114.4 (2 glyphs x 600/1000 x 12pt after
|
||||
// x=100), flush against "AB", so the merge pass joins them. Without
|
||||
// the advance "CD" overlaps "AB" at x=100 and the items stay apart.
|
||||
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (AB) ' (CD) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let merged = items.iter().find(|item| item.text == "ABCD").unwrap();
|
||||
|
||||
assert!((merged.x - 100.0).abs() < 0.1);
|
||||
assert!((merged.width - 28.8).abs() < 0.1);
|
||||
assert!((merged.y - 500.0).abs() < 0.1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_rise_shifts_item_baseline() {
|
||||
// Ts displaces the glyph origin vertically without touching the
|
||||
// advance; the next run at rise 0 must return to the original
|
||||
// baseline and follow the raised run horizontally.
|
||||
let content =
|
||||
b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (base) Tj 5 Ts (super) Tj 0 Ts (after) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let base = items.iter().find(|item| item.text == "base").unwrap();
|
||||
let raised = items.iter().find(|item| item.text == "super").unwrap();
|
||||
let after = items.iter().find(|item| item.text == "after").unwrap();
|
||||
|
||||
assert!((base.y - 500.0).abs() < 0.1);
|
||||
assert!((raised.y - 505.0).abs() < 0.1);
|
||||
assert!((after.y - 500.0).abs() < 0.1);
|
||||
assert!(after.x > raised.x);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_text_item_uses_glyph_rise() {
|
||||
// The ActualText replacement item must render at the rise in
|
||||
// effect when its glyphs were drawn — not the unshifted BDC
|
||||
// baseline, and not whatever rise is set by EMC time.
|
||||
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm \
|
||||
/Span <</ActualText (super) >> BDC 5 Ts (sup) Tj 0 Ts EMC (after) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let sup = items.iter().find(|item| item.text == "super").unwrap();
|
||||
let after = items.iter().find(|item| item.text == "after").unwrap();
|
||||
|
||||
assert!((sup.y - 505.0).abs() < 0.1);
|
||||
assert!((after.y - 500.0).abs() < 0.1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_text_shown_with_quote_op_uses_moved_risen_baseline() {
|
||||
// When the tagged span's show op is `'`, the glyph position is
|
||||
// only known AFTER its line move — falling back to the BDC-entry
|
||||
// matrix would place the item on the previous line, unrisen.
|
||||
let content = b"BT /F1 12 Tf 14 TL 1 0 0 1 100 500 Tm \
|
||||
/Span <</ActualText (replaced) >> BDC 3 Ts (raw) ' 0 Ts EMC ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let item = items.iter().find(|item| item.text == "replaced").unwrap();
|
||||
|
||||
// Line move: 500 - 14 = 486; rise: +3 -> 489.
|
||||
assert!((item.y - 489.0).abs() < 0.1);
|
||||
assert!(item.width > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strikeout_detected_on_risen_text() {
|
||||
// The rule crosses the glyphs at their risen position; without the
|
||||
// rise in item.y the strike window sits 4pt too low and misses.
|
||||
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm 4 Ts (struck) Tj ET
|
||||
1 w
|
||||
99 507 m 145 507 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let struck = items.iter().find(|item| item.text == "struck").unwrap();
|
||||
|
||||
assert!((struck.y - 504.0).abs() < 0.1);
|
||||
assert!(struck.is_strikeout);
|
||||
assert!(!struck.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skip_excessive_operations() {
|
||||
use crate::tounicode::FontCMaps;
|
||||
@@ -1711,15 +1496,7 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
doc.add_object(catalog);
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let result = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let ((items, rects, lines), _has_gid, _coords_rotated) = result;
|
||||
assert!(items.is_empty());
|
||||
assert!(rects.is_empty());
|
||||
@@ -1801,15 +1578,8 @@ BT 30 700 Tm <41> Tj ET";
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let text = items
|
||||
.iter()
|
||||
.map(|item| item.text.as_str())
|
||||
|
||||
+1
-332
@@ -4,7 +4,7 @@ use crate::glyph_names::glyph_to_char;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{FontEncodingMap, FontWidthInfo, PageFontEncodings, PageFontWidths};
|
||||
use log::debug;
|
||||
use lopdf::{Document, Encoding, Object, ObjectId};
|
||||
use lopdf::{Document, Encoding, Object};
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
@@ -739,171 +739,6 @@ pub(crate) fn get_font_file2_obj_num(doc: &Document, font_dict: &lopdf::Dictiona
|
||||
.map(|r| r.0)
|
||||
}
|
||||
|
||||
/// Document-scoped memo of embedded-font style flags, keyed by the
|
||||
/// FontFile2/FontFile3 stream's object id. The same font program is
|
||||
/// referenced from every page that uses the font, and decompressing +
|
||||
/// parsing it dominates `descriptor_style_flags` — without the memo that
|
||||
/// cost repeats per page whenever the descriptor leaves a flag unset
|
||||
/// (the common case: regular fonts report neither italic nor bold).
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct FontStyleCache {
|
||||
by_font_file: HashMap<ObjectId, (bool, bool)>,
|
||||
}
|
||||
|
||||
impl FontStyleCache {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Style flags from the FontDescriptor, which survive subset fonts whose
|
||||
/// BaseFont names are opaque tags ("Tc1", "ABCDEF+F1") that defeat the
|
||||
/// name-based bold/italic heuristics.
|
||||
///
|
||||
/// Italic: `ItalicAngle` beyond a few degrees, or Flags bit 7 (Italic,
|
||||
/// value 64). Bold: Flags bit 19 (ForceBold, value 1<<18). The small
|
||||
/// ItalicAngle threshold skips fonts that declare a token slant.
|
||||
pub(crate) fn descriptor_style_flags(
|
||||
doc: &Document,
|
||||
font_dict: &lopdf::Dictionary,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> (bool, bool) {
|
||||
let descriptor = font_dict
|
||||
.get(b"FontDescriptor")
|
||||
.ok()
|
||||
.and_then(|obj| resolve_dict(doc, obj))
|
||||
.or_else(|| {
|
||||
// Type0 fonts hang the descriptor off DescendantFonts[0].
|
||||
let desc_fonts = font_dict.get(b"DescendantFonts").ok()?;
|
||||
let desc_fonts = resolve_array(doc, desc_fonts)?;
|
||||
let cid_font_dict = resolve_dict(doc, desc_fonts.first()?)?;
|
||||
resolve_dict(doc, cid_font_dict.get(b"FontDescriptor").ok()?)
|
||||
});
|
||||
let Some(descriptor) = descriptor else {
|
||||
return (false, false);
|
||||
};
|
||||
|
||||
let italic_angle = descriptor
|
||||
.get(b"ItalicAngle")
|
||||
.ok()
|
||||
.and_then(|obj| match obj {
|
||||
Object::Integer(i) => Some(*i as f32),
|
||||
Object::Real(r) => Some(*r),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(0.0);
|
||||
let flags = descriptor
|
||||
.get(b"Flags")
|
||||
.ok()
|
||||
.and_then(|obj| obj.as_i64().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let mut italic = italic_angle.abs() >= 4.0 || flags & (1 << 6) != 0;
|
||||
let mut bold = flags & (1 << 18) != 0;
|
||||
|
||||
// Descriptors lie: subset generators write ItalicAngle 0 for genuinely
|
||||
// italic faces. The embedded font file keeps the truth — OS/2
|
||||
// fsSelection (via `Face::is_italic`) and the post table's italicAngle.
|
||||
if !italic || !bold {
|
||||
if let Some(ff_ref) = font_file_ref(descriptor) {
|
||||
let (emb_italic, emb_bold) = *style_cache
|
||||
.by_font_file
|
||||
.entry(ff_ref)
|
||||
.or_insert_with(|| embedded_style_flags(doc, ff_ref));
|
||||
italic = italic || emb_italic;
|
||||
bold = bold || emb_bold;
|
||||
}
|
||||
}
|
||||
(italic, bold)
|
||||
}
|
||||
|
||||
/// Style flags parsed from an embedded font program stream.
|
||||
fn embedded_style_flags(doc: &Document, ff_ref: ObjectId) -> (bool, bool) {
|
||||
let Some(data) = font_file_data(doc, ff_ref) else {
|
||||
return (false, false);
|
||||
};
|
||||
if let Ok(face) = ttf_parser::Face::parse(&data, 0) {
|
||||
(
|
||||
face.is_italic() || face.italic_angle().abs() >= 4.0,
|
||||
face.is_bold(),
|
||||
)
|
||||
} else if let Some(name) = cff_font_name(&data) {
|
||||
// FontFile3 is bare CFF (no sfnt container) — ttf_parser
|
||||
// can't open it, but the CFF Name INDEX keeps the real
|
||||
// PostScript name ("XXXXXX+Amplitude-LightItalic") even
|
||||
// when the descriptor was rewritten to claim upright.
|
||||
(
|
||||
crate::text_utils::is_italic_font(&name),
|
||||
crate::text_utils::is_bold_font(&name),
|
||||
)
|
||||
} else {
|
||||
(false, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// First PostScript name from a bare CFF font's Name INDEX (CFF spec §7).
|
||||
fn cff_font_name(data: &[u8]) -> Option<String> {
|
||||
// Header: major(1) minor(1) hdrSize(1) offSize(1); major must be 1.
|
||||
if data.len() < 4 || data[0] != 1 {
|
||||
return None;
|
||||
}
|
||||
let hdr_size = data[2] as usize;
|
||||
// Name INDEX: count(u16) offSize(u8) offsets[count+1] data
|
||||
let count = u16::from_be_bytes([*data.get(hdr_size)?, *data.get(hdr_size + 1)?]) as usize;
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
let off_size = *data.get(hdr_size + 2)? as usize;
|
||||
if !(1..=4).contains(&off_size) {
|
||||
return None;
|
||||
}
|
||||
let read_offset = |idx: usize| -> Option<usize> {
|
||||
let at = hdr_size + 3 + idx * off_size;
|
||||
let bytes = data.get(at..at + off_size)?;
|
||||
let mut v = 0usize;
|
||||
for b in bytes {
|
||||
v = (v << 8) | *b as usize;
|
||||
}
|
||||
Some(v)
|
||||
};
|
||||
let start = read_offset(0)?;
|
||||
let end = read_offset(1)?;
|
||||
if start == 0 || end < start {
|
||||
return None;
|
||||
}
|
||||
// Offsets are 1-based from the byte before the object data.
|
||||
let objects_base = hdr_size + 3 + (count + 1) * off_size - 1;
|
||||
let name = data.get(objects_base + start..objects_base + end)?;
|
||||
Some(String::from_utf8_lossy(name).to_string())
|
||||
}
|
||||
|
||||
/// FontFile2/FontFile3 stream reference from a FontDescriptor.
|
||||
fn font_file_ref(descriptor: &lopdf::Dictionary) -> Option<ObjectId> {
|
||||
descriptor
|
||||
.get(b"FontFile2")
|
||||
.ok()
|
||||
.and_then(|o| o.as_reference().ok())
|
||||
.or_else(|| {
|
||||
descriptor
|
||||
.get(b"FontFile3")
|
||||
.ok()
|
||||
.and_then(|o| o.as_reference().ok())
|
||||
})
|
||||
}
|
||||
|
||||
/// Decompressed embedded font program bytes.
|
||||
fn font_file_data(doc: &Document, ff_ref: ObjectId) -> Option<Vec<u8>> {
|
||||
let stream = doc
|
||||
.get_object(ff_ref)
|
||||
.and_then(lopdf::Object::as_stream)
|
||||
.ok()?;
|
||||
Some(
|
||||
stream
|
||||
.decompressed_content()
|
||||
.unwrap_or_else(|_| stream.content.clone()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Decode text from a PDF string operand using font CMaps, encodings, and fallbacks.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn extract_text_from_operand(
|
||||
@@ -1427,172 +1262,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn doc_with_descriptor(descriptor: lopdf::Dictionary) -> (Document, lopdf::Dictionary) {
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let desc_id = doc.add_object(descriptor);
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "TrueType",
|
||||
"BaseFont" => "Tc1",
|
||||
"FontDescriptor" => desc_id,
|
||||
};
|
||||
(doc, font_dict)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_italic_angle_sets_italic() {
|
||||
// Subset font with an opaque BaseFont name ("Tc1") — the name
|
||||
// heuristic sees nothing, the descriptor carries the truth.
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => -12,
|
||||
"Flags" => 32,
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_italic_flag_bit_sets_italic() {
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 64, // bit 7: Italic
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_force_bold_flag_sets_bold() {
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 1 << 18, // ForceBold
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_italic_angle_is_not_italic() {
|
||||
// A token 1-degree slant is optical correction, not italic.
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => lopdf::Object::Real(-1.0),
|
||||
"Flags" => 32,
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_descriptor_yields_no_flags() {
|
||||
let doc = Document::with_version("1.4");
|
||||
let font_dict = dictionary! { "Type" => "Font", "BaseFont" => "Tc1" };
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type0_descendant_descriptor_is_resolved() {
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let desc_id = doc.add_object(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "ABCDEF+F1",
|
||||
"ItalicAngle" => -15,
|
||||
});
|
||||
let cid_id = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "CIDFontType2",
|
||||
"FontDescriptor" => desc_id,
|
||||
});
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type0",
|
||||
"BaseFont" => "ABCDEF+F1",
|
||||
"DescendantFonts" => vec![lopdf::Object::Reference(cid_id)],
|
||||
};
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
/// Bare CFF: header + Name INDEX only — enough for `cff_font_name`.
|
||||
fn bare_cff_with_name(name: &str) -> Vec<u8> {
|
||||
let mut data = vec![1, 0, 4, 1]; // major, minor, hdrSize, offSize
|
||||
data.extend_from_slice(&1u16.to_be_bytes()); // Name INDEX count
|
||||
data.push(1); // offSize
|
||||
data.push(1); // offset of first name
|
||||
data.push(1 + name.len() as u8); // offset past last name
|
||||
data.extend_from_slice(name.as_bytes());
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_font_style_is_cached_by_font_file_object() {
|
||||
use lopdf::{Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let ff_id = doc.add_object(Object::Stream(Stream::new(
|
||||
dictionary! {},
|
||||
bare_cff_with_name("ABCDEF+Test-BoldItalic"),
|
||||
)));
|
||||
let desc_id = doc.add_object(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "ABCDEF+Test-BoldItalic",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 32,
|
||||
"FontFile3" => ff_id,
|
||||
});
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Tc1",
|
||||
"FontDescriptor" => desc_id,
|
||||
};
|
||||
|
||||
let mut cache = FontStyleCache::new();
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut cache),
|
||||
(true, true)
|
||||
);
|
||||
assert_eq!(cache.by_font_file.len(), 1);
|
||||
|
||||
// Replace the font program with garbage: a repeat call must serve
|
||||
// the memo instead of re-reading the stream — repeated per-page
|
||||
// decompression is exactly what the cache exists to avoid.
|
||||
doc.objects.insert(
|
||||
ff_id,
|
||||
Object::Stream(Stream::new(dictionary! {}, vec![0u8; 4])),
|
||||
);
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut cache),
|
||||
(true, true)
|
||||
);
|
||||
// A cold cache parses the (now garbage) stream, proving the warm
|
||||
// call above answered from the memo.
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_string_width_ts_no_tc_tw() {
|
||||
// Without Tc/Tw (both 0), width = glyph widths only
|
||||
|
||||
@@ -1495,7 +1495,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1626,7 +1625,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -79,7 +79,6 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Link(url),
|
||||
mcid: None,
|
||||
});
|
||||
@@ -319,7 +318,6 @@ pub(crate) fn walk_form_fields(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::FormField,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+4
-71
@@ -24,7 +24,6 @@ use links::{extract_form_fields, extract_page_links};
|
||||
// Re-export public types so existing `crate::extractor::X` paths keep working.
|
||||
pub use crate::text_utils::{is_bold_font, is_italic_font};
|
||||
pub use crate::types::{ItemType, TextLine};
|
||||
pub(crate) use fonts::FontStyleCache;
|
||||
pub(crate) use layout::detect_columns;
|
||||
pub use layout::group_into_lines;
|
||||
pub(crate) use layout::group_into_lines_with_thresholds;
|
||||
@@ -162,9 +161,6 @@ fn extract_positioned_text_impl(
|
||||
let mut all_lines = Vec::new();
|
||||
let mut page_thresholds: PageThresholds = HashMap::new();
|
||||
let mut gid_encoded_pages: HashSet<u32> = HashSet::new();
|
||||
// Embedded-font style flags are document-scoped: the same font program
|
||||
// is shared across pages, so parse it once, not once per page.
|
||||
let mut style_cache = FontStyleCache::new();
|
||||
|
||||
// Build page ObjectId → page number map for form field extraction
|
||||
let page_id_to_num: HashMap<ObjectId, u32> =
|
||||
@@ -176,14 +172,8 @@ fn extract_positioned_text_impl(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) =
|
||||
extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?;
|
||||
if has_gid_fonts {
|
||||
gid_encoded_pages.insert(*page_num);
|
||||
}
|
||||
@@ -244,10 +234,7 @@ fn suppress_table_underlines(
|
||||
lines: &[PdfLine],
|
||||
page: u32,
|
||||
) {
|
||||
if !items
|
||||
.iter()
|
||||
.any(|item| item.is_underline || item.is_strikeout)
|
||||
{
|
||||
if !items.iter().any(|item| item.is_underline) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -269,7 +256,6 @@ fn suppress_table_underlines(
|
||||
for index in table_item_indices {
|
||||
if let Some(item) = items.get_mut(index) {
|
||||
item.is_underline = false;
|
||||
item.is_strikeout = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -590,7 +576,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if next.is_bold != first.is_bold
|
||||
|| next.is_italic != first.is_italic
|
||||
|| next.is_underline != first.is_underline
|
||||
|| next.is_strikeout != first.is_strikeout
|
||||
{
|
||||
break;
|
||||
}
|
||||
@@ -653,7 +638,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
is_bold: first.is_bold,
|
||||
is_italic: first.is_italic,
|
||||
is_underline: first.is_underline,
|
||||
is_strikeout: first.is_strikeout,
|
||||
item_type: first.item_type.clone(),
|
||||
mcid: first.mcid,
|
||||
});
|
||||
@@ -731,9 +715,7 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
.chars()
|
||||
.last()
|
||||
.is_some_and(|c| c.is_alphabetic());
|
||||
let same_marks = parent.is_underline == item.is_underline
|
||||
&& parent.is_strikeout == item.is_strikeout;
|
||||
if parent.font_size >= sub_threshold && ends_with_letter && same_marks {
|
||||
if parent.font_size >= sub_threshold && ends_with_letter {
|
||||
let parent_right = parent.x + parent.width;
|
||||
let gap = item.x - parent_right;
|
||||
// Subscripts must be tightly adjacent (within ~1pt)
|
||||
@@ -807,7 +789,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1010,7 +991,6 @@ mod tests {
|
||||
items[3].y = 470.0;
|
||||
for item in &mut items {
|
||||
item.is_underline = true;
|
||||
item.is_strikeout = true;
|
||||
}
|
||||
let lines = vec![
|
||||
make_line(100.0, 500.0, 300.0, 500.0),
|
||||
@@ -1024,30 +1004,6 @@ mod tests {
|
||||
suppress_table_underlines(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
assert!(items.iter().all(|item| !item.is_strikeout));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subscript_digit_with_different_marks_is_not_absorbed() {
|
||||
// A struck-out word followed by an unmarked footnote digit: merging
|
||||
// would widen the parent's strikeout claim over the digit (and the
|
||||
// reverse would drop the digit's own mark). Style boundaries break
|
||||
// the merge, as in merge_text_items.
|
||||
let mut word = make_merge_item("word", 100.0, 24.0);
|
||||
word.font_size = 10.0;
|
||||
word.is_strikeout = true;
|
||||
let mut digit = make_merge_item("2", 124.5, 4.0);
|
||||
digit.font_size = 6.0;
|
||||
digit.y = word.y + 3.0;
|
||||
|
||||
let merged = merge_subscript_items(vec![word.clone(), digit.clone()]);
|
||||
assert_eq!(merged.len(), 2);
|
||||
|
||||
// Same marks still merge (footnote ref inside the strike).
|
||||
digit.is_strikeout = true;
|
||||
let merged = merge_subscript_items(vec![word, digit]);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert!(merged[0].text.starts_with("word"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1065,7 +1021,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1081,7 +1036,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1097,7 +1051,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1153,7 +1106,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1169,7 +1121,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1185,7 +1136,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1212,7 +1162,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1228,7 +1177,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1244,7 +1192,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1273,7 +1220,6 @@ mod tests {
|
||||
is_bold: true,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1309,7 +1255,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1346,7 +1291,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1362,7 +1306,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1378,7 +1321,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1402,7 +1344,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1516,7 +1457,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1532,7 +1472,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1558,7 +1497,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1574,7 +1512,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1616,7 +1553,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1662,7 +1598,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1708,7 +1643,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1747,7 +1681,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -231,27 +231,8 @@ fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
overlap >= min_overlap
|
||||
}
|
||||
|
||||
/// Strikeout window: a rule crossing the glyphs. Strikethroughs sit at
|
||||
/// roughly 20-35% of the em above the baseline (about half the x-height);
|
||||
/// accept a band well inside the glyph body so baseline underlines and
|
||||
/// overlines never qualify.
|
||||
fn rule_strikes_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
let y_min = item.y + item.font_size * 0.12;
|
||||
let y_max = item.y + item.font_size * 0.55;
|
||||
if rule.y < y_min || rule.y > y_max {
|
||||
return false;
|
||||
}
|
||||
|
||||
let ix1 = item.x;
|
||||
let ix2 = item.x + item.width;
|
||||
let min_overlap = item.width * MIN_X_OVERLAP;
|
||||
let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
|
||||
overlap >= min_overlap
|
||||
}
|
||||
|
||||
/// Mark `is_underline` on text items that have a horizontal rule just
|
||||
/// below their baseline, and `is_strikeout` on items whose glyphs a rule
|
||||
/// crosses at mid x-height. `items`, `rects`, and `lines` are a single
|
||||
/// below their baseline. `items`, `rects`, and `lines` are a single
|
||||
/// page's extraction output (all in PDF coordinates, y-up, where
|
||||
/// `TextItem::y` is the text baseline).
|
||||
pub(crate) fn mark_underlined_items(
|
||||
@@ -277,11 +258,6 @@ pub(crate) fn mark_underlined_items(
|
||||
}
|
||||
if rule_matches_item(rule, item) {
|
||||
item.is_underline = true;
|
||||
}
|
||||
if rule_strikes_item(rule, item) {
|
||||
item.is_strikeout = true;
|
||||
}
|
||||
if item.is_underline && item.is_strikeout {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -306,7 +282,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -383,44 +358,6 @@ mod tests {
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_glyph_rule_marks_strikeout_not_underline() {
|
||||
// Rule at ~30% of the em above the baseline crosses the glyphs.
|
||||
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 503.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_strikeout);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn baseline_rule_marks_underline_not_strikeout() {
|
||||
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 498.5)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_underline);
|
||||
assert!(!items[0].is_strikeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overline_is_neither_underline_nor_strikeout() {
|
||||
// Rule just above the cap height (overline / next line's rule).
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 507.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
assert!(!items[0].is_strikeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thin_filled_rect_at_mid_glyph_marks_strikeout() {
|
||||
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![thin_rect(100.0, 502.6, 60.0)];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(items[0].is_strikeout);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_above_baseline_is_not_an_underline() {
|
||||
// Strikethrough / overline geometry must not mark.
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
//! Form XObject and image XObject extraction.
|
||||
|
||||
use super::fonts::descriptor_style_flags;
|
||||
use crate::text_utils::{effective_font_size, expand_ligatures, is_bold_font, is_italic_font};
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{ItemType, TextItem};
|
||||
@@ -9,7 +8,7 @@ use std::collections::HashMap;
|
||||
|
||||
use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, FontStyleCache,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
|
||||
@@ -115,7 +114,6 @@ pub(crate) fn extract_form_xobject_text(
|
||||
font_cmaps: &FontCMaps,
|
||||
parent_ctm: &[f32; 6],
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> Vec<TextItem> {
|
||||
extract_form_xobject_text_inner(
|
||||
doc,
|
||||
@@ -124,12 +122,10 @@ pub(crate) fn extract_form_xobject_text(
|
||||
font_cmaps,
|
||||
parent_ctm,
|
||||
cmap_decisions,
|
||||
style_cache,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn extract_form_xobject_text_inner(
|
||||
doc: &Document,
|
||||
form_id: ObjectId,
|
||||
@@ -137,7 +133,6 @@ fn extract_form_xobject_text_inner(
|
||||
font_cmaps: &FontCMaps,
|
||||
parent_ctm: &[f32; 6],
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
style_cache: &mut FontStyleCache,
|
||||
depth: u8,
|
||||
) -> Vec<TextItem> {
|
||||
use lopdf::content::Content;
|
||||
@@ -172,7 +167,6 @@ fn extract_form_xobject_text_inner(
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
let mut inline_cmaps: HashMap<String, crate::tounicode::CMapEntry> = HashMap::new();
|
||||
|
||||
let mut font_style_flags: HashMap<String, (bool, bool)> = HashMap::new();
|
||||
for (font_name, font_dict) in &form_fonts {
|
||||
let resource_name = String::from_utf8_lossy(font_name).to_string();
|
||||
if let Ok(base_font) = font_dict.get(b"BaseFont") {
|
||||
@@ -181,10 +175,6 @@ fn extract_form_xobject_text_inner(
|
||||
font_base_names.insert(resource_name.clone(), base_name);
|
||||
}
|
||||
}
|
||||
let style = descriptor_style_flags(doc, font_dict, style_cache);
|
||||
if style != (false, false) {
|
||||
font_style_flags.insert(resource_name.clone(), style);
|
||||
}
|
||||
match font_dict.get(b"ToUnicode") {
|
||||
Ok(tounicode) => {
|
||||
if let Ok(obj_ref) = tounicode.as_reference() {
|
||||
@@ -282,7 +272,6 @@ fn extract_form_xobject_text_inner(
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
style_cache,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
@@ -306,7 +295,6 @@ fn extract_form_xobject_text_inner(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -440,10 +428,6 @@ fn extract_form_xobject_text_inner(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
@@ -453,10 +437,9 @@ fn extract_form_xobject_text_inner(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -577,10 +560,6 @@ fn extract_form_xobject_text_inner(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
|
||||
for (text, start_w, end_w) in &sub_items {
|
||||
let offset_tm = [
|
||||
@@ -607,10 +586,9 @@ fn extract_form_xobject_text_inner(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+4
-29
@@ -579,7 +579,6 @@ pub fn extract_text_in_regions_mem(
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -598,7 +597,6 @@ pub fn extract_text_in_regions_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -701,7 +699,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -717,7 +714,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -1023,7 +1019,6 @@ pub fn detect_vector_grid_in_region_mem(
|
||||
page_1idx,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut extractor::FontStyleCache::new(),
|
||||
)?;
|
||||
text_utils::fix_letterspaced_items(&mut items);
|
||||
|
||||
@@ -1210,15 +1205,8 @@ mod vector_grid_tests {
|
||||
let &page_id = pages.get(&1).unwrap();
|
||||
let needed: HashSet<u32> = HashSet::from([1]);
|
||||
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&cmaps,
|
||||
false,
|
||||
&mut crate::extractor::FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let ((items, rects, _lines), _has_gid, _rotated) =
|
||||
extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
|
||||
|
||||
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
|
||||
assert_eq!(rect_tables.len(), 1, "expected one rect-detected table");
|
||||
@@ -1252,15 +1240,8 @@ mod vector_grid_tests {
|
||||
let &page_id = pages.get(&page_num).unwrap();
|
||||
let needed: HashSet<u32> = HashSet::from([page_num]);
|
||||
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
page_num,
|
||||
&cmaps,
|
||||
false,
|
||||
&mut crate::extractor::FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let ((items, rects, _lines), _has_gid, _rotated) =
|
||||
extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap();
|
||||
|
||||
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num);
|
||||
rect_tables
|
||||
@@ -1985,7 +1966,6 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -2001,7 +1981,6 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -2803,7 +2782,6 @@ fn detect_tsr_quality_issue(
|
||||
page_1idx,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut extractor::FontStyleCache::new(),
|
||||
)?;
|
||||
let adaptive_threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
let coords = if coords_rotated {
|
||||
@@ -5078,7 +5056,6 @@ mod text_cluster_column_undercount_tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -5354,7 +5331,6 @@ mod table_candidate_selection_tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -6102,7 +6078,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -130,29 +130,6 @@ pub(crate) fn has_dot_leaders(text: &str) -> bool {
|
||||
dot_groups >= 2
|
||||
}
|
||||
|
||||
/// Detect a table-of-contents entry: a line ending in a page number preceded by
|
||||
/// a dot-leader group (e.g. "Measurement Lab worksheet ... 3"). `has_dot_leaders`
|
||||
/// misses single-group leaders ("..."), but a trailing "<dots> <number>" is a
|
||||
/// strong TOC signal on its own. Such lines must never be promoted to headings.
|
||||
pub(crate) fn is_toc_entry_line(text: &str) -> bool {
|
||||
let trimmed = text.trim_end();
|
||||
let digits = trimmed
|
||||
.chars()
|
||||
.rev()
|
||||
.take_while(|c| c.is_ascii_digit())
|
||||
.count();
|
||||
if digits == 0 || digits > 4 {
|
||||
return false;
|
||||
}
|
||||
let before_number = trimmed[..trimmed.len() - digits].trim_end();
|
||||
let dots = before_number
|
||||
.chars()
|
||||
.rev()
|
||||
.take_while(|c| *c == '.')
|
||||
.count();
|
||||
dots >= 3
|
||||
}
|
||||
|
||||
/// Compute the Y-gap threshold for paragraph break detection.
|
||||
///
|
||||
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
||||
@@ -343,28 +320,3 @@ pub(crate) fn detect_header_level(
|
||||
Some(4)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn toc_entry_with_single_dot_group() {
|
||||
assert!(is_toc_entry_line("Measurement Lab worksheet ... 3"));
|
||||
assert!(is_toc_entry_line("Results ........ 12"));
|
||||
assert!(is_toc_entry_line("Appendix B...42"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_toc_lines_pass() {
|
||||
assert!(!is_toc_entry_line(
|
||||
"6.2. Expectations for Re-Hiring Employees"
|
||||
));
|
||||
assert!(!is_toc_entry_line("What happened in 2020"));
|
||||
assert!(!is_toc_entry_line("IMPLEMENTATION"));
|
||||
// Ellipsis without a trailing page number
|
||||
assert!(!is_toc_entry_line("and so it goes ..."));
|
||||
// Long numbers are data, not page refs
|
||||
assert!(!is_toc_entry_line("ISBN ... 97814"));
|
||||
}
|
||||
}
|
||||
|
||||
+3
-12
@@ -7,7 +7,7 @@ use crate::types::TextLine;
|
||||
|
||||
use super::analysis::{
|
||||
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders, is_toc_entry_line,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders,
|
||||
};
|
||||
use super::classify::{
|
||||
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
||||
@@ -703,7 +703,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !starts_with_bullet_marker(plain_trimmed)
|
||||
&& !is_toc_entry_line(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
||||
@@ -739,11 +738,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// paragraph continuity and minor font-size variation
|
||||
// inflates rarity scores.
|
||||
let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
|
||||
// Single-word headings ("IMPLEMENTATION", "CONTENTS") are common;
|
||||
// accept them only with the strongest signal combination.
|
||||
let enough_words =
|
||||
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||
if score >= 0.5 && standalone && enough_words && has_strong_signal {
|
||||
if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
|
||||
Some(bold_heading_level(&heading_tiers))
|
||||
} else {
|
||||
None
|
||||
@@ -1042,7 +1037,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if options.detect_headers
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !is_toc_entry_line(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
if let Some(header_level) =
|
||||
@@ -1065,9 +1059,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
+ if all_bold { 0.3 } else { 0.0 }
|
||||
+ if standalone { 0.2 } else { 0.0 }
|
||||
+ if isolated { 0.3 } else { 0.0 };
|
||||
let enough_words =
|
||||
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||
if score >= 0.5 && standalone && enough_words {
|
||||
if score >= 0.5 && standalone && word_count >= 2 {
|
||||
return Some(bold_heading_level(&heading_tiers));
|
||||
}
|
||||
None
|
||||
@@ -1197,7 +1189,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -1221,7 +1221,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -543,7 +543,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -268,8 +268,6 @@ pub struct PyTextItem {
|
||||
#[pyo3(get)]
|
||||
pub is_underline: bool,
|
||||
#[pyo3(get)]
|
||||
pub is_strikeout: bool,
|
||||
#[pyo3(get)]
|
||||
pub item_type: String,
|
||||
}
|
||||
|
||||
@@ -354,7 +352,6 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item_type_str(&item.item_type),
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -105,7 +105,6 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
|
||||
is_bold: first_item.is_bold,
|
||||
is_italic: first_item.is_italic,
|
||||
is_underline: first_item.is_underline,
|
||||
is_strikeout: first_item.is_strikeout,
|
||||
item_type: first_item.item_type.clone(),
|
||||
mcid: first_item.mcid,
|
||||
});
|
||||
|
||||
@@ -394,7 +394,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -2392,7 +2392,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -3367,7 +3366,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -3678,7 +3676,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -587,7 +587,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -109,7 +109,6 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
|
||||
@@ -521,7 +521,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -887,7 +886,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
@@ -925,7 +923,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
|
||||
@@ -236,7 +236,6 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
@@ -258,7 +257,6 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
@@ -1434,7 +1432,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1453,7 +1450,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -884,7 +884,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1005,7 +1004,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -1083,7 +1081,6 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -120,10 +120,6 @@ pub struct TextItem {
|
||||
/// baseline — PDFs have no underline font flag, so this is detected
|
||||
/// geometrically after extraction; see `extractor::underline`).
|
||||
pub is_underline: bool,
|
||||
/// Whether the text is struck out (drawn rule/thin rect crossing the
|
||||
/// glyphs at mid x-height). Same geometric detection as underline,
|
||||
/// different vertical window; see `extractor::underline`.
|
||||
pub is_strikeout: bool,
|
||||
/// Type of item (text, image, link)
|
||||
pub item_type: ItemType,
|
||||
/// Marked Content ID from the content stream's BDC/BMC operator.
|
||||
|
||||
@@ -105,7 +105,6 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -132,7 +131,6 @@ fn make_text_item_with_font(
|
||||
is_bold: is_bold_font(font),
|
||||
is_italic: is_italic_font(font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user