Compare commits

..
Author SHA1 Message Date
Abimael MartellandClaude Fable 5 cc6ebd8e70 docs: add MIT LICENSE file and license badge
Cargo.toml and pyproject.toml already declare MIT but the repo had no
LICENSE file, so GitHub and package registries couldn't display it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 14:20:40 -07:00
27 changed files with 40 additions and 874 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.10.0",
"version": "1.9.11",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
-4
View File
@@ -83,9 +83,6 @@ pub struct TextItem {
/// Underline detected geometrically (drawn rule/thin rect under the
/// baseline) — PDFs carry no underline font flag.
pub is_underline: bool,
/// Strikeout detected geometrically (rule crossing the glyphs at mid
/// x-height).
pub is_strikeout: bool,
pub item_type: ItemType,
/// URL for link items, `None` for other types.
pub link_url: Option<String>,
@@ -297,7 +294,6 @@ pub fn extract_text_with_positions(
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
is_strikeout: item.is_strikeout,
item_type,
link_url,
}
-1
View File
@@ -39,7 +39,6 @@ class TextItem:
is_bold: bool
is_italic: bool
is_underline: bool
is_strikeout: bool
item_type: str
class RegionText:
+1 -3
View File
@@ -74,7 +74,7 @@ fn format_items_json(items: &[TextItem]) -> String {
_ => String::new(),
};
format!(
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"is_strikeout":{},"item_type":"{}","mcid":{}{}}}"#,
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
json_escape(&item.text),
item.page,
item.x,
@@ -86,7 +86,6 @@ fn format_items_json(items: &[TextItem]) -> String {
item.is_bold,
item.is_italic,
item.is_underline,
item.is_strikeout,
item_type_label(&item.item_type),
mcid,
link_url,
@@ -123,7 +122,6 @@ mod tests {
is_bold: false,
is_italic: true,
is_underline: true,
is_strikeout: true,
item_type: ItemType::Text,
mcid: Some(7),
}];
+20 -250
View File
@@ -14,9 +14,8 @@ use lopdf::{Document, Encoding, Object, ObjectId};
use std::collections::HashMap;
use super::fonts::{
build_font_encodings, build_font_widths, compute_string_width_ts, descriptor_style_flags,
extract_text_from_operand, get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
FontStyleCache,
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
};
use super::underline::UnderlineLine;
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
@@ -107,25 +106,6 @@ fn transformed_stroke_width(
user_width * (ndx * ndx + ndy * ndy).sqrt()
}
/// Text rise (Ts) displaces the glyph origin by (0, rise) in unscaled text
/// space — per the rendering-matrix definition it sits left of Tm, so the
/// offset maps through the text matrix's y column. Rise never contributes
/// to the advance, so callers apply it only to the rendering position and
/// keep advancing the unshifted text matrix.
fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] {
if rise == 0.0 {
return *tm;
}
[
tm[0],
tm[1],
tm[2],
tm[3],
tm[4] + rise * tm[2],
tm[5] + rise * tm[3],
]
}
/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
/// the page uses fonts with unresolvable gid-encoded glyphs.
pub(crate) fn extract_page_text_items(
@@ -134,7 +114,6 @@ pub(crate) fn extract_page_text_items(
page_num: u32,
font_cmaps: &FontCMaps,
include_invisible: bool,
style_cache: &mut FontStyleCache,
) -> Result<(PageExtraction, bool, bool), PdfError> {
use lopdf::content::Content;
@@ -174,8 +153,6 @@ pub(crate) fn extract_page_text_items(
std::collections::HashMap::new();
let mut inline_cmaps: std::collections::HashMap<String, crate::tounicode::CMapEntry> =
std::collections::HashMap::new();
let mut font_style_flags: std::collections::HashMap<String, (bool, bool)> =
std::collections::HashMap::new();
for (font_name, font_dict) in &fonts {
let resource_name = String::from_utf8_lossy(font_name).to_string();
if let Ok(base_font) = font_dict.get(b"BaseFont") {
@@ -184,12 +161,6 @@ pub(crate) fn extract_page_text_items(
font_base_names.insert(resource_name.clone(), base_name);
}
}
// Descriptor style flags rescue subset fonts whose BaseFont names
// are opaque tags the name heuristics can't read.
let style = descriptor_style_flags(doc, font_dict, style_cache);
if style != (false, false) {
font_style_flags.insert(resource_name.clone(), style);
}
// Track ToUnicode object reference, with FontFile2 fallback for Identity-H/V.
// Also handle inline ToUnicode streams.
match font_dict.get(b"ToUnicode") {
@@ -264,7 +235,6 @@ pub(crate) fn extract_page_text_items(
line_width: f32,
char_spacing: f32,
word_spacing: f32,
text_rise: f32,
text_leading: f32,
current_font: String,
current_font_size: f32,
@@ -277,7 +247,6 @@ pub(crate) fn extract_page_text_items(
let mut text_leading: f32 = 0.0; // TL parameter (in text-space units)
let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled)
let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled)
let mut text_rise: f32 = 0.0; // Ts parameter (baseline shift for super/subscripts, unscaled)
let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut in_text_block = false;
@@ -299,10 +268,6 @@ pub(crate) fn extract_page_text_items(
let mut suppress_glyph_extraction = false;
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
// Text rise in effect at each captured matrix — the item must render at
// the rise of its GLYPHS, not whatever rise is set by EMC time.
let mut actual_text_start_rise: f32 = 0.0;
let mut actual_text_glyph_rise: Option<f32> = None;
/// Get the innermost MCID from the marked content stack.
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
stack.iter().rev().find_map(|e| e.mcid)
@@ -319,7 +284,6 @@ pub(crate) fn extract_page_text_items(
line_width,
char_spacing,
word_spacing,
text_rise,
text_leading,
current_font: current_font.clone(),
current_font_size,
@@ -333,7 +297,6 @@ pub(crate) fn extract_page_text_items(
line_width = saved.line_width;
char_spacing = saved.char_spacing;
word_spacing = saved.word_spacing;
text_rise = saved.text_rise;
text_leading = saved.text_leading;
current_font = saved.current_font;
current_font_size = saved.current_font_size;
@@ -406,12 +369,6 @@ pub(crate) fn extract_page_text_items(
word_spacing = tw;
}
}
"Ts" => {
// Set text rise (baseline shift for superscripts/subscripts)
if let Some(ts) = op.operands.first().and_then(get_number) {
text_rise = ts;
}
}
"Td" | "TD" => {
// Move text position: TLM = T(tx,ty) × TLM; Tm = TLM
// tx,ty are in text space — must be scaled by the text line matrix
@@ -470,7 +427,6 @@ pub(crate) fn extract_page_text_items(
if suppress_glyph_extraction {
if actual_text_glyph_tm.is_none() {
actual_text_glyph_tm = Some(text_matrix);
actual_text_glyph_rise = Some(text_rise);
}
if let Some(w_ts) = w_ts_opt {
text_matrix[4] += w_ts * text_matrix[0];
@@ -500,8 +456,7 @@ pub(crate) fn extract_page_text_items(
&mut cmap_decisions,
&font_widths,
) {
let combined =
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
let combined = multiply_matrices(&text_matrix, &ctm);
let rendered_size = effective_font_size(current_font_size, &combined);
let (x, y) = (combined[4], combined[5]);
if combined[0].abs() >= combined[1].abs() {
@@ -523,10 +478,6 @@ pub(crate) fn extract_page_text_items(
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
items.push(TextItem {
text: expand_ligatures(&text),
x,
@@ -536,10 +487,9 @@ pub(crate) fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
@@ -557,7 +507,6 @@ pub(crate) fn extract_page_text_items(
// Capture first-glyph position for ActualText
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
actual_text_glyph_tm = Some(text_matrix);
actual_text_glyph_rise = Some(text_rise);
}
// Compute space threshold based on font metrics when available
@@ -678,10 +627,6 @@ pub(crate) fn extract_page_text_items(
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
for (text, start_w, end_w) in &sub_items {
let offset_tm = [
@@ -692,8 +637,7 @@ pub(crate) fn extract_page_text_items(
text_matrix[4] + start_w * text_matrix[0],
text_matrix[5] + start_w * text_matrix[1],
];
let combined =
multiply_matrices(&rise_adjusted(&offset_tm, text_rise), &ctm);
let combined = multiply_matrices(&offset_tm, &ctm);
let (x, y) = (combined[4], combined[5]);
let width = if font_info.is_some() {
((end_w - start_w) * scale_x).abs()
@@ -709,10 +653,9 @@ pub(crate) fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
@@ -736,26 +679,6 @@ pub(crate) fn extract_page_text_items(
line_matrix[4] += (-tl) * line_matrix[2];
line_matrix[5] += (-tl) * line_matrix[3];
text_matrix = line_matrix;
// Capture first-glyph position for ActualText AFTER the
// line move — the BDC-entry matrix is on the previous line.
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
actual_text_glyph_tm = Some(text_matrix);
actual_text_glyph_rise = Some(text_rise);
}
// Advance width, as for Tj — without it the item stays
// zero-width and geometric underline/strikeout detection
// rejects it (`is_underline_candidate` needs width > 0).
let w_ts_opt = font_widths.get(&current_font).and_then(|fi| {
op.operands.first().and_then(get_operand_bytes).map(|raw| {
compute_string_width_ts(
raw,
fi,
current_font_size,
char_spacing,
word_spacing,
)
})
});
if !((text_rendering_mode == 3 && !include_invisible)
|| suppress_glyph_extraction
|| op.operands.is_empty())
@@ -773,8 +696,7 @@ pub(crate) fn extract_page_text_items(
&font_widths,
) {
if !text.trim().is_empty() {
let combined =
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
let combined = multiply_matrices(&text_matrix, &ctm);
if combined[0].abs() >= combined[1].abs() {
rotation_votes.horizontal += 1;
} else {
@@ -782,45 +704,28 @@ pub(crate) fn extract_page_text_items(
}
let rendered_size = effective_font_size(current_font_size, &combined);
let (x, y) = (combined[4], combined[5]);
let width = w_ts_opt
.map(|w_ts| {
(w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]))
.abs()
})
.unwrap_or(0.0);
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
items.push(TextItem {
text: expand_ligatures(&text),
x,
y,
width,
width: 0.0,
height: rendered_size,
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
}
}
}
// Advance regardless of visibility so later show-text
// operators on the same line stay positioned (as for Tj).
if let Some(w_ts) = w_ts_opt {
text_matrix[4] += w_ts * text_matrix[0];
text_matrix[5] += w_ts * text_matrix[1];
}
}
"Do" => {
// XObject invocation - could be an image or form
@@ -852,7 +757,6 @@ pub(crate) fn extract_page_text_items(
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Image,
mcid: current_mcid(&marked_content_stack),
});
@@ -866,7 +770,6 @@ pub(crate) fn extract_page_text_items(
font_cmaps,
&ctm,
&mut cmap_decisions,
style_cache,
);
items.extend(form_items);
}
@@ -907,9 +810,7 @@ pub(crate) fn extract_page_text_items(
if actual_text.is_some() {
suppress_glyph_extraction = true;
actual_text_start_tm = Some(text_matrix);
actual_text_start_rise = text_rise;
actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
actual_text_glyph_rise = None;
}
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
}
@@ -922,11 +823,9 @@ pub(crate) fn extract_page_text_items(
// Tj may have moved the text position to the correct line —
// the BDC-entry position can be on the previous line.
let glyph_tm = actual_text_glyph_tm.take();
let glyph_rise = actual_text_glyph_rise.take();
let entry_tm = actual_text_start_tm.take();
if let Some(start_tm) = glyph_tm.or(entry_tm) {
let rise = glyph_rise.unwrap_or(actual_text_start_rise);
let combined = multiply_matrices(&rise_adjusted(&start_tm, rise), &ctm);
let combined = multiply_matrices(&start_tm, &ctm);
if combined[0].abs() >= combined[1].abs() {
rotation_votes.horizontal += 1;
} else {
@@ -943,10 +842,6 @@ pub(crate) fn extract_page_text_items(
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
items.push(TextItem {
text: expand_ligatures(&at),
x,
@@ -956,10 +851,9 @@ pub(crate) fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: entry
.mcid
@@ -1482,15 +1376,8 @@ mod tests {
let (doc, page_id) = simple_doc_with_content(content);
let font_cmaps = FontCMaps::from_doc(&doc);
let ((items, _, _), _, _) = extract_page_text_items(
&doc,
page_id,
1,
&font_cmaps,
false,
&mut FontStyleCache::new(),
)
.unwrap();
let ((items, _, _), _, _) =
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
items
}
@@ -1575,108 +1462,6 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
assert!(!world.is_underline);
}
#[test]
fn quote_operator_text_carries_advance_width() {
// `'` (move-to-next-line-and-show-text) must retain the string's
// advance width like Tj — zero-width items are invisible to
// geometric underline/strikeout detection.
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (first) Tj (struck) ' ET
1 w
99 503 m 145 503 l S";
let items = extract_simple_items(content);
let struck = items.iter().find(|item| item.text == "struck").unwrap();
// 6 glyphs x 600/1000 x 12pt = 43.2pt, drawn one leading below Tm.
assert!((struck.width - 43.2).abs() < 0.1);
assert!((struck.y - 500.0).abs() < 0.1);
assert!(struck.is_strikeout);
assert!(!struck.is_underline);
}
#[test]
fn quote_operator_advances_text_matrix() {
// Text shown after `'` on the same line must start past the shown
// string: "CD" lands at x=114.4 (2 glyphs x 600/1000 x 12pt after
// x=100), flush against "AB", so the merge pass joins them. Without
// the advance "CD" overlaps "AB" at x=100 and the items stay apart.
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (AB) ' (CD) Tj ET";
let items = extract_simple_items(content);
let merged = items.iter().find(|item| item.text == "ABCD").unwrap();
assert!((merged.x - 100.0).abs() < 0.1);
assert!((merged.width - 28.8).abs() < 0.1);
assert!((merged.y - 500.0).abs() < 0.1);
}
#[test]
fn text_rise_shifts_item_baseline() {
// Ts displaces the glyph origin vertically without touching the
// advance; the next run at rise 0 must return to the original
// baseline and follow the raised run horizontally.
let content =
b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (base) Tj 5 Ts (super) Tj 0 Ts (after) Tj ET";
let items = extract_simple_items(content);
let base = items.iter().find(|item| item.text == "base").unwrap();
let raised = items.iter().find(|item| item.text == "super").unwrap();
let after = items.iter().find(|item| item.text == "after").unwrap();
assert!((base.y - 500.0).abs() < 0.1);
assert!((raised.y - 505.0).abs() < 0.1);
assert!((after.y - 500.0).abs() < 0.1);
assert!(after.x > raised.x);
}
#[test]
fn actual_text_item_uses_glyph_rise() {
// The ActualText replacement item must render at the rise in
// effect when its glyphs were drawn — not the unshifted BDC
// baseline, and not whatever rise is set by EMC time.
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm \
/Span <</ActualText (super) >> BDC 5 Ts (sup) Tj 0 Ts EMC (after) Tj ET";
let items = extract_simple_items(content);
let sup = items.iter().find(|item| item.text == "super").unwrap();
let after = items.iter().find(|item| item.text == "after").unwrap();
assert!((sup.y - 505.0).abs() < 0.1);
assert!((after.y - 500.0).abs() < 0.1);
}
#[test]
fn actual_text_shown_with_quote_op_uses_moved_risen_baseline() {
// When the tagged span's show op is `'`, the glyph position is
// only known AFTER its line move — falling back to the BDC-entry
// matrix would place the item on the previous line, unrisen.
let content = b"BT /F1 12 Tf 14 TL 1 0 0 1 100 500 Tm \
/Span <</ActualText (replaced) >> BDC 3 Ts (raw) ' 0 Ts EMC ET";
let items = extract_simple_items(content);
let item = items.iter().find(|item| item.text == "replaced").unwrap();
// Line move: 500 - 14 = 486; rise: +3 -> 489.
assert!((item.y - 489.0).abs() < 0.1);
assert!(item.width > 0.0);
}
#[test]
fn strikeout_detected_on_risen_text() {
// The rule crosses the glyphs at their risen position; without the
// rise in item.y the strike window sits 4pt too low and misses.
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm 4 Ts (struck) Tj ET
1 w
99 507 m 145 507 l S";
let items = extract_simple_items(content);
let struck = items.iter().find(|item| item.text == "struck").unwrap();
assert!((struck.y - 504.0).abs() < 0.1);
assert!(struck.is_strikeout);
assert!(!struck.is_underline);
}
#[test]
fn test_skip_excessive_operations() {
use crate::tounicode::FontCMaps;
@@ -1711,15 +1496,7 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
doc.add_object(catalog);
let font_cmaps = FontCMaps::from_doc(&doc);
let result = extract_page_text_items(
&doc,
page_id,
1,
&font_cmaps,
false,
&mut FontStyleCache::new(),
)
.unwrap();
let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
let ((items, rects, lines), _has_gid, _coords_rotated) = result;
assert!(items.is_empty());
assert!(rects.is_empty());
@@ -1801,15 +1578,8 @@ BT 30 700 Tm <41> Tj ET";
doc.trailer.set("Root", Object::Reference(catalog_id));
let font_cmaps = FontCMaps::from_doc(&doc);
let ((items, _, _), _, _) = extract_page_text_items(
&doc,
page_id,
1,
&font_cmaps,
false,
&mut FontStyleCache::new(),
)
.unwrap();
let ((items, _, _), _, _) =
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
let text = items
.iter()
.map(|item| item.text.as_str())
+1 -332
View File
@@ -4,7 +4,7 @@ use crate::glyph_names::glyph_to_char;
use crate::tounicode::FontCMaps;
use crate::types::{FontEncodingMap, FontWidthInfo, PageFontEncodings, PageFontWidths};
use log::debug;
use lopdf::{Document, Encoding, Object, ObjectId};
use lopdf::{Document, Encoding, Object};
use std::collections::HashMap;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -739,171 +739,6 @@ pub(crate) fn get_font_file2_obj_num(doc: &Document, font_dict: &lopdf::Dictiona
.map(|r| r.0)
}
/// Document-scoped memo of embedded-font style flags, keyed by the
/// FontFile2/FontFile3 stream's object id. The same font program is
/// referenced from every page that uses the font, and decompressing +
/// parsing it dominates `descriptor_style_flags` — without the memo that
/// cost repeats per page whenever the descriptor leaves a flag unset
/// (the common case: regular fonts report neither italic nor bold).
#[derive(Debug, Default)]
pub(crate) struct FontStyleCache {
by_font_file: HashMap<ObjectId, (bool, bool)>,
}
impl FontStyleCache {
pub(crate) fn new() -> Self {
Self::default()
}
}
/// Style flags from the FontDescriptor, which survive subset fonts whose
/// BaseFont names are opaque tags ("Tc1", "ABCDEF+F1") that defeat the
/// name-based bold/italic heuristics.
///
/// Italic: `ItalicAngle` beyond a few degrees, or Flags bit 7 (Italic,
/// value 64). Bold: Flags bit 19 (ForceBold, value 1<<18). The small
/// ItalicAngle threshold skips fonts that declare a token slant.
pub(crate) fn descriptor_style_flags(
doc: &Document,
font_dict: &lopdf::Dictionary,
style_cache: &mut FontStyleCache,
) -> (bool, bool) {
let descriptor = font_dict
.get(b"FontDescriptor")
.ok()
.and_then(|obj| resolve_dict(doc, obj))
.or_else(|| {
// Type0 fonts hang the descriptor off DescendantFonts[0].
let desc_fonts = font_dict.get(b"DescendantFonts").ok()?;
let desc_fonts = resolve_array(doc, desc_fonts)?;
let cid_font_dict = resolve_dict(doc, desc_fonts.first()?)?;
resolve_dict(doc, cid_font_dict.get(b"FontDescriptor").ok()?)
});
let Some(descriptor) = descriptor else {
return (false, false);
};
let italic_angle = descriptor
.get(b"ItalicAngle")
.ok()
.and_then(|obj| match obj {
Object::Integer(i) => Some(*i as f32),
Object::Real(r) => Some(*r),
_ => None,
})
.unwrap_or(0.0);
let flags = descriptor
.get(b"Flags")
.ok()
.and_then(|obj| obj.as_i64().ok())
.unwrap_or(0);
let mut italic = italic_angle.abs() >= 4.0 || flags & (1 << 6) != 0;
let mut bold = flags & (1 << 18) != 0;
// Descriptors lie: subset generators write ItalicAngle 0 for genuinely
// italic faces. The embedded font file keeps the truth — OS/2
// fsSelection (via `Face::is_italic`) and the post table's italicAngle.
if !italic || !bold {
if let Some(ff_ref) = font_file_ref(descriptor) {
let (emb_italic, emb_bold) = *style_cache
.by_font_file
.entry(ff_ref)
.or_insert_with(|| embedded_style_flags(doc, ff_ref));
italic = italic || emb_italic;
bold = bold || emb_bold;
}
}
(italic, bold)
}
/// Style flags parsed from an embedded font program stream.
fn embedded_style_flags(doc: &Document, ff_ref: ObjectId) -> (bool, bool) {
let Some(data) = font_file_data(doc, ff_ref) else {
return (false, false);
};
if let Ok(face) = ttf_parser::Face::parse(&data, 0) {
(
face.is_italic() || face.italic_angle().abs() >= 4.0,
face.is_bold(),
)
} else if let Some(name) = cff_font_name(&data) {
// FontFile3 is bare CFF (no sfnt container) — ttf_parser
// can't open it, but the CFF Name INDEX keeps the real
// PostScript name ("XXXXXX+Amplitude-LightItalic") even
// when the descriptor was rewritten to claim upright.
(
crate::text_utils::is_italic_font(&name),
crate::text_utils::is_bold_font(&name),
)
} else {
(false, false)
}
}
/// First PostScript name from a bare CFF font's Name INDEX (CFF spec §7).
fn cff_font_name(data: &[u8]) -> Option<String> {
// Header: major(1) minor(1) hdrSize(1) offSize(1); major must be 1.
if data.len() < 4 || data[0] != 1 {
return None;
}
let hdr_size = data[2] as usize;
// Name INDEX: count(u16) offSize(u8) offsets[count+1] data
let count = u16::from_be_bytes([*data.get(hdr_size)?, *data.get(hdr_size + 1)?]) as usize;
if count == 0 {
return None;
}
let off_size = *data.get(hdr_size + 2)? as usize;
if !(1..=4).contains(&off_size) {
return None;
}
let read_offset = |idx: usize| -> Option<usize> {
let at = hdr_size + 3 + idx * off_size;
let bytes = data.get(at..at + off_size)?;
let mut v = 0usize;
for b in bytes {
v = (v << 8) | *b as usize;
}
Some(v)
};
let start = read_offset(0)?;
let end = read_offset(1)?;
if start == 0 || end < start {
return None;
}
// Offsets are 1-based from the byte before the object data.
let objects_base = hdr_size + 3 + (count + 1) * off_size - 1;
let name = data.get(objects_base + start..objects_base + end)?;
Some(String::from_utf8_lossy(name).to_string())
}
/// FontFile2/FontFile3 stream reference from a FontDescriptor.
fn font_file_ref(descriptor: &lopdf::Dictionary) -> Option<ObjectId> {
descriptor
.get(b"FontFile2")
.ok()
.and_then(|o| o.as_reference().ok())
.or_else(|| {
descriptor
.get(b"FontFile3")
.ok()
.and_then(|o| o.as_reference().ok())
})
}
/// Decompressed embedded font program bytes.
fn font_file_data(doc: &Document, ff_ref: ObjectId) -> Option<Vec<u8>> {
let stream = doc
.get_object(ff_ref)
.and_then(lopdf::Object::as_stream)
.ok()?;
Some(
stream
.decompressed_content()
.unwrap_or_else(|_| stream.content.clone()),
)
}
/// Decode text from a PDF string operand using font CMaps, encodings, and fallbacks.
#[allow(clippy::too_many_arguments)]
pub(crate) fn extract_text_from_operand(
@@ -1427,172 +1262,6 @@ mod tests {
}
}
fn doc_with_descriptor(descriptor: lopdf::Dictionary) -> (Document, lopdf::Dictionary) {
let mut doc = Document::with_version("1.4");
let desc_id = doc.add_object(descriptor);
let font_dict = dictionary! {
"Type" => "Font",
"Subtype" => "TrueType",
"BaseFont" => "Tc1",
"FontDescriptor" => desc_id,
};
(doc, font_dict)
}
#[test]
fn descriptor_italic_angle_sets_italic() {
// Subset font with an opaque BaseFont name ("Tc1") — the name
// heuristic sees nothing, the descriptor carries the truth.
let (doc, font_dict) = doc_with_descriptor(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "Tc1",
"ItalicAngle" => -12,
"Flags" => 32,
});
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(true, false)
);
}
#[test]
fn descriptor_italic_flag_bit_sets_italic() {
let (doc, font_dict) = doc_with_descriptor(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "Tc1",
"ItalicAngle" => 0,
"Flags" => 64, // bit 7: Italic
});
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(true, false)
);
}
#[test]
fn descriptor_force_bold_flag_sets_bold() {
let (doc, font_dict) = doc_with_descriptor(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "Tc1",
"ItalicAngle" => 0,
"Flags" => 1 << 18, // ForceBold
});
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(false, true)
);
}
#[test]
fn tiny_italic_angle_is_not_italic() {
// A token 1-degree slant is optical correction, not italic.
let (doc, font_dict) = doc_with_descriptor(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "Tc1",
"ItalicAngle" => lopdf::Object::Real(-1.0),
"Flags" => 32,
});
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(false, false)
);
}
#[test]
fn missing_descriptor_yields_no_flags() {
let doc = Document::with_version("1.4");
let font_dict = dictionary! { "Type" => "Font", "BaseFont" => "Tc1" };
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(false, false)
);
}
#[test]
fn type0_descendant_descriptor_is_resolved() {
let mut doc = Document::with_version("1.4");
let desc_id = doc.add_object(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "ABCDEF+F1",
"ItalicAngle" => -15,
});
let cid_id = doc.add_object(dictionary! {
"Type" => "Font",
"Subtype" => "CIDFontType2",
"FontDescriptor" => desc_id,
});
let font_dict = dictionary! {
"Type" => "Font",
"Subtype" => "Type0",
"BaseFont" => "ABCDEF+F1",
"DescendantFonts" => vec![lopdf::Object::Reference(cid_id)],
};
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(true, false)
);
}
/// Bare CFF: header + Name INDEX only — enough for `cff_font_name`.
fn bare_cff_with_name(name: &str) -> Vec<u8> {
let mut data = vec![1, 0, 4, 1]; // major, minor, hdrSize, offSize
data.extend_from_slice(&1u16.to_be_bytes()); // Name INDEX count
data.push(1); // offSize
data.push(1); // offset of first name
data.push(1 + name.len() as u8); // offset past last name
data.extend_from_slice(name.as_bytes());
data
}
#[test]
fn embedded_font_style_is_cached_by_font_file_object() {
use lopdf::{Object, Stream};
let mut doc = Document::with_version("1.4");
let ff_id = doc.add_object(Object::Stream(Stream::new(
dictionary! {},
bare_cff_with_name("ABCDEF+Test-BoldItalic"),
)));
let desc_id = doc.add_object(dictionary! {
"Type" => "FontDescriptor",
"FontName" => "ABCDEF+Test-BoldItalic",
"ItalicAngle" => 0,
"Flags" => 32,
"FontFile3" => ff_id,
});
let font_dict = dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Tc1",
"FontDescriptor" => desc_id,
};
let mut cache = FontStyleCache::new();
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut cache),
(true, true)
);
assert_eq!(cache.by_font_file.len(), 1);
// Replace the font program with garbage: a repeat call must serve
// the memo instead of re-reading the stream — repeated per-page
// decompression is exactly what the cache exists to avoid.
doc.objects.insert(
ff_id,
Object::Stream(Stream::new(dictionary! {}, vec![0u8; 4])),
);
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut cache),
(true, true)
);
// A cold cache parses the (now garbage) stream, proving the warm
// call above answered from the memo.
assert_eq!(
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
(false, false)
);
}
#[test]
fn compute_string_width_ts_no_tc_tw() {
// Without Tc/Tw (both 0), width = glyph widths only
-2
View File
@@ -1495,7 +1495,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1626,7 +1625,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
});
-2
View File
@@ -79,7 +79,6 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Link(url),
mcid: None,
});
@@ -319,7 +318,6 @@ pub(crate) fn walk_form_fields(
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::FormField,
mcid: None,
});
+4 -71
View File
@@ -24,7 +24,6 @@ use links::{extract_form_fields, extract_page_links};
// Re-export public types so existing `crate::extractor::X` paths keep working.
pub use crate::text_utils::{is_bold_font, is_italic_font};
pub use crate::types::{ItemType, TextLine};
pub(crate) use fonts::FontStyleCache;
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;
pub(crate) use layout::group_into_lines_with_thresholds;
@@ -162,9 +161,6 @@ fn extract_positioned_text_impl(
let mut all_lines = Vec::new();
let mut page_thresholds: PageThresholds = HashMap::new();
let mut gid_encoded_pages: HashSet<u32> = HashSet::new();
// Embedded-font style flags are document-scoped: the same font program
// is shared across pages, so parse it once, not once per page.
let mut style_cache = FontStyleCache::new();
// Build page ObjectId → page number map for form field extraction
let page_id_to_num: HashMap<ObjectId, u32> =
@@ -176,14 +172,8 @@ fn extract_positioned_text_impl(
continue;
}
}
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items(
doc,
page_id,
*page_num,
font_cmaps,
include_invisible,
&mut style_cache,
)?;
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) =
extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?;
if has_gid_fonts {
gid_encoded_pages.insert(*page_num);
}
@@ -244,10 +234,7 @@ fn suppress_table_underlines(
lines: &[PdfLine],
page: u32,
) {
if !items
.iter()
.any(|item| item.is_underline || item.is_strikeout)
{
if !items.iter().any(|item| item.is_underline) {
return;
}
@@ -269,7 +256,6 @@ fn suppress_table_underlines(
for index in table_item_indices {
if let Some(item) = items.get_mut(index) {
item.is_underline = false;
item.is_strikeout = false;
}
}
}
@@ -590,7 +576,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
if next.is_bold != first.is_bold
|| next.is_italic != first.is_italic
|| next.is_underline != first.is_underline
|| next.is_strikeout != first.is_strikeout
{
break;
}
@@ -653,7 +638,6 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
is_bold: first.is_bold,
is_italic: first.is_italic,
is_underline: first.is_underline,
is_strikeout: first.is_strikeout,
item_type: first.item_type.clone(),
mcid: first.mcid,
});
@@ -731,9 +715,7 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
.chars()
.last()
.is_some_and(|c| c.is_alphabetic());
let same_marks = parent.is_underline == item.is_underline
&& parent.is_strikeout == item.is_strikeout;
if parent.font_size >= sub_threshold && ends_with_letter && same_marks {
if parent.font_size >= sub_threshold && ends_with_letter {
let parent_right = parent.x + parent.width;
let gap = item.x - parent_right;
// Subscripts must be tightly adjacent (within ~1pt)
@@ -807,7 +789,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1010,7 +991,6 @@ mod tests {
items[3].y = 470.0;
for item in &mut items {
item.is_underline = true;
item.is_strikeout = true;
}
let lines = vec![
make_line(100.0, 500.0, 300.0, 500.0),
@@ -1024,30 +1004,6 @@ mod tests {
suppress_table_underlines(&mut items, &[], &lines, 1);
assert!(items.iter().all(|item| !item.is_underline));
assert!(items.iter().all(|item| !item.is_strikeout));
}
#[test]
fn subscript_digit_with_different_marks_is_not_absorbed() {
// A struck-out word followed by an unmarked footnote digit: merging
// would widen the parent's strikeout claim over the digit (and the
// reverse would drop the digit's own mark). Style boundaries break
// the merge, as in merge_text_items.
let mut word = make_merge_item("word", 100.0, 24.0);
word.font_size = 10.0;
word.is_strikeout = true;
let mut digit = make_merge_item("2", 124.5, 4.0);
digit.font_size = 6.0;
digit.y = word.y + 3.0;
let merged = merge_subscript_items(vec![word.clone(), digit.clone()]);
assert_eq!(merged.len(), 2);
// Same marks still merge (footnote ref inside the strike).
digit.is_strikeout = true;
let merged = merge_subscript_items(vec![word, digit]);
assert_eq!(merged.len(), 1);
assert!(merged[0].text.starts_with("word"));
}
#[test]
@@ -1065,7 +1021,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1081,7 +1036,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1097,7 +1051,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1153,7 +1106,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1169,7 +1121,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1185,7 +1136,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1212,7 +1162,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1228,7 +1177,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1244,7 +1192,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1273,7 +1220,6 @@ mod tests {
is_bold: true,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1309,7 +1255,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1346,7 +1291,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1362,7 +1306,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1378,7 +1321,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1402,7 +1344,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1516,7 +1457,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1532,7 +1472,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1558,7 +1497,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1574,7 +1512,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1616,7 +1553,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1662,7 +1598,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1708,7 +1643,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1747,7 +1681,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
+1 -64
View File
@@ -231,27 +231,8 @@ fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
overlap >= min_overlap
}
/// Strikeout window: a rule crossing the glyphs. Strikethroughs sit at
/// roughly 20-35% of the em above the baseline (about half the x-height);
/// accept a band well inside the glyph body so baseline underlines and
/// overlines never qualify.
fn rule_strikes_item(rule: &Rule, item: &TextItem) -> bool {
let y_min = item.y + item.font_size * 0.12;
let y_max = item.y + item.font_size * 0.55;
if rule.y < y_min || rule.y > y_max {
return false;
}
let ix1 = item.x;
let ix2 = item.x + item.width;
let min_overlap = item.width * MIN_X_OVERLAP;
let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
overlap >= min_overlap
}
/// Mark `is_underline` on text items that have a horizontal rule just
/// below their baseline, and `is_strikeout` on items whose glyphs a rule
/// crosses at mid x-height. `items`, `rects`, and `lines` are a single
/// below their baseline. `items`, `rects`, and `lines` are a single
/// page's extraction output (all in PDF coordinates, y-up, where
/// `TextItem::y` is the text baseline).
pub(crate) fn mark_underlined_items(
@@ -277,11 +258,6 @@ pub(crate) fn mark_underlined_items(
}
if rule_matches_item(rule, item) {
item.is_underline = true;
}
if rule_strikes_item(rule, item) {
item.is_strikeout = true;
}
if item.is_underline && item.is_strikeout {
break;
}
}
@@ -306,7 +282,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -383,44 +358,6 @@ mod tests {
assert!(!items[0].is_underline);
}
#[test]
fn mid_glyph_rule_marks_strikeout_not_underline() {
// Rule at ~30% of the em above the baseline crosses the glyphs.
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(99.0, 161.0, 503.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items[0].is_strikeout);
assert!(!items[0].is_underline);
}
#[test]
fn baseline_rule_marks_underline_not_strikeout() {
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(99.0, 161.0, 498.5)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items[0].is_underline);
assert!(!items[0].is_strikeout);
}
#[test]
fn overline_is_neither_underline_nor_strikeout() {
// Rule just above the cap height (overline / next line's rule).
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(99.0, 161.0, 507.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(!items[0].is_underline);
assert!(!items[0].is_strikeout);
}
#[test]
fn thin_filled_rect_at_mid_glyph_marks_strikeout() {
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
let rects = vec![thin_rect(100.0, 502.6, 60.0)];
mark_underlined_items(&mut items, &rects, &[], 1);
assert!(items[0].is_strikeout);
assert!(!items[0].is_underline);
}
#[test]
fn line_above_baseline_is_not_an_underline() {
// Strikethrough / overline geometry must not mark.
+5 -27
View File
@@ -1,6 +1,5 @@
//! Form XObject and image XObject extraction.
use super::fonts::descriptor_style_flags;
use crate::text_utils::{effective_font_size, expand_ligatures, is_bold_font, is_italic_font};
use crate::tounicode::FontCMaps;
use crate::types::{ItemType, TextItem};
@@ -9,7 +8,7 @@ use std::collections::HashMap;
use super::fonts::{
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, FontStyleCache,
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
};
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
@@ -115,7 +114,6 @@ pub(crate) fn extract_form_xobject_text(
font_cmaps: &FontCMaps,
parent_ctm: &[f32; 6],
cmap_decisions: &mut CMapDecisionCache,
style_cache: &mut FontStyleCache,
) -> Vec<TextItem> {
extract_form_xobject_text_inner(
doc,
@@ -124,12 +122,10 @@ pub(crate) fn extract_form_xobject_text(
font_cmaps,
parent_ctm,
cmap_decisions,
style_cache,
0,
)
}
#[allow(clippy::too_many_arguments)]
fn extract_form_xobject_text_inner(
doc: &Document,
form_id: ObjectId,
@@ -137,7 +133,6 @@ fn extract_form_xobject_text_inner(
font_cmaps: &FontCMaps,
parent_ctm: &[f32; 6],
cmap_decisions: &mut CMapDecisionCache,
style_cache: &mut FontStyleCache,
depth: u8,
) -> Vec<TextItem> {
use lopdf::content::Content;
@@ -172,7 +167,6 @@ fn extract_form_xobject_text_inner(
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
let mut inline_cmaps: HashMap<String, crate::tounicode::CMapEntry> = HashMap::new();
let mut font_style_flags: HashMap<String, (bool, bool)> = HashMap::new();
for (font_name, font_dict) in &form_fonts {
let resource_name = String::from_utf8_lossy(font_name).to_string();
if let Ok(base_font) = font_dict.get(b"BaseFont") {
@@ -181,10 +175,6 @@ fn extract_form_xobject_text_inner(
font_base_names.insert(resource_name.clone(), base_name);
}
}
let style = descriptor_style_flags(doc, font_dict, style_cache);
if style != (false, false) {
font_style_flags.insert(resource_name.clone(), style);
}
match font_dict.get(b"ToUnicode") {
Ok(tounicode) => {
if let Ok(obj_ref) = tounicode.as_reference() {
@@ -282,7 +272,6 @@ fn extract_form_xobject_text_inner(
font_cmaps,
&ctm,
cmap_decisions,
style_cache,
depth + 1,
);
items.extend(nested_items);
@@ -306,7 +295,6 @@ fn extract_form_xobject_text_inner(
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Image,
mcid: None,
});
@@ -440,10 +428,6 @@ fn extract_form_xobject_text_inner(
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
items.push(TextItem {
text: expand_ligatures(&text),
x,
@@ -453,10 +437,9 @@ fn extract_form_xobject_text_inner(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
});
@@ -577,10 +560,6 @@ fn extract_form_xobject_text_inner(
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
let (desc_italic, desc_bold) = font_style_flags
.get(&current_font)
.copied()
.unwrap_or((false, false));
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
for (text, start_w, end_w) in &sub_items {
let offset_tm = [
@@ -607,10 +586,9 @@ fn extract_form_xobject_text_inner(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
is_italic: is_italic_font(base_font) || desc_italic,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
});
+4 -29
View File
@@ -579,7 +579,6 @@ pub fn extract_text_in_regions_mem(
let mut gid_pages: HashSet<u32> = HashSet::new();
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
let mut rotated_pages: HashSet<u32> = HashSet::new();
let mut style_cache = extractor::FontStyleCache::new();
for (page_num, &page_id) in pages.iter() {
if !needed_pages.contains(page_num) {
@@ -598,7 +597,6 @@ pub fn extract_text_in_regions_mem(
*page_num,
&font_cmaps,
false,
&mut style_cache,
)?;
let threshold = text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
@@ -701,7 +699,6 @@ pub fn extract_tables_in_regions_mem(
let mut gid_pages: HashSet<u32> = HashSet::new();
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
let mut rotated_pages: HashSet<u32> = HashSet::new();
let mut style_cache = extractor::FontStyleCache::new();
for (page_num, &page_id) in pages.iter() {
if !needed_pages.contains(page_num) {
@@ -717,7 +714,6 @@ pub fn extract_tables_in_regions_mem(
*page_num,
&font_cmaps,
false,
&mut style_cache,
)?;
let threshold = text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
@@ -1023,7 +1019,6 @@ pub fn detect_vector_grid_in_region_mem(
page_1idx,
&font_cmaps,
false,
&mut extractor::FontStyleCache::new(),
)?;
text_utils::fix_letterspaced_items(&mut items);
@@ -1210,15 +1205,8 @@ mod vector_grid_tests {
let &page_id = pages.get(&1).unwrap();
let needed: HashSet<u32> = HashSet::from([1]);
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
&doc,
page_id,
1,
&cmaps,
false,
&mut crate::extractor::FontStyleCache::new(),
)
.unwrap();
let ((items, rects, _lines), _has_gid, _rotated) =
extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
assert_eq!(rect_tables.len(), 1, "expected one rect-detected table");
@@ -1252,15 +1240,8 @@ mod vector_grid_tests {
let &page_id = pages.get(&page_num).unwrap();
let needed: HashSet<u32> = HashSet::from([page_num]);
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
&doc,
page_id,
page_num,
&cmaps,
false,
&mut crate::extractor::FontStyleCache::new(),
)
.unwrap();
let ((items, rects, _lines), _has_gid, _rotated) =
extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap();
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num);
rect_tables
@@ -1985,7 +1966,6 @@ pub fn extract_tables_with_structure_cells_mem(
let mut page_heights: HashMap<u32, f32> = HashMap::new();
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
let mut rotated_pages: HashSet<u32> = HashSet::new();
let mut style_cache = extractor::FontStyleCache::new();
for (page_num, &page_id) in pages.iter() {
if !needed_pages.contains(page_num) {
@@ -2001,7 +1981,6 @@ pub fn extract_tables_with_structure_cells_mem(
*page_num,
&font_cmaps,
false,
&mut style_cache,
)?;
let threshold = text_utils::fix_letterspaced_items(&mut items);
if threshold > 0.10 {
@@ -2803,7 +2782,6 @@ fn detect_tsr_quality_issue(
page_1idx,
&font_cmaps,
false,
&mut extractor::FontStyleCache::new(),
)?;
let adaptive_threshold = text_utils::fix_letterspaced_items(&mut items);
let coords = if coords_rotated {
@@ -5078,7 +5056,6 @@ mod text_cluster_column_undercount_tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -5354,7 +5331,6 @@ mod table_candidate_selection_tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -6102,7 +6078,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
-48
View File
@@ -130,29 +130,6 @@ pub(crate) fn has_dot_leaders(text: &str) -> bool {
dot_groups >= 2
}
/// Detect a table-of-contents entry: a line ending in a page number preceded by
/// a dot-leader group (e.g. "Measurement Lab worksheet ... 3"). `has_dot_leaders`
/// misses single-group leaders ("..."), but a trailing "<dots> <number>" is a
/// strong TOC signal on its own. Such lines must never be promoted to headings.
pub(crate) fn is_toc_entry_line(text: &str) -> bool {
let trimmed = text.trim_end();
let digits = trimmed
.chars()
.rev()
.take_while(|c| c.is_ascii_digit())
.count();
if digits == 0 || digits > 4 {
return false;
}
let before_number = trimmed[..trimmed.len() - digits].trim_end();
let dots = before_number
.chars()
.rev()
.take_while(|c| *c == '.')
.count();
dots >= 3
}
/// Compute the Y-gap threshold for paragraph break detection.
///
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
@@ -343,28 +320,3 @@ pub(crate) fn detect_header_level(
Some(4)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn toc_entry_with_single_dot_group() {
assert!(is_toc_entry_line("Measurement Lab worksheet ... 3"));
assert!(is_toc_entry_line("Results ........ 12"));
assert!(is_toc_entry_line("Appendix B...42"));
}
#[test]
fn non_toc_lines_pass() {
assert!(!is_toc_entry_line(
"6.2. Expectations for Re-Hiring Employees"
));
assert!(!is_toc_entry_line("What happened in 2020"));
assert!(!is_toc_entry_line("IMPLEMENTATION"));
// Ellipsis without a trailing page number
assert!(!is_toc_entry_line("and so it goes ..."));
// Long numbers are data, not page refs
assert!(!is_toc_entry_line("ISBN ... 97814"));
}
}
+3 -12
View File
@@ -7,7 +7,7 @@ use crate::types::TextLine;
use super::analysis::{
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
detect_header_level, font_size_rarity, has_dot_leaders, is_toc_entry_line,
detect_header_level, font_size_rarity, has_dot_leaders,
};
use super::classify::{
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
@@ -703,7 +703,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
&& plain_trimmed.len() > 3
&& plain_trimmed.split_whitespace().count() <= 15
&& !starts_with_bullet_marker(plain_trimmed)
&& !is_toc_entry_line(plain_trimmed)
{
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
@@ -739,11 +738,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
// paragraph continuity and minor font-size variation
// inflates rarity scores.
let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
// Single-word headings ("IMPLEMENTATION", "CONTENTS") are common;
// accept them only with the strongest signal combination.
let enough_words =
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
if score >= 0.5 && standalone && enough_words && has_strong_signal {
if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
Some(bold_heading_level(&heading_tiers))
} else {
None
@@ -1042,7 +1037,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
if options.detect_headers
&& plain_trimmed.len() > 3
&& plain_trimmed.split_whitespace().count() <= 15
&& !is_toc_entry_line(plain_trimmed)
{
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
if let Some(header_level) =
@@ -1065,9 +1059,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
+ if all_bold { 0.3 } else { 0.0 }
+ if standalone { 0.2 } else { 0.0 }
+ if isolated { 0.3 } else { 0.0 };
let enough_words =
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
if score >= 0.5 && standalone && enough_words {
if score >= 0.5 && standalone && word_count >= 2 {
return Some(bold_heading_level(&heading_tiers));
}
None
@@ -1197,7 +1189,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: crate::types::ItemType::Text,
mcid,
}
-1
View File
@@ -1221,7 +1221,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: crate::types::ItemType::Text,
mcid: None,
}
-1
View File
@@ -543,7 +543,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid,
}
-3
View File
@@ -268,8 +268,6 @@ pub struct PyTextItem {
#[pyo3(get)]
pub is_underline: bool,
#[pyo3(get)]
pub is_strikeout: bool,
#[pyo3(get)]
pub item_type: String,
}
@@ -354,7 +352,6 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
is_strikeout: item.is_strikeout,
item_type: item_type_str(&item.item_type),
})
.collect()
-1
View File
@@ -105,7 +105,6 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
is_bold: first_item.is_bold,
is_italic: first_item.is_italic,
is_underline: first_item.is_underline,
is_strikeout: first_item.is_strikeout,
item_type: first_item.item_type.clone(),
mcid: first_item.mcid,
});
-1
View File
@@ -394,7 +394,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
-3
View File
@@ -2392,7 +2392,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -3367,7 +3366,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: crate::types::ItemType::Text,
mcid: None,
});
@@ -3678,7 +3676,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: crate::types::ItemType::Text,
mcid: None,
});
-1
View File
@@ -587,7 +587,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid,
}
-1
View File
@@ -109,7 +109,6 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
is_strikeout: item.is_strikeout,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
-3
View File
@@ -521,7 +521,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -887,7 +886,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
page: 1,
@@ -925,7 +923,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
page: 1,
-4
View File
@@ -236,7 +236,6 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
is_strikeout: item.is_strikeout,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
@@ -258,7 +257,6 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
is_strikeout: item.is_strikeout,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
@@ -1434,7 +1432,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1453,7 +1450,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
-3
View File
@@ -884,7 +884,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1005,7 +1004,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
});
@@ -1083,7 +1081,6 @@ mod tests {
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
-4
View File
@@ -120,10 +120,6 @@ pub struct TextItem {
/// baseline — PDFs have no underline font flag, so this is detected
/// geometrically after extraction; see `extractor::underline`).
pub is_underline: bool,
/// Whether the text is struck out (drawn rule/thin rect crossing the
/// glyphs at mid x-height). Same geometric detection as underline,
/// different vertical window; see `extractor::underline`.
pub is_strikeout: bool,
/// Type of item (text, image, link)
pub item_type: ItemType,
/// Marked Content ID from the content stream's BDC/BMC operator.
-2
View File
@@ -105,7 +105,6 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -132,7 +131,6 @@ fn make_text_item_with_font(
is_bold: is_bold_font(font),
is_italic: is_italic_font(font),
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
}