Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
46e23fb910 | ||
|
|
7afe4ac04d | ||
|
|
e7487be94b | ||
|
|
f2f49bdac2 | ||
|
|
d28594ef94 | ||
|
|
b4d401241e | ||
|
|
57335f8bcf | ||
|
|
15bc7894a4 |
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Firecrawl
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
[](LICENSE)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
|
||||
@@ -242,4 +243,4 @@ See [docs/debugging.md](docs/debugging.md) for `RUST_LOG` environment variable u
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
[MIT](LICENSE)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.11",
|
||||
"version": "1.10.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -83,6 +83,9 @@ pub struct TextItem {
|
||||
/// Underline detected geometrically (drawn rule/thin rect under the
|
||||
/// baseline) — PDFs carry no underline font flag.
|
||||
pub is_underline: bool,
|
||||
/// Strikeout detected geometrically (rule crossing the glyphs at mid
|
||||
/// x-height).
|
||||
pub is_strikeout: bool,
|
||||
pub item_type: ItemType,
|
||||
/// URL for link items, `None` for other types.
|
||||
pub link_url: Option<String>,
|
||||
@@ -294,6 +297,7 @@ pub fn extract_text_with_positions(
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type,
|
||||
link_url,
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ class TextItem:
|
||||
is_bold: bool
|
||||
is_italic: bool
|
||||
is_underline: bool
|
||||
is_strikeout: bool
|
||||
item_type: str
|
||||
|
||||
class RegionText:
|
||||
|
||||
+3
-1
@@ -74,7 +74,7 @@ fn format_items_json(items: &[TextItem]) -> String {
|
||||
_ => String::new(),
|
||||
};
|
||||
format!(
|
||||
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
|
||||
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"is_strikeout":{},"item_type":"{}","mcid":{}{}}}"#,
|
||||
json_escape(&item.text),
|
||||
item.page,
|
||||
item.x,
|
||||
@@ -86,6 +86,7 @@ fn format_items_json(items: &[TextItem]) -> String {
|
||||
item.is_bold,
|
||||
item.is_italic,
|
||||
item.is_underline,
|
||||
item.is_strikeout,
|
||||
item_type_label(&item.item_type),
|
||||
mcid,
|
||||
link_url,
|
||||
@@ -122,6 +123,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: true,
|
||||
is_underline: true,
|
||||
is_strikeout: true,
|
||||
item_type: ItemType::Text,
|
||||
mcid: Some(7),
|
||||
}];
|
||||
|
||||
+250
-20
@@ -14,8 +14,9 @@ use lopdf::{Document, Encoding, Object, ObjectId};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, descriptor_style_flags,
|
||||
extract_text_from_operand, get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
FontStyleCache,
|
||||
};
|
||||
use super::underline::UnderlineLine;
|
||||
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
|
||||
@@ -106,6 +107,25 @@ fn transformed_stroke_width(
|
||||
user_width * (ndx * ndx + ndy * ndy).sqrt()
|
||||
}
|
||||
|
||||
/// Text rise (Ts) displaces the glyph origin by (0, rise) in unscaled text
|
||||
/// space — per the rendering-matrix definition it sits left of Tm, so the
|
||||
/// offset maps through the text matrix's y column. Rise never contributes
|
||||
/// to the advance, so callers apply it only to the rendering position and
|
||||
/// keep advancing the unshifted text matrix.
|
||||
fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] {
|
||||
if rise == 0.0 {
|
||||
return *tm;
|
||||
}
|
||||
[
|
||||
tm[0],
|
||||
tm[1],
|
||||
tm[2],
|
||||
tm[3],
|
||||
tm[4] + rise * tm[2],
|
||||
tm[5] + rise * tm[3],
|
||||
]
|
||||
}
|
||||
|
||||
/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
|
||||
/// the page uses fonts with unresolvable gid-encoded glyphs.
|
||||
pub(crate) fn extract_page_text_items(
|
||||
@@ -114,6 +134,7 @@ pub(crate) fn extract_page_text_items(
|
||||
page_num: u32,
|
||||
font_cmaps: &FontCMaps,
|
||||
include_invisible: bool,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> Result<(PageExtraction, bool, bool), PdfError> {
|
||||
use lopdf::content::Content;
|
||||
|
||||
@@ -153,6 +174,8 @@ pub(crate) fn extract_page_text_items(
|
||||
std::collections::HashMap::new();
|
||||
let mut inline_cmaps: std::collections::HashMap<String, crate::tounicode::CMapEntry> =
|
||||
std::collections::HashMap::new();
|
||||
let mut font_style_flags: std::collections::HashMap<String, (bool, bool)> =
|
||||
std::collections::HashMap::new();
|
||||
for (font_name, font_dict) in &fonts {
|
||||
let resource_name = String::from_utf8_lossy(font_name).to_string();
|
||||
if let Ok(base_font) = font_dict.get(b"BaseFont") {
|
||||
@@ -161,6 +184,12 @@ pub(crate) fn extract_page_text_items(
|
||||
font_base_names.insert(resource_name.clone(), base_name);
|
||||
}
|
||||
}
|
||||
// Descriptor style flags rescue subset fonts whose BaseFont names
|
||||
// are opaque tags the name heuristics can't read.
|
||||
let style = descriptor_style_flags(doc, font_dict, style_cache);
|
||||
if style != (false, false) {
|
||||
font_style_flags.insert(resource_name.clone(), style);
|
||||
}
|
||||
// Track ToUnicode object reference, with FontFile2 fallback for Identity-H/V.
|
||||
// Also handle inline ToUnicode streams.
|
||||
match font_dict.get(b"ToUnicode") {
|
||||
@@ -235,6 +264,7 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width: f32,
|
||||
char_spacing: f32,
|
||||
word_spacing: f32,
|
||||
text_rise: f32,
|
||||
text_leading: f32,
|
||||
current_font: String,
|
||||
current_font_size: f32,
|
||||
@@ -247,6 +277,7 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut text_leading: f32 = 0.0; // TL parameter (in text-space units)
|
||||
let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled)
|
||||
let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled)
|
||||
let mut text_rise: f32 = 0.0; // Ts parameter (baseline shift for super/subscripts, unscaled)
|
||||
let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
|
||||
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
|
||||
let mut in_text_block = false;
|
||||
@@ -268,6 +299,10 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut suppress_glyph_extraction = false;
|
||||
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
|
||||
let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
|
||||
// Text rise in effect at each captured matrix — the item must render at
|
||||
// the rise of its GLYPHS, not whatever rise is set by EMC time.
|
||||
let mut actual_text_start_rise: f32 = 0.0;
|
||||
let mut actual_text_glyph_rise: Option<f32> = None;
|
||||
/// Get the innermost MCID from the marked content stack.
|
||||
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
|
||||
stack.iter().rev().find_map(|e| e.mcid)
|
||||
@@ -284,6 +319,7 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
text_rise,
|
||||
text_leading,
|
||||
current_font: current_font.clone(),
|
||||
current_font_size,
|
||||
@@ -297,6 +333,7 @@ pub(crate) fn extract_page_text_items(
|
||||
line_width = saved.line_width;
|
||||
char_spacing = saved.char_spacing;
|
||||
word_spacing = saved.word_spacing;
|
||||
text_rise = saved.text_rise;
|
||||
text_leading = saved.text_leading;
|
||||
current_font = saved.current_font;
|
||||
current_font_size = saved.current_font_size;
|
||||
@@ -369,6 +406,12 @@ pub(crate) fn extract_page_text_items(
|
||||
word_spacing = tw;
|
||||
}
|
||||
}
|
||||
"Ts" => {
|
||||
// Set text rise (baseline shift for superscripts/subscripts)
|
||||
if let Some(ts) = op.operands.first().and_then(get_number) {
|
||||
text_rise = ts;
|
||||
}
|
||||
}
|
||||
"Td" | "TD" => {
|
||||
// Move text position: TLM = T(tx,ty) × TLM; Tm = TLM
|
||||
// tx,ty are in text space — must be scaled by the text line matrix
|
||||
@@ -427,6 +470,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if suppress_glyph_extraction {
|
||||
if actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
@@ -456,7 +500,8 @@ pub(crate) fn extract_page_text_items(
|
||||
&mut cmap_decisions,
|
||||
&font_widths,
|
||||
) {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
@@ -478,6 +523,10 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
@@ -487,9 +536,10 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -507,6 +557,7 @@ pub(crate) fn extract_page_text_items(
|
||||
// Capture first-glyph position for ActualText
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
|
||||
// Compute space threshold based on font metrics when available
|
||||
@@ -627,6 +678,10 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
|
||||
for (text, start_w, end_w) in &sub_items {
|
||||
let offset_tm = [
|
||||
@@ -637,7 +692,8 @@ pub(crate) fn extract_page_text_items(
|
||||
text_matrix[4] + start_w * text_matrix[0],
|
||||
text_matrix[5] + start_w * text_matrix[1],
|
||||
];
|
||||
let combined = multiply_matrices(&offset_tm, &ctm);
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&offset_tm, text_rise), &ctm);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
let width = if font_info.is_some() {
|
||||
((end_w - start_w) * scale_x).abs()
|
||||
@@ -653,9 +709,10 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -679,6 +736,26 @@ pub(crate) fn extract_page_text_items(
|
||||
line_matrix[4] += (-tl) * line_matrix[2];
|
||||
line_matrix[5] += (-tl) * line_matrix[3];
|
||||
text_matrix = line_matrix;
|
||||
// Capture first-glyph position for ActualText AFTER the
|
||||
// line move — the BDC-entry matrix is on the previous line.
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
actual_text_glyph_rise = Some(text_rise);
|
||||
}
|
||||
// Advance width, as for Tj — without it the item stays
|
||||
// zero-width and geometric underline/strikeout detection
|
||||
// rejects it (`is_underline_candidate` needs width > 0).
|
||||
let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| {
|
||||
op.operands.first().and_then(get_operand_bytes).map(|raw| {
|
||||
compute_string_width_ts(
|
||||
raw,
|
||||
fi,
|
||||
current_font_size,
|
||||
char_spacing,
|
||||
word_spacing,
|
||||
)
|
||||
})
|
||||
});
|
||||
if !((text_rendering_mode == 3 && !include_invisible)
|
||||
|| suppress_glyph_extraction
|
||||
|| op.operands.is_empty())
|
||||
@@ -696,7 +773,8 @@ pub(crate) fn extract_page_text_items(
|
||||
&font_widths,
|
||||
) {
|
||||
if !text.trim().is_empty() {
|
||||
let combined = multiply_matrices(&text_matrix, &ctm);
|
||||
let combined =
|
||||
multiply_matrices(&rise_adjusted(&text_matrix, text_rise), &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
} else {
|
||||
@@ -704,28 +782,45 @@ pub(crate) fn extract_page_text_items(
|
||||
}
|
||||
let rendered_size = effective_font_size(current_font_size, &combined);
|
||||
let (x, y) = (combined[4], combined[5]);
|
||||
let width = w_ts_opt
|
||||
.map(|w_ts| {
|
||||
(w_ts * (text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2]))
|
||||
.abs()
|
||||
})
|
||||
.unwrap_or(0.0);
|
||||
let base_font = font_base_names
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
y,
|
||||
width: 0.0,
|
||||
width,
|
||||
height: rendered_size,
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Advance regardless of visibility so later show-text
|
||||
// operators on the same line stay positioned (as for Tj).
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
text_matrix[5] += w_ts * text_matrix[1];
|
||||
}
|
||||
}
|
||||
"Do" => {
|
||||
// XObject invocation - could be an image or form
|
||||
@@ -757,6 +852,7 @@ pub(crate) fn extract_page_text_items(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: current_mcid(&marked_content_stack),
|
||||
});
|
||||
@@ -770,6 +866,7 @@ pub(crate) fn extract_page_text_items(
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
&mut cmap_decisions,
|
||||
style_cache,
|
||||
);
|
||||
items.extend(form_items);
|
||||
}
|
||||
@@ -810,7 +907,9 @@ pub(crate) fn extract_page_text_items(
|
||||
if actual_text.is_some() {
|
||||
suppress_glyph_extraction = true;
|
||||
actual_text_start_tm = Some(text_matrix);
|
||||
actual_text_start_rise = text_rise;
|
||||
actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
|
||||
actual_text_glyph_rise = None;
|
||||
}
|
||||
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
|
||||
}
|
||||
@@ -823,9 +922,11 @@ pub(crate) fn extract_page_text_items(
|
||||
// Tj may have moved the text position to the correct line —
|
||||
// the BDC-entry position can be on the previous line.
|
||||
let glyph_tm = actual_text_glyph_tm.take();
|
||||
let glyph_rise = actual_text_glyph_rise.take();
|
||||
let entry_tm = actual_text_start_tm.take();
|
||||
if let Some(start_tm) = glyph_tm.or(entry_tm) {
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
let rise = glyph_rise.unwrap_or(actual_text_start_rise);
|
||||
let combined = multiply_matrices(&rise_adjusted(&start_tm, rise), &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
} else {
|
||||
@@ -842,6 +943,10 @@ pub(crate) fn extract_page_text_items(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&at),
|
||||
x,
|
||||
@@ -851,9 +956,10 @@ pub(crate) fn extract_page_text_items(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: entry
|
||||
.mcid
|
||||
@@ -1376,8 +1482,15 @@ mod tests {
|
||||
|
||||
let (doc, page_id) = simple_doc_with_content(content);
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let ((items, _, _), _, _) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
items
|
||||
}
|
||||
|
||||
@@ -1462,6 +1575,108 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
assert!(!world.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_operator_text_carries_advance_width() {
|
||||
// `'` (move-to-next-line-and-show-text) must retain the string's
|
||||
// advance width like Tj — zero-width items are invisible to
|
||||
// geometric underline/strikeout detection.
|
||||
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (first) Tj (struck) ' ET
|
||||
1 w
|
||||
99 503 m 145 503 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let struck = items.iter().find(|item| item.text == "struck").unwrap();
|
||||
|
||||
// 6 glyphs x 600/1000 x 12pt = 43.2pt, drawn one leading below Tm.
|
||||
assert!((struck.width - 43.2).abs() < 0.1);
|
||||
assert!((struck.y - 500.0).abs() < 0.1);
|
||||
assert!(struck.is_strikeout);
|
||||
assert!(!struck.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_operator_advances_text_matrix() {
|
||||
// Text shown after `'` on the same line must start past the shown
|
||||
// string: "CD" lands at x=114.4 (2 glyphs x 600/1000 x 12pt after
|
||||
// x=100), flush against "AB", so the merge pass joins them. Without
|
||||
// the advance "CD" overlaps "AB" at x=100 and the items stay apart.
|
||||
let content = b"BT /F1 12 Tf 12 TL 1 0 0 1 100 512 Tm (AB) ' (CD) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let merged = items.iter().find(|item| item.text == "ABCD").unwrap();
|
||||
|
||||
assert!((merged.x - 100.0).abs() < 0.1);
|
||||
assert!((merged.width - 28.8).abs() < 0.1);
|
||||
assert!((merged.y - 500.0).abs() < 0.1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_rise_shifts_item_baseline() {
|
||||
// Ts displaces the glyph origin vertically without touching the
|
||||
// advance; the next run at rise 0 must return to the original
|
||||
// baseline and follow the raised run horizontally.
|
||||
let content =
|
||||
b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (base) Tj 5 Ts (super) Tj 0 Ts (after) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let base = items.iter().find(|item| item.text == "base").unwrap();
|
||||
let raised = items.iter().find(|item| item.text == "super").unwrap();
|
||||
let after = items.iter().find(|item| item.text == "after").unwrap();
|
||||
|
||||
assert!((base.y - 500.0).abs() < 0.1);
|
||||
assert!((raised.y - 505.0).abs() < 0.1);
|
||||
assert!((after.y - 500.0).abs() < 0.1);
|
||||
assert!(after.x > raised.x);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_text_item_uses_glyph_rise() {
|
||||
// The ActualText replacement item must render at the rise in
|
||||
// effect when its glyphs were drawn — not the unshifted BDC
|
||||
// baseline, and not whatever rise is set by EMC time.
|
||||
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm \
|
||||
/Span <</ActualText (super) >> BDC 5 Ts (sup) Tj 0 Ts EMC (after) Tj ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let sup = items.iter().find(|item| item.text == "super").unwrap();
|
||||
let after = items.iter().find(|item| item.text == "after").unwrap();
|
||||
|
||||
assert!((sup.y - 505.0).abs() < 0.1);
|
||||
assert!((after.y - 500.0).abs() < 0.1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_text_shown_with_quote_op_uses_moved_risen_baseline() {
|
||||
// When the tagged span's show op is `'`, the glyph position is
|
||||
// only known AFTER its line move — falling back to the BDC-entry
|
||||
// matrix would place the item on the previous line, unrisen.
|
||||
let content = b"BT /F1 12 Tf 14 TL 1 0 0 1 100 500 Tm \
|
||||
/Span <</ActualText (replaced) >> BDC 3 Ts (raw) ' 0 Ts EMC ET";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let item = items.iter().find(|item| item.text == "replaced").unwrap();
|
||||
|
||||
// Line move: 500 - 14 = 486; rise: +3 -> 489.
|
||||
assert!((item.y - 489.0).abs() < 0.1);
|
||||
assert!(item.width > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strikeout_detected_on_risen_text() {
|
||||
// The rule crosses the glyphs at their risen position; without the
|
||||
// rise in item.y the strike window sits 4pt too low and misses.
|
||||
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm 4 Ts (struck) Tj ET
|
||||
1 w
|
||||
99 507 m 145 507 l S";
|
||||
|
||||
let items = extract_simple_items(content);
|
||||
let struck = items.iter().find(|item| item.text == "struck").unwrap();
|
||||
|
||||
assert!((struck.y - 504.0).abs() < 0.1);
|
||||
assert!(struck.is_strikeout);
|
||||
assert!(!struck.is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skip_excessive_operations() {
|
||||
use crate::tounicode::FontCMaps;
|
||||
@@ -1496,7 +1711,15 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
|
||||
doc.add_object(catalog);
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let result = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let ((items, rects, lines), _has_gid, _coords_rotated) = result;
|
||||
assert!(items.is_empty());
|
||||
assert!(rects.is_empty());
|
||||
@@ -1578,8 +1801,15 @@ BT 30 700 Tm <41> Tj ET";
|
||||
doc.trailer.set("Root", Object::Reference(catalog_id));
|
||||
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let ((items, _, _), _, _) =
|
||||
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
|
||||
let ((items, _, _), _, _) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let text = items
|
||||
.iter()
|
||||
.map(|item| item.text.as_str())
|
||||
|
||||
+332
-1
@@ -4,7 +4,7 @@ use crate::glyph_names::glyph_to_char;
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{FontEncodingMap, FontWidthInfo, PageFontEncodings, PageFontWidths};
|
||||
use log::debug;
|
||||
use lopdf::{Document, Encoding, Object};
|
||||
use lopdf::{Document, Encoding, Object, ObjectId};
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
@@ -739,6 +739,171 @@ pub(crate) fn get_font_file2_obj_num(doc: &Document, font_dict: &lopdf::Dictiona
|
||||
.map(|r| r.0)
|
||||
}
|
||||
|
||||
/// Document-scoped memo of embedded-font style flags, keyed by the
|
||||
/// FontFile2/FontFile3 stream's object id. The same font program is
|
||||
/// referenced from every page that uses the font, and decompressing +
|
||||
/// parsing it dominates `descriptor_style_flags` — without the memo that
|
||||
/// cost repeats per page whenever the descriptor leaves a flag unset
|
||||
/// (the common case: regular fonts report neither italic nor bold).
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct FontStyleCache {
|
||||
by_font_file: HashMap<ObjectId, (bool, bool)>,
|
||||
}
|
||||
|
||||
impl FontStyleCache {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Style flags from the FontDescriptor, which survive subset fonts whose
|
||||
/// BaseFont names are opaque tags ("Tc1", "ABCDEF+F1") that defeat the
|
||||
/// name-based bold/italic heuristics.
|
||||
///
|
||||
/// Italic: `ItalicAngle` beyond a few degrees, or Flags bit 7 (Italic,
|
||||
/// value 64). Bold: Flags bit 19 (ForceBold, value 1<<18). The small
|
||||
/// ItalicAngle threshold skips fonts that declare a token slant.
|
||||
pub(crate) fn descriptor_style_flags(
|
||||
doc: &Document,
|
||||
font_dict: &lopdf::Dictionary,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> (bool, bool) {
|
||||
let descriptor = font_dict
|
||||
.get(b"FontDescriptor")
|
||||
.ok()
|
||||
.and_then(|obj| resolve_dict(doc, obj))
|
||||
.or_else(|| {
|
||||
// Type0 fonts hang the descriptor off DescendantFonts[0].
|
||||
let desc_fonts = font_dict.get(b"DescendantFonts").ok()?;
|
||||
let desc_fonts = resolve_array(doc, desc_fonts)?;
|
||||
let cid_font_dict = resolve_dict(doc, desc_fonts.first()?)?;
|
||||
resolve_dict(doc, cid_font_dict.get(b"FontDescriptor").ok()?)
|
||||
});
|
||||
let Some(descriptor) = descriptor else {
|
||||
return (false, false);
|
||||
};
|
||||
|
||||
let italic_angle = descriptor
|
||||
.get(b"ItalicAngle")
|
||||
.ok()
|
||||
.and_then(|obj| match obj {
|
||||
Object::Integer(i) => Some(*i as f32),
|
||||
Object::Real(r) => Some(*r),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or(0.0);
|
||||
let flags = descriptor
|
||||
.get(b"Flags")
|
||||
.ok()
|
||||
.and_then(|obj| obj.as_i64().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let mut italic = italic_angle.abs() >= 4.0 || flags & (1 << 6) != 0;
|
||||
let mut bold = flags & (1 << 18) != 0;
|
||||
|
||||
// Descriptors lie: subset generators write ItalicAngle 0 for genuinely
|
||||
// italic faces. The embedded font file keeps the truth — OS/2
|
||||
// fsSelection (via `Face::is_italic`) and the post table's italicAngle.
|
||||
if !italic || !bold {
|
||||
if let Some(ff_ref) = font_file_ref(descriptor) {
|
||||
let (emb_italic, emb_bold) = *style_cache
|
||||
.by_font_file
|
||||
.entry(ff_ref)
|
||||
.or_insert_with(|| embedded_style_flags(doc, ff_ref));
|
||||
italic = italic || emb_italic;
|
||||
bold = bold || emb_bold;
|
||||
}
|
||||
}
|
||||
(italic, bold)
|
||||
}
|
||||
|
||||
/// Style flags parsed from an embedded font program stream.
|
||||
fn embedded_style_flags(doc: &Document, ff_ref: ObjectId) -> (bool, bool) {
|
||||
let Some(data) = font_file_data(doc, ff_ref) else {
|
||||
return (false, false);
|
||||
};
|
||||
if let Ok(face) = ttf_parser::Face::parse(&data, 0) {
|
||||
(
|
||||
face.is_italic() || face.italic_angle().abs() >= 4.0,
|
||||
face.is_bold(),
|
||||
)
|
||||
} else if let Some(name) = cff_font_name(&data) {
|
||||
// FontFile3 is bare CFF (no sfnt container) — ttf_parser
|
||||
// can't open it, but the CFF Name INDEX keeps the real
|
||||
// PostScript name ("XXXXXX+Amplitude-LightItalic") even
|
||||
// when the descriptor was rewritten to claim upright.
|
||||
(
|
||||
crate::text_utils::is_italic_font(&name),
|
||||
crate::text_utils::is_bold_font(&name),
|
||||
)
|
||||
} else {
|
||||
(false, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// First PostScript name from a bare CFF font's Name INDEX (CFF spec §7).
|
||||
fn cff_font_name(data: &[u8]) -> Option<String> {
|
||||
// Header: major(1) minor(1) hdrSize(1) offSize(1); major must be 1.
|
||||
if data.len() < 4 || data[0] != 1 {
|
||||
return None;
|
||||
}
|
||||
let hdr_size = data[2] as usize;
|
||||
// Name INDEX: count(u16) offSize(u8) offsets[count+1] data
|
||||
let count = u16::from_be_bytes([*data.get(hdr_size)?, *data.get(hdr_size + 1)?]) as usize;
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
let off_size = *data.get(hdr_size + 2)? as usize;
|
||||
if !(1..=4).contains(&off_size) {
|
||||
return None;
|
||||
}
|
||||
let read_offset = |idx: usize| -> Option<usize> {
|
||||
let at = hdr_size + 3 + idx * off_size;
|
||||
let bytes = data.get(at..at + off_size)?;
|
||||
let mut v = 0usize;
|
||||
for b in bytes {
|
||||
v = (v << 8) | *b as usize;
|
||||
}
|
||||
Some(v)
|
||||
};
|
||||
let start = read_offset(0)?;
|
||||
let end = read_offset(1)?;
|
||||
if start == 0 || end < start {
|
||||
return None;
|
||||
}
|
||||
// Offsets are 1-based from the byte before the object data.
|
||||
let objects_base = hdr_size + 3 + (count + 1) * off_size - 1;
|
||||
let name = data.get(objects_base + start..objects_base + end)?;
|
||||
Some(String::from_utf8_lossy(name).to_string())
|
||||
}
|
||||
|
||||
/// FontFile2/FontFile3 stream reference from a FontDescriptor.
|
||||
fn font_file_ref(descriptor: &lopdf::Dictionary) -> Option<ObjectId> {
|
||||
descriptor
|
||||
.get(b"FontFile2")
|
||||
.ok()
|
||||
.and_then(|o| o.as_reference().ok())
|
||||
.or_else(|| {
|
||||
descriptor
|
||||
.get(b"FontFile3")
|
||||
.ok()
|
||||
.and_then(|o| o.as_reference().ok())
|
||||
})
|
||||
}
|
||||
|
||||
/// Decompressed embedded font program bytes.
|
||||
fn font_file_data(doc: &Document, ff_ref: ObjectId) -> Option<Vec<u8>> {
|
||||
let stream = doc
|
||||
.get_object(ff_ref)
|
||||
.and_then(lopdf::Object::as_stream)
|
||||
.ok()?;
|
||||
Some(
|
||||
stream
|
||||
.decompressed_content()
|
||||
.unwrap_or_else(|_| stream.content.clone()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Decode text from a PDF string operand using font CMaps, encodings, and fallbacks.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn extract_text_from_operand(
|
||||
@@ -1262,6 +1427,172 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn doc_with_descriptor(descriptor: lopdf::Dictionary) -> (Document, lopdf::Dictionary) {
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let desc_id = doc.add_object(descriptor);
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "TrueType",
|
||||
"BaseFont" => "Tc1",
|
||||
"FontDescriptor" => desc_id,
|
||||
};
|
||||
(doc, font_dict)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_italic_angle_sets_italic() {
|
||||
// Subset font with an opaque BaseFont name ("Tc1") — the name
|
||||
// heuristic sees nothing, the descriptor carries the truth.
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => -12,
|
||||
"Flags" => 32,
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_italic_flag_bit_sets_italic() {
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 64, // bit 7: Italic
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_force_bold_flag_sets_bold() {
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 1 << 18, // ForceBold
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_italic_angle_is_not_italic() {
|
||||
// A token 1-degree slant is optical correction, not italic.
|
||||
let (doc, font_dict) = doc_with_descriptor(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "Tc1",
|
||||
"ItalicAngle" => lopdf::Object::Real(-1.0),
|
||||
"Flags" => 32,
|
||||
});
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_descriptor_yields_no_flags() {
|
||||
let doc = Document::with_version("1.4");
|
||||
let font_dict = dictionary! { "Type" => "Font", "BaseFont" => "Tc1" };
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type0_descendant_descriptor_is_resolved() {
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let desc_id = doc.add_object(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "ABCDEF+F1",
|
||||
"ItalicAngle" => -15,
|
||||
});
|
||||
let cid_id = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "CIDFontType2",
|
||||
"FontDescriptor" => desc_id,
|
||||
});
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type0",
|
||||
"BaseFont" => "ABCDEF+F1",
|
||||
"DescendantFonts" => vec![lopdf::Object::Reference(cid_id)],
|
||||
};
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(true, false)
|
||||
);
|
||||
}
|
||||
|
||||
/// Bare CFF: header + Name INDEX only — enough for `cff_font_name`.
|
||||
fn bare_cff_with_name(name: &str) -> Vec<u8> {
|
||||
let mut data = vec![1, 0, 4, 1]; // major, minor, hdrSize, offSize
|
||||
data.extend_from_slice(&1u16.to_be_bytes()); // Name INDEX count
|
||||
data.push(1); // offSize
|
||||
data.push(1); // offset of first name
|
||||
data.push(1 + name.len() as u8); // offset past last name
|
||||
data.extend_from_slice(name.as_bytes());
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_font_style_is_cached_by_font_file_object() {
|
||||
use lopdf::{Object, Stream};
|
||||
|
||||
let mut doc = Document::with_version("1.4");
|
||||
let ff_id = doc.add_object(Object::Stream(Stream::new(
|
||||
dictionary! {},
|
||||
bare_cff_with_name("ABCDEF+Test-BoldItalic"),
|
||||
)));
|
||||
let desc_id = doc.add_object(dictionary! {
|
||||
"Type" => "FontDescriptor",
|
||||
"FontName" => "ABCDEF+Test-BoldItalic",
|
||||
"ItalicAngle" => 0,
|
||||
"Flags" => 32,
|
||||
"FontFile3" => ff_id,
|
||||
});
|
||||
let font_dict = dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Tc1",
|
||||
"FontDescriptor" => desc_id,
|
||||
};
|
||||
|
||||
let mut cache = FontStyleCache::new();
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut cache),
|
||||
(true, true)
|
||||
);
|
||||
assert_eq!(cache.by_font_file.len(), 1);
|
||||
|
||||
// Replace the font program with garbage: a repeat call must serve
|
||||
// the memo instead of re-reading the stream — repeated per-page
|
||||
// decompression is exactly what the cache exists to avoid.
|
||||
doc.objects.insert(
|
||||
ff_id,
|
||||
Object::Stream(Stream::new(dictionary! {}, vec![0u8; 4])),
|
||||
);
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut cache),
|
||||
(true, true)
|
||||
);
|
||||
// A cold cache parses the (now garbage) stream, proving the warm
|
||||
// call above answered from the memo.
|
||||
assert_eq!(
|
||||
descriptor_style_flags(&doc, &font_dict, &mut FontStyleCache::new()),
|
||||
(false, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_string_width_ts_no_tc_tw() {
|
||||
// Without Tc/Tw (both 0), width = glyph widths only
|
||||
|
||||
@@ -1495,6 +1495,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1625,6 +1626,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Link(url),
|
||||
mcid: None,
|
||||
});
|
||||
@@ -318,6 +319,7 @@ pub(crate) fn walk_form_fields(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::FormField,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+71
-4
@@ -24,6 +24,7 @@ use links::{extract_form_fields, extract_page_links};
|
||||
// Re-export public types so existing `crate::extractor::X` paths keep working.
|
||||
pub use crate::text_utils::{is_bold_font, is_italic_font};
|
||||
pub use crate::types::{ItemType, TextLine};
|
||||
pub(crate) use fonts::FontStyleCache;
|
||||
pub(crate) use layout::detect_columns;
|
||||
pub use layout::group_into_lines;
|
||||
pub(crate) use layout::group_into_lines_with_thresholds;
|
||||
@@ -161,6 +162,9 @@ fn extract_positioned_text_impl(
|
||||
let mut all_lines = Vec::new();
|
||||
let mut page_thresholds: PageThresholds = HashMap::new();
|
||||
let mut gid_encoded_pages: HashSet<u32> = HashSet::new();
|
||||
// Embedded-font style flags are document-scoped: the same font program
|
||||
// is shared across pages, so parse it once, not once per page.
|
||||
let mut style_cache = FontStyleCache::new();
|
||||
|
||||
// Build page ObjectId → page number map for form field extraction
|
||||
let page_id_to_num: HashMap<ObjectId, u32> =
|
||||
@@ -172,8 +176,14 @@ fn extract_positioned_text_impl(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) =
|
||||
extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?;
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
if has_gid_fonts {
|
||||
gid_encoded_pages.insert(*page_num);
|
||||
}
|
||||
@@ -234,7 +244,10 @@ fn suppress_table_underlines(
|
||||
lines: &[PdfLine],
|
||||
page: u32,
|
||||
) {
|
||||
if !items.iter().any(|item| item.is_underline) {
|
||||
if !items
|
||||
.iter()
|
||||
.any(|item| item.is_underline || item.is_strikeout)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -256,6 +269,7 @@ fn suppress_table_underlines(
|
||||
for index in table_item_indices {
|
||||
if let Some(item) = items.get_mut(index) {
|
||||
item.is_underline = false;
|
||||
item.is_strikeout = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -576,6 +590,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
if next.is_bold != first.is_bold
|
||||
|| next.is_italic != first.is_italic
|
||||
|| next.is_underline != first.is_underline
|
||||
|| next.is_strikeout != first.is_strikeout
|
||||
{
|
||||
break;
|
||||
}
|
||||
@@ -638,6 +653,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
is_bold: first.is_bold,
|
||||
is_italic: first.is_italic,
|
||||
is_underline: first.is_underline,
|
||||
is_strikeout: first.is_strikeout,
|
||||
item_type: first.item_type.clone(),
|
||||
mcid: first.mcid,
|
||||
});
|
||||
@@ -715,7 +731,9 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
.chars()
|
||||
.last()
|
||||
.is_some_and(|c| c.is_alphabetic());
|
||||
if parent.font_size >= sub_threshold && ends_with_letter {
|
||||
let same_marks = parent.is_underline == item.is_underline
|
||||
&& parent.is_strikeout == item.is_strikeout;
|
||||
if parent.font_size >= sub_threshold && ends_with_letter && same_marks {
|
||||
let parent_right = parent.x + parent.width;
|
||||
let gap = item.x - parent_right;
|
||||
// Subscripts must be tightly adjacent (within ~1pt)
|
||||
@@ -789,6 +807,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -991,6 +1010,7 @@ mod tests {
|
||||
items[3].y = 470.0;
|
||||
for item in &mut items {
|
||||
item.is_underline = true;
|
||||
item.is_strikeout = true;
|
||||
}
|
||||
let lines = vec![
|
||||
make_line(100.0, 500.0, 300.0, 500.0),
|
||||
@@ -1004,6 +1024,30 @@ mod tests {
|
||||
suppress_table_underlines(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
assert!(items.iter().all(|item| !item.is_strikeout));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subscript_digit_with_different_marks_is_not_absorbed() {
|
||||
// A struck-out word followed by an unmarked footnote digit: merging
|
||||
// would widen the parent's strikeout claim over the digit (and the
|
||||
// reverse would drop the digit's own mark). Style boundaries break
|
||||
// the merge, as in merge_text_items.
|
||||
let mut word = make_merge_item("word", 100.0, 24.0);
|
||||
word.font_size = 10.0;
|
||||
word.is_strikeout = true;
|
||||
let mut digit = make_merge_item("2", 124.5, 4.0);
|
||||
digit.font_size = 6.0;
|
||||
digit.y = word.y + 3.0;
|
||||
|
||||
let merged = merge_subscript_items(vec![word.clone(), digit.clone()]);
|
||||
assert_eq!(merged.len(), 2);
|
||||
|
||||
// Same marks still merge (footnote ref inside the strike).
|
||||
digit.is_strikeout = true;
|
||||
let merged = merge_subscript_items(vec![word, digit]);
|
||||
assert_eq!(merged.len(), 1);
|
||||
assert!(merged[0].text.starts_with("word"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1021,6 +1065,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1036,6 +1081,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1051,6 +1097,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1106,6 +1153,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1121,6 +1169,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1136,6 +1185,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1162,6 +1212,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1177,6 +1228,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1192,6 +1244,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1220,6 +1273,7 @@ mod tests {
|
||||
is_bold: true,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1255,6 +1309,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1291,6 +1346,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1306,6 +1362,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1321,6 +1378,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1344,6 +1402,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1457,6 +1516,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1472,6 +1532,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1497,6 +1558,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1512,6 +1574,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
},
|
||||
@@ -1553,6 +1616,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1598,6 +1662,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1643,6 +1708,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}],
|
||||
@@ -1681,6 +1747,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -231,8 +231,27 @@ fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
overlap >= min_overlap
|
||||
}
|
||||
|
||||
/// Strikeout window: a rule crossing the glyphs. Strikethroughs sit at
|
||||
/// roughly 20-35% of the em above the baseline (about half the x-height);
|
||||
/// accept a band well inside the glyph body so baseline underlines and
|
||||
/// overlines never qualify.
|
||||
fn rule_strikes_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
let y_min = item.y + item.font_size * 0.12;
|
||||
let y_max = item.y + item.font_size * 0.55;
|
||||
if rule.y < y_min || rule.y > y_max {
|
||||
return false;
|
||||
}
|
||||
|
||||
let ix1 = item.x;
|
||||
let ix2 = item.x + item.width;
|
||||
let min_overlap = item.width * MIN_X_OVERLAP;
|
||||
let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
|
||||
overlap >= min_overlap
|
||||
}
|
||||
|
||||
/// Mark `is_underline` on text items that have a horizontal rule just
|
||||
/// below their baseline. `items`, `rects`, and `lines` are a single
|
||||
/// below their baseline, and `is_strikeout` on items whose glyphs a rule
|
||||
/// crosses at mid x-height. `items`, `rects`, and `lines` are a single
|
||||
/// page's extraction output (all in PDF coordinates, y-up, where
|
||||
/// `TextItem::y` is the text baseline).
|
||||
pub(crate) fn mark_underlined_items(
|
||||
@@ -258,6 +277,11 @@ pub(crate) fn mark_underlined_items(
|
||||
}
|
||||
if rule_matches_item(rule, item) {
|
||||
item.is_underline = true;
|
||||
}
|
||||
if rule_strikes_item(rule, item) {
|
||||
item.is_strikeout = true;
|
||||
}
|
||||
if item.is_underline && item.is_strikeout {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -282,6 +306,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -358,6 +383,44 @@ mod tests {
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_glyph_rule_marks_strikeout_not_underline() {
|
||||
// Rule at ~30% of the em above the baseline crosses the glyphs.
|
||||
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 503.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_strikeout);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn baseline_rule_marks_underline_not_strikeout() {
|
||||
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 498.5)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(items[0].is_underline);
|
||||
assert!(!items[0].is_strikeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overline_is_neither_underline_nor_strikeout() {
|
||||
// Rule just above the cap height (overline / next line's rule).
|
||||
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
|
||||
let lines = vec![hline(99.0, 161.0, 507.0)];
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
assert!(!items[0].is_underline);
|
||||
assert!(!items[0].is_strikeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thin_filled_rect_at_mid_glyph_marks_strikeout() {
|
||||
let mut items = vec![item("struck out", 100.0, 500.0, 60.0, 10.0)];
|
||||
let rects = vec![thin_rect(100.0, 502.6, 60.0)];
|
||||
mark_underlined_items(&mut items, &rects, &[], 1);
|
||||
assert!(items[0].is_strikeout);
|
||||
assert!(!items[0].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_above_baseline_is_not_an_underline() {
|
||||
// Strikethrough / overline geometry must not mark.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Form XObject and image XObject extraction.
|
||||
|
||||
use super::fonts::descriptor_style_flags;
|
||||
use crate::text_utils::{effective_font_size, expand_ligatures, is_bold_font, is_italic_font};
|
||||
use crate::tounicode::FontCMaps;
|
||||
use crate::types::{ItemType, TextItem};
|
||||
@@ -8,7 +9,7 @@ use std::collections::HashMap;
|
||||
|
||||
use super::fonts::{
|
||||
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
|
||||
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, FontStyleCache,
|
||||
};
|
||||
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
|
||||
|
||||
@@ -114,6 +115,7 @@ pub(crate) fn extract_form_xobject_text(
|
||||
font_cmaps: &FontCMaps,
|
||||
parent_ctm: &[f32; 6],
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
style_cache: &mut FontStyleCache,
|
||||
) -> Vec<TextItem> {
|
||||
extract_form_xobject_text_inner(
|
||||
doc,
|
||||
@@ -122,10 +124,12 @@ pub(crate) fn extract_form_xobject_text(
|
||||
font_cmaps,
|
||||
parent_ctm,
|
||||
cmap_decisions,
|
||||
style_cache,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn extract_form_xobject_text_inner(
|
||||
doc: &Document,
|
||||
form_id: ObjectId,
|
||||
@@ -133,6 +137,7 @@ fn extract_form_xobject_text_inner(
|
||||
font_cmaps: &FontCMaps,
|
||||
parent_ctm: &[f32; 6],
|
||||
cmap_decisions: &mut CMapDecisionCache,
|
||||
style_cache: &mut FontStyleCache,
|
||||
depth: u8,
|
||||
) -> Vec<TextItem> {
|
||||
use lopdf::content::Content;
|
||||
@@ -167,6 +172,7 @@ fn extract_form_xobject_text_inner(
|
||||
let mut font_tounicode_refs: HashMap<String, u32> = HashMap::new();
|
||||
let mut inline_cmaps: HashMap<String, crate::tounicode::CMapEntry> = HashMap::new();
|
||||
|
||||
let mut font_style_flags: HashMap<String, (bool, bool)> = HashMap::new();
|
||||
for (font_name, font_dict) in &form_fonts {
|
||||
let resource_name = String::from_utf8_lossy(font_name).to_string();
|
||||
if let Ok(base_font) = font_dict.get(b"BaseFont") {
|
||||
@@ -175,6 +181,10 @@ fn extract_form_xobject_text_inner(
|
||||
font_base_names.insert(resource_name.clone(), base_name);
|
||||
}
|
||||
}
|
||||
let style = descriptor_style_flags(doc, font_dict, style_cache);
|
||||
if style != (false, false) {
|
||||
font_style_flags.insert(resource_name.clone(), style);
|
||||
}
|
||||
match font_dict.get(b"ToUnicode") {
|
||||
Ok(tounicode) => {
|
||||
if let Ok(obj_ref) = tounicode.as_reference() {
|
||||
@@ -272,6 +282,7 @@ fn extract_form_xobject_text_inner(
|
||||
font_cmaps,
|
||||
&ctm,
|
||||
cmap_decisions,
|
||||
style_cache,
|
||||
depth + 1,
|
||||
);
|
||||
items.extend(nested_items);
|
||||
@@ -295,6 +306,7 @@ fn extract_form_xobject_text_inner(
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Image,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -428,6 +440,10 @@ fn extract_form_xobject_text_inner(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
items.push(TextItem {
|
||||
text: expand_ligatures(&text),
|
||||
x,
|
||||
@@ -437,9 +453,10 @@ fn extract_form_xobject_text_inner(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -560,6 +577,10 @@ fn extract_form_xobject_text_inner(
|
||||
.get(¤t_font)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or(¤t_font);
|
||||
let (desc_italic, desc_bold) = font_style_flags
|
||||
.get(¤t_font)
|
||||
.copied()
|
||||
.unwrap_or((false, false));
|
||||
let scale_x = text_matrix[0] * ctm[0] + text_matrix[1] * ctm[2];
|
||||
for (text, start_w, end_w) in &sub_items {
|
||||
let offset_tm = [
|
||||
@@ -586,9 +607,10 @@ fn extract_form_xobject_text_inner(
|
||||
font: current_font.clone(),
|
||||
font_size: rendered_size,
|
||||
page: page_num,
|
||||
is_bold: is_bold_font(base_font),
|
||||
is_italic: is_italic_font(base_font),
|
||||
is_bold: is_bold_font(base_font) || desc_bold,
|
||||
is_italic: is_italic_font(base_font) || desc_italic,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
+29
-4
@@ -579,6 +579,7 @@ pub fn extract_text_in_regions_mem(
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -597,6 +598,7 @@ pub fn extract_text_in_regions_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -699,6 +701,7 @@ pub fn extract_tables_in_regions_mem(
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -714,6 +717,7 @@ pub fn extract_tables_in_regions_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -1019,6 +1023,7 @@ pub fn detect_vector_grid_in_region_mem(
|
||||
page_1idx,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut extractor::FontStyleCache::new(),
|
||||
)?;
|
||||
text_utils::fix_letterspaced_items(&mut items);
|
||||
|
||||
@@ -1205,8 +1210,15 @@ mod vector_grid_tests {
|
||||
let &page_id = pages.get(&1).unwrap();
|
||||
let needed: HashSet<u32> = HashSet::from([1]);
|
||||
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((items, rects, _lines), _has_gid, _rotated) =
|
||||
extract_page_text_items(&doc, page_id, 1, &cmaps, false).unwrap();
|
||||
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
1,
|
||||
&cmaps,
|
||||
false,
|
||||
&mut crate::extractor::FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, 1);
|
||||
assert_eq!(rect_tables.len(), 1, "expected one rect-detected table");
|
||||
@@ -1240,8 +1252,15 @@ mod vector_grid_tests {
|
||||
let &page_id = pages.get(&page_num).unwrap();
|
||||
let needed: HashSet<u32> = HashSet::from([page_num]);
|
||||
let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed));
|
||||
let ((items, rects, _lines), _has_gid, _rotated) =
|
||||
extract_page_text_items(&doc, page_id, page_num, &cmaps, false).unwrap();
|
||||
let ((items, rects, _lines), _has_gid, _rotated) = extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
page_num,
|
||||
&cmaps,
|
||||
false,
|
||||
&mut crate::extractor::FontStyleCache::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (rect_tables, _) = detect_tables_from_rects(&items, &rects, page_num);
|
||||
rect_tables
|
||||
@@ -1966,6 +1985,7 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
let mut style_cache = extractor::FontStyleCache::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
@@ -1981,6 +2001,7 @@ pub fn extract_tables_with_structure_cells_mem(
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
@@ -2782,6 +2803,7 @@ fn detect_tsr_quality_issue(
|
||||
page_1idx,
|
||||
&font_cmaps,
|
||||
false,
|
||||
&mut extractor::FontStyleCache::new(),
|
||||
)?;
|
||||
let adaptive_threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
let coords = if coords_rotated {
|
||||
@@ -5056,6 +5078,7 @@ mod text_cluster_column_undercount_tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -5331,6 +5354,7 @@ mod table_candidate_selection_tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -6078,6 +6102,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -130,6 +130,29 @@ pub(crate) fn has_dot_leaders(text: &str) -> bool {
|
||||
dot_groups >= 2
|
||||
}
|
||||
|
||||
/// Detect a table-of-contents entry: a line ending in a page number preceded by
|
||||
/// a dot-leader group (e.g. "Measurement Lab worksheet ... 3"). `has_dot_leaders`
|
||||
/// misses single-group leaders ("..."), but a trailing "<dots> <number>" is a
|
||||
/// strong TOC signal on its own. Such lines must never be promoted to headings.
|
||||
pub(crate) fn is_toc_entry_line(text: &str) -> bool {
|
||||
let trimmed = text.trim_end();
|
||||
let digits = trimmed
|
||||
.chars()
|
||||
.rev()
|
||||
.take_while(|c| c.is_ascii_digit())
|
||||
.count();
|
||||
if digits == 0 || digits > 4 {
|
||||
return false;
|
||||
}
|
||||
let before_number = trimmed[..trimmed.len() - digits].trim_end();
|
||||
let dots = before_number
|
||||
.chars()
|
||||
.rev()
|
||||
.take_while(|c| *c == '.')
|
||||
.count();
|
||||
dots >= 3
|
||||
}
|
||||
|
||||
/// Compute the Y-gap threshold for paragraph break detection.
|
||||
///
|
||||
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
||||
@@ -320,3 +343,28 @@ pub(crate) fn detect_header_level(
|
||||
Some(4)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn toc_entry_with_single_dot_group() {
|
||||
assert!(is_toc_entry_line("Measurement Lab worksheet ... 3"));
|
||||
assert!(is_toc_entry_line("Results ........ 12"));
|
||||
assert!(is_toc_entry_line("Appendix B...42"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_toc_lines_pass() {
|
||||
assert!(!is_toc_entry_line(
|
||||
"6.2. Expectations for Re-Hiring Employees"
|
||||
));
|
||||
assert!(!is_toc_entry_line("What happened in 2020"));
|
||||
assert!(!is_toc_entry_line("IMPLEMENTATION"));
|
||||
// Ellipsis without a trailing page number
|
||||
assert!(!is_toc_entry_line("and so it goes ..."));
|
||||
// Long numbers are data, not page refs
|
||||
assert!(!is_toc_entry_line("ISBN ... 97814"));
|
||||
}
|
||||
}
|
||||
|
||||
+12
-3
@@ -7,7 +7,7 @@ use crate::types::TextLine;
|
||||
|
||||
use super::analysis::{
|
||||
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders,
|
||||
detect_header_level, font_size_rarity, has_dot_leaders, is_toc_entry_line,
|
||||
};
|
||||
use super::classify::{
|
||||
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
||||
@@ -703,6 +703,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !starts_with_bullet_marker(plain_trimmed)
|
||||
&& !is_toc_entry_line(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
||||
@@ -738,7 +739,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// paragraph continuity and minor font-size variation
|
||||
// inflates rarity scores.
|
||||
let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
|
||||
if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
|
||||
// Single-word headings ("IMPLEMENTATION", "CONTENTS") are common;
|
||||
// accept them only with the strongest signal combination.
|
||||
let enough_words =
|
||||
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||
if score >= 0.5 && standalone && enough_words && has_strong_signal {
|
||||
Some(bold_heading_level(&heading_tiers))
|
||||
} else {
|
||||
None
|
||||
@@ -1037,6 +1042,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if options.detect_headers
|
||||
&& plain_trimmed.len() > 3
|
||||
&& plain_trimmed.split_whitespace().count() <= 15
|
||||
&& !is_toc_entry_line(plain_trimmed)
|
||||
{
|
||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||
if let Some(header_level) =
|
||||
@@ -1059,7 +1065,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
+ if all_bold { 0.3 } else { 0.0 }
|
||||
+ if standalone { 0.2 } else { 0.0 }
|
||||
+ if isolated { 0.3 } else { 0.0 };
|
||||
if score >= 0.5 && standalone && word_count >= 2 {
|
||||
let enough_words =
|
||||
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||
if score >= 0.5 && standalone && enough_words {
|
||||
return Some(bold_heading_level(&heading_tiers));
|
||||
}
|
||||
None
|
||||
@@ -1189,6 +1197,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -1221,6 +1221,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -543,6 +543,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -268,6 +268,8 @@ pub struct PyTextItem {
|
||||
#[pyo3(get)]
|
||||
pub is_underline: bool,
|
||||
#[pyo3(get)]
|
||||
pub is_strikeout: bool,
|
||||
#[pyo3(get)]
|
||||
pub item_type: String,
|
||||
}
|
||||
|
||||
@@ -352,6 +354,7 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item_type_str(&item.item_type),
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -105,6 +105,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
|
||||
is_bold: first_item.is_bold,
|
||||
is_italic: first_item.is_italic,
|
||||
is_underline: first_item.is_underline,
|
||||
is_strikeout: first_item.is_strikeout,
|
||||
item_type: first_item.item_type.clone(),
|
||||
mcid: first_item.mcid,
|
||||
});
|
||||
|
||||
@@ -394,6 +394,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -1477,6 +1477,10 @@ fn detect_row_stripe_table(
|
||||
debug!(" row-stripe rejected: sparse outline/prose continuation shape");
|
||||
return None;
|
||||
}
|
||||
if has_dominant_prose_cell(&cells) {
|
||||
debug!(" row-stripe rejected: dominant prose cell (chart/figure region over body text)");
|
||||
return None;
|
||||
}
|
||||
|
||||
let column_centers: Vec<f32> = (0..num_cols)
|
||||
.map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0)
|
||||
@@ -1495,6 +1499,34 @@ fn detect_row_stripe_table(
|
||||
Some(Table::new(column_centers, row_centers, cells, item_indices))
|
||||
}
|
||||
|
||||
/// Detect a grid that swallowed body text instead of tabular data.
|
||||
///
|
||||
/// Charts (bar graphs, axis gridlines) emit fields of drawing rects that can
|
||||
/// pass the row-stripe shape test; the resulting "table" then captures the
|
||||
/// page's prose. The signature: one cell holds an entire paragraph — ≥60 words
|
||||
/// AND at least a third of all words in the table.
|
||||
///
|
||||
/// There is deliberately no row-count exemption. A small table whose single
|
||||
/// long cell dominates its word count is indistinguishable by content from a
|
||||
/// phantom grid over body text, and across the regression corpora every such
|
||||
/// grid observed has been swallowed prose, never a real note table. The costs
|
||||
/// are also asymmetric: rejecting a real table degrades it to readable prose,
|
||||
/// while accepting a phantom scrambles the page into Y-interleaved cells.
|
||||
/// Larger legitimate tables are safe because the one-third-of-total threshold
|
||||
/// scales with table size.
|
||||
fn has_dominant_prose_cell(cells: &[Vec<String>]) -> bool {
|
||||
let mut total_words = 0usize;
|
||||
let mut max_cell_words = 0usize;
|
||||
for row in cells {
|
||||
for cell in row {
|
||||
let words = cell.split_whitespace().count();
|
||||
total_words += words;
|
||||
max_cell_words = max_cell_words.max(words);
|
||||
}
|
||||
}
|
||||
max_cell_words >= 60 && max_cell_words * 3 >= total_words
|
||||
}
|
||||
|
||||
fn row_stripe_is_sparse_prose_outline(cells: &[Vec<String>]) -> bool {
|
||||
let Some(num_cols) = cells.first().map(|row| row.len()) else {
|
||||
return false;
|
||||
@@ -2294,6 +2326,12 @@ fn detect_merged_cluster_table(
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if has_dominant_prose_cell(&cells) {
|
||||
debug!(
|
||||
" merged-cluster rejected: dominant prose cell (chart/figure region over body text)"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// No empty columns
|
||||
for col in 0..num_cols {
|
||||
@@ -2392,11 +2430,95 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
}
|
||||
|
||||
// --- has_dominant_prose_cell ---
|
||||
|
||||
fn cells_of(rows: &[&[&str]]) -> Vec<Vec<String>> {
|
||||
rows.iter()
|
||||
.map(|r| r.iter().map(|c| c.to_string()).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dominant_prose_cell_rejects_swallowed_paragraph() {
|
||||
// Two cells hold paragraphs (the shape every observed phantom grid
|
||||
// has: swallowed body text spans multiple cells), rest are chart labels
|
||||
let para = ["word"; 70].join(" ");
|
||||
let para2 = ["word"; 35].join(" ");
|
||||
let cells = cells_of(&[
|
||||
&[para.as_str(), "81", "76"],
|
||||
&[para2.as_str(), "56", "9"],
|
||||
&["2019", "2020", ""],
|
||||
]);
|
||||
assert!(has_dominant_prose_cell(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dominant_prose_cell_rejects_small_table_dominated_by_one_cell() {
|
||||
// Boundary case, documented as INTENDED: a small grid whose single
|
||||
// long cell dominates the word count is rejected even at 4+ rows.
|
||||
// By content alone this shape is indistinguishable from a phantom
|
||||
// grid over body text, and every observed instance in the regression
|
||||
// corpora was swallowed prose (chart/figure regions), not a real
|
||||
// note table. Rejection degrades gracefully — the text is still
|
||||
// extracted as prose — while accepting a phantom scrambles reading
|
||||
// order.
|
||||
let note = ["word"; 70].join(" ");
|
||||
let cells = cells_of(&[
|
||||
&["Purpose", note.as_str()],
|
||||
&["Owner", "Facilities team"],
|
||||
&["Date", "2024-06-01"],
|
||||
&["Status", "Active"],
|
||||
]);
|
||||
assert!(has_dominant_prose_cell(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dominant_prose_cell_allows_description_column() {
|
||||
// Long-ish description cells, but text is spread across the table
|
||||
let desc = ["word"; 25].join(" ");
|
||||
let cells = cells_of(&[
|
||||
&["Item A", desc.as_str(), "100"],
|
||||
&["Item B", desc.as_str(), "200"],
|
||||
&["Item C", desc.as_str(), "300"],
|
||||
&["Item D", desc.as_str(), "400"],
|
||||
]);
|
||||
assert!(!has_dominant_prose_cell(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dominant_prose_cell_allows_short_tables() {
|
||||
let cells = cells_of(&[&["Name", "Value"], &["Total", "42"]]);
|
||||
assert!(!has_dominant_prose_cell(&cells));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dominant_prose_cell_allows_data_table_with_long_note() {
|
||||
// A real 4+ row table with one verbose remark cell: the note is ≥60
|
||||
// words but the table's other content carries more than 2× its word
|
||||
// count, so concentration stays below the 1/3 threshold. The
|
||||
// denominator scales with table size — this is what keeps large
|
||||
// legitimate tables safe where a bare length cap would not.
|
||||
let note = ["word"; 60].join(" ");
|
||||
let row_text = ["data"; 12].join(" ");
|
||||
let mut rows: Vec<Vec<String>> = (0..11)
|
||||
.map(|i| {
|
||||
vec![
|
||||
format!("Item {i}"),
|
||||
row_text.clone(),
|
||||
format!("{}", i * 100),
|
||||
]
|
||||
})
|
||||
.collect();
|
||||
rows.push(vec!["Note".into(), note, String::new()]);
|
||||
assert!(!has_dominant_prose_cell(&rows));
|
||||
}
|
||||
|
||||
// --- rects_overlap ---
|
||||
|
||||
#[test]
|
||||
@@ -3366,6 +3488,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -3676,6 +3799,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: crate::types::ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
|
||||
@@ -587,6 +587,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid,
|
||||
}
|
||||
|
||||
@@ -109,6 +109,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
|
||||
@@ -521,6 +521,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -886,6 +887,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
@@ -923,6 +925,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
page: 1,
|
||||
|
||||
@@ -236,6 +236,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
@@ -257,6 +258,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
|
||||
is_bold: item.is_bold,
|
||||
is_italic: item.is_italic,
|
||||
is_underline: item.is_underline,
|
||||
is_strikeout: item.is_strikeout,
|
||||
item_type: item.item_type.clone(),
|
||||
mcid: item.mcid,
|
||||
});
|
||||
@@ -1432,6 +1434,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1450,6 +1453,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -93,6 +93,9 @@ pub fn is_bold_font(font_name: &str) -> bool {
|
||||
|| lower.contains("extrabold")
|
||||
|| lower.contains("ultrabold")
|
||||
|| lower.contains("medium") && !lower.contains("mediumitalic") // Some fonts use Medium for semi-bold
|
||||
// URW Type 1 fonts abbreviate Medium as "Medi" (e.g. NimbusRomNo9L-Medi,
|
||||
// the Times-Bold substitute in LaTeX documents; -MediItal is bold italic).
|
||||
|| lower.contains("-medi") && !lower.contains("mediumital")
|
||||
}
|
||||
|
||||
/// Detect if a font name indicates italic/oblique style
|
||||
@@ -762,6 +765,17 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::types::ItemType;
|
||||
|
||||
#[test]
|
||||
fn bold_font_urw_medi_abbreviation() {
|
||||
// URW Type 1 fonts (LaTeX default Times) abbreviate Medium as "Medi"
|
||||
assert!(is_bold_font("NROFIU+NimbusRomNo9L-Medi"));
|
||||
assert!(is_bold_font("NimbusRomNo9L-MediItal"));
|
||||
assert!(!is_bold_font("DSSZWN+NimbusRomNo9L-Regu"));
|
||||
assert!(!is_bold_font("NimbusRomNo9L-ReguItal"));
|
||||
// Medium-Italic exclusion still holds
|
||||
assert!(!is_bold_font("Foo-MediumItalic"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_soft_hyphen() {
|
||||
assert_eq!(expand_ligatures("con\u{00AD}tent"), "content");
|
||||
@@ -884,6 +898,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -1004,6 +1019,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
});
|
||||
@@ -1081,6 +1097,7 @@ mod tests {
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
@@ -120,6 +120,10 @@ pub struct TextItem {
|
||||
/// baseline — PDFs have no underline font flag, so this is detected
|
||||
/// geometrically after extraction; see `extractor::underline`).
|
||||
pub is_underline: bool,
|
||||
/// Whether the text is struck out (drawn rule/thin rect crossing the
|
||||
/// glyphs at mid x-height). Same geometric detection as underline,
|
||||
/// different vertical window; see `extractor::underline`.
|
||||
pub is_strikeout: bool,
|
||||
/// Type of item (text, image, link)
|
||||
pub item_type: ItemType,
|
||||
/// Marked Content ID from the content stream's BDC/BMC operator.
|
||||
|
||||
@@ -105,6 +105,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
|
||||
is_bold: false,
|
||||
is_italic: false,
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
@@ -131,6 +132,7 @@ fn make_text_item_with_font(
|
||||
is_bold: is_bold_font(font),
|
||||
is_italic: is_italic_font(font),
|
||||
is_underline: false,
|
||||
is_strikeout: false,
|
||||
item_type: ItemType::Text,
|
||||
mcid: None,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user