Compare commits

...
Author SHA1 Message Date
Abimael Martell 2d4cade1d4 feat(cli): export positioned text item json 2026-07-08 10:06:09 -07:00
Abimael Martell 26ad531857 fix(extractor): harden underline detection 2026-07-08 08:40:41 -07:00
Abimael MartellandCursor 2d97f5aadc fix(extractor): underline rules only from painted rects, normalized extents (review)
Two review fixes: (1) normalize rect extents before the thickness/width
checks — `re` operands pass through the CTM so width/height can be
negative, which missed negative-width rules and let negative-height
bands pass as thin; (2) only feed painted rects to underline detection —
`re` rects now wait in a pending list until a paint operator (S/s, f/F/
f*, B/B*/b/b*) confirms them, and `re W n` clip-only paths are discarded
at `n`, so invisible clip boundaries no longer underline nearby text.
Marking moved into content_stream where paint state lives (pre-rotation,
consistent device space).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-07 22:38:58 -07:00
Abimael MartellandCursor 28decf15d6 feat(extractor): geometric underline detection on TextItem (ENG-5015)
PDFs carry no underline font flag — underlines are stroked horizontal
lines or thin filled rects drawn under the baseline. Correlate those
graphics (already parsed from the content stream) with text items in a
post-pass: a rule within ~0.35em below the baseline covering >=60% of
an item's width marks is_underline.

Exposed through the napi and python bindings. Verified on real docs:
4/4 underlined sentences flagged on a Japanese report, links/headings
flagged on 8 of 10 underline-bearing eval docs, zero flags on docs
without underlines. Known FP source (table cell borders) documented —
downstream applies inline styling only to plain-text regions.

napi 1.9.8 -> 1.9.9.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-07 22:24:15 -07:00
26 changed files with 978 additions and 16 deletions
+3
View File
@@ -111,6 +111,9 @@ pdf2md document.pdf
# JSON output (for piping)
pdf2md document.pdf --json
# Positioned TextItem JSON, including is_underline metadata
pdf2md document.pdf --items-json
# Raw markdown only (no headers)
pdf2md document.pdf --raw
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@firecrawl/pdf-inspector",
"version": "1.9.8",
"version": "1.9.9",
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
"main": "index.js",
"types": "index.d.ts",
+4
View File
@@ -80,6 +80,9 @@ pub struct TextItem {
pub page: u32,
pub is_bold: bool,
pub is_italic: bool,
/// Underline detected geometrically (drawn rule/thin rect under the
/// baseline) — PDFs carry no underline font flag.
pub is_underline: bool,
pub item_type: ItemType,
/// URL for link items, `None` for other types.
pub link_url: Option<String>,
@@ -290,6 +293,7 @@ pub fn extract_text_with_positions(
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
item_type,
link_url,
}
+1
View File
@@ -38,6 +38,7 @@ class TextItem:
page: int
is_bold: bool
is_italic: bool
is_underline: bool
item_type: str
class RegionText:
+105 -1
View File
@@ -1,6 +1,10 @@
//! CLI tool for PDF to Markdown conversion
use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode};
use pdf_inspector::extractor::ItemType;
use pdf_inspector::{
extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions,
PdfType, ProcessMode, TextItem,
};
use std::collections::HashSet;
use std::env;
use std::fmt::Write;
@@ -47,6 +51,92 @@ fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> Stri
.join(",")
}
fn item_type_label(item_type: &ItemType) -> &'static str {
match item_type {
ItemType::Text => "text",
ItemType::Image => "image",
ItemType::Link(_) => "link",
ItemType::FormField => "form_field",
}
}
fn format_items_json(items: &[TextItem]) -> String {
let underlined_count = items.iter().filter(|item| item.is_underline).count();
let items_json = items
.iter()
.map(|item| {
let mcid = item
.mcid
.map(|value| value.to_string())
.unwrap_or_else(|| "null".to_string());
let link_url = match &item.item_type {
ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)),
_ => String::new(),
};
format!(
r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
json_escape(&item.text),
item.page,
item.x,
item.y,
item.width,
item.height,
json_escape(&item.font),
item.font_size,
item.is_bold,
item.is_italic,
item.is_underline,
item_type_label(&item.item_type),
mcid,
link_url,
)
})
.collect::<Vec<_>>()
.join(",");
format!(
r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#,
items.len(),
underlined_count,
items_json
)
}
#[cfg(test)]
mod tests {
use super::format_items_json;
use pdf_inspector::extractor::ItemType;
use pdf_inspector::TextItem;
#[test]
fn items_json_includes_position_and_underline_metadata() {
let items = vec![TextItem {
text: "A \"quoted\" item".to_string(),
x: 12.345,
y: 67.891,
width: 23.456,
height: 9.876,
font: "F1".to_string(),
font_size: 10.0,
page: 2,
is_bold: false,
is_italic: true,
is_underline: true,
item_type: ItemType::Text,
mcid: Some(7),
}];
let json = format_items_json(&items);
assert!(json.contains(r#""text":"A \"quoted\" item""#));
assert!(json.contains(r#""page":2"#));
assert!(json.contains(r#""x":12.35"#));
assert!(json.contains(r#""is_underline":true"#));
assert!(json.contains(r#""item_type":"text""#));
assert!(json.contains(r#""mcid":7"#));
}
}
/// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
let mut pages = HashSet::new();
@@ -104,6 +194,7 @@ fn main() {
if args.len() < 2 {
eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
eprintln!(" {} <pdf_file> --json", args[0]);
eprintln!(" {} <pdf_file> --items-json", args[0]);
eprintln!(" {} <pdf_file> --raw", args[0]);
eprintln!();
eprintln!("Converts PDF to Markdown with smart type detection.");
@@ -111,6 +202,7 @@ fn main() {
eprintln!();
eprintln!("Options:");
eprintln!(" --json Output result as JSON");
eprintln!(" --items-json Output positioned TextItem JSON");
eprintln!(" --raw Output only markdown (no headers)");
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
@@ -121,6 +213,7 @@ fn main() {
let pdf_path = &args[1];
let json_output = args.iter().any(|a| a == "--json");
let items_json_output = args.iter().any(|a| a == "--items-json");
let raw_output = args.iter().any(|a| a == "--raw");
let page_numbers = args.iter().any(|a| a == "--pages");
let detect_only = args.iter().any(|a| a == "--detect-only");
@@ -145,6 +238,17 @@ fn main() {
})
});
if items_json_output {
match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) {
Ok(items) => println!("{}", format_items_json(&items)),
Err(e) => {
println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
process::exit(1);
}
}
return;
}
let output_file = args
.get(2)
.filter(|a| !a.starts_with("--"))
+208 -12
View File
@@ -17,6 +17,7 @@ use super::fonts::{
build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
};
use super::underline::UnderlineLine;
use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
use super::{get_number, image_bbox_from_ctm, multiply_matrices};
@@ -74,6 +75,37 @@ fn strip_pdf_comments(data: &[u8]) -> Vec<u8> {
result
}
fn transform_path_point(x: f32, y: f32, ctm: &[f32; 6]) -> (f32, f32) {
(
x * ctm[0] + y * ctm[2] + ctm[4],
x * ctm[1] + y * ctm[3] + ctm[5],
)
}
fn transformed_stroke_width(
line_width: f32,
ctm: &[f32; 6],
x1: f32,
y1: f32,
x2: f32,
y2: f32,
) -> f32 {
let user_width = line_width.abs();
let dx = x2 - x1;
let dy = y2 - y1;
let len = (dx * dx + dy * dy).sqrt();
if len <= f32::EPSILON {
return user_width;
}
// PDF stroke width scales perpendicular to the path direction.
let nx = -dy / len;
let ny = dx / len;
let ndx = nx * ctm[0] + ny * ctm[2];
let ndy = nx * ctm[1] + ny * ctm[3];
user_width * (ndx * ndx + ndy * ndy).sqrt()
}
/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
/// the page uses fonts with unresolvable gid-encoded glyphs.
pub(crate) fn extract_page_text_items(
@@ -89,6 +121,7 @@ pub(crate) fn extract_page_text_items(
let mut rects: Vec<PdfRect> = Vec::new();
let mut clip_rects: Vec<PdfRect> = Vec::new();
let mut lines: Vec<PdfLine> = Vec::new();
let mut underline_lines: Vec<UnderlineLine> = Vec::new();
// Path construction state for m/l/h → S/s line extraction
let mut path_subpath_start: Option<(f32, f32)> = None;
@@ -97,6 +130,12 @@ pub(crate) fn extract_page_text_items(
// Completed subpaths (each a vec of line segments) for f/f* rect extraction
let mut pending_subpaths: Vec<Vec<(f32, f32, f32, f32)>> = Vec::new();
let mut fill_rects: Vec<PdfRect> = Vec::new();
// `re` rects awaiting a paint operator. Underline detection must only
// see painted rects: a `re W n` clip path or `re n` no-op draws nothing
// on the page, so treating every `re` as ink would underline text that
// merely sits near an invisible clip boundary.
let mut pending_re_rects: Vec<PdfRect> = Vec::new();
let mut painted_rects: Vec<PdfRect> = Vec::new();
// Get fonts for encoding
let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
@@ -188,10 +227,12 @@ pub(crate) fn extract_page_text_items(
// Graphics state tracking
let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
let mut line_width: f32 = 1.0;
#[derive(Clone)]
struct SavedGraphicsState {
ctm: [f32; 6],
text_rendering_mode: i32,
line_width: f32,
char_spacing: f32,
word_spacing: f32,
text_leading: f32,
@@ -240,6 +281,7 @@ pub(crate) fn extract_page_text_items(
gstate_stack.push(SavedGraphicsState {
ctm,
text_rendering_mode,
line_width,
char_spacing,
word_spacing,
text_leading,
@@ -252,6 +294,7 @@ pub(crate) fn extract_page_text_items(
if let Some(saved) = gstate_stack.pop() {
ctm = saved.ctm;
text_rendering_mode = saved.text_rendering_mode;
line_width = saved.line_width;
char_spacing = saved.char_spacing;
word_spacing = saved.word_spacing;
text_leading = saved.text_leading;
@@ -273,6 +316,11 @@ pub(crate) fn extract_page_text_items(
ctm = multiply_matrices(&new_matrix, &ctm);
}
}
"w" => {
if let Some(width) = op.operands.first().and_then(get_number) {
line_width = width;
}
}
"BT" => {
// Begin text block
in_text_block = true;
@@ -441,6 +489,7 @@ pub(crate) fn extract_page_text_items(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
@@ -606,6 +655,7 @@ pub(crate) fn extract_page_text_items(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
@@ -669,6 +719,7 @@ pub(crate) fn extract_page_text_items(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: current_mcid(&marked_content_stack),
});
@@ -705,6 +756,7 @@ pub(crate) fn extract_page_text_items(
page: page_num,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Image,
mcid: current_mcid(&marked_content_stack),
});
@@ -801,6 +853,7 @@ pub(crate) fn extract_page_text_items(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: entry
.mcid
@@ -825,13 +878,19 @@ pub(crate) fn extract_page_text_items(
let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5];
let w_dev = rw * ctm[0];
let h_dev = rh * ctm[3];
rects.push(PdfRect {
let rect = PdfRect {
x: x_dev,
y: y_dev,
width: w_dev,
height: h_dev,
page: page_num,
});
};
// Underline detection must only see rects that are
// actually painted — a `re` used purely as a clip path
// (`re W n`) or discarded (`re n`) draws nothing. Hold
// the rect as pending until a paint operator confirms it.
pending_re_rects.push(rect.clone());
rects.push(rect);
}
}
// ── Path construction operators ──────────────────────
@@ -881,10 +940,8 @@ pub(crate) fn extract_page_text_items(
}
}
for (x1, y1, x2, y2) in pending_lines.drain(..) {
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
lines.push(PdfLine {
x1: x1d,
y1: y1d,
@@ -892,7 +949,16 @@ pub(crate) fn extract_page_text_items(
y2: y2d,
page: page_num,
});
underline_lines.push(UnderlineLine {
x1: x1d,
y1: y1d,
x2: x2d,
y2: y2d,
stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
page: page_num,
});
}
painted_rects.append(&mut pending_re_rects);
pending_subpaths.clear();
path_subpath_start = None;
path_current = None;
@@ -908,10 +974,8 @@ pub(crate) fn extract_page_text_items(
}
}
for (x1, y1, x2, y2) in pending_lines.drain(..) {
let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
lines.push(PdfLine {
x1: x1d,
y1: y1d,
@@ -919,7 +983,16 @@ pub(crate) fn extract_page_text_items(
y2: y2d,
page: page_num,
});
underline_lines.push(UnderlineLine {
x1: x1d,
y1: y1d,
x2: x2d,
y2: y2d,
stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
page: page_num,
});
}
painted_rects.append(&mut pending_re_rects);
pending_subpaths.clear();
path_subpath_start = None;
path_current = None;
@@ -977,6 +1050,7 @@ pub(crate) fn extract_page_text_items(
}
}
}
painted_rects.append(&mut pending_re_rects);
pending_lines.clear();
path_subpath_start = None;
path_current = None;
@@ -1042,7 +1116,10 @@ pub(crate) fn extract_page_text_items(
// Do NOT clear pending_lines — the following `n` does that
}
"n" => {
// end path (no-op): discard
// end path (no-op): discard — including any `re` rects that
// were only ever part of a clip path (`re W n`), which draw
// no ink and must not feed underline detection.
pending_re_rects.clear();
pending_lines.clear();
pending_subpaths.clear();
path_subpath_start = None;
@@ -1052,6 +1129,12 @@ pub(crate) fn extract_page_text_items(
}
}
// Underline detection reads only painted ink: `re` rects confirmed by
// a paint operator plus filled-subpath rects — never clip-only rects,
// which draw nothing.
let mut underline_rects = painted_rects;
underline_rects.extend(fill_rects.iter().cloned());
// Only use clip/fill rects when no `re` rects exist on this page.
// Clip rects take priority over fill rects, but first we deduplicate
// them: some PDFs wrap every text block in a full-page W* clip path,
@@ -1081,8 +1164,17 @@ pub(crate) fn extract_page_text_items(
// Some PDFs embed landscape content in portrait pages using a rotated text
// matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW). The layout engine
// assumes x=horizontal, y=vertical — so we swap coordinates to match.
let (items, rects, lines, coords_rotated) =
let (mut items, rects, lines, coords_rotated) =
correct_rotated_page(items, rects, lines, &rotation_votes);
if coords_rotated {
rotate_underline_graphics(&mut underline_rects, &mut underline_lines);
}
super::underline::mark_underlined_items(
&mut items,
&underline_rects,
&underline_lines,
page_num,
);
let items = super::merge_text_items(items);
let items = super::merge_subscript_items(items);
@@ -1168,6 +1260,27 @@ fn correct_rotated_page(
(items, rects, lines, true)
}
fn rotate_underline_graphics(rects: &mut [PdfRect], lines: &mut [UnderlineLine]) {
for rect in rects {
let new_x = rect.y;
let new_y = -(rect.x + rect.width.abs());
rect.x = new_x;
rect.y = new_y;
std::mem::swap(&mut rect.width, &mut rect.height);
}
for line in lines {
let new_x1 = line.y1;
let new_y1 = -line.x1;
let new_x2 = line.y2;
let new_y2 = -line.x2;
line.x1 = new_x1;
line.y1 = new_y1;
line.x2 = new_x2;
line.y2 = new_y2;
}
}
/// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance).
/// Some PDFs emit a full-page clip path for every text block, producing
/// thousands of identical rects. After dedup these collapse to one rect,
@@ -1217,6 +1330,57 @@ mod tests {
}
}
fn simple_doc_with_content(content: &[u8]) -> (lopdf::Document, lopdf::ObjectId) {
use lopdf::{dictionary, Object, Stream};
let mut doc = lopdf::Document::new();
let widths: Vec<Object> = (0..=255).map(|_| 600.into()).collect();
let font_id = doc.add_object(dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
"FirstChar" => 0,
"LastChar" => 255,
"Widths" => Object::Array(widths),
});
let content_id = doc.add_object(Object::Stream(Stream::new(
dictionary! {},
content.to_vec(),
)));
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Contents" => Object::Reference(content_id),
"Resources" => dictionary! {
"Font" => dictionary! {
"F1" => Object::Reference(font_id),
},
},
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
});
let pages_id = doc.add_object(dictionary! {
"Type" => "Pages",
"Count" => Object::Integer(1),
"Kids" => vec![Object::Reference(page_id)],
});
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => Object::Reference(pages_id),
});
doc.trailer.set("Root", Object::Reference(catalog_id));
(doc, page_id)
}
fn extract_simple_items(content: &[u8]) -> Vec<TextItem> {
use crate::tounicode::FontCMaps;
let (doc, page_id) = simple_doc_with_content(content);
let font_cmaps = FontCMaps::from_doc(&doc);
let ((items, _, _), _, _) =
extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
items
}
#[test]
fn test_dedup_rects_identical() {
let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759];
@@ -1266,6 +1430,38 @@ mod tests {
assert_eq!(single.len(), 1);
}
#[test]
fn thick_stroked_rule_does_not_mark_underline() {
let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (THICK) Tj ET
4 w
100 498 m 170 498 l S
BT /F1 12 Tf 1 0 0 1 100 480 Tm (THIN) Tj ET
1 w
100 478 m 160 478 l S";
let items = extract_simple_items(content);
let thick = items.iter().find(|item| item.text == "THICK").unwrap();
let thin = items.iter().find(|item| item.text == "THIN").unwrap();
assert!(!thick.is_underline);
assert!(thin.is_underline);
}
#[test]
fn rotated_page_underline_is_detected_after_coordinate_correction() {
let content = b"BT /F1 12 Tf 0 1 -1 0 200 100 Tm (HELLO) Tj ET
BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
1 w
202 100 m 202 170 l S";
let items = extract_simple_items(content);
let hello = items.iter().find(|item| item.text == "HELLO").unwrap();
let world = items.iter().find(|item| item.text == "WORLD").unwrap();
assert!(hello.is_underline);
assert!(!world.is_underline);
}
#[test]
fn test_skip_excessive_operations() {
use crate::tounicode::FontCMaps;
+2
View File
@@ -1494,6 +1494,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1623,6 +1624,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
});
+2
View File
@@ -78,6 +78,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
page: page_num,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Link(url),
mcid: None,
});
@@ -316,6 +317,7 @@ pub(crate) fn walk_form_fields(
page: page_num,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::FormField,
mcid: None,
});
+117 -2
View File
@@ -6,11 +6,12 @@ pub(crate) mod content_stream;
mod fonts;
mod layout;
mod links;
pub(crate) mod underline;
mod xobjects;
use crate::text_utils::is_rtl_text;
use crate::tounicode::FontCMaps;
use crate::types::{PageExtraction, TextItem};
use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
use crate::PdfError;
use log::debug;
use lopdf::{Document, Object, ObjectId};
@@ -180,6 +181,7 @@ fn extract_positioned_text_impl(
if threshold > 0.10 {
page_thresholds.insert(*page_num, threshold);
}
suppress_table_underlines(&mut items, &rects, &lines, *page_num);
debug!(
"page {}: {} text items, {} rects, {} lines{}",
page_num,
@@ -226,6 +228,38 @@ fn extract_positioned_text_impl(
))
}
fn suppress_table_underlines(
items: &mut [TextItem],
rects: &[PdfRect],
lines: &[PdfLine],
page: u32,
) {
if !items.iter().any(|item| item.is_underline) {
return;
}
let mut table_item_indices: HashSet<usize> = HashSet::new();
if !rects.is_empty() {
let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page);
for table in rect_tables {
table_item_indices.extend(table.item_indices);
}
}
if !lines.is_empty() {
for table in crate::tables::detect_tables_from_lines(items, lines, page) {
table_item_indices.extend(table.item_indices);
}
}
for index in table_item_indices {
if let Some(item) = items.get_mut(index) {
item.is_underline = false;
}
}
}
// ---------------------------------------------------------------------------
// Shared helpers (used by submodules via `super::`)
// ---------------------------------------------------------------------------
@@ -525,6 +559,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
let first = group[i];
let mut text = first.text.clone();
let mut end_x = first.x + effective_merge_width(first);
let mut is_underline = first.is_underline;
let mut j = i + 1;
while j < group.len() {
@@ -571,6 +606,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
text.push(' ');
}
text.push_str(&next.text);
is_underline |= next.is_underline;
let next_end = next.x + effective_merge_width(next);
end_x = if *preserve_stream_order {
end_x.max(next_end)
@@ -591,6 +627,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
page: first.page,
is_bold: first.is_bold,
is_italic: first.is_italic,
is_underline,
item_type: first.item_type.clone(),
mcid: first.mcid,
});
@@ -701,7 +738,7 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
mod tests {
use super::*;
use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items};
use crate::types::{ItemType, TextLine};
use crate::types::{ItemType, PdfLine, TextLine};
use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
@@ -716,6 +753,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -726,6 +764,16 @@ mod tests {
item
}
fn make_line(x1: f32, y1: f32, x2: f32, y2: f32) -> PdfLine {
PdfLine {
x1,
y1,
x2,
y2,
page: 1,
}
}
#[test]
fn trace_text_preview_truncates_on_char_boundary() {
let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}');
@@ -774,6 +822,21 @@ mod tests {
assert_eq!(merged[0].text, "hello world");
}
#[test]
fn merge_items_preserves_underline_from_later_fragment() {
let mut items = vec![
make_merge_item("pre", 100.0, 18.0),
make_merge_item("fix", 119.0, 18.0),
];
items[1].is_underline = true;
let merged = merge_text_items(items);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].text, "prefix");
assert!(merged[0].is_underline);
}
#[test]
fn merge_items_preserves_stream_order_for_backtracking_heading() {
// Some tagged PDFs emit first-letter ActualText fragments, then reset
@@ -851,6 +914,35 @@ mod tests {
assert_eq!(texts, vec!["", "Distant item"]);
}
#[test]
fn suppress_table_underlines_clears_line_detected_table_items() {
let mut items = vec![
make_merge_item("H1", 125.0, 20.0),
make_merge_item("H2", 225.0, 20.0),
make_merge_item("A", 125.0, 20.0),
make_merge_item("B", 225.0, 20.0),
];
items[0].y = 490.0;
items[1].y = 490.0;
items[2].y = 470.0;
items[3].y = 470.0;
for item in &mut items {
item.is_underline = true;
}
let lines = vec![
make_line(100.0, 500.0, 300.0, 500.0),
make_line(100.0, 480.0, 300.0, 480.0),
make_line(100.0, 460.0, 300.0, 460.0),
make_line(100.0, 460.0, 100.0, 500.0),
make_line(200.0, 460.0, 200.0, 500.0),
make_line(300.0, 460.0, 300.0, 500.0),
];
suppress_table_underlines(&mut items, &[], &lines, 1);
assert!(items.iter().all(|item| !item.is_underline));
}
#[test]
fn test_group_into_lines() {
let items = vec![
@@ -865,6 +957,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -879,6 +972,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -893,6 +987,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -947,6 +1042,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -961,6 +1057,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -975,6 +1072,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1000,6 +1098,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1014,6 +1113,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1028,6 +1128,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1055,6 +1156,7 @@ mod tests {
page: 1,
is_bold: true,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1089,6 +1191,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1124,6 +1227,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1138,6 +1242,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1152,6 +1257,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1174,6 +1280,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1286,6 +1393,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1300,6 +1408,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1324,6 +1433,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1338,6 +1448,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
},
@@ -1378,6 +1489,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1422,6 +1534,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1466,6 +1579,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}],
@@ -1503,6 +1617,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
+500
View File
@@ -0,0 +1,500 @@
//! Geometric underline detection.
//!
//! PDFs have no underline font flag — underlines are drawn as separate
//! graphics: stroked horizontal lines (`l`/`S` operators) or thin filled
//! rectangles (`re`/`f`). This pass correlates those graphics with text
//! items after extraction: an item is underlined when a horizontal
//! line/thin rect sits just below its baseline and covers most of its
//! horizontal extent.
//!
//! Repeated same-span rules are treated as table/form rulings rather than
//! underlines, which avoids marking every cell in ruled tables.
use std::collections::HashSet;
use crate::types::{ItemType, PdfRect, TextItem};
/// Max thickness (pt) for a stroked line / filled rect to count as an
/// underline rule rather than a border or decorative band.
const MAX_RULE_THICKNESS: f32 = 2.0;
/// Fraction of the item's width that the rule must cover horizontally.
const MIN_X_OVERLAP: f32 = 0.6;
/// Same-span rules repeated at this many y-levels are usually table/form
/// rulings, not semantic underlines.
const MIN_REPEATED_RULE_LEVELS: usize = 3;
/// Vertical tolerance for considering two rules to be on the same row edge.
const RULE_Y_DEDUP_EPS: f32 = 2.0;
/// Horizontal span similarity required when clustering repeated rulings.
const RULE_SPAN_OVERLAP_RATIO: f32 = 0.8;
const RULE_SPAN_WIDTH_RATIO: f32 = 1.5;
/// Multiple separated rule segments on one row are usually per-column table
/// header/body separators.
const MIN_SEGMENTED_ROW_RULES: usize = 3;
const MIN_SEGMENTED_ROW_GAPS: usize = 2;
const SEGMENTED_ROW_GAP_MIN: f32 = 12.0;
/// A single rule under several widely separated items is usually a table
/// header/body separator, not a sentence underline.
const MIN_TABULAR_RULE_ITEMS: usize = 3;
const MIN_TABULAR_RULE_GAPS: usize = 2;
const TABULAR_RULE_GAP_EM: f32 = 2.0;
#[derive(Clone)]
pub(crate) struct UnderlineLine {
pub(crate) x1: f32,
pub(crate) y1: f32,
pub(crate) x2: f32,
pub(crate) y2: f32,
pub(crate) stroke_width: f32,
pub(crate) page: u32,
}
/// A horizontal rule candidate in page coordinates (PDF y-up).
#[derive(Clone)]
struct Rule {
x1: f32,
x2: f32,
y: f32,
}
impl Rule {
fn width(&self) -> f32 {
self.x2 - self.x1
}
}
fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) -> Vec<Rule> {
let mut rules: Vec<Rule> = Vec::new();
for l in lines {
if l.page != page {
continue;
}
// Horizontal stroked line (tolerate slight skew).
if l.stroke_width <= MAX_RULE_THICKNESS && (l.y1 - l.y2).abs() <= MAX_RULE_THICKNESS {
let (x1, x2) = if l.x1 <= l.x2 {
(l.x1, l.x2)
} else {
(l.x2, l.x1)
};
if x2 - x1 > 1.0 {
rules.push(Rule {
x1,
x2,
y: (l.y1 + l.y2) / 2.0,
});
}
}
}
for r in rects {
if r.page != page {
continue;
}
// Thin filled rect used as an underline rule. Extents are
// normalized first: `re` operands pass through the CTM, so
// width/height can be negative (flipped axes / negative scale) —
// without normalization negative-width rules are missed and
// negative-height bands sneak past the thickness check.
let (x1, x2) = if r.width >= 0.0 {
(r.x, r.x + r.width)
} else {
(r.x + r.width, r.x)
};
if r.height.abs() <= MAX_RULE_THICKNESS && x2 - x1 > 1.0 {
rules.push(Rule {
x1,
x2,
y: r.y + r.height / 2.0,
});
}
}
rules
}
fn discard_repeated_ruling_rules(rules: Vec<Rule>) -> Vec<Rule> {
if rules.len() < MIN_REPEATED_RULE_LEVELS {
return rules;
}
rules
.iter()
.filter(|rule| {
!is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules)
})
.cloned()
.collect()
}
fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
let mut y_levels: Vec<f32> = rules
.iter()
.filter(|other| has_similar_span(rule, other))
.map(|other| other.y)
.collect();
y_levels.sort_by(|a, b| a.total_cmp(b));
y_levels.dedup_by(|a, b| (*a - *b).abs() <= RULE_Y_DEDUP_EPS);
y_levels.len() >= MIN_REPEATED_RULE_LEVELS
}
fn is_segmented_row_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
let mut row_rules: Vec<&Rule> = rules
.iter()
.filter(|other| (other.y - rule.y).abs() <= RULE_Y_DEDUP_EPS)
.collect();
if row_rules.len() < MIN_SEGMENTED_ROW_RULES {
return false;
}
row_rules.sort_by(|a, b| a.x1.total_cmp(&b.x1));
let large_gaps = row_rules
.windows(2)
.filter(|pair| pair[1].x1 - pair[0].x2 > SEGMENTED_ROW_GAP_MIN)
.count();
large_gaps >= MIN_SEGMENTED_ROW_GAPS
}
fn has_similar_span(a: &Rule, b: &Rule) -> bool {
let a_width = a.width();
let b_width = b.width();
if a_width <= 1.0 || b_width <= 1.0 {
return false;
}
let width_ratio = a_width.max(b_width) / a_width.min(b_width);
if width_ratio > RULE_SPAN_WIDTH_RATIO {
return false;
}
let overlap = a.x2.min(b.x2) - a.x1.max(b.x1);
overlap >= a_width.min(b_width) * RULE_SPAN_OVERLAP_RATIO
}
fn tabular_row_separator_rule_indices(rules: &[Rule], items: &[TextItem]) -> HashSet<usize> {
let mut tabular_rules = HashSet::new();
for (rule_idx, rule) in rules.iter().enumerate() {
let mut matched_items: Vec<&TextItem> = items
.iter()
.filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item))
.collect();
if matched_items.len() < MIN_TABULAR_RULE_ITEMS {
continue;
}
matched_items.sort_by(|a, b| a.x.total_cmp(&b.x));
let large_gaps = matched_items
.windows(2)
.filter(|pair| {
let left = pair[0];
let right = pair[1];
let gap = right.x - (left.x + left.width);
let font_size = left.font_size.max(right.font_size).max(1.0);
gap > font_size * TABULAR_RULE_GAP_EM
})
.count();
if large_gaps >= MIN_TABULAR_RULE_GAPS {
tabular_rules.insert(rule_idx);
}
}
tabular_rules
}
fn is_underline_candidate(item: &TextItem) -> bool {
matches!(item.item_type, ItemType::Text) && !item.text.trim().is_empty() && item.width > 0.0
}
fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
// Vertical window: underlines sit at or slightly below the baseline.
// Fonts draw them at roughly 5-15% of the em below; allow up to 35%
// (min 3pt) below and 1pt above for rounding.
let below = (item.font_size * 0.35).max(3.0);
let y_min = item.y - below;
let y_max = item.y + 1.0;
if rule.y < y_min || rule.y > y_max {
return false;
}
let ix1 = item.x;
let ix2 = item.x + item.width;
let min_overlap = item.width * MIN_X_OVERLAP;
let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
overlap >= min_overlap
}
/// Mark `is_underline` on text items that have a horizontal rule just
/// below their baseline. `items`, `rects`, and `lines` are a single
/// page's extraction output (all in PDF coordinates, y-up, where
/// `TextItem::y` is the text baseline).
pub(crate) fn mark_underlined_items(
items: &mut [TextItem],
rects: &[PdfRect],
lines: &[UnderlineLine],
page: u32,
) {
let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page));
if rules.is_empty() {
return;
}
let tabular_rules = tabular_row_separator_rule_indices(&rules, items);
for item in items.iter_mut() {
if !is_underline_candidate(item) {
continue;
}
for (rule_idx, rule) in rules.iter().enumerate() {
if tabular_rules.contains(&rule_idx) {
continue;
}
if rule_matches_item(rule, item) {
item.is_underline = true;
break;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ItemType;
fn item(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem {
TextItem {
text: text.to_string(),
x,
y,
width,
height: font_size,
font: "F1".to_string(),
font_size,
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
}
fn hline(x1: f32, x2: f32, y: f32) -> UnderlineLine {
UnderlineLine {
x1,
y1: y,
x2,
y2: y,
stroke_width: 1.0,
page: 1,
}
}
fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect {
PdfRect {
x,
y,
width,
height: 0.8,
page: 1,
}
}
#[test]
fn stroked_line_under_baseline_marks_underline() {
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(99.0, 161.0, 498.5)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items[0].is_underline);
}
#[test]
fn thin_filled_rect_under_baseline_marks_underline() {
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
let rects = vec![thin_rect(100.0, 497.8, 60.0)];
mark_underlined_items(&mut items, &rects, &[], 1);
assert!(items[0].is_underline);
}
#[test]
fn long_rule_under_multiple_items_marks_each() {
// One underline drawn under a whole sentence: every overlapped
// item gets the flag.
let mut items = vec![
item("first", 100.0, 500.0, 40.0, 10.0),
item("second", 145.0, 500.0, 50.0, 10.0),
];
let lines = vec![hline(98.0, 200.0, 498.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items[0].is_underline);
assert!(items[1].is_underline);
}
#[test]
fn line_far_below_baseline_is_not_an_underline() {
// A horizontal rule 30pt below (section divider) must not mark.
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(90.0, 300.0, 470.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(!items[0].is_underline);
}
#[test]
fn thick_stroked_line_is_not_an_underline() {
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let mut line = hline(99.0, 161.0, 498.5);
line.stroke_width = 4.0;
mark_underlined_items(&mut items, &[], &[line], 1);
assert!(!items[0].is_underline);
}
#[test]
fn line_above_baseline_is_not_an_underline() {
// Strikethrough / overline geometry must not mark.
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![hline(90.0, 300.0, 505.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(!items[0].is_underline);
}
#[test]
fn insufficient_horizontal_overlap_is_not_an_underline() {
// Rule under only a quarter of the item (e.g. neighboring cell
// border) must not mark.
let mut items = vec![item("wide text item", 100.0, 500.0, 100.0, 10.0)];
let lines = vec![hline(100.0, 125.0, 498.5)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(!items[0].is_underline);
}
#[test]
fn negative_width_rect_is_normalized_and_marks_underline() {
// A CTM with negative x-scale (or negative `re` operands) produces
// rects whose width is negative; the rule extents must normalize.
let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
let rects = vec![PdfRect {
x: 160.0,
y: 497.8,
width: -60.0,
height: 0.8,
page: 1,
}];
mark_underlined_items(&mut items, &rects, &[], 1);
assert!(items[0].is_underline);
}
#[test]
fn negative_height_band_is_not_an_underline() {
// A 14pt band expressed with negative height must not pass the
// thickness check via sign trickery.
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let rects = vec![PdfRect {
x: 95.0,
y: 509.0,
width: 80.0,
height: -14.0,
page: 1,
}];
mark_underlined_items(&mut items, &rects, &[], 1);
assert!(!items[0].is_underline);
}
#[test]
fn thick_band_is_not_an_underline() {
// A highlight bar / filled cell background (tall rect) must not mark.
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let rects = vec![PdfRect {
x: 95.0,
y: 495.0,
width: 80.0,
height: 14.0,
page: 1,
}];
mark_underlined_items(&mut items, &rects, &[], 1);
assert!(!items[0].is_underline);
}
#[test]
fn vertical_line_is_not_an_underline() {
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let lines = vec![UnderlineLine {
x1: 120.0,
y1: 498.0,
x2: 120.0,
y2: 400.0,
stroke_width: 1.0,
page: 1,
}];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(!items[0].is_underline);
}
#[test]
fn other_pages_graphics_do_not_mark() {
let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
let mut line = hline(99.0, 161.0, 498.5);
line.page = 2;
mark_underlined_items(&mut items, &[], &[line], 1);
assert!(!items[0].is_underline);
}
#[test]
fn repeated_table_row_rules_do_not_mark_cell_text() {
let mut items = vec![
item("A", 110.0, 500.0, 20.0, 10.0),
item("B", 110.0, 480.0, 20.0, 10.0),
item("C", 110.0, 460.0, 20.0, 10.0),
];
let lines = vec![
hline(100.0, 150.0, 498.0),
hline(100.0, 150.0, 478.0),
hline(100.0, 150.0, 458.0),
];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items.iter().all(|item| !item.is_underline));
}
#[test]
fn row_separator_under_spaced_column_labels_is_not_an_underline() {
let mut items = vec![
item("Date", 100.0, 500.0, 25.0, 10.0),
item("Rate", 200.0, 500.0, 25.0, 10.0),
item("Yield", 300.0, 500.0, 30.0, 10.0),
];
let lines = vec![hline(90.0, 340.0, 498.0)];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items.iter().all(|item| !item.is_underline));
}
#[test]
fn same_row_spaced_rule_segments_do_not_mark_column_labels() {
let mut items = vec![
item("Date", 100.0, 500.0, 25.0, 10.0),
item("Rate", 200.0, 500.0, 25.0, 10.0),
item("Yield", 300.0, 500.0, 30.0, 10.0),
];
let lines = vec![
hline(98.0, 128.0, 498.0),
hline(198.0, 228.0, 498.0),
hline(298.0, 333.0, 498.0),
];
mark_underlined_items(&mut items, &[], &lines, 1);
assert!(items.iter().all(|item| !item.is_underline));
}
}
+3
View File
@@ -294,6 +294,7 @@ fn extract_form_xobject_text_inner(
page: page_num,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Image,
mcid: None,
});
@@ -438,6 +439,7 @@ fn extract_form_xobject_text_inner(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: None,
});
@@ -586,6 +588,7 @@ fn extract_form_xobject_text_inner(
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
is_underline: false,
item_type: ItemType::Text,
mcid: None,
});
+3
View File
@@ -4882,6 +4882,7 @@ mod text_cluster_column_undercount_tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -5156,6 +5157,7 @@ mod table_candidate_selection_tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -5902,6 +5904,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
+1
View File
@@ -1169,6 +1169,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: crate::types::ItemType::Text,
mcid,
}
+1
View File
@@ -1217,6 +1217,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: crate::types::ItemType::Text,
mcid: None,
}
+1
View File
@@ -542,6 +542,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid,
}
+3
View File
@@ -266,6 +266,8 @@ pub struct PyTextItem {
#[pyo3(get)]
pub is_italic: bool,
#[pyo3(get)]
pub is_underline: bool,
#[pyo3(get)]
pub item_type: String,
}
@@ -349,6 +351,7 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
item_type: item_type_str(&item.item_type),
})
.collect()
+1
View File
@@ -104,6 +104,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
page: first_item.page,
is_bold: first_item.is_bold,
is_italic: first_item.is_italic,
is_underline: first_item.is_underline,
item_type: first_item.item_type.clone(),
mcid: first_item.mcid,
});
+1
View File
@@ -393,6 +393,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
+3
View File
@@ -2314,6 +2314,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -3272,6 +3273,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: crate::types::ItemType::Text,
mcid: None,
});
@@ -3581,6 +3583,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: crate::types::ItemType::Text,
mcid: None,
});
+1
View File
@@ -586,6 +586,7 @@ mod tests {
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid,
}
+1
View File
@@ -108,6 +108,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
+3
View File
@@ -514,6 +514,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -866,6 +867,7 @@ mod tests {
font: String::new(),
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
page: 1,
@@ -902,6 +904,7 @@ mod tests {
font: String::new(),
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
page: 1,
+4
View File
@@ -235,6 +235,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
@@ -255,6 +256,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
page: item.page,
is_bold: item.is_bold,
is_italic: item.is_italic,
is_underline: item.is_underline,
item_type: item.item_type.clone(),
mcid: item.mcid,
});
@@ -1429,6 +1431,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1446,6 +1449,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
+3
View File
@@ -883,6 +883,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -1002,6 +1003,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
});
@@ -1078,6 +1080,7 @@ mod tests {
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
+4
View File
@@ -116,6 +116,10 @@ pub struct TextItem {
pub is_bold: bool,
/// Whether the font is italic
pub is_italic: bool,
/// Whether the text is underlined (drawn rule/thin rect under the
/// baseline — PDFs have no underline font flag, so this is detected
/// geometrically after extraction; see `extractor::underline`).
pub is_underline: bool,
/// Type of item (text, image, link)
pub item_type: ItemType,
/// Marked Content ID from the content stream's BDC/BMC operator.
+2
View File
@@ -104,6 +104,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
page,
is_bold: false,
is_italic: false,
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}
@@ -129,6 +130,7 @@ fn make_text_item_with_font(
page,
is_bold: is_bold_font(font),
is_italic: is_italic_font(font),
is_underline: false,
item_type: ItemType::Text,
mcid: None,
}