From 422a2ff118af6371733514606b8043e6a67d0b35 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:20:18 -0700 Subject: [PATCH] feat(extractor): geometric underline detection on TextItem (#116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(extractor): geometric underline detection on TextItem (ENG-5015) PDFs carry no underline font flag — underlines are stroked horizontal lines or thin filled rects drawn under the baseline. Correlate those graphics (already parsed from the content stream) with text items in a post-pass: a rule within ~0.35em below the baseline covering >=60% of an item's width marks is_underline. Exposed through the napi and python bindings. Verified on real docs: 4/4 underlined sentences flagged on a Japanese report, links/headings flagged on 8 of 10 underline-bearing eval docs, zero flags on docs without underlines. Known FP source (table cell borders) documented — downstream applies inline styling only to plain-text regions. napi 1.9.8 -> 1.9.9. Co-authored-by: Cursor * fix(extractor): underline rules only from painted rects, normalized extents (review) Two review fixes: (1) normalize rect extents before the thickness/width checks — `re` operands pass through the CTM so width/height can be negative, which missed negative-width rules and let negative-height bands pass as thin; (2) only feed painted rects to underline detection — `re` rects now wait in a pending list until a paint operator (S/s, f/F/ f*, B/B*/b/b*) confirms them, and `re W n` clip-only paths are discarded at `n`, so invisible clip boundaries no longer underline nearby text. Marking moved into content_stream where paint state lives (pre-rotation, consistent device space). Co-authored-by: Cursor * fix(extractor): harden underline detection * feat(cli): export positioned text item json --------- Co-authored-by: Cursor --- README.md | 3 + napi/package.json | 2 +- napi/src/lib.rs | 4 + pdf_inspector.pyi | 1 + src/bin/pdf2md.rs | 106 ++++++- src/extractor/content_stream.rs | 220 +++++++++++++- src/extractor/layout.rs | 2 + src/extractor/links.rs | 2 + src/extractor/mod.rs | 119 +++++++- src/extractor/underline.rs | 500 ++++++++++++++++++++++++++++++++ src/extractor/xobjects.rs | 3 + src/lib.rs | 3 + src/markdown/convert.rs | 1 + src/markdown/mod.rs | 1 + src/markdown/preprocess.rs | 1 + src/python.rs | 3 + src/tables/detect_heuristic.rs | 1 + src/tables/detect_lines.rs | 1 + src/tables/detect_rects.rs | 3 + src/tables/detect_struct.rs | 1 + src/tables/financial.rs | 1 + src/tables/grid.rs | 3 + src/tables/mod.rs | 4 + src/text_utils.rs | 3 + src/types.rs | 4 + tests/integration_tests.rs | 2 + 26 files changed, 978 insertions(+), 16 deletions(-) create mode 100644 src/extractor/underline.rs diff --git a/README.md b/README.md index c3a5b7a..d08d0f0 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ pdf2md document.pdf # JSON output (for piping) pdf2md document.pdf --json +# Positioned TextItem JSON, including is_underline metadata +pdf2md document.pdf --items-json + # Raw markdown only (no headers) pdf2md document.pdf --raw diff --git a/napi/package.json b/napi/package.json index bdfc195..60a85d2 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.9.8", + "version": "1.9.9", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 3f8301f..4ec4bc0 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -80,6 +80,9 @@ pub struct TextItem { pub page: u32, pub is_bold: bool, pub is_italic: bool, + /// Underline detected geometrically (drawn rule/thin rect under the + /// baseline) — PDFs carry no underline font flag. + pub is_underline: bool, pub item_type: ItemType, /// URL for link items, `None` for other types. pub link_url: Option, @@ -290,6 +293,7 @@ pub fn extract_text_with_positions( page: item.page, is_bold: item.is_bold, is_italic: item.is_italic, + is_underline: item.is_underline, item_type, link_url, } diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index fa89917..21b1153 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -38,6 +38,7 @@ class TextItem: page: int is_bold: bool is_italic: bool + is_underline: bool item_type: str class RegionText: diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index 7fe6b62..a87303f 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -1,6 +1,10 @@ //! CLI tool for PDF to Markdown conversion -use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode}; +use pdf_inspector::extractor::ItemType; +use pdf_inspector::{ + extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions, + PdfType, ProcessMode, TextItem, +}; use std::collections::HashSet; use std::env; use std::fmt::Write; @@ -47,6 +51,92 @@ fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> Stri .join(",") } +fn item_type_label(item_type: &ItemType) -> &'static str { + match item_type { + ItemType::Text => "text", + ItemType::Image => "image", + ItemType::Link(_) => "link", + ItemType::FormField => "form_field", + } +} + +fn format_items_json(items: &[TextItem]) -> String { + let underlined_count = items.iter().filter(|item| item.is_underline).count(); + let items_json = items + .iter() + .map(|item| { + let mcid = item + .mcid + .map(|value| value.to_string()) + .unwrap_or_else(|| "null".to_string()); + let link_url = match &item.item_type { + ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)), + _ => String::new(), + }; + format!( + r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#, + json_escape(&item.text), + item.page, + item.x, + item.y, + item.width, + item.height, + json_escape(&item.font), + item.font_size, + item.is_bold, + item.is_italic, + item.is_underline, + item_type_label(&item.item_type), + mcid, + link_url, + ) + }) + .collect::>() + .join(","); + + format!( + r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#, + items.len(), + underlined_count, + items_json + ) +} + +#[cfg(test)] +mod tests { + use super::format_items_json; + use pdf_inspector::extractor::ItemType; + use pdf_inspector::TextItem; + + #[test] + fn items_json_includes_position_and_underline_metadata() { + let items = vec![TextItem { + text: "A \"quoted\" item".to_string(), + x: 12.345, + y: 67.891, + width: 23.456, + height: 9.876, + font: "F1".to_string(), + font_size: 10.0, + page: 2, + is_bold: false, + is_italic: true, + is_underline: true, + item_type: ItemType::Text, + mcid: Some(7), + }]; + + let json = format_items_json(&items); + + assert!(json.contains(r#""text":"A \"quoted\" item""#)); + assert!(json.contains(r#""page":2"#)); + assert!(json.contains(r#""x":12.35"#)); + assert!(json.contains(r#""is_underline":true"#)); + assert!(json.contains(r#""item_type":"text""#)); + assert!(json.contains(r#""mcid":7"#)); + } +} + /// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers. fn parse_page_spec(spec: &str) -> Result, String> { let mut pages = HashSet::new(); @@ -104,6 +194,7 @@ fn main() { if args.len() < 2 { eprintln!("Usage: {} [output_file]", args[0]); eprintln!(" {} --json", args[0]); + eprintln!(" {} --items-json", args[0]); eprintln!(" {} --raw", args[0]); eprintln!(); eprintln!("Converts PDF to Markdown with smart type detection."); @@ -111,6 +202,7 @@ fn main() { eprintln!(); eprintln!("Options:"); eprintln!(" --json Output result as JSON"); + eprintln!(" --items-json Output positioned TextItem JSON"); eprintln!(" --raw Output only markdown (no headers)"); eprintln!(" --pages Insert page break markers ()"); eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)"); @@ -121,6 +213,7 @@ fn main() { let pdf_path = &args[1]; let json_output = args.iter().any(|a| a == "--json"); + let items_json_output = args.iter().any(|a| a == "--items-json"); let raw_output = args.iter().any(|a| a == "--raw"); let page_numbers = args.iter().any(|a| a == "--pages"); let detect_only = args.iter().any(|a| a == "--detect-only"); @@ -145,6 +238,17 @@ fn main() { }) }); + if items_json_output { + match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) { + Ok(items) => println!("{}", format_items_json(&items)), + Err(e) => { + println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string())); + process::exit(1); + } + } + return; + } + let output_file = args .get(2) .filter(|a| !a.starts_with("--")) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 85229e7..abd3ae7 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -17,6 +17,7 @@ use super::fonts::{ build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand, get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache, }; +use super::underline::UnderlineLine; use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; use super::{get_number, image_bbox_from_ctm, multiply_matrices}; @@ -74,6 +75,37 @@ fn strip_pdf_comments(data: &[u8]) -> Vec { result } +fn transform_path_point(x: f32, y: f32, ctm: &[f32; 6]) -> (f32, f32) { + ( + x * ctm[0] + y * ctm[2] + ctm[4], + x * ctm[1] + y * ctm[3] + ctm[5], + ) +} + +fn transformed_stroke_width( + line_width: f32, + ctm: &[f32; 6], + x1: f32, + y1: f32, + x2: f32, + y2: f32, +) -> f32 { + let user_width = line_width.abs(); + let dx = x2 - x1; + let dy = y2 - y1; + let len = (dx * dx + dy * dy).sqrt(); + if len <= f32::EPSILON { + return user_width; + } + + // PDF stroke width scales perpendicular to the path direction. + let nx = -dy / len; + let ny = dx / len; + let ndx = nx * ctm[0] + ny * ctm[2]; + let ndy = nx * ctm[1] + ny * ctm[3]; + user_width * (ndx * ndx + ndy * ndy).sqrt() +} + /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates /// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( @@ -89,6 +121,7 @@ pub(crate) fn extract_page_text_items( let mut rects: Vec = Vec::new(); let mut clip_rects: Vec = Vec::new(); let mut lines: Vec = Vec::new(); + let mut underline_lines: Vec = Vec::new(); // Path construction state for m/l/h → S/s line extraction let mut path_subpath_start: Option<(f32, f32)> = None; @@ -97,6 +130,12 @@ pub(crate) fn extract_page_text_items( // Completed subpaths (each a vec of line segments) for f/f* rect extraction let mut pending_subpaths: Vec> = Vec::new(); let mut fill_rects: Vec = Vec::new(); + // `re` rects awaiting a paint operator. Underline detection must only + // see painted rects: a `re W n` clip path or `re n` no-op draws nothing + // on the page, so treating every `re` as ink would underline text that + // merely sits near an invisible clip boundary. + let mut pending_re_rects: Vec = Vec::new(); + let mut painted_rects: Vec = Vec::new(); // Get fonts for encoding let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); @@ -188,10 +227,12 @@ pub(crate) fn extract_page_text_items( // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible + let mut line_width: f32 = 1.0; #[derive(Clone)] struct SavedGraphicsState { ctm: [f32; 6], text_rendering_mode: i32, + line_width: f32, char_spacing: f32, word_spacing: f32, text_leading: f32, @@ -240,6 +281,7 @@ pub(crate) fn extract_page_text_items( gstate_stack.push(SavedGraphicsState { ctm, text_rendering_mode, + line_width, char_spacing, word_spacing, text_leading, @@ -252,6 +294,7 @@ pub(crate) fn extract_page_text_items( if let Some(saved) = gstate_stack.pop() { ctm = saved.ctm; text_rendering_mode = saved.text_rendering_mode; + line_width = saved.line_width; char_spacing = saved.char_spacing; word_spacing = saved.word_spacing; text_leading = saved.text_leading; @@ -273,6 +316,11 @@ pub(crate) fn extract_page_text_items( ctm = multiply_matrices(&new_matrix, &ctm); } } + "w" => { + if let Some(width) = op.operands.first().and_then(get_number) { + line_width = width; + } + } "BT" => { // Begin text block in_text_block = true; @@ -441,6 +489,7 @@ pub(crate) fn extract_page_text_items( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); @@ -606,6 +655,7 @@ pub(crate) fn extract_page_text_items( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); @@ -669,6 +719,7 @@ pub(crate) fn extract_page_text_items( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: current_mcid(&marked_content_stack), }); @@ -705,6 +756,7 @@ pub(crate) fn extract_page_text_items( page: page_num, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Image, mcid: current_mcid(&marked_content_stack), }); @@ -801,6 +853,7 @@ pub(crate) fn extract_page_text_items( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: entry .mcid @@ -825,13 +878,19 @@ pub(crate) fn extract_page_text_items( let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5]; let w_dev = rw * ctm[0]; let h_dev = rh * ctm[3]; - rects.push(PdfRect { + let rect = PdfRect { x: x_dev, y: y_dev, width: w_dev, height: h_dev, page: page_num, - }); + }; + // Underline detection must only see rects that are + // actually painted — a `re` used purely as a clip path + // (`re W n`) or discarded (`re n`) draws nothing. Hold + // the rect as pending until a paint operator confirms it. + pending_re_rects.push(rect.clone()); + rects.push(rect); } } // ── Path construction operators ────────────────────── @@ -881,10 +940,8 @@ pub(crate) fn extract_page_text_items( } } for (x1, y1, x2, y2) in pending_lines.drain(..) { - let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4]; - let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5]; - let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4]; - let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5]; + let (x1d, y1d) = transform_path_point(x1, y1, &ctm); + let (x2d, y2d) = transform_path_point(x2, y2, &ctm); lines.push(PdfLine { x1: x1d, y1: y1d, @@ -892,7 +949,16 @@ pub(crate) fn extract_page_text_items( y2: y2d, page: page_num, }); + underline_lines.push(UnderlineLine { + x1: x1d, + y1: y1d, + x2: x2d, + y2: y2d, + stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2), + page: page_num, + }); } + painted_rects.append(&mut pending_re_rects); pending_subpaths.clear(); path_subpath_start = None; path_current = None; @@ -908,10 +974,8 @@ pub(crate) fn extract_page_text_items( } } for (x1, y1, x2, y2) in pending_lines.drain(..) { - let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4]; - let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5]; - let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4]; - let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5]; + let (x1d, y1d) = transform_path_point(x1, y1, &ctm); + let (x2d, y2d) = transform_path_point(x2, y2, &ctm); lines.push(PdfLine { x1: x1d, y1: y1d, @@ -919,7 +983,16 @@ pub(crate) fn extract_page_text_items( y2: y2d, page: page_num, }); + underline_lines.push(UnderlineLine { + x1: x1d, + y1: y1d, + x2: x2d, + y2: y2d, + stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2), + page: page_num, + }); } + painted_rects.append(&mut pending_re_rects); pending_subpaths.clear(); path_subpath_start = None; path_current = None; @@ -977,6 +1050,7 @@ pub(crate) fn extract_page_text_items( } } } + painted_rects.append(&mut pending_re_rects); pending_lines.clear(); path_subpath_start = None; path_current = None; @@ -1042,7 +1116,10 @@ pub(crate) fn extract_page_text_items( // Do NOT clear pending_lines — the following `n` does that } "n" => { - // end path (no-op): discard + // end path (no-op): discard — including any `re` rects that + // were only ever part of a clip path (`re W n`), which draw + // no ink and must not feed underline detection. + pending_re_rects.clear(); pending_lines.clear(); pending_subpaths.clear(); path_subpath_start = None; @@ -1052,6 +1129,12 @@ pub(crate) fn extract_page_text_items( } } + // Underline detection reads only painted ink: `re` rects confirmed by + // a paint operator plus filled-subpath rects — never clip-only rects, + // which draw nothing. + let mut underline_rects = painted_rects; + underline_rects.extend(fill_rects.iter().cloned()); + // Only use clip/fill rects when no `re` rects exist on this page. // Clip rects take priority over fill rects, but first we deduplicate // them: some PDFs wrap every text block in a full-page W* clip path, @@ -1081,8 +1164,17 @@ pub(crate) fn extract_page_text_items( // Some PDFs embed landscape content in portrait pages using a rotated text // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW). The layout engine // assumes x=horizontal, y=vertical — so we swap coordinates to match. - let (items, rects, lines, coords_rotated) = + let (mut items, rects, lines, coords_rotated) = correct_rotated_page(items, rects, lines, &rotation_votes); + if coords_rotated { + rotate_underline_graphics(&mut underline_rects, &mut underline_lines); + } + super::underline::mark_underlined_items( + &mut items, + &underline_rects, + &underline_lines, + page_num, + ); let items = super::merge_text_items(items); let items = super::merge_subscript_items(items); @@ -1168,6 +1260,27 @@ fn correct_rotated_page( (items, rects, lines, true) } +fn rotate_underline_graphics(rects: &mut [PdfRect], lines: &mut [UnderlineLine]) { + for rect in rects { + let new_x = rect.y; + let new_y = -(rect.x + rect.width.abs()); + rect.x = new_x; + rect.y = new_y; + std::mem::swap(&mut rect.width, &mut rect.height); + } + + for line in lines { + let new_x1 = line.y1; + let new_y1 = -line.x1; + let new_x2 = line.y2; + let new_y2 = -line.x2; + line.x1 = new_x1; + line.y1 = new_y1; + line.x2 = new_x2; + line.y2 = new_y2; + } +} + /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance). /// Some PDFs emit a full-page clip path for every text block, producing /// thousands of identical rects. After dedup these collapse to one rect, @@ -1217,6 +1330,57 @@ mod tests { } } + fn simple_doc_with_content(content: &[u8]) -> (lopdf::Document, lopdf::ObjectId) { + use lopdf::{dictionary, Object, Stream}; + + let mut doc = lopdf::Document::new(); + let widths: Vec = (0..=255).map(|_| 600.into()).collect(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "FirstChar" => 0, + "LastChar" => 255, + "Widths" => Object::Array(widths), + }); + let content_id = doc.add_object(Object::Stream(Stream::new( + dictionary! {}, + content.to_vec(), + ))); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Contents" => Object::Reference(content_id), + "Resources" => dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(font_id), + }, + }, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + let pages_id = doc.add_object(dictionary! { + "Type" => "Pages", + "Count" => Object::Integer(1), + "Kids" => vec![Object::Reference(page_id)], + }); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => Object::Reference(pages_id), + }); + doc.trailer.set("Root", Object::Reference(catalog_id)); + + (doc, page_id) + } + + fn extract_simple_items(content: &[u8]) -> Vec { + use crate::tounicode::FontCMaps; + + let (doc, page_id) = simple_doc_with_content(content); + let font_cmaps = FontCMaps::from_doc(&doc); + let ((items, _, _), _, _) = + extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + items + } + #[test] fn test_dedup_rects_identical() { let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759]; @@ -1266,6 +1430,38 @@ mod tests { assert_eq!(single.len(), 1); } + #[test] + fn thick_stroked_rule_does_not_mark_underline() { + let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (THICK) Tj ET +4 w +100 498 m 170 498 l S +BT /F1 12 Tf 1 0 0 1 100 480 Tm (THIN) Tj ET +1 w +100 478 m 160 478 l S"; + + let items = extract_simple_items(content); + let thick = items.iter().find(|item| item.text == "THICK").unwrap(); + let thin = items.iter().find(|item| item.text == "THIN").unwrap(); + + assert!(!thick.is_underline); + assert!(thin.is_underline); + } + + #[test] + fn rotated_page_underline_is_detected_after_coordinate_correction() { + let content = b"BT /F1 12 Tf 0 1 -1 0 200 100 Tm (HELLO) Tj ET +BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET +1 w +202 100 m 202 170 l S"; + + let items = extract_simple_items(content); + let hello = items.iter().find(|item| item.text == "HELLO").unwrap(); + let world = items.iter().find(|item| item.text == "WORLD").unwrap(); + + assert!(hello.is_underline); + assert!(!world.is_underline); + } + #[test] fn test_skip_excessive_operations() { use crate::tounicode::FontCMaps; diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index 0a4530e..7c017e6 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -1494,6 +1494,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -1623,6 +1624,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }); diff --git a/src/extractor/links.rs b/src/extractor/links.rs index 74a1fae..b08e97a 100644 --- a/src/extractor/links.rs +++ b/src/extractor/links.rs @@ -78,6 +78,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V page: page_num, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Link(url), mcid: None, }); @@ -316,6 +317,7 @@ pub(crate) fn walk_form_fields( page: page_num, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::FormField, mcid: None, }); diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index 42215c0..75d7b0d 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -6,11 +6,12 @@ pub(crate) mod content_stream; mod fonts; mod layout; mod links; +pub(crate) mod underline; mod xobjects; use crate::text_utils::is_rtl_text; use crate::tounicode::FontCMaps; -use crate::types::{PageExtraction, TextItem}; +use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem}; use crate::PdfError; use log::debug; use lopdf::{Document, Object, ObjectId}; @@ -180,6 +181,7 @@ fn extract_positioned_text_impl( if threshold > 0.10 { page_thresholds.insert(*page_num, threshold); } + suppress_table_underlines(&mut items, &rects, &lines, *page_num); debug!( "page {}: {} text items, {} rects, {} lines{}", page_num, @@ -226,6 +228,38 @@ fn extract_positioned_text_impl( )) } +fn suppress_table_underlines( + items: &mut [TextItem], + rects: &[PdfRect], + lines: &[PdfLine], + page: u32, +) { + if !items.iter().any(|item| item.is_underline) { + return; + } + + let mut table_item_indices: HashSet = HashSet::new(); + + if !rects.is_empty() { + let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page); + for table in rect_tables { + table_item_indices.extend(table.item_indices); + } + } + + if !lines.is_empty() { + for table in crate::tables::detect_tables_from_lines(items, lines, page) { + table_item_indices.extend(table.item_indices); + } + } + + for index in table_item_indices { + if let Some(item) = items.get_mut(index) { + item.is_underline = false; + } + } +} + // --------------------------------------------------------------------------- // Shared helpers (used by submodules via `super::`) // --------------------------------------------------------------------------- @@ -525,6 +559,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { let first = group[i]; let mut text = first.text.clone(); let mut end_x = first.x + effective_merge_width(first); + let mut is_underline = first.is_underline; let mut j = i + 1; while j < group.len() { @@ -571,6 +606,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { text.push(' '); } text.push_str(&next.text); + is_underline |= next.is_underline; let next_end = next.x + effective_merge_width(next); end_x = if *preserve_stream_order { end_x.max(next_end) @@ -591,6 +627,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { page: first.page, is_bold: first.is_bold, is_italic: first.is_italic, + is_underline, item_type: first.item_type.clone(), mcid: first.mcid, }); @@ -701,7 +738,7 @@ pub(crate) fn get_number(obj: &Object) -> Option { mod tests { use super::*; use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items}; - use crate::types::{ItemType, TextLine}; + use crate::types::{ItemType, PdfLine, TextLine}; use layout::{detect_columns, is_newspaper_layout, ColumnRegion}; fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem { @@ -716,6 +753,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -726,6 +764,16 @@ mod tests { item } + fn make_line(x1: f32, y1: f32, x2: f32, y2: f32) -> PdfLine { + PdfLine { + x1, + y1, + x2, + y2, + page: 1, + } + } + #[test] fn trace_text_preview_truncates_on_char_boundary() { let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}'); @@ -774,6 +822,21 @@ mod tests { assert_eq!(merged[0].text, "hello world"); } + #[test] + fn merge_items_preserves_underline_from_later_fragment() { + let mut items = vec![ + make_merge_item("pre", 100.0, 18.0), + make_merge_item("fix", 119.0, 18.0), + ]; + items[1].is_underline = true; + + let merged = merge_text_items(items); + + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "prefix"); + assert!(merged[0].is_underline); + } + #[test] fn merge_items_preserves_stream_order_for_backtracking_heading() { // Some tagged PDFs emit first-letter ActualText fragments, then reset @@ -851,6 +914,35 @@ mod tests { assert_eq!(texts, vec!["•", "Distant item"]); } + #[test] + fn suppress_table_underlines_clears_line_detected_table_items() { + let mut items = vec![ + make_merge_item("H1", 125.0, 20.0), + make_merge_item("H2", 225.0, 20.0), + make_merge_item("A", 125.0, 20.0), + make_merge_item("B", 225.0, 20.0), + ]; + items[0].y = 490.0; + items[1].y = 490.0; + items[2].y = 470.0; + items[3].y = 470.0; + for item in &mut items { + item.is_underline = true; + } + let lines = vec![ + make_line(100.0, 500.0, 300.0, 500.0), + make_line(100.0, 480.0, 300.0, 480.0), + make_line(100.0, 460.0, 300.0, 460.0), + make_line(100.0, 460.0, 100.0, 500.0), + make_line(200.0, 460.0, 200.0, 500.0), + make_line(300.0, 460.0, 300.0, 500.0), + ]; + + suppress_table_underlines(&mut items, &[], &lines, 1); + + assert!(items.iter().all(|item| !item.is_underline)); + } + #[test] fn test_group_into_lines() { let items = vec![ @@ -865,6 +957,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -879,6 +972,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -893,6 +987,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -947,6 +1042,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -961,6 +1057,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -975,6 +1072,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1000,6 +1098,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1014,6 +1113,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1028,6 +1128,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1055,6 +1156,7 @@ mod tests { page: 1, is_bold: true, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -1089,6 +1191,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -1124,6 +1227,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1138,6 +1242,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1152,6 +1257,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1174,6 +1280,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -1286,6 +1393,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1300,6 +1408,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1324,6 +1433,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1338,6 +1448,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }, @@ -1378,6 +1489,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }], @@ -1422,6 +1534,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }], @@ -1466,6 +1579,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, }], @@ -1503,6 +1617,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } diff --git a/src/extractor/underline.rs b/src/extractor/underline.rs new file mode 100644 index 0000000..b5ea643 --- /dev/null +++ b/src/extractor/underline.rs @@ -0,0 +1,500 @@ +//! Geometric underline detection. +//! +//! PDFs have no underline font flag — underlines are drawn as separate +//! graphics: stroked horizontal lines (`l`/`S` operators) or thin filled +//! rectangles (`re`/`f`). This pass correlates those graphics with text +//! items after extraction: an item is underlined when a horizontal +//! line/thin rect sits just below its baseline and covers most of its +//! horizontal extent. +//! +//! Repeated same-span rules are treated as table/form rulings rather than +//! underlines, which avoids marking every cell in ruled tables. + +use std::collections::HashSet; + +use crate::types::{ItemType, PdfRect, TextItem}; + +/// Max thickness (pt) for a stroked line / filled rect to count as an +/// underline rule rather than a border or decorative band. +const MAX_RULE_THICKNESS: f32 = 2.0; + +/// Fraction of the item's width that the rule must cover horizontally. +const MIN_X_OVERLAP: f32 = 0.6; + +/// Same-span rules repeated at this many y-levels are usually table/form +/// rulings, not semantic underlines. +const MIN_REPEATED_RULE_LEVELS: usize = 3; + +/// Vertical tolerance for considering two rules to be on the same row edge. +const RULE_Y_DEDUP_EPS: f32 = 2.0; + +/// Horizontal span similarity required when clustering repeated rulings. +const RULE_SPAN_OVERLAP_RATIO: f32 = 0.8; +const RULE_SPAN_WIDTH_RATIO: f32 = 1.5; + +/// Multiple separated rule segments on one row are usually per-column table +/// header/body separators. +const MIN_SEGMENTED_ROW_RULES: usize = 3; +const MIN_SEGMENTED_ROW_GAPS: usize = 2; +const SEGMENTED_ROW_GAP_MIN: f32 = 12.0; + +/// A single rule under several widely separated items is usually a table +/// header/body separator, not a sentence underline. +const MIN_TABULAR_RULE_ITEMS: usize = 3; +const MIN_TABULAR_RULE_GAPS: usize = 2; +const TABULAR_RULE_GAP_EM: f32 = 2.0; + +#[derive(Clone)] +pub(crate) struct UnderlineLine { + pub(crate) x1: f32, + pub(crate) y1: f32, + pub(crate) x2: f32, + pub(crate) y2: f32, + pub(crate) stroke_width: f32, + pub(crate) page: u32, +} + +/// A horizontal rule candidate in page coordinates (PDF y-up). +#[derive(Clone)] +struct Rule { + x1: f32, + x2: f32, + y: f32, +} + +impl Rule { + fn width(&self) -> f32 { + self.x2 - self.x1 + } +} + +fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) -> Vec { + let mut rules: Vec = Vec::new(); + for l in lines { + if l.page != page { + continue; + } + // Horizontal stroked line (tolerate slight skew). + if l.stroke_width <= MAX_RULE_THICKNESS && (l.y1 - l.y2).abs() <= MAX_RULE_THICKNESS { + let (x1, x2) = if l.x1 <= l.x2 { + (l.x1, l.x2) + } else { + (l.x2, l.x1) + }; + if x2 - x1 > 1.0 { + rules.push(Rule { + x1, + x2, + y: (l.y1 + l.y2) / 2.0, + }); + } + } + } + for r in rects { + if r.page != page { + continue; + } + // Thin filled rect used as an underline rule. Extents are + // normalized first: `re` operands pass through the CTM, so + // width/height can be negative (flipped axes / negative scale) — + // without normalization negative-width rules are missed and + // negative-height bands sneak past the thickness check. + let (x1, x2) = if r.width >= 0.0 { + (r.x, r.x + r.width) + } else { + (r.x + r.width, r.x) + }; + if r.height.abs() <= MAX_RULE_THICKNESS && x2 - x1 > 1.0 { + rules.push(Rule { + x1, + x2, + y: r.y + r.height / 2.0, + }); + } + } + rules +} + +fn discard_repeated_ruling_rules(rules: Vec) -> Vec { + if rules.len() < MIN_REPEATED_RULE_LEVELS { + return rules; + } + + rules + .iter() + .filter(|rule| { + !is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules) + }) + .cloned() + .collect() +} + +fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool { + let mut y_levels: Vec = rules + .iter() + .filter(|other| has_similar_span(rule, other)) + .map(|other| other.y) + .collect(); + + y_levels.sort_by(|a, b| a.total_cmp(b)); + y_levels.dedup_by(|a, b| (*a - *b).abs() <= RULE_Y_DEDUP_EPS); + y_levels.len() >= MIN_REPEATED_RULE_LEVELS +} + +fn is_segmented_row_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool { + let mut row_rules: Vec<&Rule> = rules + .iter() + .filter(|other| (other.y - rule.y).abs() <= RULE_Y_DEDUP_EPS) + .collect(); + + if row_rules.len() < MIN_SEGMENTED_ROW_RULES { + return false; + } + + row_rules.sort_by(|a, b| a.x1.total_cmp(&b.x1)); + let large_gaps = row_rules + .windows(2) + .filter(|pair| pair[1].x1 - pair[0].x2 > SEGMENTED_ROW_GAP_MIN) + .count(); + + large_gaps >= MIN_SEGMENTED_ROW_GAPS +} + +fn has_similar_span(a: &Rule, b: &Rule) -> bool { + let a_width = a.width(); + let b_width = b.width(); + if a_width <= 1.0 || b_width <= 1.0 { + return false; + } + + let width_ratio = a_width.max(b_width) / a_width.min(b_width); + if width_ratio > RULE_SPAN_WIDTH_RATIO { + return false; + } + + let overlap = a.x2.min(b.x2) - a.x1.max(b.x1); + overlap >= a_width.min(b_width) * RULE_SPAN_OVERLAP_RATIO +} + +fn tabular_row_separator_rule_indices(rules: &[Rule], items: &[TextItem]) -> HashSet { + let mut tabular_rules = HashSet::new(); + + for (rule_idx, rule) in rules.iter().enumerate() { + let mut matched_items: Vec<&TextItem> = items + .iter() + .filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item)) + .collect(); + + if matched_items.len() < MIN_TABULAR_RULE_ITEMS { + continue; + } + + matched_items.sort_by(|a, b| a.x.total_cmp(&b.x)); + let large_gaps = matched_items + .windows(2) + .filter(|pair| { + let left = pair[0]; + let right = pair[1]; + let gap = right.x - (left.x + left.width); + let font_size = left.font_size.max(right.font_size).max(1.0); + gap > font_size * TABULAR_RULE_GAP_EM + }) + .count(); + + if large_gaps >= MIN_TABULAR_RULE_GAPS { + tabular_rules.insert(rule_idx); + } + } + + tabular_rules +} + +fn is_underline_candidate(item: &TextItem) -> bool { + matches!(item.item_type, ItemType::Text) && !item.text.trim().is_empty() && item.width > 0.0 +} + +fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool { + // Vertical window: underlines sit at or slightly below the baseline. + // Fonts draw them at roughly 5-15% of the em below; allow up to 35% + // (min 3pt) below and 1pt above for rounding. + let below = (item.font_size * 0.35).max(3.0); + let y_min = item.y - below; + let y_max = item.y + 1.0; + if rule.y < y_min || rule.y > y_max { + return false; + } + + let ix1 = item.x; + let ix2 = item.x + item.width; + let min_overlap = item.width * MIN_X_OVERLAP; + let overlap = rule.x2.min(ix2) - rule.x1.max(ix1); + overlap >= min_overlap +} + +/// Mark `is_underline` on text items that have a horizontal rule just +/// below their baseline. `items`, `rects`, and `lines` are a single +/// page's extraction output (all in PDF coordinates, y-up, where +/// `TextItem::y` is the text baseline). +pub(crate) fn mark_underlined_items( + items: &mut [TextItem], + rects: &[PdfRect], + lines: &[UnderlineLine], + page: u32, +) { + let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page)); + if rules.is_empty() { + return; + } + let tabular_rules = tabular_row_separator_rule_indices(&rules, items); + + for item in items.iter_mut() { + if !is_underline_candidate(item) { + continue; + } + + for (rule_idx, rule) in rules.iter().enumerate() { + if tabular_rules.contains(&rule_idx) { + continue; + } + if rule_matches_item(rule, item) { + item.is_underline = true; + break; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::ItemType; + + fn item(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width, + height: font_size, + font: "F1".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + is_underline: false, + item_type: ItemType::Text, + mcid: None, + } + } + + fn hline(x1: f32, x2: f32, y: f32) -> UnderlineLine { + UnderlineLine { + x1, + y1: y, + x2, + y2: y, + stroke_width: 1.0, + page: 1, + } + } + + fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect { + PdfRect { + x, + y, + width, + height: 0.8, + page: 1, + } + } + + #[test] + fn stroked_line_under_baseline_marks_underline() { + let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(99.0, 161.0, 498.5)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(items[0].is_underline); + } + + #[test] + fn thin_filled_rect_under_baseline_marks_underline() { + let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)]; + let rects = vec![thin_rect(100.0, 497.8, 60.0)]; + mark_underlined_items(&mut items, &rects, &[], 1); + assert!(items[0].is_underline); + } + + #[test] + fn long_rule_under_multiple_items_marks_each() { + // One underline drawn under a whole sentence: every overlapped + // item gets the flag. + let mut items = vec![ + item("first", 100.0, 500.0, 40.0, 10.0), + item("second", 145.0, 500.0, 50.0, 10.0), + ]; + let lines = vec![hline(98.0, 200.0, 498.0)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(items[0].is_underline); + assert!(items[1].is_underline); + } + + #[test] + fn line_far_below_baseline_is_not_an_underline() { + // A horizontal rule 30pt below (section divider) must not mark. + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(90.0, 300.0, 470.0)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(!items[0].is_underline); + } + + #[test] + fn thick_stroked_line_is_not_an_underline() { + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let mut line = hline(99.0, 161.0, 498.5); + line.stroke_width = 4.0; + + mark_underlined_items(&mut items, &[], &[line], 1); + + assert!(!items[0].is_underline); + } + + #[test] + fn line_above_baseline_is_not_an_underline() { + // Strikethrough / overline geometry must not mark. + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![hline(90.0, 300.0, 505.0)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(!items[0].is_underline); + } + + #[test] + fn insufficient_horizontal_overlap_is_not_an_underline() { + // Rule under only a quarter of the item (e.g. neighboring cell + // border) must not mark. + let mut items = vec![item("wide text item", 100.0, 500.0, 100.0, 10.0)]; + let lines = vec![hline(100.0, 125.0, 498.5)]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(!items[0].is_underline); + } + + #[test] + fn negative_width_rect_is_normalized_and_marks_underline() { + // A CTM with negative x-scale (or negative `re` operands) produces + // rects whose width is negative; the rule extents must normalize. + let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)]; + let rects = vec![PdfRect { + x: 160.0, + y: 497.8, + width: -60.0, + height: 0.8, + page: 1, + }]; + mark_underlined_items(&mut items, &rects, &[], 1); + assert!(items[0].is_underline); + } + + #[test] + fn negative_height_band_is_not_an_underline() { + // A 14pt band expressed with negative height must not pass the + // thickness check via sign trickery. + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let rects = vec![PdfRect { + x: 95.0, + y: 509.0, + width: 80.0, + height: -14.0, + page: 1, + }]; + mark_underlined_items(&mut items, &rects, &[], 1); + assert!(!items[0].is_underline); + } + + #[test] + fn thick_band_is_not_an_underline() { + // A highlight bar / filled cell background (tall rect) must not mark. + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let rects = vec![PdfRect { + x: 95.0, + y: 495.0, + width: 80.0, + height: 14.0, + page: 1, + }]; + mark_underlined_items(&mut items, &rects, &[], 1); + assert!(!items[0].is_underline); + } + + #[test] + fn vertical_line_is_not_an_underline() { + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let lines = vec![UnderlineLine { + x1: 120.0, + y1: 498.0, + x2: 120.0, + y2: 400.0, + stroke_width: 1.0, + page: 1, + }]; + mark_underlined_items(&mut items, &[], &lines, 1); + assert!(!items[0].is_underline); + } + + #[test] + fn other_pages_graphics_do_not_mark() { + let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)]; + let mut line = hline(99.0, 161.0, 498.5); + line.page = 2; + mark_underlined_items(&mut items, &[], &[line], 1); + assert!(!items[0].is_underline); + } + + #[test] + fn repeated_table_row_rules_do_not_mark_cell_text() { + let mut items = vec![ + item("A", 110.0, 500.0, 20.0, 10.0), + item("B", 110.0, 480.0, 20.0, 10.0), + item("C", 110.0, 460.0, 20.0, 10.0), + ]; + let lines = vec![ + hline(100.0, 150.0, 498.0), + hline(100.0, 150.0, 478.0), + hline(100.0, 150.0, 458.0), + ]; + + mark_underlined_items(&mut items, &[], &lines, 1); + + assert!(items.iter().all(|item| !item.is_underline)); + } + + #[test] + fn row_separator_under_spaced_column_labels_is_not_an_underline() { + let mut items = vec![ + item("Date", 100.0, 500.0, 25.0, 10.0), + item("Rate", 200.0, 500.0, 25.0, 10.0), + item("Yield", 300.0, 500.0, 30.0, 10.0), + ]; + let lines = vec![hline(90.0, 340.0, 498.0)]; + + mark_underlined_items(&mut items, &[], &lines, 1); + + assert!(items.iter().all(|item| !item.is_underline)); + } + + #[test] + fn same_row_spaced_rule_segments_do_not_mark_column_labels() { + let mut items = vec![ + item("Date", 100.0, 500.0, 25.0, 10.0), + item("Rate", 200.0, 500.0, 25.0, 10.0), + item("Yield", 300.0, 500.0, 30.0, 10.0), + ]; + let lines = vec![ + hline(98.0, 128.0, 498.0), + hline(198.0, 228.0, 498.0), + hline(298.0, 333.0, 498.0), + ]; + + mark_underlined_items(&mut items, &[], &lines, 1); + + assert!(items.iter().all(|item| !item.is_underline)); + } +} diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 850ffce..06eea8a 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -294,6 +294,7 @@ fn extract_form_xobject_text_inner( page: page_num, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Image, mcid: None, }); @@ -438,6 +439,7 @@ fn extract_form_xobject_text_inner( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: None, }); @@ -586,6 +588,7 @@ fn extract_form_xobject_text_inner( page: page_num, is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), + is_underline: false, item_type: ItemType::Text, mcid: None, }); diff --git a/src/lib.rs b/src/lib.rs index e5a33be..393dec5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4882,6 +4882,7 @@ mod text_cluster_column_undercount_tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -5156,6 +5157,7 @@ mod table_candidate_selection_tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -5902,6 +5904,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index f69898c..d216ade 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -1169,6 +1169,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: crate::types::ItemType::Text, mcid, } diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 61f2692..c4c8116 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -1217,6 +1217,7 @@ mod tests { page, is_bold: false, is_italic: false, + is_underline: false, item_type: crate::types::ItemType::Text, mcid: None, } diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index 00ccec7..fc89cee 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -542,6 +542,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid, } diff --git a/src/python.rs b/src/python.rs index 8d59498..cdd4d65 100644 --- a/src/python.rs +++ b/src/python.rs @@ -266,6 +266,8 @@ pub struct PyTextItem { #[pyo3(get)] pub is_italic: bool, #[pyo3(get)] + pub is_underline: bool, + #[pyo3(get)] pub item_type: String, } @@ -349,6 +351,7 @@ fn convert_text_items(items: Vec) -> Vec { page: item.page, is_bold: item.is_bold, is_italic: item.is_italic, + is_underline: item.is_underline, item_type: item_type_str(&item.item_type), }) .collect() diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 78e064f..60eb0c7 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -104,6 +104,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec Option> page: item.page, is_bold: item.is_bold, is_italic: item.is_italic, + is_underline: item.is_underline, item_type: item.item_type.clone(), mcid: item.mcid, }); diff --git a/src/tables/grid.rs b/src/tables/grid.rs index 7fda970..34b5bea 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -514,6 +514,7 @@ mod tests { page: 1, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -866,6 +867,7 @@ mod tests { font: String::new(), is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, page: 1, @@ -902,6 +904,7 @@ mod tests { font: String::new(), is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, page: 1, diff --git a/src/tables/mod.rs b/src/tables/mod.rs index 83ca12c..270247f 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -235,6 +235,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec Vec Text page, is_bold: false, is_italic: false, + is_underline: false, item_type: ItemType::Text, mcid: None, } @@ -129,6 +130,7 @@ fn make_text_item_with_font( page, is_bold: is_bold_font(font), is_italic: is_italic_font(font), + is_underline: false, item_type: ItemType::Text, mcid: None, }