feat(cli): export positioned text item json

Two review fixes: (1) normalize rect extents before the thickness/width
checks — `re` operands pass through the CTM so width/height can be
negative, which missed negative-width rules and let negative-height
bands pass as thin; (2) only feed painted rects to underline detection —
`re` rects now wait in a pending list until a paint operator (S/s, f/F/
f*, B/B*/b/b*) confirms them, and `re W n` clip-only paths are discarded
at `n`, so invisible clip boundaries no longer underline nearby text.
Marking moved into content_stream where paint state lives (pre-rotation,
consistent device space).

Co-authored-by: Cursor <cursoragent@cursor.com>

README.md

@@ -111,6 +111,9 @@ pdf2md document.pdf
 # JSON output (for piping)
 pdf2md document.pdf --json
 
+# Positioned TextItem JSON, including is_underline metadata
+pdf2md document.pdf --items-json
+
 # Raw markdown only (no headers)
 pdf2md document.pdf --raw
 

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.8",
+  "version": "1.9.9",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -80,6 +80,9 @@ pub struct TextItem {
     pub page: u32,
     pub is_bold: bool,
     pub is_italic: bool,
+    /// Underline detected geometrically (drawn rule/thin rect under the
+    /// baseline) — PDFs carry no underline font flag.
+    pub is_underline: bool,
     pub item_type: ItemType,
     /// URL for link items, `None` for other types.
     pub link_url: Option<String>,
@@ -290,6 +293,7 @@ pub fn extract_text_with_positions(
                     page: item.page,
                     is_bold: item.is_bold,
                     is_italic: item.is_italic,
+                    is_underline: item.is_underline,
                     item_type,
                     link_url,
                 }

pdf_inspector.pyi

@@ -38,6 +38,7 @@ class TextItem:
     page: int
     is_bold: bool
     is_italic: bool
+    is_underline: bool
     item_type: str
 
 class RegionText:

src/bin/pdf2md.rs

@@ -1,6 +1,10 @@
 //! CLI tool for PDF to Markdown conversion
 
-use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode};
+use pdf_inspector::extractor::ItemType;
+use pdf_inspector::{
+    extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions,
+    PdfType, ProcessMode, TextItem,
+};
 use std::collections::HashSet;
 use std::env;
 use std::fmt::Write;
@@ -47,6 +51,92 @@ fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> Stri
         .join(",")
 }
 
+fn item_type_label(item_type: &ItemType) -> &'static str {
+    match item_type {
+        ItemType::Text => "text",
+        ItemType::Image => "image",
+        ItemType::Link(_) => "link",
+        ItemType::FormField => "form_field",
+    }
+}
+
+fn format_items_json(items: &[TextItem]) -> String {
+    let underlined_count = items.iter().filter(|item| item.is_underline).count();
+    let items_json = items
+        .iter()
+        .map(|item| {
+            let mcid = item
+                .mcid
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "null".to_string());
+            let link_url = match &item.item_type {
+                ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)),
+                _ => String::new(),
+            };
+            format!(
+                r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
+                json_escape(&item.text),
+                item.page,
+                item.x,
+                item.y,
+                item.width,
+                item.height,
+                json_escape(&item.font),
+                item.font_size,
+                item.is_bold,
+                item.is_italic,
+                item.is_underline,
+                item_type_label(&item.item_type),
+                mcid,
+                link_url,
+            )
+        })
+        .collect::<Vec<_>>()
+        .join(",");
+
+    format!(
+        r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#,
+        items.len(),
+        underlined_count,
+        items_json
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::format_items_json;
+    use pdf_inspector::extractor::ItemType;
+    use pdf_inspector::TextItem;
+
+    #[test]
+    fn items_json_includes_position_and_underline_metadata() {
+        let items = vec![TextItem {
+            text: "A \"quoted\" item".to_string(),
+            x: 12.345,
+            y: 67.891,
+            width: 23.456,
+            height: 9.876,
+            font: "F1".to_string(),
+            font_size: 10.0,
+            page: 2,
+            is_bold: false,
+            is_italic: true,
+            is_underline: true,
+            item_type: ItemType::Text,
+            mcid: Some(7),
+        }];
+
+        let json = format_items_json(&items);
+
+        assert!(json.contains(r#""text":"A \"quoted\" item""#));
+        assert!(json.contains(r#""page":2"#));
+        assert!(json.contains(r#""x":12.35"#));
+        assert!(json.contains(r#""is_underline":true"#));
+        assert!(json.contains(r#""item_type":"text""#));
+        assert!(json.contains(r#""mcid":7"#));
+    }
+}
+
 /// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
 fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
     let mut pages = HashSet::new();
@@ -104,6 +194,7 @@ fn main() {
     if args.len() < 2 {
         eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
         eprintln!("       {} <pdf_file> --json", args[0]);
+        eprintln!("       {} <pdf_file> --items-json", args[0]);
         eprintln!("       {} <pdf_file> --raw", args[0]);
         eprintln!();
         eprintln!("Converts PDF to Markdown with smart type detection.");
@@ -111,6 +202,7 @@ fn main() {
         eprintln!();
         eprintln!("Options:");
         eprintln!("  --json              Output result as JSON");
+        eprintln!("  --items-json        Output positioned TextItem JSON");
         eprintln!("  --raw               Output only markdown (no headers)");
         eprintln!("  --pages             Insert page break markers (<!-- Page N -->)");
         eprintln!("  --select-pages N    Only process specified pages (e.g. 1,3,5-10)");
@@ -121,6 +213,7 @@ fn main() {
 
     let pdf_path = &args[1];
     let json_output = args.iter().any(|a| a == "--json");
+    let items_json_output = args.iter().any(|a| a == "--items-json");
     let raw_output = args.iter().any(|a| a == "--raw");
     let page_numbers = args.iter().any(|a| a == "--pages");
     let detect_only = args.iter().any(|a| a == "--detect-only");
@@ -145,6 +238,17 @@ fn main() {
             })
         });
 
+    if items_json_output {
+        match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) {
+            Ok(items) => println!("{}", format_items_json(&items)),
+            Err(e) => {
+                println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
+                process::exit(1);
+            }
+        }
+        return;
+    }
+
     let output_file = args
         .get(2)
         .filter(|a| !a.starts_with("--"))

src/extractor/content_stream.rs

@@ -17,6 +17,7 @@ use super::fonts::{
     build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
     get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
 };
+use super::underline::UnderlineLine;
 use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
 use super::{get_number, image_bbox_from_ctm, multiply_matrices};
 
@@ -74,6 +75,37 @@ fn strip_pdf_comments(data: &[u8]) -> Vec<u8> {
     result
 }
 
+fn transform_path_point(x: f32, y: f32, ctm: &[f32; 6]) -> (f32, f32) {
+    (
+        x * ctm[0] + y * ctm[2] + ctm[4],
+        x * ctm[1] + y * ctm[3] + ctm[5],
+    )
+}
+
+fn transformed_stroke_width(
+    line_width: f32,
+    ctm: &[f32; 6],
+    x1: f32,
+    y1: f32,
+    x2: f32,
+    y2: f32,
+) -> f32 {
+    let user_width = line_width.abs();
+    let dx = x2 - x1;
+    let dy = y2 - y1;
+    let len = (dx * dx + dy * dy).sqrt();
+    if len <= f32::EPSILON {
+        return user_width;
+    }
+
+    // PDF stroke width scales perpendicular to the path direction.
+    let nx = -dy / len;
+    let ny = dx / len;
+    let ndx = nx * ctm[0] + ny * ctm[2];
+    let ndy = nx * ctm[1] + ny * ctm[3];
+    user_width * (ndx * ndx + ndy * ndy).sqrt()
+}
+
 /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
 /// the page uses fonts with unresolvable gid-encoded glyphs.
 pub(crate) fn extract_page_text_items(
@@ -89,6 +121,7 @@ pub(crate) fn extract_page_text_items(
     let mut rects: Vec<PdfRect> = Vec::new();
     let mut clip_rects: Vec<PdfRect> = Vec::new();
     let mut lines: Vec<PdfLine> = Vec::new();
+    let mut underline_lines: Vec<UnderlineLine> = Vec::new();
 
     // Path construction state for m/l/h → S/s line extraction
     let mut path_subpath_start: Option<(f32, f32)> = None;
@@ -97,6 +130,12 @@ pub(crate) fn extract_page_text_items(
     // Completed subpaths (each a vec of line segments) for f/f* rect extraction
     let mut pending_subpaths: Vec<Vec<(f32, f32, f32, f32)>> = Vec::new();
     let mut fill_rects: Vec<PdfRect> = Vec::new();
+    // `re` rects awaiting a paint operator. Underline detection must only
+    // see painted rects: a `re W n` clip path or `re n` no-op draws nothing
+    // on the page, so treating every `re` as ink would underline text that
+    // merely sits near an invisible clip boundary.
+    let mut pending_re_rects: Vec<PdfRect> = Vec::new();
+    let mut painted_rects: Vec<PdfRect> = Vec::new();
 
     // Get fonts for encoding
     let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
@@ -188,10 +227,12 @@ pub(crate) fn extract_page_text_items(
     // Graphics state tracking
     let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
     let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
+    let mut line_width: f32 = 1.0;
     #[derive(Clone)]
     struct SavedGraphicsState {
         ctm: [f32; 6],
         text_rendering_mode: i32,
+        line_width: f32,
         char_spacing: f32,
         word_spacing: f32,
         text_leading: f32,
@@ -240,6 +281,7 @@ pub(crate) fn extract_page_text_items(
                 gstate_stack.push(SavedGraphicsState {
                     ctm,
                     text_rendering_mode,
+                    line_width,
                     char_spacing,
                     word_spacing,
                     text_leading,
@@ -252,6 +294,7 @@ pub(crate) fn extract_page_text_items(
                 if let Some(saved) = gstate_stack.pop() {
                     ctm = saved.ctm;
                     text_rendering_mode = saved.text_rendering_mode;
+                    line_width = saved.line_width;
                     char_spacing = saved.char_spacing;
                     word_spacing = saved.word_spacing;
                     text_leading = saved.text_leading;
@@ -273,6 +316,11 @@ pub(crate) fn extract_page_text_items(
                     ctm = multiply_matrices(&new_matrix, &ctm);
                 }
             }
+            "w" => {
+                if let Some(width) = op.operands.first().and_then(get_number) {
+                    line_width = width;
+                }
+            }
             "BT" => {
                 // Begin text block
                 in_text_block = true;
@@ -441,6 +489,7 @@ pub(crate) fn extract_page_text_items(
                                 page: page_num,
                                 is_bold: is_bold_font(base_font),
                                 is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                 item_type: ItemType::Text,
                                 mcid: current_mcid(&marked_content_stack),
                             });
@@ -606,6 +655,7 @@ pub(crate) fn extract_page_text_items(
                                     page: page_num,
                                     is_bold: is_bold_font(base_font),
                                     is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                     item_type: ItemType::Text,
                                     mcid: current_mcid(&marked_content_stack),
                                 });
@@ -669,6 +719,7 @@ pub(crate) fn extract_page_text_items(
                                 page: page_num,
                                 is_bold: is_bold_font(base_font),
                                 is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                 item_type: ItemType::Text,
                                 mcid: current_mcid(&marked_content_stack),
                             });
@@ -705,6 +756,7 @@ pub(crate) fn extract_page_text_items(
                                         page: page_num,
                                         is_bold: false,
                                         is_italic: false,
+                                        is_underline: false,
                                         item_type: ItemType::Image,
                                         mcid: current_mcid(&marked_content_stack),
                                     });
@@ -801,6 +853,7 @@ pub(crate) fn extract_page_text_items(
                                     page: page_num,
                                     is_bold: is_bold_font(base_font),
                                     is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                     item_type: ItemType::Text,
                                     mcid: entry
                                         .mcid
@@ -825,13 +878,19 @@ pub(crate) fn extract_page_text_items(
                     let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5];
                     let w_dev = rw * ctm[0];
                     let h_dev = rh * ctm[3];
-                    rects.push(PdfRect {
+                    let rect = PdfRect {
                         x: x_dev,
                         y: y_dev,
                         width: w_dev,
                         height: h_dev,
                         page: page_num,
-                    });
+                    };
+                    // Underline detection must only see rects that are
+                    // actually painted — a `re` used purely as a clip path
+                    // (`re W n`) or discarded (`re n`) draws nothing. Hold
+                    // the rect as pending until a paint operator confirms it.
+                    pending_re_rects.push(rect.clone());
+                    rects.push(rect);
                 }
             }
             // ── Path construction operators ──────────────────────
@@ -881,10 +940,8 @@ pub(crate) fn extract_page_text_items(
                     }
                 }
                 for (x1, y1, x2, y2) in pending_lines.drain(..) {
-                    let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
-                    let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
-                    let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
-                    let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
+                    let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
+                    let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
                     lines.push(PdfLine {
                         x1: x1d,
                         y1: y1d,
@@ -892,7 +949,16 @@ pub(crate) fn extract_page_text_items(
                         y2: y2d,
                         page: page_num,
                     });
+                    underline_lines.push(UnderlineLine {
+                        x1: x1d,
+                        y1: y1d,
+                        x2: x2d,
+                        y2: y2d,
+                        stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
+                        page: page_num,
+                    });
                 }
+                painted_rects.append(&mut pending_re_rects);
                 pending_subpaths.clear();
                 path_subpath_start = None;
                 path_current = None;
@@ -908,10 +974,8 @@ pub(crate) fn extract_page_text_items(
                     }
                 }
                 for (x1, y1, x2, y2) in pending_lines.drain(..) {
-                    let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
-                    let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
-                    let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
-                    let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
+                    let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
+                    let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
                     lines.push(PdfLine {
                         x1: x1d,
                         y1: y1d,
@@ -919,7 +983,16 @@ pub(crate) fn extract_page_text_items(
                         y2: y2d,
                         page: page_num,
                     });
+                    underline_lines.push(UnderlineLine {
+                        x1: x1d,
+                        y1: y1d,
+                        x2: x2d,
+                        y2: y2d,
+                        stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
+                        page: page_num,
+                    });
                 }
+                painted_rects.append(&mut pending_re_rects);
                 pending_subpaths.clear();
                 path_subpath_start = None;
                 path_current = None;
@@ -977,6 +1050,7 @@ pub(crate) fn extract_page_text_items(
                         }
                     }
                 }
+                painted_rects.append(&mut pending_re_rects);
                 pending_lines.clear();
                 path_subpath_start = None;
                 path_current = None;
@@ -1042,7 +1116,10 @@ pub(crate) fn extract_page_text_items(
                 // Do NOT clear pending_lines — the following `n` does that
             }
             "n" => {
-                // end path (no-op): discard
+                // end path (no-op): discard — including any `re` rects that
+                // were only ever part of a clip path (`re W n`), which draw
+                // no ink and must not feed underline detection.
+                pending_re_rects.clear();
                 pending_lines.clear();
                 pending_subpaths.clear();
                 path_subpath_start = None;
@@ -1052,6 +1129,12 @@ pub(crate) fn extract_page_text_items(
         }
     }
 
+    // Underline detection reads only painted ink: `re` rects confirmed by
+    // a paint operator plus filled-subpath rects — never clip-only rects,
+    // which draw nothing.
+    let mut underline_rects = painted_rects;
+    underline_rects.extend(fill_rects.iter().cloned());
+
     // Only use clip/fill rects when no `re` rects exist on this page.
     // Clip rects take priority over fill rects, but first we deduplicate
     // them: some PDFs wrap every text block in a full-page W* clip path,
@@ -1081,8 +1164,17 @@ pub(crate) fn extract_page_text_items(
     // Some PDFs embed landscape content in portrait pages using a rotated text
     // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW).  The layout engine
     // assumes x=horizontal, y=vertical — so we swap coordinates to match.
-    let (items, rects, lines, coords_rotated) =
+    let (mut items, rects, lines, coords_rotated) =
         correct_rotated_page(items, rects, lines, &rotation_votes);
+    if coords_rotated {
+        rotate_underline_graphics(&mut underline_rects, &mut underline_lines);
+    }
+    super::underline::mark_underlined_items(
+        &mut items,
+        &underline_rects,
+        &underline_lines,
+        page_num,
+    );
 
     let items = super::merge_text_items(items);
     let items = super::merge_subscript_items(items);
@@ -1168,6 +1260,27 @@ fn correct_rotated_page(
     (items, rects, lines, true)
 }
 
+fn rotate_underline_graphics(rects: &mut [PdfRect], lines: &mut [UnderlineLine]) {
+    for rect in rects {
+        let new_x = rect.y;
+        let new_y = -(rect.x + rect.width.abs());
+        rect.x = new_x;
+        rect.y = new_y;
+        std::mem::swap(&mut rect.width, &mut rect.height);
+    }
+
+    for line in lines {
+        let new_x1 = line.y1;
+        let new_y1 = -line.x1;
+        let new_x2 = line.y2;
+        let new_y2 = -line.x2;
+        line.x1 = new_x1;
+        line.y1 = new_y1;
+        line.x2 = new_x2;
+        line.y2 = new_y2;
+    }
+}
+
 /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance).
 /// Some PDFs emit a full-page clip path for every text block, producing
 /// thousands of identical rects. After dedup these collapse to one rect,
@@ -1217,6 +1330,57 @@ mod tests {
         }
     }
 
+    fn simple_doc_with_content(content: &[u8]) -> (lopdf::Document, lopdf::ObjectId) {
+        use lopdf::{dictionary, Object, Stream};
+
+        let mut doc = lopdf::Document::new();
+        let widths: Vec<Object> = (0..=255).map(|_| 600.into()).collect();
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+            "FirstChar" => 0,
+            "LastChar" => 255,
+            "Widths" => Object::Array(widths),
+        });
+        let content_id = doc.add_object(Object::Stream(Stream::new(
+            dictionary! {},
+            content.to_vec(),
+        )));
+        let page_id = doc.add_object(dictionary! {
+            "Type" => "Page",
+            "Contents" => Object::Reference(content_id),
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => Object::Reference(font_id),
+                },
+            },
+            "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
+        });
+        let pages_id = doc.add_object(dictionary! {
+            "Type" => "Pages",
+            "Count" => Object::Integer(1),
+            "Kids" => vec![Object::Reference(page_id)],
+        });
+        let catalog_id = doc.add_object(dictionary! {
+            "Type" => "Catalog",
+            "Pages" => Object::Reference(pages_id),
+        });
+        doc.trailer.set("Root", Object::Reference(catalog_id));
+
+        (doc, page_id)
+    }
+
+    fn extract_simple_items(content: &[u8]) -> Vec<TextItem> {
+        use crate::tounicode::FontCMaps;
+
+        let (doc, page_id) = simple_doc_with_content(content);
+        let font_cmaps = FontCMaps::from_doc(&doc);
+        let ((items, _, _), _, _) =
+            extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
+        items
+    }
+
     #[test]
     fn test_dedup_rects_identical() {
         let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759];
@@ -1266,6 +1430,38 @@ mod tests {
         assert_eq!(single.len(), 1);
     }
 
+    #[test]
+    fn thick_stroked_rule_does_not_mark_underline() {
+        let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (THICK) Tj ET
+4 w
+100 498 m 170 498 l S
+BT /F1 12 Tf 1 0 0 1 100 480 Tm (THIN) Tj ET
+1 w
+100 478 m 160 478 l S";
+
+        let items = extract_simple_items(content);
+        let thick = items.iter().find(|item| item.text == "THICK").unwrap();
+        let thin = items.iter().find(|item| item.text == "THIN").unwrap();
+
+        assert!(!thick.is_underline);
+        assert!(thin.is_underline);
+    }
+
+    #[test]
+    fn rotated_page_underline_is_detected_after_coordinate_correction() {
+        let content = b"BT /F1 12 Tf 0 1 -1 0 200 100 Tm (HELLO) Tj ET
+BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
+1 w
+202 100 m 202 170 l S";
+
+        let items = extract_simple_items(content);
+        let hello = items.iter().find(|item| item.text == "HELLO").unwrap();
+        let world = items.iter().find(|item| item.text == "WORLD").unwrap();
+
+        assert!(hello.is_underline);
+        assert!(!world.is_underline);
+    }
+
     #[test]
     fn test_skip_excessive_operations() {
         use crate::tounicode::FontCMaps;

src/extractor/layout.rs

@@ -1494,6 +1494,7 @@ mod tests {
             page,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -1623,6 +1624,7 @@ mod tests {
                     page,
                     is_bold: false,
                     is_italic: false,
+                    is_underline: false,
                     item_type: ItemType::Text,
                     mcid: None,
                 });

src/extractor/links.rs

@@ -78,6 +78,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
                             page: page_num,
                             is_bold: false,
                             is_italic: false,
+                            is_underline: false,
                             item_type: ItemType::Link(url),
                             mcid: None,
                         });
@@ -316,6 +317,7 @@ pub(crate) fn walk_form_fields(
         page: page_num,
         is_bold: false,
         is_italic: false,
+        is_underline: false,
         item_type: ItemType::FormField,
         mcid: None,
     });

src/extractor/mod.rs

@@ -6,11 +6,12 @@ pub(crate) mod content_stream;
 mod fonts;
 mod layout;
 mod links;
+pub(crate) mod underline;
 mod xobjects;
 
 use crate::text_utils::is_rtl_text;
 use crate::tounicode::FontCMaps;
-use crate::types::{PageExtraction, TextItem};
+use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
 use crate::PdfError;
 use log::debug;
 use lopdf::{Document, Object, ObjectId};
@@ -180,6 +181,7 @@ fn extract_positioned_text_impl(
         if threshold > 0.10 {
             page_thresholds.insert(*page_num, threshold);
         }
+        suppress_table_underlines(&mut items, &rects, &lines, *page_num);
         debug!(
             "page {}: {} text items, {} rects, {} lines{}",
             page_num,
@@ -226,6 +228,38 @@ fn extract_positioned_text_impl(
     ))
 }
 
+fn suppress_table_underlines(
+    items: &mut [TextItem],
+    rects: &[PdfRect],
+    lines: &[PdfLine],
+    page: u32,
+) {
+    if !items.iter().any(|item| item.is_underline) {
+        return;
+    }
+
+    let mut table_item_indices: HashSet<usize> = HashSet::new();
+
+    if !rects.is_empty() {
+        let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page);
+        for table in rect_tables {
+            table_item_indices.extend(table.item_indices);
+        }
+    }
+
+    if !lines.is_empty() {
+        for table in crate::tables::detect_tables_from_lines(items, lines, page) {
+            table_item_indices.extend(table.item_indices);
+        }
+    }
+
+    for index in table_item_indices {
+        if let Some(item) = items.get_mut(index) {
+            item.is_underline = false;
+        }
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Shared helpers (used by submodules via `super::`)
 // ---------------------------------------------------------------------------
@@ -525,6 +559,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
             let first = group[i];
             let mut text = first.text.clone();
             let mut end_x = first.x + effective_merge_width(first);
+            let mut is_underline = first.is_underline;
 
             let mut j = i + 1;
             while j < group.len() {
@@ -571,6 +606,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                     text.push(' ');
                 }
                 text.push_str(&next.text);
+                is_underline |= next.is_underline;
                 let next_end = next.x + effective_merge_width(next);
                 end_x = if *preserve_stream_order {
                     end_x.max(next_end)
@@ -591,6 +627,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                 page: first.page,
                 is_bold: first.is_bold,
                 is_italic: first.is_italic,
+                is_underline,
                 item_type: first.item_type.clone(),
                 mcid: first.mcid,
             });
@@ -701,7 +738,7 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
 mod tests {
     use super::*;
     use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items};
-    use crate::types::{ItemType, TextLine};
+    use crate::types::{ItemType, PdfLine, TextLine};
     use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
 
     fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
@@ -716,6 +753,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -726,6 +764,16 @@ mod tests {
         item
     }
 
+    fn make_line(x1: f32, y1: f32, x2: f32, y2: f32) -> PdfLine {
+        PdfLine {
+            x1,
+            y1,
+            x2,
+            y2,
+            page: 1,
+        }
+    }
+
     #[test]
     fn trace_text_preview_truncates_on_char_boundary() {
         let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}');
@@ -774,6 +822,21 @@ mod tests {
         assert_eq!(merged[0].text, "hello world");
     }
 
+    #[test]
+    fn merge_items_preserves_underline_from_later_fragment() {
+        let mut items = vec![
+            make_merge_item("pre", 100.0, 18.0),
+            make_merge_item("fix", 119.0, 18.0),
+        ];
+        items[1].is_underline = true;
+
+        let merged = merge_text_items(items);
+
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "prefix");
+        assert!(merged[0].is_underline);
+    }
+
     #[test]
     fn merge_items_preserves_stream_order_for_backtracking_heading() {
         // Some tagged PDFs emit first-letter ActualText fragments, then reset
@@ -851,6 +914,35 @@ mod tests {
         assert_eq!(texts, vec!["•", "Distant item"]);
     }
 
+    #[test]
+    fn suppress_table_underlines_clears_line_detected_table_items() {
+        let mut items = vec![
+            make_merge_item("H1", 125.0, 20.0),
+            make_merge_item("H2", 225.0, 20.0),
+            make_merge_item("A", 125.0, 20.0),
+            make_merge_item("B", 225.0, 20.0),
+        ];
+        items[0].y = 490.0;
+        items[1].y = 490.0;
+        items[2].y = 470.0;
+        items[3].y = 470.0;
+        for item in &mut items {
+            item.is_underline = true;
+        }
+        let lines = vec![
+            make_line(100.0, 500.0, 300.0, 500.0),
+            make_line(100.0, 480.0, 300.0, 480.0),
+            make_line(100.0, 460.0, 300.0, 460.0),
+            make_line(100.0, 460.0, 100.0, 500.0),
+            make_line(200.0, 460.0, 200.0, 500.0),
+            make_line(300.0, 460.0, 300.0, 500.0),
+        ];
+
+        suppress_table_underlines(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
     #[test]
     fn test_group_into_lines() {
         let items = vec![
@@ -865,6 +957,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -879,6 +972,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -893,6 +987,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -947,6 +1042,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -961,6 +1057,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -975,6 +1072,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1000,6 +1098,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1014,6 +1113,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1028,6 +1128,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1055,6 +1156,7 @@ mod tests {
                 page: 1,
                 is_bold: true,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             }
@@ -1089,6 +1191,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             }
@@ -1124,6 +1227,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1138,6 +1242,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1152,6 +1257,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1174,6 +1280,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -1286,6 +1393,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1300,6 +1408,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1324,6 +1433,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1338,6 +1448,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             },
@@ -1378,6 +1489,7 @@ mod tests {
                 page,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             }],
@@ -1422,6 +1534,7 @@ mod tests {
                 page,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             }],
@@ -1466,6 +1579,7 @@ mod tests {
                 page,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             }],
@@ -1503,6 +1617,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }

src/extractor/underline.rs

@@ -0,0 +1,500 @@
+//! Geometric underline detection.
+//!
+//! PDFs have no underline font flag — underlines are drawn as separate
+//! graphics: stroked horizontal lines (`l`/`S` operators) or thin filled
+//! rectangles (`re`/`f`). This pass correlates those graphics with text
+//! items after extraction: an item is underlined when a horizontal
+//! line/thin rect sits just below its baseline and covers most of its
+//! horizontal extent.
+//!
+//! Repeated same-span rules are treated as table/form rulings rather than
+//! underlines, which avoids marking every cell in ruled tables.
+
+use std::collections::HashSet;
+
+use crate::types::{ItemType, PdfRect, TextItem};
+
+/// Max thickness (pt) for a stroked line / filled rect to count as an
+/// underline rule rather than a border or decorative band.
+const MAX_RULE_THICKNESS: f32 = 2.0;
+
+/// Fraction of the item's width that the rule must cover horizontally.
+const MIN_X_OVERLAP: f32 = 0.6;
+
+/// Same-span rules repeated at this many y-levels are usually table/form
+/// rulings, not semantic underlines.
+const MIN_REPEATED_RULE_LEVELS: usize = 3;
+
+/// Vertical tolerance for considering two rules to be on the same row edge.
+const RULE_Y_DEDUP_EPS: f32 = 2.0;
+
+/// Horizontal span similarity required when clustering repeated rulings.
+const RULE_SPAN_OVERLAP_RATIO: f32 = 0.8;
+const RULE_SPAN_WIDTH_RATIO: f32 = 1.5;
+
+/// Multiple separated rule segments on one row are usually per-column table
+/// header/body separators.
+const MIN_SEGMENTED_ROW_RULES: usize = 3;
+const MIN_SEGMENTED_ROW_GAPS: usize = 2;
+const SEGMENTED_ROW_GAP_MIN: f32 = 12.0;
+
+/// A single rule under several widely separated items is usually a table
+/// header/body separator, not a sentence underline.
+const MIN_TABULAR_RULE_ITEMS: usize = 3;
+const MIN_TABULAR_RULE_GAPS: usize = 2;
+const TABULAR_RULE_GAP_EM: f32 = 2.0;
+
+#[derive(Clone)]
+pub(crate) struct UnderlineLine {
+    pub(crate) x1: f32,
+    pub(crate) y1: f32,
+    pub(crate) x2: f32,
+    pub(crate) y2: f32,
+    pub(crate) stroke_width: f32,
+    pub(crate) page: u32,
+}
+
+/// A horizontal rule candidate in page coordinates (PDF y-up).
+#[derive(Clone)]
+struct Rule {
+    x1: f32,
+    x2: f32,
+    y: f32,
+}
+
+impl Rule {
+    fn width(&self) -> f32 {
+        self.x2 - self.x1
+    }
+}
+
+fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) -> Vec<Rule> {
+    let mut rules: Vec<Rule> = Vec::new();
+    for l in lines {
+        if l.page != page {
+            continue;
+        }
+        // Horizontal stroked line (tolerate slight skew).
+        if l.stroke_width <= MAX_RULE_THICKNESS && (l.y1 - l.y2).abs() <= MAX_RULE_THICKNESS {
+            let (x1, x2) = if l.x1 <= l.x2 {
+                (l.x1, l.x2)
+            } else {
+                (l.x2, l.x1)
+            };
+            if x2 - x1 > 1.0 {
+                rules.push(Rule {
+                    x1,
+                    x2,
+                    y: (l.y1 + l.y2) / 2.0,
+                });
+            }
+        }
+    }
+    for r in rects {
+        if r.page != page {
+            continue;
+        }
+        // Thin filled rect used as an underline rule. Extents are
+        // normalized first: `re` operands pass through the CTM, so
+        // width/height can be negative (flipped axes / negative scale) —
+        // without normalization negative-width rules are missed and
+        // negative-height bands sneak past the thickness check.
+        let (x1, x2) = if r.width >= 0.0 {
+            (r.x, r.x + r.width)
+        } else {
+            (r.x + r.width, r.x)
+        };
+        if r.height.abs() <= MAX_RULE_THICKNESS && x2 - x1 > 1.0 {
+            rules.push(Rule {
+                x1,
+                x2,
+                y: r.y + r.height / 2.0,
+            });
+        }
+    }
+    rules
+}
+
+fn discard_repeated_ruling_rules(rules: Vec<Rule>) -> Vec<Rule> {
+    if rules.len() < MIN_REPEATED_RULE_LEVELS {
+        return rules;
+    }
+
+    rules
+        .iter()
+        .filter(|rule| {
+            !is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules)
+        })
+        .cloned()
+        .collect()
+}
+
+fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
+    let mut y_levels: Vec<f32> = rules
+        .iter()
+        .filter(|other| has_similar_span(rule, other))
+        .map(|other| other.y)
+        .collect();
+
+    y_levels.sort_by(|a, b| a.total_cmp(b));
+    y_levels.dedup_by(|a, b| (*a - *b).abs() <= RULE_Y_DEDUP_EPS);
+    y_levels.len() >= MIN_REPEATED_RULE_LEVELS
+}
+
+fn is_segmented_row_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
+    let mut row_rules: Vec<&Rule> = rules
+        .iter()
+        .filter(|other| (other.y - rule.y).abs() <= RULE_Y_DEDUP_EPS)
+        .collect();
+
+    if row_rules.len() < MIN_SEGMENTED_ROW_RULES {
+        return false;
+    }
+
+    row_rules.sort_by(|a, b| a.x1.total_cmp(&b.x1));
+    let large_gaps = row_rules
+        .windows(2)
+        .filter(|pair| pair[1].x1 - pair[0].x2 > SEGMENTED_ROW_GAP_MIN)
+        .count();
+
+    large_gaps >= MIN_SEGMENTED_ROW_GAPS
+}
+
+fn has_similar_span(a: &Rule, b: &Rule) -> bool {
+    let a_width = a.width();
+    let b_width = b.width();
+    if a_width <= 1.0 || b_width <= 1.0 {
+        return false;
+    }
+
+    let width_ratio = a_width.max(b_width) / a_width.min(b_width);
+    if width_ratio > RULE_SPAN_WIDTH_RATIO {
+        return false;
+    }
+
+    let overlap = a.x2.min(b.x2) - a.x1.max(b.x1);
+    overlap >= a_width.min(b_width) * RULE_SPAN_OVERLAP_RATIO
+}
+
+fn tabular_row_separator_rule_indices(rules: &[Rule], items: &[TextItem]) -> HashSet<usize> {
+    let mut tabular_rules = HashSet::new();
+
+    for (rule_idx, rule) in rules.iter().enumerate() {
+        let mut matched_items: Vec<&TextItem> = items
+            .iter()
+            .filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item))
+            .collect();
+
+        if matched_items.len() < MIN_TABULAR_RULE_ITEMS {
+            continue;
+        }
+
+        matched_items.sort_by(|a, b| a.x.total_cmp(&b.x));
+        let large_gaps = matched_items
+            .windows(2)
+            .filter(|pair| {
+                let left = pair[0];
+                let right = pair[1];
+                let gap = right.x - (left.x + left.width);
+                let font_size = left.font_size.max(right.font_size).max(1.0);
+                gap > font_size * TABULAR_RULE_GAP_EM
+            })
+            .count();
+
+        if large_gaps >= MIN_TABULAR_RULE_GAPS {
+            tabular_rules.insert(rule_idx);
+        }
+    }
+
+    tabular_rules
+}
+
+fn is_underline_candidate(item: &TextItem) -> bool {
+    matches!(item.item_type, ItemType::Text) && !item.text.trim().is_empty() && item.width > 0.0
+}
+
+fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
+    // Vertical window: underlines sit at or slightly below the baseline.
+    // Fonts draw them at roughly 5-15% of the em below; allow up to 35%
+    // (min 3pt) below and 1pt above for rounding.
+    let below = (item.font_size * 0.35).max(3.0);
+    let y_min = item.y - below;
+    let y_max = item.y + 1.0;
+    if rule.y < y_min || rule.y > y_max {
+        return false;
+    }
+
+    let ix1 = item.x;
+    let ix2 = item.x + item.width;
+    let min_overlap = item.width * MIN_X_OVERLAP;
+    let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
+    overlap >= min_overlap
+}
+
+/// Mark `is_underline` on text items that have a horizontal rule just
+/// below their baseline. `items`, `rects`, and `lines` are a single
+/// page's extraction output (all in PDF coordinates, y-up, where
+/// `TextItem::y` is the text baseline).
+pub(crate) fn mark_underlined_items(
+    items: &mut [TextItem],
+    rects: &[PdfRect],
+    lines: &[UnderlineLine],
+    page: u32,
+) {
+    let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page));
+    if rules.is_empty() {
+        return;
+    }
+    let tabular_rules = tabular_row_separator_rule_indices(&rules, items);
+
+    for item in items.iter_mut() {
+        if !is_underline_candidate(item) {
+            continue;
+        }
+
+        for (rule_idx, rule) in rules.iter().enumerate() {
+            if tabular_rules.contains(&rule_idx) {
+                continue;
+            }
+            if rule_matches_item(rule, item) {
+                item.is_underline = true;
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::ItemType;
+
+    fn item(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem {
+        TextItem {
+            text: text.to_string(),
+            x,
+            y,
+            width,
+            height: font_size,
+            font: "F1".to_string(),
+            font_size,
+            page: 1,
+            is_bold: false,
+            is_italic: false,
+            is_underline: false,
+            item_type: ItemType::Text,
+            mcid: None,
+        }
+    }
+
+    fn hline(x1: f32, x2: f32, y: f32) -> UnderlineLine {
+        UnderlineLine {
+            x1,
+            y1: y,
+            x2,
+            y2: y,
+            stroke_width: 1.0,
+            page: 1,
+        }
+    }
+
+    fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect {
+        PdfRect {
+            x,
+            y,
+            width,
+            height: 0.8,
+            page: 1,
+        }
+    }
+
+    #[test]
+    fn stroked_line_under_baseline_marks_underline() {
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(99.0, 161.0, 498.5)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn thin_filled_rect_under_baseline_marks_underline() {
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![thin_rect(100.0, 497.8, 60.0)];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn long_rule_under_multiple_items_marks_each() {
+        // One underline drawn under a whole sentence: every overlapped
+        // item gets the flag.
+        let mut items = vec![
+            item("first", 100.0, 500.0, 40.0, 10.0),
+            item("second", 145.0, 500.0, 50.0, 10.0),
+        ];
+        let lines = vec![hline(98.0, 200.0, 498.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(items[0].is_underline);
+        assert!(items[1].is_underline);
+    }
+
+    #[test]
+    fn line_far_below_baseline_is_not_an_underline() {
+        // A horizontal rule 30pt below (section divider) must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(90.0, 300.0, 470.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn thick_stroked_line_is_not_an_underline() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let mut line = hline(99.0, 161.0, 498.5);
+        line.stroke_width = 4.0;
+
+        mark_underlined_items(&mut items, &[], &[line], 1);
+
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn line_above_baseline_is_not_an_underline() {
+        // Strikethrough / overline geometry must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(90.0, 300.0, 505.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn insufficient_horizontal_overlap_is_not_an_underline() {
+        // Rule under only a quarter of the item (e.g. neighboring cell
+        // border) must not mark.
+        let mut items = vec![item("wide text item", 100.0, 500.0, 100.0, 10.0)];
+        let lines = vec![hline(100.0, 125.0, 498.5)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn negative_width_rect_is_normalized_and_marks_underline() {
+        // A CTM with negative x-scale (or negative `re` operands) produces
+        // rects whose width is negative; the rule extents must normalize.
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 160.0,
+            y: 497.8,
+            width: -60.0,
+            height: 0.8,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn negative_height_band_is_not_an_underline() {
+        // A 14pt band expressed with negative height must not pass the
+        // thickness check via sign trickery.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 95.0,
+            y: 509.0,
+            width: 80.0,
+            height: -14.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn thick_band_is_not_an_underline() {
+        // A highlight bar / filled cell background (tall rect) must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 95.0,
+            y: 495.0,
+            width: 80.0,
+            height: 14.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn vertical_line_is_not_an_underline() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![UnderlineLine {
+            x1: 120.0,
+            y1: 498.0,
+            x2: 120.0,
+            y2: 400.0,
+            stroke_width: 1.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn other_pages_graphics_do_not_mark() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let mut line = hline(99.0, 161.0, 498.5);
+        line.page = 2;
+        mark_underlined_items(&mut items, &[], &[line], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn repeated_table_row_rules_do_not_mark_cell_text() {
+        let mut items = vec![
+            item("A", 110.0, 500.0, 20.0, 10.0),
+            item("B", 110.0, 480.0, 20.0, 10.0),
+            item("C", 110.0, 460.0, 20.0, 10.0),
+        ];
+        let lines = vec![
+            hline(100.0, 150.0, 498.0),
+            hline(100.0, 150.0, 478.0),
+            hline(100.0, 150.0, 458.0),
+        ];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
+    #[test]
+    fn row_separator_under_spaced_column_labels_is_not_an_underline() {
+        let mut items = vec![
+            item("Date", 100.0, 500.0, 25.0, 10.0),
+            item("Rate", 200.0, 500.0, 25.0, 10.0),
+            item("Yield", 300.0, 500.0, 30.0, 10.0),
+        ];
+        let lines = vec![hline(90.0, 340.0, 498.0)];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
+    #[test]
+    fn same_row_spaced_rule_segments_do_not_mark_column_labels() {
+        let mut items = vec![
+            item("Date", 100.0, 500.0, 25.0, 10.0),
+            item("Rate", 200.0, 500.0, 25.0, 10.0),
+            item("Yield", 300.0, 500.0, 30.0, 10.0),
+        ];
+        let lines = vec![
+            hline(98.0, 128.0, 498.0),
+            hline(198.0, 228.0, 498.0),
+            hline(298.0, 333.0, 498.0),
+        ];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+}

src/extractor/xobjects.rs

@@ -294,6 +294,7 @@ fn extract_form_xobject_text_inner(
                                     page: page_num,
                                     is_bold: false,
                                     is_italic: false,
+                                    is_underline: false,
                                     item_type: ItemType::Image,
                                     mcid: None,
                                 });
@@ -438,6 +439,7 @@ fn extract_form_xobject_text_inner(
                                 page: page_num,
                                 is_bold: is_bold_font(base_font),
                                 is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                 item_type: ItemType::Text,
                                 mcid: None,
                             });
@@ -586,6 +588,7 @@ fn extract_form_xobject_text_inner(
                                     page: page_num,
                                     is_bold: is_bold_font(base_font),
                                     is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                     item_type: ItemType::Text,
                                     mcid: None,
                                 });

src/lib.rs

@@ -4882,6 +4882,7 @@ mod text_cluster_column_undercount_tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -5156,6 +5157,7 @@ mod table_candidate_selection_tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -5902,6 +5904,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }

src/markdown/convert.rs

@@ -1169,6 +1169,7 @@ mod tests {
             page,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: crate::types::ItemType::Text,
             mcid,
         }

src/markdown/mod.rs

@@ -1217,6 +1217,7 @@ mod tests {
             page,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: crate::types::ItemType::Text,
             mcid: None,
         }

src/markdown/preprocess.rs

@@ -208,157 +208,6 @@ fn normalize_for_comparison(s: &str) -> String {
     trimmed.to_string()
 }
 
-/// Compact a comparison key for fuzzy matching of damaged running headers.
-///
-/// Some tagged PDFs emit running footer text with overlapping fragments, so one
-/// page may read "F rom ..." while later pages read "F om r ...". Exact
-/// normalized text still drives candidate discovery; this compact form is only
-/// used when deciding whether a one-off edge line is close enough to an already
-/// repeated candidate.
-fn compact_comparison_key(s: &str) -> String {
-    s.chars()
-        .filter(|c| c.is_ascii_alphanumeric())
-        .map(|c| c.to_ascii_lowercase())
-        .collect()
-}
-
-fn bounded_levenshtein(a: &str, b: &str, max_distance: usize) -> Option<usize> {
-    let a_chars: Vec<char> = a.chars().collect();
-    let b_chars: Vec<char> = b.chars().collect();
-
-    if a_chars.len().abs_diff(b_chars.len()) > max_distance {
-        return None;
-    }
-
-    let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
-    let mut curr = vec![0; b_chars.len() + 1];
-
-    for (i, a_ch) in a_chars.iter().enumerate() {
-        curr[0] = i + 1;
-        let mut row_min = curr[0];
-
-        for (j, b_ch) in b_chars.iter().enumerate() {
-            let cost = usize::from(a_ch != b_ch);
-            curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(prev[j] + cost);
-            row_min = row_min.min(curr[j + 1]);
-        }
-
-        if row_min > max_distance {
-            return None;
-        }
-
-        std::mem::swap(&mut prev, &mut curr);
-    }
-
-    let distance = prev[b_chars.len()];
-    (distance <= max_distance).then_some(distance)
-}
-
-fn matches_candidate(
-    normalized: &str,
-    candidates: &HashSet<String>,
-    compact_candidates: &[String],
-) -> bool {
-    if candidates.contains(normalized) {
-        return true;
-    }
-
-    if !has_broken_word_spacing(normalized) {
-        return false;
-    }
-
-    let compact = compact_comparison_key(normalized);
-    if compact.len() < 20 {
-        return false;
-    }
-
-    compact_candidates.iter().any(|candidate| {
-        candidate.len().abs_diff(compact.len()) <= 2
-            && bounded_levenshtein(&compact, candidate, 2).is_some()
-    })
-}
-
-fn ends_with_hyphen(raw: &str) -> bool {
-    matches!(
-        raw.chars().last(),
-        Some('-' | '\u{00ad}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}')
-    )
-}
-
-fn suspicious_short_token(raw: &str, alpha: &str, contains_equals: bool) -> bool {
-    let len = alpha.chars().count();
-    if len == 0 || len > 2 || ends_with_hyphen(raw) {
-        return false;
-    }
-
-    if contains_equals {
-        let raw_alpha: String = raw.chars().filter(|c| c.is_alphabetic()).collect();
-        if raw_alpha.chars().all(|c| c.is_uppercase()) {
-            return false;
-        }
-    }
-
-    true
-}
-
-fn is_uppercase_heavy(text: &str) -> bool {
-    let mut alpha = 0usize;
-    let mut uppercase = 0usize;
-    let mut lowercase = 0usize;
-
-    for ch in text.chars().filter(|ch| ch.is_alphabetic()) {
-        alpha += 1;
-        if ch.is_uppercase() {
-            uppercase += 1;
-        } else if ch.is_lowercase() {
-            lowercase += 1;
-        }
-    }
-
-    alpha >= 12 && lowercase == 0 && uppercase * 100 / alpha >= 80
-}
-
-fn has_broken_word_spacing(text: &str) -> bool {
-    if is_uppercase_heavy(text) {
-        return false;
-    }
-
-    let contains_equals = text.contains('=');
-    let tokens: Vec<(usize, bool)> = text
-        .split_whitespace()
-        .filter_map(|raw| {
-            let alpha: String = raw
-                .chars()
-                .filter(|c| c.is_alphabetic())
-                .flat_map(|c| c.to_lowercase())
-                .collect();
-            let len = alpha.chars().count();
-            (len > 0).then(|| (len, suspicious_short_token(raw, &alpha, contains_equals)))
-        })
-        .collect();
-
-    if tokens.len() < 4 {
-        return false;
-    }
-
-    let suspicious_tokens = tokens.iter().filter(|(_, suspicious)| *suspicious).count();
-    if suspicious_tokens < 3 {
-        return false;
-    }
-
-    let split_word_windows = tokens
-        .windows(3)
-        .filter(|window| window[0].0 >= 3 && window[1].1 && window[2].0 >= 3)
-        .count();
-    let adjacent_fragments = tokens
-        .windows(2)
-        .filter(|window| window[0].1 && window[1].1)
-        .count();
-
-    suspicious_tokens as f32 / tokens.len() as f32 >= 0.35
-        && (split_word_windows > 0 || adjacent_fragments > 0)
-}
-
 /// Returns true if the line looks like a list item or heading (should not be stripped).
 fn is_structural_line(text: &str) -> bool {
     let t = text.trim_start();
@@ -387,9 +236,7 @@ fn is_decorative_separator(text: &str) -> bool {
 /// Strip lines that repeat on many distinct pages (running headers/footers).
 ///
 /// A line is considered a repeated header/footer if:
-/// 1. Its normalized text appears on enough distinct pages. The normal threshold
-///    is document-wide; visibly broken/letter-spaced running text can use a
-///    capped chapter-level threshold in long books.
+/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
 /// 2. It is at least 10 characters long
 /// 3. It doesn't look like a structural element (heading, list item)
 /// 4. It consistently appears in the top or bottom N distinct Y positions
@@ -406,27 +253,13 @@ fn is_decorative_separator(text: &str) -> bool {
 /// Page numbers are stripped from line text before comparison, so headers like
 /// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
 pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
-    let removal_set = find_repeated_line_indices(&lines, page_count);
-    if removal_set.is_empty() {
-        return lines;
-    }
-
-    lines
-        .into_iter()
-        .enumerate()
-        .filter(|(idx, _)| !removal_set.contains(idx))
-        .map(|(_, line)| line)
-        .collect()
-}
-
-fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<usize> {
     if lines.is_empty() || page_count < 3 {
-        return HashSet::new();
+        return lines;
     }
 
     // Compute Y range per page (min_y, max_y)
     let mut page_y_range: HashMap<u32, (f32, f32)> = HashMap::new();
-    for line in lines {
+    for line in &lines {
         let entry = page_y_range.entry(line.page).or_insert((line.y, line.y));
         if line.y < entry.0 {
             entry.0 = line.y;
@@ -438,7 +271,7 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
 
     // Build sorted Y values per page, so we can check line rank (position from edge)
     let mut page_sorted_ys: HashMap<u32, Vec<f32>> = HashMap::new();
-    for line in lines {
+    for line in &lines {
         page_sorted_ys.entry(line.page).or_default().push(line.y);
     }
     for ys in page_sorted_ys.values_mut() {
@@ -454,39 +287,23 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     // page margin.
     const EDGE_LINE_COUNT: usize = 5;
 
-    fn y_position_rank(
-        y: f32,
-        page: u32,
-        page_sorted_ys: &HashMap<u32, Vec<f32>>,
-    ) -> Option<(usize, usize)> {
-        let ys = page_sorted_ys.get(&page)?;
-        let pos = ys.iter().position(|&py| (py - y).abs() < 0.1)?;
-        Some((pos, ys.len()))
-    }
-
     /// Returns true if the given Y position is among the first or last N distinct
     /// Y positions on the specified page.
     fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
-        let Some((pos, len)) = y_position_rank(y, page, page_sorted_ys) else {
-            return false;
+        let ys = match page_sorted_ys.get(&page) {
+            Some(ys) => ys,
+            None => return false,
         };
-        if len <= n * 2 {
+        if ys.len() <= n * 2 {
             // Page has very few lines — everything is near the edge
             return true;
         }
-        pos < n || pos >= len - n
-    }
-
-    fn is_y_at_strict_lower_edge(
-        y: f32,
-        page: u32,
-        page_sorted_ys: &HashMap<u32, Vec<f32>>,
-        n: usize,
-    ) -> bool {
-        let Some((pos, len)) = y_position_rank(y, page, page_sorted_ys) else {
-            return false;
+        // Check if this Y is among the first or last N
+        let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
+            Some(p) => p,
+            None => return false,
         };
-        len > n * 2 && pos < n
+        pos < n || pos >= ys.len() - n
     }
 
     // Average page span for normalizing Y variance
@@ -510,9 +327,8 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     // Build frequency maps using normalize_for_comparison.
     // Individual line text -> distinct pages
     let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
-    let mut bottom_freq: HashMap<String, HashSet<u32>> = HashMap::new();
     let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
-    for line in lines {
+    for line in &lines {
         if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
             continue;
         }
@@ -524,12 +340,6 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
         freq.entry(normalized.clone())
             .or_default()
             .insert(line.page);
-        if is_y_at_strict_lower_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            bottom_freq
-                .entry(normalized.clone())
-                .or_default()
-                .insert(line.page);
-        }
         y_positions.entry(normalized).or_default().push(line.y);
     }
 
@@ -537,7 +347,6 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     // This catches split column headers where individual fragments don't meet
     // the frequency threshold but the combined row does.
     let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
-    let mut band_bottom_freq: HashMap<String, HashSet<u32>> = HashMap::new();
     let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
     for (&(page, _), indices) in &y_bands {
         if indices.len() < 2 {
@@ -562,36 +371,11 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
             .entry(normalized.clone())
             .or_default()
             .insert(page);
-        if is_y_at_strict_lower_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
-            band_bottom_freq
-                .entry(normalized.clone())
-                .or_default()
-                .insert(page);
-        }
         band_y_positions.entry(normalized).or_default().push(band_y);
     }
 
-    // Compute thresholds. Keep the conservative document-wide threshold for
-    // clean text, and allow a lower cap only for visibly broken/letter-spaced
-    // running headers in books where each chapter has its own footer/header.
-    let document_threshold = 3u32.max(page_count * 30 / 100);
-    let garbled_chapter_threshold = 3u32.max((page_count * 30 / 100).min(8));
-    let remove_all_bottom_threshold = document_threshold.min(garbled_chapter_threshold);
-    let meets_frequency_threshold =
-        |text: &str, pages: &HashSet<u32>, bottom_pages: &HashMap<String, HashSet<u32>>| -> bool {
-            pages.len() as u32 >= document_threshold
-                || (has_broken_word_spacing(text)
-                    && bottom_pages
-                        .get(text)
-                        .is_some_and(|pages| pages.len() as u32 >= garbled_chapter_threshold))
-        };
-    let should_remove_all_occurrences =
-        |text: &str, bottom_pages: &HashMap<String, HashSet<u32>>| -> bool {
-            has_broken_word_spacing(text)
-                && bottom_pages
-                    .get(text)
-                    .is_some_and(|pages| pages.len() as u32 >= remove_all_bottom_threshold)
-        };
+    // Compute threshold
+    let threshold = 3u32.max(page_count * 30 / 100);
 
     // Check Y-position consistency: headers/footers appear at the same position
     // on every page, table content varies. Require normalized stddev < 5% of
@@ -609,61 +393,29 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     };
 
     // Identify candidates from individual frequency map
-    let mut remove_all_candidates: HashSet<String> = HashSet::new();
-    let mut candidates: HashSet<String> = HashSet::new();
-    for (text, pages) in freq {
-        if meets_frequency_threshold(&text, &pages, &bottom_freq)
-            && !is_structural_line(&text)
-            && has_consistent_y(&text, &y_positions)
-        {
-            if should_remove_all_occurrences(&text, &bottom_freq) {
-                remove_all_candidates.insert(text.clone());
-            }
-            candidates.insert(text);
-        }
-    }
-    let compact_candidates: Vec<String> = candidates
-        .iter()
-        .filter(|text| has_broken_word_spacing(text))
-        .map(|text| compact_comparison_key(text))
-        .filter(|text| text.len() >= 20)
-        .collect();
-    let compact_remove_all_candidates: Vec<String> = remove_all_candidates
-        .iter()
-        .filter(|text| has_broken_word_spacing(text))
-        .map(|text| compact_comparison_key(text))
-        .filter(|text| text.len() >= 20)
+    let candidates: HashSet<String> = freq
+        .into_iter()
+        .filter(|(text, pages)| {
+            pages.len() as u32 >= threshold
+                && !is_structural_line(text)
+                && has_consistent_y(text, &y_positions)
+        })
+        .map(|(text, _)| text)
         .collect();
 
     // Identify candidates from coalesced band frequency map
-    let mut remove_all_band_candidates: HashSet<String> = HashSet::new();
-    let mut band_candidates: HashSet<String> = HashSet::new();
-    for (text, pages) in band_freq {
-        if meets_frequency_threshold(&text, &pages, &band_bottom_freq)
-            && !is_structural_line(&text)
-            && has_consistent_y(&text, &band_y_positions)
-        {
-            if should_remove_all_occurrences(&text, &band_bottom_freq) {
-                remove_all_band_candidates.insert(text.clone());
-            }
-            band_candidates.insert(text);
-        }
-    }
-    let compact_band_candidates: Vec<String> = band_candidates
-        .iter()
-        .filter(|text| has_broken_word_spacing(text))
-        .map(|text| compact_comparison_key(text))
-        .filter(|text| text.len() >= 20)
-        .collect();
-    let compact_remove_all_band_candidates: Vec<String> = remove_all_band_candidates
-        .iter()
-        .filter(|text| has_broken_word_spacing(text))
-        .map(|text| compact_comparison_key(text))
-        .filter(|text| text.len() >= 20)
+    let band_candidates: HashSet<String> = band_freq
+        .into_iter()
+        .filter(|(text, pages)| {
+            pages.len() as u32 >= threshold
+                && !is_structural_line(text)
+                && has_consistent_y(text, &band_y_positions)
+        })
+        .map(|(text, _)| text)
         .collect();
 
     if candidates.is_empty() && band_candidates.is_empty() {
-        return HashSet::new();
+        return lines;
     }
 
     // Build removal set.
@@ -672,10 +424,8 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     //   (b) its Y-band's coalesced text matches a band candidate, OR
     //   (c) any sibling in its Y-band was removed (propagation).
     //
-    // The first occurrence (lowest page number) of each repeated line is kept
-    // so that document titles, column headers, etc. appear once. Visibly broken
-    // footers proven by repeated lower-edge placement are removed from every
-    // matching edge occurrence, including sparse first pages.
+    // The first occurrence (lowest page number) of each repeated header/footer
+    // is kept so that document titles, column headers, etc. appear once.
     let mut removal_set: HashSet<usize> = HashSet::new();
 
     // Track which page first shows each candidate (to preserve first occurrence)
@@ -686,15 +436,7 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
         }
         let text = line.text();
         let normalized = normalize_for_comparison(&text);
-        if matches_candidate(&normalized, &candidates, &compact_candidates) {
-            if matches_candidate(
-                &normalized,
-                &remove_all_candidates,
-                &compact_remove_all_candidates,
-            ) {
-                removal_set.insert(idx);
-                continue;
-            }
+        if candidates.contains(&normalized) {
             let first = first_page_individual.entry(normalized).or_insert(line.page);
             if line.page > *first {
                 removal_set.insert(idx);
@@ -723,7 +465,7 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
             .collect::<Vec<_>>()
             .join(" ");
         let normalized = normalize_for_comparison(&coalesced);
-        if matches_candidate(&normalized, &band_candidates, &compact_band_candidates) {
+        if band_candidates.contains(&normalized) {
             let first = first_page_band.entry(normalized).or_insert(page);
             if page < *first {
                 *first = page;
@@ -747,17 +489,7 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
             .collect::<Vec<_>>()
             .join(" ");
         let normalized = normalize_for_comparison(&coalesced);
-        if matches_candidate(&normalized, &band_candidates, &compact_band_candidates) {
-            if matches_candidate(
-                &normalized,
-                &remove_all_band_candidates,
-                &compact_remove_all_band_candidates,
-            ) {
-                for &idx in &sorted_indices {
-                    removal_set.insert(idx);
-                }
-                continue;
-            }
+        if band_candidates.contains(&normalized) {
             let first = first_page_band.get(&normalized).copied().unwrap_or(0);
             if page > first {
                 for &idx in &sorted_indices {
@@ -782,10 +514,15 @@ fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<us
     }
 
     if removal_set.is_empty() {
-        return HashSet::new();
+        return lines;
     }
 
-    removal_set
+    lines
+        .into_iter()
+        .enumerate()
+        .filter(|(idx, _)| !removal_set.contains(idx))
+        .map(|(_, line)| line)
+        .collect()
 }
 
 #[cfg(test)]
@@ -805,6 +542,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid,
         }
@@ -819,29 +557,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_has_broken_word_spacing_detects_split_words() {
-        assert!(has_broken_word_spacing(
-            "F rom p rese rva tion to access a nd be yond"
-        ));
-        assert!(has_broken_word_spacing("Conve rs ing w ith the pas t"));
-        assert!(has_broken_word_spacing("The Na tional Arch ives (U K)"));
-    }
-
-    #[test]
-    fn test_has_broken_word_spacing_ignores_normal_short_words() {
-        assert!(!has_broken_word_spacing(
-            "Reunir talento e empresas é um dos fatores po- sitivos para comunidades de sucesso"
-        ));
-        assert!(!has_broken_word_spacing("Witnessed on behalf of"));
-        assert!(!has_broken_word_spacing(
-            "V = Volume in m3/kg H = Enthalpy in kJ/kg S = Entropy in kJ/kg.K"
-        ));
-        assert!(!has_broken_word_spacing(
-            "TITULAR DEL PODER EJECUTIVO FEDERAL, A TRAVÉS DE LA SECRETARÍA DE ECONOMÍA, A HACER VALER EL PRINCIPIO DE"
-        ));
-    }
-
     #[test]
     fn test_merge_struct_tree_headings() {
         // Two consecutive lines tagged as H2 via struct tree, same font size as body
@@ -969,222 +684,4 @@ mod tests {
             .unwrap();
         assert_eq!(first_header.page, 1, "first occurrence should be on page 1");
     }
-
-    #[test]
-    fn test_strip_repeated_clean_bottom_footers_kept_below_document_threshold() {
-        let mut lines = Vec::new();
-        for page in 1..=8u32 {
-            for row in 0..12u32 {
-                lines.push(make_line(
-                    &format!("unique body content page {page} row {row}"),
-                    9.5,
-                    page,
-                    600.0 - row as f32 * 20.0,
-                    None,
-                ));
-            }
-            lines.push(make_line(
-                &format!("Chapter running footer {}", 90 + page),
-                7.5,
-                page,
-                39.5,
-                None,
-            ));
-        }
-
-        let result = strip_repeated_lines(lines, 200);
-
-        let footer_count = result
-            .iter()
-            .filter(|line| line.text().contains("Chapter running footer"))
-            .count();
-        assert_eq!(
-            footer_count, 8,
-            "clean repeated footer should not use the lower garbled-text threshold"
-        );
-    }
-
-    #[test]
-    fn test_strip_repeated_garbled_bottom_footers_removes_all_occurrences_in_long_doc() {
-        let mut lines = Vec::new();
-        for page in 1..=8u32 {
-            for row in 0..12u32 {
-                lines.push(make_line(
-                    &format!("unique body content page {page} row {row}"),
-                    9.5,
-                    page,
-                    600.0 - row as f32 * 20.0,
-                    None,
-                ));
-            }
-            lines.push(make_line(
-                &format!("M L a t the Na tional Libra ry of N orwa y {}", 90 + page),
-                7.5,
-                page,
-                39.5,
-                None,
-            ));
-        }
-
-        let result = strip_repeated_lines(lines, 200);
-
-        assert!(
-            result
-                .iter()
-                .all(|line| !line.text().contains("Na tional Libra")),
-            "garbled bottom running footer should be removed from every page"
-        );
-        assert!(
-            result
-                .iter()
-                .any(|line| line.text().contains("unique body content page 1 row 0")),
-            "body text should be preserved"
-        );
-    }
-
-    #[test]
-    fn test_strip_repeated_document_wide_garbled_footers_removes_all_occurrences() {
-        let mut lines = Vec::new();
-        for page in 1..=8u32 {
-            for row in 0..12u32 {
-                lines.push(make_line(
-                    &format!("unique body content page {page} row {row}"),
-                    9.5,
-                    page,
-                    600.0 - row as f32 * 20.0,
-                    None,
-                ));
-            }
-            lines.push(make_line(
-                &format!("F rom p rese rva tion to access a nd be yond {}", 90 + page),
-                7.5,
-                page,
-                39.5,
-                None,
-            ));
-        }
-
-        let result = strip_repeated_lines(lines, 8);
-
-        let footer_count = result
-            .iter()
-            .filter(|line| line.text().contains("be yond"))
-            .count();
-        assert_eq!(
-            footer_count, 0,
-            "document-wide garbled footers should be removed from every page"
-        );
-    }
-
-    #[test]
-    fn test_strip_repeated_sparse_uppercase_headers_keep_first_occurrence() {
-        let mut lines = Vec::new();
-        for page in 1..=5u32 {
-            lines.push(make_line(
-                "PROPOSICIÓN CON PUNTO DE ACUERDO POR EL QUE EL SENADO DE LA REPÚBLICA",
-                8.0,
-                page,
-                720.0,
-                None,
-            ));
-            lines.push(make_line(
-                "A TRAVÉS DE LA SECRETARÍA DE ECONOMÍA",
-                8.0,
-                page,
-                704.0,
-                None,
-            ));
-            lines.push(make_line(
-                &format!("unique sparse-page body text {page}"),
-                10.0,
-                page,
-                620.0,
-                None,
-            ));
-        }
-
-        let result = strip_repeated_lines(lines, 5);
-
-        let title_count = result
-            .iter()
-            .filter(|line| line.text().contains("PROPOSICIÓN CON PUNTO"))
-            .count();
-        assert_eq!(
-            title_count, 1,
-            "sparse repeated heading should keep the first occurrence"
-        );
-    }
-
-    #[test]
-    fn test_strip_repeated_bottom_footers_matches_minor_garbling() {
-        let mut lines = Vec::new();
-        for page in 1..=9u32 {
-            for row in 0..12u32 {
-                lines.push(make_line(
-                    &format!("distinct paragraph text page {page} row {row}"),
-                    9.5,
-                    page,
-                    600.0 - row as f32 * 20.0,
-                    None,
-                ));
-            }
-
-            let footer = if page == 1 {
-                "F rom p rese rva tion to access a nd be yond 95"
-            } else {
-                "F om r p rese rva tion to access a nd be yond 97"
-            };
-            lines.push(make_line(footer, 7.5, page, 39.5, None));
-        }
-
-        let result = strip_repeated_lines(lines, 200);
-
-        assert!(
-            result.iter().all(|line| !line.text().contains("be yond")),
-            "fuzzy footer variant should be removed once the repeated form is detected"
-        );
-        assert!(
-            result.iter().any(|line| line
-                .text()
-                .contains("distinct paragraph text page 9 row 11")),
-            "non-footer edge-adjacent body text should be preserved"
-        );
-    }
-
-    #[test]
-    fn test_strip_repeated_bottom_footers_matches_sparse_first_page_variant() {
-        let mut lines = Vec::new();
-        for page in 1..=9u32 {
-            let body_rows = if page == 1 { 3 } else { 12 };
-            for row in 0..body_rows {
-                lines.push(make_line(
-                    &format!("distinct paragraph text page {page} row {row}"),
-                    9.5,
-                    page,
-                    600.0 - row as f32 * 20.0,
-                    None,
-                ));
-            }
-
-            let footer = if page == 1 {
-                "F rom p rese rva tion to access a nd be yond 95"
-            } else {
-                "F om r p rese rva tion to access a nd be yond 97"
-            };
-            lines.push(make_line(footer, 7.5, page, 39.5, None));
-        }
-
-        let result = strip_repeated_lines(lines, 200);
-
-        assert!(
-            result.iter().all(|line| !line.text().contains("be yond")),
-            "sparse first page variant should be removed once later lower-edge footers prove the candidate"
-        );
-        assert!(
-            result
-                .iter()
-                .any(|line| line.text().contains("distinct paragraph text page 1 row 0")),
-            "sparse first page body text should be preserved"
-        );
-    }
 }

src/python.rs

@@ -266,6 +266,8 @@ pub struct PyTextItem {
     #[pyo3(get)]
     pub is_italic: bool,
     #[pyo3(get)]
+    pub is_underline: bool,
+    #[pyo3(get)]
     pub item_type: String,
 }
 
@@ -349,6 +351,7 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
             page: item.page,
             is_bold: item.is_bold,
             is_italic: item.is_italic,
+            is_underline: item.is_underline,
             item_type: item_type_str(&item.item_type),
         })
         .collect()

src/tables/detect_heuristic.rs

@@ -104,6 +104,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
                 page: first_item.page,
                 is_bold: first_item.is_bold,
                 is_italic: first_item.is_italic,
+                is_underline: first_item.is_underline,
                 item_type: first_item.item_type.clone(),
                 mcid: first_item.mcid,
             });

src/tables/detect_lines.rs

@@ -393,6 +393,7 @@ mod tests {
             page,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }

src/tables/detect_rects.rs

@@ -2314,6 +2314,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -3272,6 +3273,7 @@ mod tests {
                     page: 1,
                     is_bold: false,
                     is_italic: false,
+                    is_underline: false,
                     item_type: crate::types::ItemType::Text,
                     mcid: None,
                 });
@@ -3581,6 +3583,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: crate::types::ItemType::Text,
                 mcid: None,
             });

src/tables/detect_struct.rs

@@ -586,6 +586,7 @@ mod tests {
             page,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid,
         }

src/tables/financial.rs

@@ -108,6 +108,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
             page: item.page,
             is_bold: item.is_bold,
             is_italic: item.is_italic,
+            is_underline: item.is_underline,
             item_type: item.item_type.clone(),
             mcid: item.mcid,
         });

src/tables/grid.rs

@@ -514,6 +514,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -866,6 +867,7 @@ mod tests {
                         font: String::new(),
                         is_bold: false,
                         is_italic: false,
+                        is_underline: false,
                         item_type: ItemType::Text,
                         mcid: None,
                         page: 1,
@@ -902,6 +904,7 @@ mod tests {
                         font: String::new(),
                         is_bold: false,
                         is_italic: false,
+                        is_underline: false,
                         item_type: ItemType::Text,
                         mcid: None,
                         page: 1,

src/tables/mod.rs

@@ -235,6 +235,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
             page: item.page,
             is_bold: item.is_bold,
             is_italic: item.is_italic,
+            is_underline: item.is_underline,
             item_type: item.item_type.clone(),
             mcid: item.mcid,
         });
@@ -255,6 +256,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
             page: item.page,
             is_bold: item.is_bold,
             is_italic: item.is_italic,
+            is_underline: item.is_underline,
             item_type: item.item_type.clone(),
             mcid: item.mcid,
         });
@@ -1429,6 +1431,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -1446,6 +1449,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }

src/text_utils.rs

@@ -883,6 +883,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }
@@ -1002,6 +1003,7 @@ mod tests {
                 page: 1,
                 is_bold: false,
                 is_italic: false,
+                is_underline: false,
                 item_type: ItemType::Text,
                 mcid: None,
             });
@@ -1078,6 +1080,7 @@ mod tests {
             page: 1,
             is_bold: false,
             is_italic: false,
+            is_underline: false,
             item_type: ItemType::Text,
             mcid: None,
         }

src/types.rs

@@ -116,6 +116,10 @@ pub struct TextItem {
     pub is_bold: bool,
     /// Whether the font is italic
     pub is_italic: bool,
+    /// Whether the text is underlined (drawn rule/thin rect under the
+    /// baseline — PDFs have no underline font flag, so this is detected
+    /// geometrically after extraction; see `extractor::underline`).
+    pub is_underline: bool,
     /// Type of item (text, image, link)
     pub item_type: ItemType,
     /// Marked Content ID from the content stream's BDC/BMC operator.

tests/integration_tests.rs

@@ -104,6 +104,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
         page,
         is_bold: false,
         is_italic: false,
+        is_underline: false,
         item_type: ItemType::Text,
         mcid: None,
     }
@@ -129,6 +130,7 @@ fn make_text_item_with_font(
         page,
         is_bold: is_bold_font(font),
         is_italic: is_italic_font(font),
+        is_underline: false,
         item_type: ItemType::Text,
         mcid: None,
     }