chore: source Python package version from Cargo.toml via maturin

Address PR review (cubic P2): pyproject.toml pinned version = "0.1.0", which overrides Cargo.toml, so a maturin build produced a 0.1.0 Python artifact regardless of the crate version (it had drifted since the PyO3 bindings were added). Switch to dynamic = ["version"] so maturin sources the version from Cargo.toml [package] version and the two can no longer diverge. No workflow auto-publishes the Python package, so this is metadata hygiene rather than a release-path fix. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
fix(lib): make cipher detection case-agnostic via sorted-histogram shape
2026-07-09 12:36:59 -07:00 · 2026-07-09 12:35:24 -07:00 · 2026-07-09 12:25:10 -07:00 · 2026-07-09 12:16:35 -07:00 · 2026-07-09 12:12:20 -07:00 · 2026-07-08 23:40:08 -07:00
37 changed files with 1703 additions and 80 deletions
@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector"
-version = "0.1.3"
+version = "0.1.4"
 edition = "2021"
 autobins = false
 authors = ["Firecrawl Team"]
@@ -111,6 +111,9 @@ pdf2md document.pdf
 # JSON output (for piping)
 pdf2md document.pdf --json

+# Positioned TextItem JSON, including is_underline metadata
+pdf2md document.pdf --items-json
+
 # Raw markdown only (no headers)
 pdf2md document.pdf --raw

@@ -830,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"

 [[package]]
 name = "pdf-inspector"
-version = "0.1.3"
+version = "0.1.4"
 dependencies = [
 "env_logger",
 "log",
@@ -1,6 +1,6 @@
 {
  "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.8",
+  "version": "1.9.11",
  "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
  "main": "index.js",
  "types": "index.d.ts",
@@ -80,6 +80,9 @@ pub struct TextItem {
    pub page: u32,
    pub is_bold: bool,
    pub is_italic: bool,
+    /// Underline detected geometrically (drawn rule/thin rect under the
+    /// baseline) — PDFs carry no underline font flag.
+    pub is_underline: bool,
    pub item_type: ItemType,
    /// URL for link items, `None` for other types.
    pub link_url: Option<String>,
@@ -290,6 +293,7 @@ pub fn extract_text_with_positions(
                    page: item.page,
                    is_bold: item.is_bold,
                    is_italic: item.is_italic,
+                    is_underline: item.is_underline,
                    item_type,
                    link_url,
                }
@@ -38,6 +38,7 @@ class TextItem:
    page: int
    is_bold: bool
    is_italic: bool
+    is_underline: bool
    item_type: str

 class RegionText:
@@ -4,7 +4,9 @@ build-backend = "maturin"

 [project]
 name = "pdf-inspector"
-version = "0.1.0"
+# Version is sourced from Cargo.toml [package] version by maturin so the Python
+# artifact always tracks the crate release instead of drifting on its own.
+dynamic = ["version"]
 description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
 license = { text = "MIT" }
 requires-python = ">=3.8"
@@ -1,6 +1,10 @@
 //! CLI tool for PDF to Markdown conversion

-use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode};
+use pdf_inspector::extractor::ItemType;
+use pdf_inspector::{
+    extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions,
+    PdfType, ProcessMode, TextItem,
+};
 use std::collections::HashSet;
 use std::env;
 use std::fmt::Write;
@@ -47,6 +51,92 @@ fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> Stri
        .join(",")
 }

+fn item_type_label(item_type: &ItemType) -> &'static str {
+    match item_type {
+        ItemType::Text => "text",
+        ItemType::Image => "image",
+        ItemType::Link(_) => "link",
+        ItemType::FormField => "form_field",
+    }
+}
+
+fn format_items_json(items: &[TextItem]) -> String {
+    let underlined_count = items.iter().filter(|item| item.is_underline).count();
+    let items_json = items
+        .iter()
+        .map(|item| {
+            let mcid = item
+                .mcid
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "null".to_string());
+            let link_url = match &item.item_type {
+                ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)),
+                _ => String::new(),
+            };
+            format!(
+                r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
+                json_escape(&item.text),
+                item.page,
+                item.x,
+                item.y,
+                item.width,
+                item.height,
+                json_escape(&item.font),
+                item.font_size,
+                item.is_bold,
+                item.is_italic,
+                item.is_underline,
+                item_type_label(&item.item_type),
+                mcid,
+                link_url,
+            )
+        })
+        .collect::<Vec<_>>()
+        .join(",");
+
+    format!(
+        r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#,
+        items.len(),
+        underlined_count,
+        items_json
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::format_items_json;
+    use pdf_inspector::extractor::ItemType;
+    use pdf_inspector::TextItem;
+
+    #[test]
+    fn items_json_includes_position_and_underline_metadata() {
+        let items = vec![TextItem {
+            text: "A \"quoted\" item".to_string(),
+            x: 12.345,
+            y: 67.891,
+            width: 23.456,
+            height: 9.876,
+            font: "F1".to_string(),
+            font_size: 10.0,
+            page: 2,
+            is_bold: false,
+            is_italic: true,
+            is_underline: true,
+            item_type: ItemType::Text,
+            mcid: Some(7),
+        }];
+
+        let json = format_items_json(&items);
+
+        assert!(json.contains(r#""text":"A \"quoted\" item""#));
+        assert!(json.contains(r#""page":2"#));
+        assert!(json.contains(r#""x":12.35"#));
+        assert!(json.contains(r#""is_underline":true"#));
+        assert!(json.contains(r#""item_type":"text""#));
+        assert!(json.contains(r#""mcid":7"#));
+    }
+}
+
 /// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
 fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
    let mut pages = HashSet::new();
@@ -104,6 +194,7 @@ fn main() {
    if args.len() < 2 {
        eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
        eprintln!("       {} <pdf_file> --json", args[0]);
+        eprintln!("       {} <pdf_file> --items-json", args[0]);
        eprintln!("       {} <pdf_file> --raw", args[0]);
        eprintln!();
        eprintln!("Converts PDF to Markdown with smart type detection.");
@@ -111,6 +202,7 @@ fn main() {
        eprintln!();
        eprintln!("Options:");
        eprintln!("  --json              Output result as JSON");
+        eprintln!("  --items-json        Output positioned TextItem JSON");
        eprintln!("  --raw               Output only markdown (no headers)");
        eprintln!("  --pages             Insert page break markers (<!-- Page N -->)");
        eprintln!("  --select-pages N    Only process specified pages (e.g. 1,3,5-10)");
@@ -121,6 +213,7 @@ fn main() {

    let pdf_path = &args[1];
    let json_output = args.iter().any(|a| a == "--json");
+    let items_json_output = args.iter().any(|a| a == "--items-json");
    let raw_output = args.iter().any(|a| a == "--raw");
    let page_numbers = args.iter().any(|a| a == "--pages");
    let detect_only = args.iter().any(|a| a == "--detect-only");
@@ -145,6 +238,17 @@ fn main() {
            })
        });

+    if items_json_output {
+        match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) {
+            Ok(items) => println!("{}", format_items_json(&items)),
+            Err(e) => {
+                println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
+                process::exit(1);
+            }
+        }
+        return;
+    }
+
    let output_file = args
        .get(2)
        .filter(|a| !a.starts_with("--"))
@@ -17,6 +17,7 @@ use super::fonts::{
    build_font_encodings, build_font_widths, compute_string_width_ts, extract_text_from_operand,
    get_font_file2_obj_num, get_operand_bytes, CMapDecisionCache,
 };
+use super::underline::UnderlineLine;
 use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType};
 use super::{get_number, image_bbox_from_ctm, multiply_matrices};

@@ -74,6 +75,37 @@ fn strip_pdf_comments(data: &[u8]) -> Vec<u8> {
    result
 }

+fn transform_path_point(x: f32, y: f32, ctm: &[f32; 6]) -> (f32, f32) {
+    (
+        x * ctm[0] + y * ctm[2] + ctm[4],
+        x * ctm[1] + y * ctm[3] + ctm[5],
+    )
+}
+
+fn transformed_stroke_width(
+    line_width: f32,
+    ctm: &[f32; 6],
+    x1: f32,
+    y1: f32,
+    x2: f32,
+    y2: f32,
+) -> f32 {
+    let user_width = line_width.abs();
+    let dx = x2 - x1;
+    let dy = y2 - y1;
+    let len = (dx * dx + dy * dy).sqrt();
+    if len <= f32::EPSILON {
+        return user_width;
+    }
+
+    // PDF stroke width scales perpendicular to the path direction.
+    let nx = -dy / len;
+    let ny = dx / len;
+    let ndx = nx * ctm[0] + ny * ctm[2];
+    let ndy = nx * ctm[1] + ny * ctm[3];
+    user_width * (ndx * ndx + ndy * ndy).sqrt()
+}
+
 /// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates
 /// the page uses fonts with unresolvable gid-encoded glyphs.
 pub(crate) fn extract_page_text_items(
@@ -89,6 +121,7 @@ pub(crate) fn extract_page_text_items(
    let mut rects: Vec<PdfRect> = Vec::new();
    let mut clip_rects: Vec<PdfRect> = Vec::new();
    let mut lines: Vec<PdfLine> = Vec::new();
+    let mut underline_lines: Vec<UnderlineLine> = Vec::new();

    // Path construction state for m/l/h → S/s line extraction
    let mut path_subpath_start: Option<(f32, f32)> = None;
@@ -97,6 +130,12 @@ pub(crate) fn extract_page_text_items(
    // Completed subpaths (each a vec of line segments) for f/f* rect extraction
    let mut pending_subpaths: Vec<Vec<(f32, f32, f32, f32)>> = Vec::new();
    let mut fill_rects: Vec<PdfRect> = Vec::new();
+    // `re` rects awaiting a paint operator. Underline detection must only
+    // see painted rects: a `re W n` clip path or `re n` no-op draws nothing
+    // on the page, so treating every `re` as ink would underline text that
+    // merely sits near an invisible clip boundary.
+    let mut pending_re_rects: Vec<PdfRect> = Vec::new();
+    let mut painted_rects: Vec<PdfRect> = Vec::new();

    // Get fonts for encoding
    let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
@@ -188,10 +227,12 @@ pub(crate) fn extract_page_text_items(
    // Graphics state tracking
    let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
    let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
+    let mut line_width: f32 = 1.0;
    #[derive(Clone)]
    struct SavedGraphicsState {
        ctm: [f32; 6],
        text_rendering_mode: i32,
+        line_width: f32,
        char_spacing: f32,
        word_spacing: f32,
        text_leading: f32,
@@ -240,6 +281,7 @@ pub(crate) fn extract_page_text_items(
                gstate_stack.push(SavedGraphicsState {
                    ctm,
                    text_rendering_mode,
+                    line_width,
                    char_spacing,
                    word_spacing,
                    text_leading,
@@ -252,6 +294,7 @@ pub(crate) fn extract_page_text_items(
                if let Some(saved) = gstate_stack.pop() {
                    ctm = saved.ctm;
                    text_rendering_mode = saved.text_rendering_mode;
+                    line_width = saved.line_width;
                    char_spacing = saved.char_spacing;
                    word_spacing = saved.word_spacing;
                    text_leading = saved.text_leading;
@@ -273,6 +316,11 @@ pub(crate) fn extract_page_text_items(
                    ctm = multiply_matrices(&new_matrix, &ctm);
                }
            }
+            "w" => {
+                if let Some(width) = op.operands.first().and_then(get_number) {
+                    line_width = width;
+                }
+            }
            "BT" => {
                // Begin text block
                in_text_block = true;
@@ -441,6 +489,7 @@ pub(crate) fn extract_page_text_items(
                                page: page_num,
                                is_bold: is_bold_font(base_font),
                                is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                item_type: ItemType::Text,
                                mcid: current_mcid(&marked_content_stack),
                            });
@@ -606,6 +655,7 @@ pub(crate) fn extract_page_text_items(
                                    page: page_num,
                                    is_bold: is_bold_font(base_font),
                                    is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                    item_type: ItemType::Text,
                                    mcid: current_mcid(&marked_content_stack),
                                });
@@ -669,6 +719,7 @@ pub(crate) fn extract_page_text_items(
                                page: page_num,
                                is_bold: is_bold_font(base_font),
                                is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                item_type: ItemType::Text,
                                mcid: current_mcid(&marked_content_stack),
                            });
@@ -705,6 +756,7 @@ pub(crate) fn extract_page_text_items(
                                        page: page_num,
                                        is_bold: false,
                                        is_italic: false,
+                                        is_underline: false,
                                        item_type: ItemType::Image,
                                        mcid: current_mcid(&marked_content_stack),
                                    });
@@ -801,6 +853,7 @@ pub(crate) fn extract_page_text_items(
                                    page: page_num,
                                    is_bold: is_bold_font(base_font),
                                    is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                    item_type: ItemType::Text,
                                    mcid: entry
                                        .mcid
@@ -825,13 +878,19 @@ pub(crate) fn extract_page_text_items(
                    let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5];
                    let w_dev = rw * ctm[0];
                    let h_dev = rh * ctm[3];
-                    rects.push(PdfRect {
+                    let rect = PdfRect {
                        x: x_dev,
                        y: y_dev,
                        width: w_dev,
                        height: h_dev,
                        page: page_num,
-                    });
+                    };
+                    // Underline detection must only see rects that are
+                    // actually painted — a `re` used purely as a clip path
+                    // (`re W n`) or discarded (`re n`) draws nothing. Hold
+                    // the rect as pending until a paint operator confirms it.
+                    pending_re_rects.push(rect.clone());
+                    rects.push(rect);
                }
            }
            // ── Path construction operators ──────────────────────
@@ -881,10 +940,8 @@ pub(crate) fn extract_page_text_items(
                    }
                }
                for (x1, y1, x2, y2) in pending_lines.drain(..) {
-                    let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
-                    let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
-                    let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
-                    let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
+                    let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
+                    let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
                    lines.push(PdfLine {
                        x1: x1d,
                        y1: y1d,
@@ -892,7 +949,16 @@ pub(crate) fn extract_page_text_items(
                        y2: y2d,
                        page: page_num,
                    });
+                    underline_lines.push(UnderlineLine {
+                        x1: x1d,
+                        y1: y1d,
+                        x2: x2d,
+                        y2: y2d,
+                        stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
+                        page: page_num,
+                    });
                }
+                painted_rects.append(&mut pending_re_rects);
                pending_subpaths.clear();
                path_subpath_start = None;
                path_current = None;
@@ -908,10 +974,8 @@ pub(crate) fn extract_page_text_items(
                    }
                }
                for (x1, y1, x2, y2) in pending_lines.drain(..) {
-                    let x1d = x1 * ctm[0] + y1 * ctm[2] + ctm[4];
-                    let y1d = x1 * ctm[1] + y1 * ctm[3] + ctm[5];
-                    let x2d = x2 * ctm[0] + y2 * ctm[2] + ctm[4];
-                    let y2d = x2 * ctm[1] + y2 * ctm[3] + ctm[5];
+                    let (x1d, y1d) = transform_path_point(x1, y1, &ctm);
+                    let (x2d, y2d) = transform_path_point(x2, y2, &ctm);
                    lines.push(PdfLine {
                        x1: x1d,
                        y1: y1d,
@@ -919,7 +983,16 @@ pub(crate) fn extract_page_text_items(
                        y2: y2d,
                        page: page_num,
                    });
+                    underline_lines.push(UnderlineLine {
+                        x1: x1d,
+                        y1: y1d,
+                        x2: x2d,
+                        y2: y2d,
+                        stroke_width: transformed_stroke_width(line_width, &ctm, x1, y1, x2, y2),
+                        page: page_num,
+                    });
                }
+                painted_rects.append(&mut pending_re_rects);
                pending_subpaths.clear();
                path_subpath_start = None;
                path_current = None;
@@ -977,6 +1050,7 @@ pub(crate) fn extract_page_text_items(
                        }
                    }
                }
+                painted_rects.append(&mut pending_re_rects);
                pending_lines.clear();
                path_subpath_start = None;
                path_current = None;
@@ -1042,7 +1116,10 @@ pub(crate) fn extract_page_text_items(
                // Do NOT clear pending_lines — the following `n` does that
            }
            "n" => {
-                // end path (no-op): discard
+                // end path (no-op): discard — including any `re` rects that
+                // were only ever part of a clip path (`re W n`), which draw
+                // no ink and must not feed underline detection.
+                pending_re_rects.clear();
                pending_lines.clear();
                pending_subpaths.clear();
                path_subpath_start = None;
@@ -1052,6 +1129,12 @@ pub(crate) fn extract_page_text_items(
        }
    }

+    // Underline detection reads only painted ink: `re` rects confirmed by
+    // a paint operator plus filled-subpath rects — never clip-only rects,
+    // which draw nothing.
+    let mut underline_rects = painted_rects;
+    underline_rects.extend(fill_rects.iter().cloned());
+
    // Only use clip/fill rects when no `re` rects exist on this page.
    // Clip rects take priority over fill rects, but first we deduplicate
    // them: some PDFs wrap every text block in a full-page W* clip path,
@@ -1081,8 +1164,17 @@ pub(crate) fn extract_page_text_items(
    // Some PDFs embed landscape content in portrait pages using a rotated text
    // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW).  The layout engine
    // assumes x=horizontal, y=vertical — so we swap coordinates to match.
-    let (items, rects, lines, coords_rotated) =
+    let (mut items, rects, lines, coords_rotated) =
        correct_rotated_page(items, rects, lines, &rotation_votes);
+    if coords_rotated {
+        rotate_underline_graphics(&mut underline_rects, &mut underline_lines);
+    }
+    super::underline::mark_underlined_items(
+        &mut items,
+        &underline_rects,
+        &underline_lines,
+        page_num,
+    );

    let items = super::merge_text_items(items);
    let items = super::merge_subscript_items(items);
@@ -1168,6 +1260,27 @@ fn correct_rotated_page(
    (items, rects, lines, true)
 }

+fn rotate_underline_graphics(rects: &mut [PdfRect], lines: &mut [UnderlineLine]) {
+    for rect in rects {
+        let new_x = rect.y;
+        let new_y = -(rect.x + rect.width.abs());
+        rect.x = new_x;
+        rect.y = new_y;
+        std::mem::swap(&mut rect.width, &mut rect.height);
+    }
+
+    for line in lines {
+        let new_x1 = line.y1;
+        let new_y1 = -line.x1;
+        let new_x2 = line.y2;
+        let new_y2 = -line.x2;
+        line.x1 = new_x1;
+        line.y1 = new_y1;
+        line.x2 = new_x2;
+        line.y2 = new_y2;
+    }
+}
+
 /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance).
 /// Some PDFs emit a full-page clip path for every text block, producing
 /// thousands of identical rects. After dedup these collapse to one rect,
@@ -1217,6 +1330,57 @@ mod tests {
        }
    }

+    fn simple_doc_with_content(content: &[u8]) -> (lopdf::Document, lopdf::ObjectId) {
+        use lopdf::{dictionary, Object, Stream};
+
+        let mut doc = lopdf::Document::new();
+        let widths: Vec<Object> = (0..=255).map(|_| 600.into()).collect();
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+            "FirstChar" => 0,
+            "LastChar" => 255,
+            "Widths" => Object::Array(widths),
+        });
+        let content_id = doc.add_object(Object::Stream(Stream::new(
+            dictionary! {},
+            content.to_vec(),
+        )));
+        let page_id = doc.add_object(dictionary! {
+            "Type" => "Page",
+            "Contents" => Object::Reference(content_id),
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => Object::Reference(font_id),
+                },
+            },
+            "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
+        });
+        let pages_id = doc.add_object(dictionary! {
+            "Type" => "Pages",
+            "Count" => Object::Integer(1),
+            "Kids" => vec![Object::Reference(page_id)],
+        });
+        let catalog_id = doc.add_object(dictionary! {
+            "Type" => "Catalog",
+            "Pages" => Object::Reference(pages_id),
+        });
+        doc.trailer.set("Root", Object::Reference(catalog_id));
+
+        (doc, page_id)
+    }
+
+    fn extract_simple_items(content: &[u8]) -> Vec<TextItem> {
+        use crate::tounicode::FontCMaps;
+
+        let (doc, page_id) = simple_doc_with_content(content);
+        let font_cmaps = FontCMaps::from_doc(&doc);
+        let ((items, _, _), _, _) =
+            extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
+        items
+    }
+
    #[test]
    fn test_dedup_rects_identical() {
        let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759];
@@ -1266,6 +1430,38 @@ mod tests {
        assert_eq!(single.len(), 1);
    }

+    #[test]
+    fn thick_stroked_rule_does_not_mark_underline() {
+        let content = b"BT /F1 12 Tf 1 0 0 1 100 500 Tm (THICK) Tj ET
+4 w
+100 498 m 170 498 l S
+BT /F1 12 Tf 1 0 0 1 100 480 Tm (THIN) Tj ET
+1 w
+100 478 m 160 478 l S";
+
+        let items = extract_simple_items(content);
+        let thick = items.iter().find(|item| item.text == "THICK").unwrap();
+        let thin = items.iter().find(|item| item.text == "THIN").unwrap();
+
+        assert!(!thick.is_underline);
+        assert!(thin.is_underline);
+    }
+
+    #[test]
+    fn rotated_page_underline_is_detected_after_coordinate_correction() {
+        let content = b"BT /F1 12 Tf 0 1 -1 0 200 100 Tm (HELLO) Tj ET
+BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET
+1 w
+202 100 m 202 170 l S";
+
+        let items = extract_simple_items(content);
+        let hello = items.iter().find(|item| item.text == "HELLO").unwrap();
+        let world = items.iter().find(|item| item.text == "WORLD").unwrap();
+
+        assert!(hello.is_underline);
+        assert!(!world.is_underline);
+    }
+
    #[test]
    fn test_skip_excessive_operations() {
        use crate::tounicode::FontCMaps;
@@ -1494,6 +1494,7 @@ mod tests {
            page,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1623,6 +1624,7 @@ mod tests {
                    page,
                    is_bold: false,
                    is_italic: false,
+                    is_underline: false,
                    item_type: ItemType::Text,
                    mcid: None,
                });
@@ -78,6 +78,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V
                            page: page_num,
                            is_bold: false,
                            is_italic: false,
+                            is_underline: false,
                            item_type: ItemType::Link(url),
                            mcid: None,
                        });
@@ -316,6 +317,7 @@ pub(crate) fn walk_form_fields(
        page: page_num,
        is_bold: false,
        is_italic: false,
+        is_underline: false,
        item_type: ItemType::FormField,
        mcid: None,
    });
@@ -6,11 +6,12 @@ pub(crate) mod content_stream;
 mod fonts;
 mod layout;
 mod links;
+pub(crate) mod underline;
 mod xobjects;

 use crate::text_utils::is_rtl_text;
 use crate::tounicode::FontCMaps;
-use crate::types::{PageExtraction, TextItem};
+use crate::types::{PageExtraction, PdfLine, PdfRect, TextItem};
 use crate::PdfError;
 use log::debug;
 use lopdf::{Document, Object, ObjectId};
@@ -180,6 +181,7 @@ fn extract_positioned_text_impl(
        if threshold > 0.10 {
            page_thresholds.insert(*page_num, threshold);
        }
+        suppress_table_underlines(&mut items, &rects, &lines, *page_num);
        debug!(
            "page {}: {} text items, {} rects, {} lines{}",
            page_num,
@@ -226,6 +228,38 @@ fn extract_positioned_text_impl(
    ))
 }

+fn suppress_table_underlines(
+    items: &mut [TextItem],
+    rects: &[PdfRect],
+    lines: &[PdfLine],
+    page: u32,
+) {
+    if !items.iter().any(|item| item.is_underline) {
+        return;
+    }
+
+    let mut table_item_indices: HashSet<usize> = HashSet::new();
+
+    if !rects.is_empty() {
+        let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page);
+        for table in rect_tables {
+            table_item_indices.extend(table.item_indices);
+        }
+    }
+
+    if !lines.is_empty() {
+        for table in crate::tables::detect_tables_from_lines(items, lines, page) {
+            table_item_indices.extend(table.item_indices);
+        }
+    }
+
+    for index in table_item_indices {
+        if let Some(item) = items.get_mut(index) {
+            item.is_underline = false;
+        }
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Shared helpers (used by submodules via `super::`)
 // ---------------------------------------------------------------------------
@@ -533,6 +567,18 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                if (next.font_size - first.font_size).abs() > first.font_size * 0.20 {
                    break;
                }
+                // Never merge across style boundaries: the merged item
+                // carries `first`'s flags, so absorbing a styled run into a
+                // plain neighbor (or vice versa) silently erases the styling
+                // that markdown emission and downstream inline-styling need —
+                // and OR-ing underline instead would stretch `<u>` spans over
+                // neighboring plain text.
+                if next.is_bold != first.is_bold
+                    || next.is_italic != first.is_italic
+                    || next.is_underline != first.is_underline
+                {
+                    break;
+                }
                let gap = next.x - end_x;
                let x_gap_max = if *preserve_stream_order && is_standalone_bullet_text(&text) {
                    first.font_size * 1.2
@@ -591,6 +637,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                page: first.page,
                is_bold: first.is_bold,
                is_italic: first.is_italic,
+                is_underline: first.is_underline,
                item_type: first.item_type.clone(),
                mcid: first.mcid,
            });
@@ -673,7 +720,17 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
                        let gap = item.x - parent_right;
                        // Subscripts must be tightly adjacent (within ~1pt)
                        if gap < parent.font_size * 0.2 && gap > -parent.font_size * 0.3 {
-                            parent.text.push_str(&item.text);
+                            // Preserve the script when absorbing it: map the
+                            // digits to Unicode sub/superscript forms so the
+                            // raised/lowered rendering survives in extracted
+                            // text ("H"+"2" → "H₂", "word"+"2" → "word²").
+                            // NFKC/NFKD normalization folds these back to
+                            // plain digits, so text matching downstream is
+                            // unaffected. Direction from the baseline offset
+                            // (y-up here): raised → superscript (footnote
+                            // refs), lowered/level → subscript (chemistry).
+                            let raised = item.y > parent.y + parent.font_size * 0.1;
+                            parent.text.push_str(&map_script_digits(&item.text, raised));
                            parent.width = (item.x + item.width) - parent.x;
                            continue;
                        }
@@ -688,6 +745,21 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
    result
 }

+/// Map ASCII digits to their Unicode superscript (`raised`) or subscript
+/// forms. Callers guarantee digit-only input (see `merge_subscript_items`);
+/// anything else passes through unchanged.
+fn map_script_digits(text: &str, raised: bool) -> String {
+    const SUP: [char; 10] = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹'];
+    const SUB: [char; 10] = ['₀', '₁', '₂', '₃', '₄', '₅', '₆', '₇', '₈', '₉'];
+    text.chars()
+        .map(|c| match c.to_digit(10) {
+            Some(d) if raised => SUP[d as usize],
+            Some(d) => SUB[d as usize],
+            None => c,
+        })
+        .collect()
+}
+
 /// Helper to get f32 from Object
 pub(crate) fn get_number(obj: &Object) -> Option<f32> {
    match obj {
@@ -701,7 +773,7 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
 mod tests {
    use super::*;
    use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items};
-    use crate::types::{ItemType, TextLine};
+    use crate::types::{ItemType, PdfLine, TextLine};
    use layout::{detect_columns, is_newspaper_layout, ColumnRegion};

    fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
@@ -716,6 +788,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -726,6 +799,16 @@ mod tests {
        item
    }

+    fn make_line(x1: f32, y1: f32, x2: f32, y2: f32) -> PdfLine {
+        PdfLine {
+            x1,
+            y1,
+            x2,
+            y2,
+            page: 1,
+        }
+    }
+
    #[test]
    fn trace_text_preview_truncates_on_char_boundary() {
        let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}');
@@ -736,6 +819,28 @@ mod tests {
        assert!(preview.ends_with('\u{FFFD}'));
    }

+    #[test]
+    fn merge_items_breaks_at_style_boundaries() {
+        // A styled run adjacent to plain text must stay a separate item —
+        // merging would erase the flags (italic) or stretch the span
+        // (underline) before markdown emission sees them.
+        let mut italic = make_merge_item("emphasis", 150.0, 40.0);
+        italic.is_italic = true;
+        let mut underlined = make_merge_item("term", 195.0, 20.0);
+        underlined.is_underline = true;
+        let items = vec![
+            make_merge_item("plain lead", 100.0, 48.0),
+            italic,
+            underlined,
+            make_merge_item("plain tail", 218.0, 45.0),
+        ];
+        let merged = merge_text_items(items);
+        assert_eq!(merged.len(), 4);
+        assert!(merged[1].is_italic && !merged[1].is_underline);
+        assert!(merged[2].is_underline && !merged[2].is_italic);
+        assert!(!merged[3].is_underline && !merged[3].is_italic);
+    }
+
    #[test]
    fn merge_items_no_space_before_period() {
        // Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap
@@ -774,6 +879,27 @@ mod tests {
        assert_eq!(merged[0].text, "hello world");
    }

+    #[test]
+    fn merge_items_preserves_underline_from_later_fragment() {
+        // Fragments with differing underline stay separate items — OR-merging
+        // would stretch the eventual `<u>` span over the plain fragment.
+        // Line-level text assembly still joins them without a space (tight
+        // gap), so the rendered word is unchanged: `pre<u>fix</u>`.
+        let mut items = vec![
+            make_merge_item("pre", 100.0, 18.0),
+            make_merge_item("fix", 119.0, 18.0),
+        ];
+        items[1].is_underline = true;
+
+        let merged = merge_text_items(items);
+
+        assert_eq!(merged.len(), 2);
+        assert_eq!(merged[0].text, "pre");
+        assert!(!merged[0].is_underline);
+        assert_eq!(merged[1].text, "fix");
+        assert!(merged[1].is_underline);
+    }
+
    #[test]
    fn merge_items_preserves_stream_order_for_backtracking_heading() {
        // Some tagged PDFs emit first-letter ActualText fragments, then reset
@@ -851,6 +977,35 @@ mod tests {
        assert_eq!(texts, vec!["•", "Distant item"]);
    }

+    #[test]
+    fn suppress_table_underlines_clears_line_detected_table_items() {
+        let mut items = vec![
+            make_merge_item("H1", 125.0, 20.0),
+            make_merge_item("H2", 225.0, 20.0),
+            make_merge_item("A", 125.0, 20.0),
+            make_merge_item("B", 225.0, 20.0),
+        ];
+        items[0].y = 490.0;
+        items[1].y = 490.0;
+        items[2].y = 470.0;
+        items[3].y = 470.0;
+        for item in &mut items {
+            item.is_underline = true;
+        }
+        let lines = vec![
+            make_line(100.0, 500.0, 300.0, 500.0),
+            make_line(100.0, 480.0, 300.0, 480.0),
+            make_line(100.0, 460.0, 300.0, 460.0),
+            make_line(100.0, 460.0, 100.0, 500.0),
+            make_line(200.0, 460.0, 200.0, 500.0),
+            make_line(300.0, 460.0, 300.0, 500.0),
+        ];
+
+        suppress_table_underlines(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
    #[test]
    fn test_group_into_lines() {
        let items = vec![
@@ -865,6 +1020,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -879,6 +1035,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -893,6 +1050,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -947,6 +1105,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -961,6 +1120,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -975,6 +1135,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1000,6 +1161,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1014,6 +1176,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1028,6 +1191,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1055,6 +1219,7 @@ mod tests {
                page: 1,
                is_bold: true,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            }
@@ -1089,6 +1254,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            }
@@ -1124,6 +1290,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1138,6 +1305,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1152,6 +1320,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1174,6 +1343,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1286,6 +1456,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1300,6 +1471,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1324,6 +1496,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1338,6 +1511,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            },
@@ -1378,6 +1552,7 @@ mod tests {
                page,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            }],
@@ -1422,6 +1597,7 @@ mod tests {
                page,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            }],
@@ -1466,6 +1642,7 @@ mod tests {
                page,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            }],
@@ -1503,6 +1680,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1518,7 +1696,8 @@ mod tests {
        ];
        let merged = merge_subscript_items(items);
        assert_eq!(merged.len(), 2);
-        assert_eq!(merged[0].text, "NH3");
+        // Lowered baseline → Unicode subscript form (NFKC folds back to "NH3")
+        assert_eq!(merged[0].text, "NH₃");
        assert_eq!(merged[1].text, "Cl");
    }

@@ -1532,10 +1711,21 @@ mod tests {
        ];
        let merged = merge_subscript_items(items);
        assert_eq!(merged.len(), 2);
-        assert_eq!(merged[0].text, "H2");
+        assert_eq!(merged[0].text, "H₂");
        assert_eq!(merged[1].text, "O");
    }

+    #[test]
+    fn test_merge_subscript_items_raised_marker_becomes_superscript() {
+        // Footnote reference: "word" followed by a RAISED small "2" → word²
+        let mut marker = make_item_fs("2", 90.0, 502.5, 2.3, 4.7);
+        marker.y = 502.5; // raised above the 499.0 parent baseline
+        let items = vec![make_item_fs("word", 78.0, 499.0, 12.0, 8.0), marker];
+        let merged = merge_subscript_items(items);
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "word²");
+    }
+
    #[test]
    fn test_merge_subscript_items_no_merge_far_gap() {
        // Subscript-sized item that's far from the parent should NOT merge
@@ -0,0 +1,500 @@
+//! Geometric underline detection.
+//!
+//! PDFs have no underline font flag — underlines are drawn as separate
+//! graphics: stroked horizontal lines (`l`/`S` operators) or thin filled
+//! rectangles (`re`/`f`). This pass correlates those graphics with text
+//! items after extraction: an item is underlined when a horizontal
+//! line/thin rect sits just below its baseline and covers most of its
+//! horizontal extent.
+//!
+//! Repeated same-span rules are treated as table/form rulings rather than
+//! underlines, which avoids marking every cell in ruled tables.
+
+use std::collections::HashSet;
+
+use crate::types::{ItemType, PdfRect, TextItem};
+
+/// Max thickness (pt) for a stroked line / filled rect to count as an
+/// underline rule rather than a border or decorative band.
+const MAX_RULE_THICKNESS: f32 = 2.0;
+
+/// Fraction of the item's width that the rule must cover horizontally.
+const MIN_X_OVERLAP: f32 = 0.6;
+
+/// Same-span rules repeated at this many y-levels are usually table/form
+/// rulings, not semantic underlines.
+const MIN_REPEATED_RULE_LEVELS: usize = 3;
+
+/// Vertical tolerance for considering two rules to be on the same row edge.
+const RULE_Y_DEDUP_EPS: f32 = 2.0;
+
+/// Horizontal span similarity required when clustering repeated rulings.
+const RULE_SPAN_OVERLAP_RATIO: f32 = 0.8;
+const RULE_SPAN_WIDTH_RATIO: f32 = 1.5;
+
+/// Multiple separated rule segments on one row are usually per-column table
+/// header/body separators.
+const MIN_SEGMENTED_ROW_RULES: usize = 3;
+const MIN_SEGMENTED_ROW_GAPS: usize = 2;
+const SEGMENTED_ROW_GAP_MIN: f32 = 12.0;
+
+/// A single rule under several widely separated items is usually a table
+/// header/body separator, not a sentence underline.
+const MIN_TABULAR_RULE_ITEMS: usize = 3;
+const MIN_TABULAR_RULE_GAPS: usize = 2;
+const TABULAR_RULE_GAP_EM: f32 = 2.0;
+
+#[derive(Clone)]
+pub(crate) struct UnderlineLine {
+    pub(crate) x1: f32,
+    pub(crate) y1: f32,
+    pub(crate) x2: f32,
+    pub(crate) y2: f32,
+    pub(crate) stroke_width: f32,
+    pub(crate) page: u32,
+}
+
+/// A horizontal rule candidate in page coordinates (PDF y-up).
+#[derive(Clone)]
+struct Rule {
+    x1: f32,
+    x2: f32,
+    y: f32,
+}
+
+impl Rule {
+    fn width(&self) -> f32 {
+        self.x2 - self.x1
+    }
+}
+
+fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) -> Vec<Rule> {
+    let mut rules: Vec<Rule> = Vec::new();
+    for l in lines {
+        if l.page != page {
+            continue;
+        }
+        // Horizontal stroked line (tolerate slight skew).
+        if l.stroke_width <= MAX_RULE_THICKNESS && (l.y1 - l.y2).abs() <= MAX_RULE_THICKNESS {
+            let (x1, x2) = if l.x1 <= l.x2 {
+                (l.x1, l.x2)
+            } else {
+                (l.x2, l.x1)
+            };
+            if x2 - x1 > 1.0 {
+                rules.push(Rule {
+                    x1,
+                    x2,
+                    y: (l.y1 + l.y2) / 2.0,
+                });
+            }
+        }
+    }
+    for r in rects {
+        if r.page != page {
+            continue;
+        }
+        // Thin filled rect used as an underline rule. Extents are
+        // normalized first: `re` operands pass through the CTM, so
+        // width/height can be negative (flipped axes / negative scale) —
+        // without normalization negative-width rules are missed and
+        // negative-height bands sneak past the thickness check.
+        let (x1, x2) = if r.width >= 0.0 {
+            (r.x, r.x + r.width)
+        } else {
+            (r.x + r.width, r.x)
+        };
+        if r.height.abs() <= MAX_RULE_THICKNESS && x2 - x1 > 1.0 {
+            rules.push(Rule {
+                x1,
+                x2,
+                y: r.y + r.height / 2.0,
+            });
+        }
+    }
+    rules
+}
+
+fn discard_repeated_ruling_rules(rules: Vec<Rule>) -> Vec<Rule> {
+    if rules.len() < MIN_REPEATED_RULE_LEVELS {
+        return rules;
+    }
+
+    rules
+        .iter()
+        .filter(|rule| {
+            !is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules)
+        })
+        .cloned()
+        .collect()
+}
+
+fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
+    let mut y_levels: Vec<f32> = rules
+        .iter()
+        .filter(|other| has_similar_span(rule, other))
+        .map(|other| other.y)
+        .collect();
+
+    y_levels.sort_by(|a, b| a.total_cmp(b));
+    y_levels.dedup_by(|a, b| (*a - *b).abs() <= RULE_Y_DEDUP_EPS);
+    y_levels.len() >= MIN_REPEATED_RULE_LEVELS
+}
+
+fn is_segmented_row_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
+    let mut row_rules: Vec<&Rule> = rules
+        .iter()
+        .filter(|other| (other.y - rule.y).abs() <= RULE_Y_DEDUP_EPS)
+        .collect();
+
+    if row_rules.len() < MIN_SEGMENTED_ROW_RULES {
+        return false;
+    }
+
+    row_rules.sort_by(|a, b| a.x1.total_cmp(&b.x1));
+    let large_gaps = row_rules
+        .windows(2)
+        .filter(|pair| pair[1].x1 - pair[0].x2 > SEGMENTED_ROW_GAP_MIN)
+        .count();
+
+    large_gaps >= MIN_SEGMENTED_ROW_GAPS
+}
+
+fn has_similar_span(a: &Rule, b: &Rule) -> bool {
+    let a_width = a.width();
+    let b_width = b.width();
+    if a_width <= 1.0 || b_width <= 1.0 {
+        return false;
+    }
+
+    let width_ratio = a_width.max(b_width) / a_width.min(b_width);
+    if width_ratio > RULE_SPAN_WIDTH_RATIO {
+        return false;
+    }
+
+    let overlap = a.x2.min(b.x2) - a.x1.max(b.x1);
+    overlap >= a_width.min(b_width) * RULE_SPAN_OVERLAP_RATIO
+}
+
+fn tabular_row_separator_rule_indices(rules: &[Rule], items: &[TextItem]) -> HashSet<usize> {
+    let mut tabular_rules = HashSet::new();
+
+    for (rule_idx, rule) in rules.iter().enumerate() {
+        let mut matched_items: Vec<&TextItem> = items
+            .iter()
+            .filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item))
+            .collect();
+
+        if matched_items.len() < MIN_TABULAR_RULE_ITEMS {
+            continue;
+        }
+
+        matched_items.sort_by(|a, b| a.x.total_cmp(&b.x));
+        let large_gaps = matched_items
+            .windows(2)
+            .filter(|pair| {
+                let left = pair[0];
+                let right = pair[1];
+                let gap = right.x - (left.x + left.width);
+                let font_size = left.font_size.max(right.font_size).max(1.0);
+                gap > font_size * TABULAR_RULE_GAP_EM
+            })
+            .count();
+
+        if large_gaps >= MIN_TABULAR_RULE_GAPS {
+            tabular_rules.insert(rule_idx);
+        }
+    }
+
+    tabular_rules
+}
+
+fn is_underline_candidate(item: &TextItem) -> bool {
+    matches!(item.item_type, ItemType::Text) && !item.text.trim().is_empty() && item.width > 0.0
+}
+
+fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
+    // Vertical window: underlines sit at or slightly below the baseline.
+    // Fonts draw them at roughly 5-15% of the em below; allow up to 35%
+    // (min 3pt) below and 1pt above for rounding.
+    let below = (item.font_size * 0.35).max(3.0);
+    let y_min = item.y - below;
+    let y_max = item.y + 1.0;
+    if rule.y < y_min || rule.y > y_max {
+        return false;
+    }
+
+    let ix1 = item.x;
+    let ix2 = item.x + item.width;
+    let min_overlap = item.width * MIN_X_OVERLAP;
+    let overlap = rule.x2.min(ix2) - rule.x1.max(ix1);
+    overlap >= min_overlap
+}
+
+/// Mark `is_underline` on text items that have a horizontal rule just
+/// below their baseline. `items`, `rects`, and `lines` are a single
+/// page's extraction output (all in PDF coordinates, y-up, where
+/// `TextItem::y` is the text baseline).
+pub(crate) fn mark_underlined_items(
+    items: &mut [TextItem],
+    rects: &[PdfRect],
+    lines: &[UnderlineLine],
+    page: u32,
+) {
+    let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page));
+    if rules.is_empty() {
+        return;
+    }
+    let tabular_rules = tabular_row_separator_rule_indices(&rules, items);
+
+    for item in items.iter_mut() {
+        if !is_underline_candidate(item) {
+            continue;
+        }
+
+        for (rule_idx, rule) in rules.iter().enumerate() {
+            if tabular_rules.contains(&rule_idx) {
+                continue;
+            }
+            if rule_matches_item(rule, item) {
+                item.is_underline = true;
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::ItemType;
+
+    fn item(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem {
+        TextItem {
+            text: text.to_string(),
+            x,
+            y,
+            width,
+            height: font_size,
+            font: "F1".to_string(),
+            font_size,
+            page: 1,
+            is_bold: false,
+            is_italic: false,
+            is_underline: false,
+            item_type: ItemType::Text,
+            mcid: None,
+        }
+    }
+
+    fn hline(x1: f32, x2: f32, y: f32) -> UnderlineLine {
+        UnderlineLine {
+            x1,
+            y1: y,
+            x2,
+            y2: y,
+            stroke_width: 1.0,
+            page: 1,
+        }
+    }
+
+    fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect {
+        PdfRect {
+            x,
+            y,
+            width,
+            height: 0.8,
+            page: 1,
+        }
+    }
+
+    #[test]
+    fn stroked_line_under_baseline_marks_underline() {
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(99.0, 161.0, 498.5)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn thin_filled_rect_under_baseline_marks_underline() {
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![thin_rect(100.0, 497.8, 60.0)];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn long_rule_under_multiple_items_marks_each() {
+        // One underline drawn under a whole sentence: every overlapped
+        // item gets the flag.
+        let mut items = vec![
+            item("first", 100.0, 500.0, 40.0, 10.0),
+            item("second", 145.0, 500.0, 50.0, 10.0),
+        ];
+        let lines = vec![hline(98.0, 200.0, 498.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(items[0].is_underline);
+        assert!(items[1].is_underline);
+    }
+
+    #[test]
+    fn line_far_below_baseline_is_not_an_underline() {
+        // A horizontal rule 30pt below (section divider) must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(90.0, 300.0, 470.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn thick_stroked_line_is_not_an_underline() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let mut line = hline(99.0, 161.0, 498.5);
+        line.stroke_width = 4.0;
+
+        mark_underlined_items(&mut items, &[], &[line], 1);
+
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn line_above_baseline_is_not_an_underline() {
+        // Strikethrough / overline geometry must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![hline(90.0, 300.0, 505.0)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn insufficient_horizontal_overlap_is_not_an_underline() {
+        // Rule under only a quarter of the item (e.g. neighboring cell
+        // border) must not mark.
+        let mut items = vec![item("wide text item", 100.0, 500.0, 100.0, 10.0)];
+        let lines = vec![hline(100.0, 125.0, 498.5)];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn negative_width_rect_is_normalized_and_marks_underline() {
+        // A CTM with negative x-scale (or negative `re` operands) produces
+        // rects whose width is negative; the rule extents must normalize.
+        let mut items = vec![item("underlined", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 160.0,
+            y: 497.8,
+            width: -60.0,
+            height: 0.8,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(items[0].is_underline);
+    }
+
+    #[test]
+    fn negative_height_band_is_not_an_underline() {
+        // A 14pt band expressed with negative height must not pass the
+        // thickness check via sign trickery.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 95.0,
+            y: 509.0,
+            width: 80.0,
+            height: -14.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn thick_band_is_not_an_underline() {
+        // A highlight bar / filled cell background (tall rect) must not mark.
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let rects = vec![PdfRect {
+            x: 95.0,
+            y: 495.0,
+            width: 80.0,
+            height: 14.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &rects, &[], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn vertical_line_is_not_an_underline() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let lines = vec![UnderlineLine {
+            x1: 120.0,
+            y1: 498.0,
+            x2: 120.0,
+            y2: 400.0,
+            stroke_width: 1.0,
+            page: 1,
+        }];
+        mark_underlined_items(&mut items, &[], &lines, 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn other_pages_graphics_do_not_mark() {
+        let mut items = vec![item("text", 100.0, 500.0, 60.0, 10.0)];
+        let mut line = hline(99.0, 161.0, 498.5);
+        line.page = 2;
+        mark_underlined_items(&mut items, &[], &[line], 1);
+        assert!(!items[0].is_underline);
+    }
+
+    #[test]
+    fn repeated_table_row_rules_do_not_mark_cell_text() {
+        let mut items = vec![
+            item("A", 110.0, 500.0, 20.0, 10.0),
+            item("B", 110.0, 480.0, 20.0, 10.0),
+            item("C", 110.0, 460.0, 20.0, 10.0),
+        ];
+        let lines = vec![
+            hline(100.0, 150.0, 498.0),
+            hline(100.0, 150.0, 478.0),
+            hline(100.0, 150.0, 458.0),
+        ];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
+    #[test]
+    fn row_separator_under_spaced_column_labels_is_not_an_underline() {
+        let mut items = vec![
+            item("Date", 100.0, 500.0, 25.0, 10.0),
+            item("Rate", 200.0, 500.0, 25.0, 10.0),
+            item("Yield", 300.0, 500.0, 30.0, 10.0),
+        ];
+        let lines = vec![hline(90.0, 340.0, 498.0)];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+
+    #[test]
+    fn same_row_spaced_rule_segments_do_not_mark_column_labels() {
+        let mut items = vec![
+            item("Date", 100.0, 500.0, 25.0, 10.0),
+            item("Rate", 200.0, 500.0, 25.0, 10.0),
+            item("Yield", 300.0, 500.0, 30.0, 10.0),
+        ];
+        let lines = vec![
+            hline(98.0, 128.0, 498.0),
+            hline(198.0, 228.0, 498.0),
+            hline(298.0, 333.0, 498.0),
+        ];
+
+        mark_underlined_items(&mut items, &[], &lines, 1);
+
+        assert!(items.iter().all(|item| !item.is_underline));
+    }
+}
@@ -294,6 +294,7 @@ fn extract_form_xobject_text_inner(
                                    page: page_num,
                                    is_bold: false,
                                    is_italic: false,
+                                    is_underline: false,
                                    item_type: ItemType::Image,
                                    mcid: None,
                                });
@@ -438,6 +439,7 @@ fn extract_form_xobject_text_inner(
                                page: page_num,
                                is_bold: is_bold_font(base_font),
                                is_italic: is_italic_font(base_font),
+                                is_underline: false,
                                item_type: ItemType::Text,
                                mcid: None,
                            });
@@ -586,6 +588,7 @@ fn extract_form_xobject_text_inner(
                                    page: page_num,
                                    is_bold: is_bold_font(base_font),
                                    is_italic: is_italic_font(base_font),
+                                    is_underline: false,
                                    item_type: ItemType::Text,
                                    mcid: None,
                                });
@@ -1288,6 +1288,19 @@ mod vector_grid_tests {
        );
    }

+    #[test]
+    fn td9264_insurance_prose_not_rect_table() {
+        let tables = detect_rect_tables_in_fixture_page("tests/fixtures/td9264.pdf", 4);
+        assert!(
+            tables.is_empty(),
+            "expected no rect-detected tables for the insurance-company prose; got {:?}",
+            tables
+                .iter()
+                .map(|t| (t.rows.len(), t.columns.len(), t.cells.clone()))
+                .collect::<Vec<_>>()
+        );
+    }
+
    /// Wireless table regression: decorative/text-region rects may provide row
    /// bands, but without a real rect-derived column scaffold they must not be
    /// accepted as a vector grid.
@@ -3684,7 +3697,14 @@ fn detect_encoding_issues(markdown: &str) -> bool {
    }

    // Heuristic 2: dollar-as-space pattern
-    has_dollar_as_space_pattern(markdown)
+    if has_dollar_as_space_pattern(markdown) {
+        return true;
+    }
+
+    // Heuristic 3: substitution-cipher letter statistics (broken ToUnicode)
+    let mut stats = CipherGarbleStats::default();
+    stats.add_text(markdown);
+    stats.looks_garbled()
 }

 fn has_dollar_as_space_pattern(markdown: &str) -> bool {
@@ -3708,6 +3728,156 @@ fn has_dollar_as_space_pattern(markdown: &str) -> bool {
    false
 }

+/// English letter frequencies (percent, a–z). Used as a natural-language
+/// reference: every Latin-script language in the eval corpus (Swedish,
+/// Finnish, Turkish, German, romaji) scores ≥ 0.80 cosine similarity against
+/// it, while substitution-cipher text scores ~0.53.
+const ENGLISH_LETTER_FREQ: [f64; 26] = [
+    8.2, 1.5, 2.8, 4.3, 12.7, 2.2, 2.0, 6.1, 7.0, 0.15, 0.8, 4.0, 2.4, 6.7, 7.5, 1.9, 0.1, 6.0,
+    6.3, 9.1, 2.8, 1.0, 2.4, 0.15, 2.0, 0.07,
+];
+
+/// Letter statistics for detecting substitution-cipher garbling: broken
+/// ToUnicode CMaps that shift every character by a per-range constant (e.g.
+/// `Certificate` extracted as `8VceZWZTReV`). Such text is 100% printable
+/// ASCII with word-like token lengths, so it defeats `is_garbage_text` and
+/// produces no replacement characters — it needs its own discriminator.
+#[derive(Debug, Default)]
+struct CipherGarbleStats {
+    /// Case-folded ASCII letter histogram.
+    letter_counts: [u32; 26],
+    ascii_letters: usize,
+    ascii_vowels: usize,
+    /// Accented Latin letters (Latin-1 Supplement through Latin Extended-B,
+    /// plus Latin Extended Additional). Count toward Latin dominance only.
+    latin_ext_letters: usize,
+    non_latin_letters: usize,
+    /// Adjacent ASCII-letter pairs, and how many of them switch from
+    /// lowercase straight to uppercase mid-word.
+    letter_bigrams: usize,
+    case_shift_bigrams: usize,
+}
+
+impl CipherGarbleStats {
+    fn add_text(&mut self, text: &str) {
+        let mut prev: Option<char> = None;
+        for ch in text.chars() {
+            if ch.is_ascii_alphabetic() {
+                let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize;
+                self.letter_counts[idx] += 1;
+                self.ascii_letters += 1;
+                if matches!(ch.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u') {
+                    self.ascii_vowels += 1;
+                }
+                if let Some(p) = prev {
+                    self.letter_bigrams += 1;
+                    if p.is_ascii_lowercase() && ch.is_ascii_uppercase() {
+                        self.case_shift_bigrams += 1;
+                    }
+                }
+                prev = Some(ch);
+            } else {
+                if ch.is_alphabetic() {
+                    if matches!(ch as u32, 0xC0..=0x24F | 0x1E00..=0x1EFF) {
+                        self.latin_ext_letters += 1;
+                    } else {
+                        self.non_latin_letters += 1;
+                    }
+                }
+                prev = None;
+            }
+        }
+    }
+
+    /// Cosine similarity between the observed letter histogram and English
+    /// letter frequencies. A shifted alphabet permutes the histogram, which
+    /// destroys the similarity regardless of the shift amount.
+    fn english_cosine(&self) -> f64 {
+        if self.ascii_letters == 0 {
+            return 1.0;
+        }
+        let n = self.ascii_letters as f64;
+        let mut dot = 0.0;
+        let mut norm_obs = 0.0;
+        for (count, freq) in self.letter_counts.iter().zip(ENGLISH_LETTER_FREQ) {
+            let p = *count as f64 / n;
+            dot += p * freq;
+            norm_obs += p * p;
+        }
+        let norm_en = ENGLISH_LETTER_FREQ
+            .iter()
+            .map(|f| f * f)
+            .sum::<f64>()
+            .sqrt();
+        dot / (norm_obs.sqrt() * norm_en)
+    }
+
+    /// Cosine similarity between the observed histogram and English
+    /// frequencies after sorting BOTH descending — i.e. comparing the *shape*
+    /// of the frequency profile, ignoring which letter sits where. A
+    /// substitution cipher is a bijection, so it preserves this shape exactly
+    /// (att10k 0.97, arbitrary shifts 0.99) regardless of case or offset.
+    /// Non-linguistic ASCII has a different profile: a small alphabet is far
+    /// steeper (random DNA 0.74, hex dumps 0.81), so the shape diverges.
+    fn english_shape_cosine(&self) -> f64 {
+        if self.ascii_letters == 0 {
+            return 1.0;
+        }
+        let n = self.ascii_letters as f64;
+        let mut obs: [f64; 26] = std::array::from_fn(|i| self.letter_counts[i] as f64 / n);
+        obs.sort_unstable_by(|a, b| b.total_cmp(a));
+        let mut en = ENGLISH_LETTER_FREQ;
+        en.sort_unstable_by(|a, b| b.total_cmp(a));
+
+        let dot: f64 = obs.iter().zip(en).map(|(o, e)| o * e).sum();
+        let norm_obs = obs.iter().map(|o| o * o).sum::<f64>().sqrt();
+        let norm_en = en.iter().map(|e| e * e).sum::<f64>().sqrt();
+        dot / (norm_obs * norm_en)
+    }
+
+    /// Thresholds validated against the 380-document pdf-evals snapshot
+    /// corpus (0 false positives) and the garbled ParseBench `att10k` page
+    /// (vowel ratio 0.245, case-shift rate 0.225, cosine 0.532). Closest
+    /// legitimate document on each axis: vowel ratio 0.264 (circuit
+    /// schematic), case-shift rate 0.021, cosine 0.801.
+    fn looks_garbled(&self) -> bool {
+        // Need a statistically meaningful, Latin-dominant sample.
+        if self.ascii_letters < 200
+            || self.non_latin_letters > self.ascii_letters + self.latin_ext_letters
+        {
+            return false;
+        }
+
+        // Real Latin-script text keeps vowels above ~30% of letters even in
+        // acronym- and part-number-heavy documents; shifted text starves them.
+        let vowel_ratio = self.ascii_vowels as f64 / self.ascii_letters as f64;
+        if vowel_ratio > 0.30 {
+            return false;
+        }
+
+        // Signal 1: lowercase→uppercase transitions inside words. A shifted
+        // lowercase alphabet straddles the ASCII uppercase block ('i'→'Z',
+        // 't'→'e'), so garbled words flip case constantly. Real documents
+        // stay ≤ 0.02 even with camelCase identifiers.
+        let case_shifts = self.letter_bigrams >= 100
+            && self.case_shift_bigrams as f64 >= self.letter_bigrams as f64 * 0.10;
+
+        // Signal 2: the histogram is a permutation of natural language — an
+        // English-like frequency SHAPE (sorted cosine high) but with letters
+        // in the wrong POSITIONS (unsorted cosine low). This is the signature
+        // of a substitution cipher and is case-independent, so it catches
+        // all-lowercase and all-uppercase shifts as well as case-straddling
+        // ones. Genuinely non-linguistic ASCII that is merely "unlike English"
+        // fails one of the two halves: DNA/hex dumps have too steep a profile
+        // (shape cosine < 0.90), while protein sequences, ticker symbols and
+        // base64 are not sufficiently unlike English in position (unsorted
+        // cosine ≥ 0.60) — so none of them are routed to OCR.
+        let permuted_language = self.english_cosine() < 0.60 && self.english_shape_cosine() >= 0.90;
+
+        case_shifts || permuted_language
+    }
+}
+
 #[derive(Debug, Default)]
 struct TextQualityReport {
    pages_needing_ocr: Vec<u32>,
@@ -3721,6 +3891,7 @@ struct PageTextQualityEvidence {
    replacement_chars: usize,
    replacement_spans: usize,
    longest_replacement_run: usize,
+    cipher_garble: CipherGarbleStats,
 }

 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -3740,6 +3911,7 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {

        let evidence = evidence_by_page.entry(item.page).or_default();
        evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
+        evidence.cipher_garble.add_text(&item.text);

        match text_span_decoding_issue_kind(&item.text) {
            Some(TextSpanIssueKind::Strong) => {
@@ -3763,7 +3935,8 @@ fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
        if reasons_by_page.contains_key(&page) {
            continue;
        }
-        if page_replacement_evidence_needs_ocr(&evidence) {
+        if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
+        {
            add_ocr_reason(
                &mut reasons_by_page,
                page,
@@ -4882,6 +5055,7 @@ mod text_cluster_column_undercount_tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -5156,6 +5330,7 @@ mod table_candidate_selection_tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -5902,6 +6077,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -5949,6 +6125,171 @@ mod tests {
        assert!(!detect_encoding_issues(text));
    }

+    /// Real garbled output from ParseBench `text_simple__att10k.pdf`: a broken
+    /// ToUnicode CMap shifts every character by a per-range constant, so
+    /// "Certificate of Designations with respect to Series C" extracts as
+    /// pure-ASCII ciphertext (issue #118).
+    const SHIFTED_CIPHER_TEXT: &str =
+        "8VceZWZTReVW9VdZXReZdhZeYcVdaVTeeHVcZVd8EcVWVccVUHeT:iYZSZe-(,e2'WZ]V \
+        ;VScfRcj2&,*,*$ .'R CZdecfVehYZTYUVWZVdeYVcZXYedWY]UVcdW]X'eVcUVSeYVcVXZdecReRUR]]ed \
+        TCBGC@ZUReVUdfSdZUZRcZVdZdWZ]VUYVcVhZeYafcdfReeVXf]ReZV0*S$.$ZZZ$6$&ViTVae \
+        WceYVZdecfVedcVWVccVUe.'S&.'T&.'U&.'V&.'WSV]h(EfcdfReeYZdcVXf]ReZYV \
+        cVXZdecReYVcVSjRXcVVdeWfcZdYRTajWRjdfTYZdecfVeWZ]VUYVcVhZeYeYVH:8 \
+        cVbfVde( .'S fRcRejWTVceRZZXReZTZWZT7V]]IV]VaYV8(RUHfeYhVdeVc7V]]IV]VaYV8 \
+        :iYZSZe.'TeYVaVcZUVUZX9VTVSVc-&,*$";
+
+    #[test]
+    fn test_detect_encoding_issues_shifted_cipher_text() {
+        assert!(detect_encoding_issues(SHIFTED_CIPHER_TEXT));
+    }
+
+    #[test]
+    fn test_shifted_cipher_below_sample_threshold_not_flagged() {
+        // Fewer than 200 ASCII letters: not enough evidence to condemn.
+        assert!(!detect_encoding_issues("8VceZWZTReVW9VdZXReZdhZeY"));
+    }
+
+    #[test]
+    fn test_clean_prose_not_flagged_as_cipher() {
+        let prose = "Certificate of Designations with respect to Series C Preferred Stock, \
+            filed February 18, 2020. No instrument which defines the rights of holders of \
+            long-term debt of the registrant and all of its consolidated subsidiaries is \
+            filed herewith pursuant to Regulation S-K, Item 601. Pursuant to this regulation, \
+            the registrant hereby agrees to furnish a copy of any such instrument to the SEC \
+            upon request.";
+        assert!(!detect_encoding_issues(prose));
+    }
+
+    #[test]
+    fn test_camel_case_code_not_flagged_as_cipher() {
+        let code = "The getElementById and querySelectorAll methods return DOM nodes. Use \
+            addEventListener with removeEventListener, requestAnimationFrame with \
+            cancelAnimationFrame, and setTimeout with clearTimeout. The XMLHttpRequest \
+            object exposes onreadystatechange, responseText and getAllResponseHeaders. \
+            Prefer createElement, appendChild, insertBefore and replaceChild for DOM \
+            manipulation, and getBoundingClientRect for layout measurement.";
+        assert!(!detect_encoding_issues(code));
+    }
+
+    #[test]
+    fn test_accented_european_text_not_flagged_as_cipher() {
+        let swedish = "Regeringen föreslår att riksdagen antar förslaget till lag om ändring \
+            i skatteförfarandelagen. Bestämmelserna föreslås träda i kraft den första januari. \
+            Förslaget innebär att företag med säte i utlandet måste lämna särskilda uppgifter \
+            till Skatteverket varje kvartal, och att avgiften höjs för överträdelser av de nya \
+            bestämmelserna om rapporteringsskyldighet för gränsöverskridande arrangemang.";
+        assert!(!detect_encoding_issues(swedish));
+    }
+
+    #[test]
+    fn test_all_caps_text_not_flagged_as_cipher() {
+        let caps = "EXHIBIT INDEX PURSUANT TO ITEM 601 OF REGULATION SK CERTIFICATE OF \
+            DESIGNATIONS WITH RESPECT TO SERIES C PREFERRED STOCK FILED FEBRUARY EIGHTEEN \
+            TWENTY TWENTY AND INCORPORATED HEREIN BY REFERENCE TO THE ANNUAL REPORT ON FORM \
+            TENK FOR THE PERIOD ENDED DECEMBER THIRTYFIRST TWENTY NINETEEN AS AMENDED";
+        assert!(!detect_encoding_issues(caps));
+    }
+
+    // A Caesar shift of prose that stays within a single case block does not
+    // trigger the case-shift signal, but it permutes the letter histogram: the
+    // frequency SHAPE stays English-like while letter POSITIONS scramble, so
+    // the permutation signal catches it regardless of case.
+    const CAESAR_PROSE: &str =
+        "The registrant hereby agrees to furnish a copy of any such instrument to the \
+        Commission upon request. This certificate of designations was filed February with \
+        respect to Series Preferred Stock and incorporated herein by reference to the annual \
+        report on form for the period ended December as amended and restated thereafter.";
+
+    fn caesar_shift(text: &str, k: u8) -> String {
+        text.chars()
+            .map(|c| match c {
+                'a'..='z' => (((c as u8 - b'a' + k) % 26) + b'a') as char,
+                'A'..='Z' => (((c as u8 - b'A' + k) % 26) + b'A') as char,
+                _ => c,
+            })
+            .collect()
+    }
+
+    #[test]
+    fn test_mixed_case_caesar_shift_flagged() {
+        assert!(detect_encoding_issues(&caesar_shift(CAESAR_PROSE, 3)));
+    }
+
+    #[test]
+    fn test_all_lowercase_caesar_shift_flagged() {
+        // Uniform all-lowercase garbled prose: the earlier mixed-case guard
+        // would have exempted this, so it must be caught by the case-agnostic
+        // permutation signal instead.
+        assert!(detect_encoding_issues(&caesar_shift(
+            &CAESAR_PROSE.to_lowercase(),
+            5
+        )));
+    }
+
+    #[test]
+    fn test_all_uppercase_caesar_shift_flagged() {
+        assert!(detect_encoding_issues(&caesar_shift(
+            &CAESAR_PROSE.to_uppercase(),
+            7
+        )));
+    }
+
+    #[test]
+    fn test_dna_sequence_not_flagged_as_cipher() {
+        // Non-linguistic ASCII: unlike English (low vowel ratio, low cosine)
+        // but not garbled. Its 4-letter alphabet makes the frequency profile
+        // too steep, so the shape cosine falls below the permutation threshold.
+        let dna = "ACGT".repeat(120);
+        assert!(!detect_encoding_issues(&dna));
+    }
+
+    #[test]
+    fn test_protein_sequence_not_flagged_as_cipher() {
+        let protein =
+            "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQAPILSRVGDGTQDNLSGAEKAVQVKVKALPDAQFEVVHSLAKWKR"
+                .repeat(6);
+        assert!(!detect_encoding_issues(&protein));
+    }
+
+    #[test]
+    fn test_ticker_list_not_flagged_as_cipher() {
+        let tickers = "AAPL MSFT GOOG TSLA NVDA AMZN META NFLX AMD INTC CSCO ORCL CRM ADBE QCOM \
+            TXN AVGO MU LRCX KLAC ASML SNPS CDNS FTNT PANW "
+            .repeat(4);
+        assert!(!detect_encoding_issues(&tickers));
+    }
+
+    #[test]
+    fn test_text_quality_flags_shifted_cipher_page() {
+        let items = vec![
+            test_text_item_on_page(1, SHIFTED_CIPHER_TEXT),
+            test_text_item_on_page(2, "A clean second page should not be routed to OCR."),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(quality.has_encoding_issues);
+        assert_eq!(quality.pages_needing_ocr, vec![1]);
+        assert_eq!(
+            quality.reasons_by_page.get(&1).cloned(),
+            Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
+        );
+    }
+
+    #[test]
+    fn test_text_quality_cipher_stats_accumulate_across_items() {
+        // The garbled page arrives as many short spans; no single span has
+        // enough letters to flag on its own.
+        let items: Vec<TextItem> = SHIFTED_CIPHER_TEXT
+            .split_whitespace()
+            .map(|chunk| test_text_item_on_page(1, chunk))
+            .collect();
+
+        let quality = analyze_text_quality(&items);
+
+        assert_eq!(quality.pages_needing_ocr, vec![1]);
+    }
+
    #[test]
    fn test_text_quality_flags_localized_cid_mojibake_span() {
        let items = vec![
@@ -131,10 +131,11 @@ pub(crate) fn format_list_item(text: &str) -> String {
        if let Some(rest) = trimmed.strip_prefix(*bullet) {
            return format!("- {}", rest.trim_start());
        }
-        // Bullet inside a leading bold/italic run (e.g. "**● Label:** rest").
-        // The run wraps both the marker and the following label because both
-        // use a bold font in the PDF.
-        for wrapper in ["**", "*"] {
+        // Bullet inside a leading style run (e.g. "**● Label:** rest" or
+        // "<u>● Label</u>"). The run wraps both the marker and the following
+        // label because both carry the style in the PDF. The marker must move
+        // outside the wrapper so markdown still sees a list item.
+        for wrapper in ["**", "*", "<u>"] {
            if let Some(after_open) = trimmed.strip_prefix(wrapper) {
                if let Some(rest) = after_open.strip_prefix(*bullet) {
                    return format!("- {}{}", wrapper, rest.trim_start());
@@ -235,6 +236,13 @@ mod tests {
        assert_eq!(format_list_item("• Item"), "- Item");
    }

+    #[test]
+    fn format_list_item_bullet_inside_underline() {
+        // Fully-underlined bullet line: the marker must move outside the
+        // <u> wrapper so markdown still renders a list item.
+        assert_eq!(format_list_item("<u>● Item text</u>"), "- <u>Item text</u>");
+    }
+
    #[test]
    fn format_list_item_bullet_inside_bold() {
        // PDF that uses bold font for both the marker and the label produces
@@ -625,7 +625,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
        prev_x = line_x;

        // Get text with optional bold/italic formatting
-        let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
+        let text = line.text_with_formatting(
+            options.detect_bold,
+            options.detect_italic,
+            options.detect_underline,
+        );
        let trimmed = text.trim();

        // Also get plain text for pattern matching (list detection, captions, etc.)
@@ -751,8 +755,14 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
                paragraph_in_wrapped_bold_run = false;
            }
            let prefix = "#".repeat(level);
-            // Use plain text for headers to avoid redundant formatting
-            output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
+            // Plain text for headers (no redundant bold/italic inside `#`),
+            // but underline is preserved: `<u>` carries meaning `#` doesn't.
+            let heading_text = if options.detect_underline {
+                line.text_with_formatting(false, false, true)
+            } else {
+                plain_text.clone()
+            };
+            output.push_str(&format!("{} {}\n\n", prefix, heading_text.trim()));
            in_list = false;
            continue;
        }
@@ -994,7 +1004,11 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
        prev_y = line.y;

        // Get text with optional bold/italic formatting
-        let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
+        let text = line.text_with_formatting(
+            options.detect_bold,
+            options.detect_italic,
+            options.detect_underline,
+        );
        let trimmed = text.trim();

        // Also get plain text for pattern matching
@@ -1057,8 +1071,13 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
                    paragraph_in_wrapped_bold_run = false;
                }
                let prefix = "#".repeat(header_level);
-                // Use plain text for headers to avoid redundant formatting
-                output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
+                // Plain text for headers, except underline (see above).
+                let heading_text = if options.detect_underline {
+                    line.text_with_formatting(false, false, true)
+                } else {
+                    plain_text.clone()
+                };
+                output.push_str(&format!("{} {}\n\n", prefix, heading_text.trim()));
                in_list = false;
                continue;
            }
@@ -1169,6 +1188,7 @@ mod tests {
            page,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: crate::types::ItemType::Text,
            mcid,
        }
@@ -400,6 +400,8 @@ pub struct MarkdownOptions {
    pub detect_bold: bool,
    /// Detect and format italic text from font names
    pub detect_italic: bool,
+    /// Emit `<u>` runs for text with a geometrically-detected underline
+    pub detect_underline: bool,
    /// Include image placeholders in output
    pub include_images: bool,
    /// Include extracted hyperlinks
@@ -422,6 +424,7 @@ impl Default for MarkdownOptions {
            fix_hyphenation: true,
            detect_bold: true,
            detect_italic: true,
+            detect_underline: true,
            // `include_images: false` is intentional. The content-stream walker
            // now emits `ItemType::Image` `TextItem`s for every Image XObject
            // it encounters (see `extractor/content_stream.rs`). If we rendered
@@ -1217,6 +1220,7 @@ mod tests {
            page,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: crate::types::ItemType::Text,
            mcid: None,
        }
@@ -30,6 +30,7 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
    // double spaces ("Vice  President" instead of "Vice President").
    collapse_consecutive_spaces(&mut text);
    remove_spaces_before_closing_brackets(&mut text);
+    remove_spaces_before_sentence_punctuation(&mut text);

    // Remove excessive newlines (more than 2 in a row)
    while text.contains("\n\n\n") {
@@ -86,6 +87,32 @@ fn remove_spaces_before_closing_brackets(text: &mut String) {
    *text = result;
 }

+/// Remove a stray space before sentence punctuation ("word ." → "word.").
+/// Style-boundary item splits (bold/italic/underline runs) can strand a
+/// trailing period or comma in its own fragment, and several assembly paths
+/// join fragments with spaces. Only fires when the punctuation ends the
+/// token (followed by whitespace or end of text), so decimals ("3 .14" stays
+/// untouched — no such input exists, but the guard is cheap) and dot leaders
+/// (" ... ") are unaffected.
+fn remove_spaces_before_sentence_punctuation(text: &mut String) {
+    let chars: Vec<char> = text.chars().collect();
+    let mut result = String::with_capacity(text.len());
+    for (i, &ch) in chars.iter().enumerate() {
+        if matches!(ch, '.' | ',' | ';') && result.ends_with(' ') {
+            let next = chars.get(i + 1);
+            // `|` counts as a token end so table cells get the same fix.
+            let token_ends = next.is_none_or(|c| c.is_whitespace() || *c == '|');
+            // Never touch runs of dots (ellipsis / dot leaders).
+            let in_dot_run = ch == '.' && next == Some(&'.');
+            if token_ends && !in_dot_run {
+                result.pop();
+            }
+        }
+        result.push(ch);
+    }
+    *text = result;
+}
+
 /// Collapse dot leaders (runs of 4+ dots) into " ... "
 /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
 fn collapse_dot_leaders(text: &str) -> String {
@@ -369,6 +396,36 @@ mod tests {
        );
    }

+    // --- remove_spaces_before_sentence_punctuation ---
+
+    #[test]
+    fn strips_space_before_trailing_period() {
+        let mut t = "Foreign insurance companies . The provisions".to_string();
+        remove_spaces_before_sentence_punctuation(&mut t);
+        assert_eq!(t, "Foreign insurance companies. The provisions");
+    }
+
+    #[test]
+    fn strips_space_before_period_at_cell_boundary() {
+        let mut t = "|Applicability date .|This section|".to_string();
+        remove_spaces_before_sentence_punctuation(&mut t);
+        assert_eq!(t, "|Applicability date.|This section|");
+    }
+
+    #[test]
+    fn keeps_dot_leaders_and_ellipses() {
+        let mut t = "Introduction ... 1".to_string();
+        remove_spaces_before_sentence_punctuation(&mut t);
+        assert_eq!(t, "Introduction ... 1");
+    }
+
+    #[test]
+    fn keeps_mid_token_periods() {
+        let mut t = "version 3 .14 released".to_string();
+        remove_spaces_before_sentence_punctuation(&mut t);
+        assert_eq!(t, "version 3 .14 released");
+    }
+
    // --- fix_hyphenation ---

    #[test]
@@ -542,6 +542,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid,
        }
@@ -266,6 +266,8 @@ pub struct PyTextItem {
    #[pyo3(get)]
    pub is_italic: bool,
    #[pyo3(get)]
+    pub is_underline: bool,
+    #[pyo3(get)]
    pub item_type: String,
 }

@@ -349,6 +351,7 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
            page: item.page,
            is_bold: item.is_bold,
            is_italic: item.is_italic,
+            is_underline: item.is_underline,
            item_type: item_type_str(&item.item_type),
        })
        .collect()
@@ -104,6 +104,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec<TextItem>, Vec<Ve
                page: first_item.page,
                is_bold: first_item.is_bold,
                is_italic: first_item.is_italic,
+                is_underline: first_item.is_underline,
                item_type: first_item.item_type.clone(),
                mcid: first_item.mcid,
            });
@@ -393,6 +393,7 @@ mod tests {
            page,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1124,12 +1124,13 @@ pub(crate) fn assign_items_to_grid(
                            .unwrap_or(std::cmp::Ordering::Equal)
                    })
            });
-            let text: String = col_items
+            let text = col_items
                .iter()
                .map(|(_, item)| item.text.trim())
                .filter(|t| !t.is_empty())
                .collect::<Vec<_>>()
                .join(" ");
+            let text = remove_inner_delimiter_spaces(&text);
            row_cells.push(text);
        }
        cells.push(row_cells);
@@ -1138,6 +1139,27 @@ pub(crate) fn assign_items_to_grid(
    (cells, indices)
 }

+fn remove_inner_delimiter_spaces(text: &str) -> String {
+    let chars: Vec<char> = text.chars().collect();
+    let mut result = String::with_capacity(text.len());
+
+    for (i, &ch) in chars.iter().enumerate() {
+        if ch == ' ' {
+            let after_open =
+                result.ends_with('(') || result.ends_with('[') || result.ends_with('{');
+            let before_close = chars
+                .get(i + 1)
+                .is_some_and(|next| matches!(next, ')' | ']' | '}'));
+            if after_open || before_close {
+                continue;
+            }
+        }
+        result.push(ch);
+    }
+
+    result
+}
+
 /// Consolidate text in vertically-merged cells.
 ///
 /// When a single rect spans multiple grid rows (e.g. a "Classification" label
@@ -1451,6 +1473,10 @@ fn detect_row_stripe_table(
        (col_edges, cells)
    };
    let num_cols = col_edges.len() - 1;
+    if row_stripe_is_sparse_prose_outline(&cells) {
+        debug!("  row-stripe rejected: sparse outline/prose continuation shape");
+        return None;
+    }

    let column_centers: Vec<f32> = (0..num_cols)
        .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0)
@@ -1469,6 +1495,57 @@ fn detect_row_stripe_table(
    Some(Table::new(column_centers, row_centers, cells, item_indices))
 }

+fn row_stripe_is_sparse_prose_outline(cells: &[Vec<String>]) -> bool {
+    let Some(num_cols) = cells.first().map(|row| row.len()) else {
+        return false;
+    };
+    if num_cols != 2 || cells.len() < 4 {
+        return false;
+    }
+
+    let non_empty_rows = cells
+        .iter()
+        .filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
+        .count();
+    if non_empty_rows < 4 {
+        return false;
+    }
+
+    let mut col_counts = [0usize; 2];
+    for row in cells {
+        for (idx, cell) in row.iter().enumerate() {
+            if !cell.trim().is_empty() {
+                col_counts[idx] += 1;
+            }
+        }
+    }
+
+    let (sparse_col, dense_col) = if col_counts[0] <= col_counts[1] {
+        (0usize, 1usize)
+    } else {
+        (1usize, 0usize)
+    };
+    let sparse_count = col_counts[sparse_col];
+    let dense_count = col_counts[dense_col];
+    if sparse_count * 2 >= non_empty_rows || dense_count * 3 < non_empty_rows * 2 {
+        return false;
+    }
+
+    let blank_sparse_dense_rows = cells
+        .iter()
+        .filter(|row| row[sparse_col].trim().is_empty() && !row[dense_col].trim().is_empty())
+        .count();
+    if blank_sparse_dense_rows * 2 < non_empty_rows {
+        return false;
+    }
+
+    let long_dense_cells = cells
+        .iter()
+        .filter(|row| row[dense_col].split_whitespace().count() >= 6)
+        .count();
+    long_dense_cells * 2 >= dense_count
+}
+
 /// Detect a table from cell-background rects that failed grid detection.
 ///
 /// Uses rect Y-edges for row boundaries and text X-position clustering for
@@ -2314,6 +2391,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -2523,6 +2601,21 @@ mod tests {
        assert!(cells[0][0].contains("World"));
    }

+    #[test]
+    fn test_assign_items_parenthetical_no_inner_spaces() {
+        let items = vec![
+            make_item("The first sentence", 15.0, 85.0, 10.0),
+            make_item("(", 90.0, 85.0, 10.0),
+            make_item("twice", 95.0, 85.0, 10.0),
+            make_item(")", 120.0, 85.0, 10.0),
+        ];
+        let col_edges = vec![10.0, 150.0];
+        let row_edges = vec![90.0, 70.0];
+        let (cells, indices) = assign_items_to_grid(&items, &col_edges, &row_edges, 1);
+        assert_eq!(indices.len(), 4);
+        assert_eq!(cells[0][0], "The first sentence (twice)");
+    }
+
    #[test]
    fn test_assign_items_boundary_tolerance() {
        // Item right at edge with ±2pt tolerance
@@ -3272,6 +3365,7 @@ mod tests {
                    page: 1,
                    is_bold: false,
                    is_italic: false,
+                    is_underline: false,
                    item_type: crate::types::ItemType::Text,
                    mcid: None,
                });
@@ -3581,6 +3675,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: crate::types::ItemType::Text,
                mcid: None,
            });
@@ -586,6 +586,7 @@ mod tests {
            page,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid,
        }
@@ -108,6 +108,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option<Vec<TextItem>>
            page: item.page,
            is_bold: item.is_bold,
            is_italic: item.is_italic,
+            is_underline: item.is_underline,
            item_type: item.item_type.clone(),
            mcid: item.mcid,
        });
@@ -369,6 +369,10 @@ pub(crate) fn join_cell_items(items: &[&TextItem]) -> String {
            let prev_ends_with_hyphen = result.ends_with('-');
            let curr_is_hyphen = text == "-";
            let curr_starts_with_hyphen = text.starts_with('-');
+            let prev_ends_with_open_delimiter =
+                result.ends_with('(') || result.ends_with('[') || result.ends_with('{');
+            let curr_starts_with_close_delimiter =
+                text.starts_with(')') || text.starts_with(']') || text.starts_with('}');

            // Detect subscript/superscript: smaller font size and/or Y offset
            let font_ratio = item.font_size / prev_item.font_size;
@@ -385,6 +389,8 @@ pub(crate) fn join_cell_items(items: &[&TextItem]) -> String {
                || curr_starts_with_hyphen
                || is_sub_super
                || was_sub_super
+                || prev_ends_with_open_delimiter
+                || curr_starts_with_close_delimiter
            {
                result.push_str(text);
            } else {
@@ -514,6 +520,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -727,6 +734,18 @@ mod tests {
        assert_eq!(join_cell_items(&[&a, &b, &c]), "pre-fix");
    }

+    #[test]
+    fn test_join_cell_items_parenthetical_no_inner_spaces() {
+        let a = make_item("The first sentence", 100.0, 500.0, 10.0);
+        let b = make_item("(", 190.0, 500.0, 10.0);
+        let c = make_item("twice", 195.0, 500.0, 10.0);
+        let d = make_item(")", 220.0, 500.0, 10.0);
+        assert_eq!(
+            join_cell_items(&[&a, &b, &c, &d]),
+            "The first sentence (twice)"
+        );
+    }
+
    #[test]
    fn test_join_cell_items_subscript_no_space() {
        let a = make_item("H", 100.0, 500.0, 12.0);
@@ -866,6 +885,7 @@ mod tests {
                        font: String::new(),
                        is_bold: false,
                        is_italic: false,
+                        is_underline: false,
                        item_type: ItemType::Text,
                        mcid: None,
                        page: 1,
@@ -902,6 +922,7 @@ mod tests {
                        font: String::new(),
                        is_bold: false,
                        is_italic: false,
+                        is_underline: false,
                        item_type: ItemType::Text,
                        mcid: None,
                        page: 1,
@@ -235,6 +235,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
            page: item.page,
            is_bold: item.is_bold,
            is_italic: item.is_italic,
+            is_underline: item.is_underline,
            item_type: item.item_type.clone(),
            mcid: item.mcid,
        });
@@ -255,6 +256,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec<TextItem
            page: item.page,
            is_bold: item.is_bold,
            is_italic: item.is_italic,
+            is_underline: item.is_underline,
            item_type: item.item_type.clone(),
            mcid: item.mcid,
        });
@@ -1429,6 +1431,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1446,6 +1449,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -883,6 +883,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -1002,6 +1003,7 @@ mod tests {
                page: 1,
                is_bold: false,
                is_italic: false,
+                is_underline: false,
                item_type: ItemType::Text,
                mcid: None,
            });
@@ -1078,6 +1080,7 @@ mod tests {
            page: 1,
            is_bold: false,
            is_italic: false,
+            is_underline: false,
            item_type: ItemType::Text,
            mcid: None,
        }
@@ -116,6 +116,10 @@ pub struct TextItem {
    pub is_bold: bool,
    /// Whether the font is italic
    pub is_italic: bool,
+    /// Whether the text is underlined (drawn rule/thin rect under the
+    /// baseline — PDFs have no underline font flag, so this is detected
+    /// geometrically after extraction; see `extractor::underline`).
+    pub is_underline: bool,
    /// Type of item (text, image, link)
    pub item_type: ItemType,
    /// Marked Content ID from the content stream's BDC/BMC operator.
@@ -137,12 +141,17 @@ pub struct TextLine {

 impl TextLine {
    pub fn text(&self) -> String {
-        self.text_with_formatting(false, false)
+        self.text_with_formatting(false, false, false)
    }

-    /// Get text with optional bold/italic markdown formatting
-    pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String {
-        if !format_bold && !format_italic {
+    /// Get text with optional bold/italic/underline markdown formatting
+    pub fn text_with_formatting(
+        &self,
+        format_bold: bool,
+        format_italic: bool,
+        format_underline: bool,
+    ) -> String {
+        if !format_bold && !format_italic && !format_underline {
            return self.text_plain();
        }

@@ -151,6 +160,7 @@ impl TextLine {
        let mut result = String::new();
        let mut current_bold = false;
        let mut current_italic = false;
+        let mut current_underline = false;

        for (i, item) in self.items.iter().enumerate() {
            let text = item.text.as_str();
@@ -176,9 +186,13 @@ impl TextLine {
            // we push text_trimmed below (which strips it).
            let has_leading_space = text.starts_with(' ');

-            // Check for style changes
-            let item_bold = format_bold && item.is_bold;
-            let item_italic = format_italic && item.is_italic;
+            // Check for style changes. Underline is exclusive: `<u>` content
+            // stays free of `**`/`*` markers — consumers (and the eval
+            // harnesses this feeds) match the tag content literally, and
+            // mixed `<u>**x**</u>` nesting breaks that.
+            let item_underline = format_underline && item.is_underline;
+            let item_bold = format_bold && item.is_bold && !item_underline;
+            let item_italic = format_italic && item.is_italic && !item_underline;

            // Close previous styles if they change
            if current_italic && !item_italic {
@@ -189,6 +203,10 @@ impl TextLine {
                result.push_str("**");
                current_bold = false;
            }
+            if current_underline && !item_underline {
+                result.push_str("</u>");
+                current_underline = false;
+            }

            // Add space: either from spacing logic or preserved from item text
            if needs_space || (has_leading_space && !result.is_empty() && !result.ends_with(' ')) {
@@ -196,6 +214,10 @@ impl TextLine {
            }

            // Open new styles
+            if item_underline && !current_underline {
+                result.push_str("<u>");
+                current_underline = true;
+            }
            if item_bold && !current_bold {
                result.push_str("**");
                current_bold = true;
@@ -215,6 +237,9 @@ impl TextLine {
        if current_bold {
            result.push_str("**");
        }
+        if current_underline {
+            result.push_str("</u>");
+        }

        result
    }
@@ -104,6 +104,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
        page,
        is_bold: false,
        is_italic: false,
+        is_underline: false,
        item_type: ItemType::Text,
        mcid: None,
    }
@@ -129,6 +130,7 @@ fn make_text_item_with_font(
        page,
        is_bold: is_bold_font(font),
        is_italic: is_italic_font(font),
+        is_underline: false,
        item_type: ItemType::Text,
        mcid: None,
    }
@@ -1354,6 +1356,32 @@ fn test_extract_regions_mem_identity_h_needs_ocr() {
    );
 }

+/// ParseBench `text_simple__att10k.pdf` (issue #118): the producer authored a
+/// broken ToUnicode CMap that shifts every character by a per-range constant,
+/// and the embedded subset font has no `cmap` table to recover from. The
+/// resulting ciphertext is 100% printable ASCII, so it must be caught by the
+/// substitution-cipher statistics and routed to OCR instead of served silently.
+#[test]
+fn test_extract_pages_mem_shifted_cipher_tounicode_needs_ocr() {
+    let buf = std::fs::read("tests/fixtures/shifted_cipher_tounicode.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, None).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert!(
+        result.pages[0].needs_ocr,
+        "shifted-cipher garbled page should be flagged needs_ocr"
+    );
+    assert!(
+        result.pages[0].markdown.is_empty(),
+        "garbled markdown should be suppressed"
+    );
+    assert_eq!(result.pages_needing_ocr, vec![1]);
+    assert_eq!(
+        result.pages[0].ocr_reason.as_deref(),
+        Some("suspected_garbled_text")
+    );
+}
+
 #[test]
 fn test_extract_regions_mem_multiple_regions_per_page() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
@@ -188,7 +188,7 @@
 |156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75|
 |157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00|
 |158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00|
-|159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00|
+|159|30/7|Proposed walkway, Drain, rock armoring, road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00|
 |160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00|
 |161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00|
 |162 AUG|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00|
@@ -237,7 +237,7 @@
 |201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00|
 ||OCT|||||
 |202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00|
-|203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00|
+|203|1/10||Proposed Walkway, Drain, rock armoring, road and Bridge widening at Anse Talbot(Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00|
 |204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00|
 |205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00|
 |206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11|
@@ -10,7 +10,7 @@ Department of the Treasury **Internal Revenue Service**

 ### This publication contains:

-**Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer
+**Form 4070A,** Employee’s Daily Record of Tips **Form 4070,** Employee’s Report of Tips to Employer

 For the period

@@ -22,7 +22,7 @@ Name and address of employee

 **Publication 1244 (Rev. 7-96)** Cat. No. 44472W

-**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A, Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.—If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages.
+**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use **Form 4070A**, Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.—**If you receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use **Form 4070**, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages.

 *(continued on inside of back cover)*

@@ -30,14 +30,14 @@ Form **4070A** Employee’s Daily Record of Tips (Rev. July 1996) **This is a vo

 Establishment name (if different)

-Date Date **a. Tips received**
+Date Date **a.** Tips received

-**b. Credit card tips c. Tips paid out to d. Names of employees to whom you**
+**b.** Credit card tips **c.** Tips paid out to **d.** Names of employees to whom you
 tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1**

-Date Date **a. Tips received**
+Date Date **a.** Tips received

-**b. Credit card tips c. Tips paid out to d. Names of employees to whom you**
+**b.** Credit card tips **c.** Tips paid out to **d.** Names of employees to whom you
 tips of directly from customers received other employees paid tips rec’d. entry and other employees

 7 8 9 10 11 12 13 14 15 **Subtotals**
@@ -50,9 +50,9 @@ tips of directly from customers received other employees paid tips rec’d. entr

 27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals**

-**1.** Report total cash tips (col. a) on Form 4070, line 1.
-**2.** Report total credit card tips (col. b) on Form 4070, line 2.
-**3.** Report total tips paid out (col. c) on Form 4070, line 3. **Page 4**
+**1.** Report total cash tips (col. **a**) on Form 4070, line **1.**
+**2.** Report total credit card tips (col. **b**) on Form 4070, line **2.**
+**3.** Report total tips paid out (col. **c**) on Form 4070, line **3.** **Page 4**

 Form Employee’s Report (Rev. July 1996)

@@ -66,17 +66,17 @@ Employer’s name and address (include establishment name, if different) **1** C

 **3** Tips paid out

-Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3) from, 19, to, 19 Signature Date
+Month or shorter period in which tips were received **4** Net tips (lines **1 + 2 - 3**) from, 19, to, 19 Signature Date

-**Paperwork Reduction Act Notice.—We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; Learning **about the law—each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these
+**Paperwork Reduction Act Notice.—**We ask for the information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: **Recordkeeping**—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law**—each form, 2 min.; **Preparing** Form 4070, 13 min.; Form 4070A, 55 min.; and **Copying and** **providing** Form 4070, 10 min.; Form 4070A, 14 min. If you have comments concerning the accuracy of these time estimates or suggestions for making these

-forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676).
+forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—**Use this form to report tips you receive to your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See **Pub. 531**, Reporting Tip Income, for more information. You can get additional copies of **Pub. 1244**, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676).

-**Instructions (continued)**
+**Instructions** *(continued)*

-**Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law.
+**Unreported Tips.—**If you received tips of $20 or more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you **must** use Form 1040 and **Form 4137,** Social Security and Medicare Tax on Unreported Tip Income, to report them. You may **not** use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act **cannot** use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—**Get **Pub. 531,** Reporting Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—**If you do not keep a daily record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law.

-### Instructions (continued)
+**Instructions** *(continued)*

 Use this space to total your tips for the year

@@ -6,7 +6,7 @@

 8 4 Z E L L / L U R I E R E A L E S T A T E C E N T E R

-**Table I: Cap rate correlations** **Cap Rate Correlation With:*** **BBB Corp** **10-Year Bond Yield S&P Dividend** **Treasury (10-15 yr) Yield** Multifamily 0.187 0.771 0.068 Industrial-0.221 0.748-0.307 CBD Office-0.449 0.694-0.458 Retail-0.181 0.649-02.58
+**Table I:** Cap rate correlations **Cap Rate Correlation With:*** **BBB Corp** **10-Year Bond Yield S&P Dividend** **Treasury (10-15 yr) Yield** Multifamily 0.187 0.771 0.068 Industrial-0.221 0.748-0.307 CBD Office-0.449 0.694-0.458 Retail-0.181 0.649-02.58

 * Based on 25 years of data for the 10-yrT & S&P DivYld; and 14 years for BBB.
 **Figure 1:** NCREIF cap rates vs. 10-yearTreasury
@@ -20,7 +20,9 @@ R E V I E W 8 5

 **Figure 2:** Capratespreadsover10-yearTreasury

-**Basis Points -200** -400
+**Basis Points** -200
+
+-400

 -600

@@ -32,7 +34,7 @@ R E V I E W 8 5

 1982 1986 1990 1994 1998 2002 2006

-**Table II: Correlationsofspreadsbypropertytype** **Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office**
+**Table II:** Correlationsofspreadsbypropertytype **Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office**

 ||Multifamily|Industrial|CBD Office|
 |---|---|---|---|
@@ -1,8 +1,8 @@
 (e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows:
-§1.1563-3T Rules for determining stock ownership (temporary).
+<u>§1.1563-3T Rules for determining stock ownership (temporary)</u>.

 (a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a)
-through (d)(2)(iii). (iv) Statement. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include--
+through (d)(2)(iii). (iv) <u>Statement</u>. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include--

 (A) A description of each of the controlled groups in which the corporation
 could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and
@@ -10,7 +10,7 @@ could be included. The description must include the name and employer identifica
 (B) The following representation: [INSERT NAME AND EMPLOYER
 IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER OF THE [INSERT DESIGNATION OF GROUP].

-(v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of
+(v) <u>Election</u>-- (A) <u>Election filed</u>. An election filed under paragraph (d)(2)(iv) of
 this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in

 |termination of membership in the controlled group in which such corporation has||
@@ -30,47 +30,47 @@ Federal income tax return (including any amended return filed on or before the d

 2006.
 (2) Expiration date. The applicability of this section will expire on May 26,
-2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: §1.6012-2 Corporations required to make returns of income.
+2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: <u>§1.6012-2 Corporations required to make returns of income</u>.
 * * * * *
 (c) [Reserved]. For further guidance, see §1.6012-2T(c).
 * * * * *
 (k) [Reserved]. For further guidance, see §1.6012-2T(k)(1).

-Par. 52. Section 1.6012-2T is added to read as follows: §1.6012-2T Corporations required to make returns of income (temporary).
+Par. 52. Section 1.6012-2T is added to read as follows: <u>§1.6012-2T Corporations required to make returns of income (temporary)</u>.

 (a) through (b) [Reserved]. For further guidance, see §1.6012-2(a) through
 (b).
 (c) Insurance companies-- (1) Domestic life insurance companies-- (i) In
-general. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return--
+<u>general</u>. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return--

 (A) A copy of its annual statement which shows the reserves used by the
 company in computing the taxable income reported on its return; and

 (B) A copy of Schedule A (real estate) and of Schedule D (bonds and stocks),
-or any successor thereto, of such annual statement. (ii) Mutual savings banks. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L.
+or any successor thereto, of such annual statement. (ii) <u>Mutual savings banks</u>. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L.

-(2) Domestic nonlife insurance companies. Every domestic insurance
+(2) <u>Domestic nonlife insurance companies</u>. Every domestic insurance
 company other than a life insurance company shall make a return on Form 1120PC. This includes organizations described in section 501(m)(1) that provide commercial- type insurance and organizations described in section 833. Except as provided in paragraph (c)(4) of this section, such company shall file with its return a copy of its

 annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return.

-(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and
+(3) <u>Foreign insurance companies</u>. The provisions of paragraphs (c)(1) and
 (c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company.
-(4) Exception for insurance companies filing their Federal income tax returns
-electronically. If an insurance company described in paragraph (c)(1), (c)(2), or
+(4) <u>Exception for insurance companies filing their Federal income tax returns</u>
+<u>electronically</u>. If an insurance company described in paragraph (c)(1), (c)(2), or

 (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e).
-(5) Definition. For purposes of this section, the term annual statement means
+(5) <u>Definition</u>. For purposes of this section, the term <u>annual statement</u> means
 the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of

 Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement.

 (d) through (j) [Reserved]. For further guidance, see §1.6012-2(d) through (j).
-(k) Effective date-- (1) Applicability date. This section applies to any original
+(k) <u>Effective date</u>-- (1) <u>Applicability date</u>. This section applies to any original
 Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30,

 2006.
-(2) Expiration date. The applicability of this section will expire on May 26,
+(2) <u>Expiration date</u>. The applicability of this section will expire on May 26,
 2009.

 |||Par. 53. For each entry in the “Location” column of the following table,|
@@ -165,7 +165,7 @@ section and paragraph
 PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The authority citation for part 602 continues to read as follows: Authority: 26 U.S.C. 7805. Par. 55. In §602.101, paragraph (b) is amended to read as follows:

 1. The following entries to the table are removed:
-§602.101 OMB Control numbers.
+<u>§602.101 OMB Control numbers</u>.

 * * * * *
 (b) * * *
@@ -180,7 +180,7 @@ CFR part or section where Current OMB identified or described control No.
 1.1081-11………………………………………………………………. 1545-2019
 * * * * * **______________________________________________________________**
 2. The following entries are added in numerical order to the table:
-§602.101 OMB Control numbers.
+<u>§602.101 OMB Control numbers</u>.

 * * * * *
 (b) * * *
@@ -26,7 +26,7 @@ A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST the

 ##### Physical Properties

-|Chemical Formula|CCl2F2|
+|Chemical Formula|CCl₂F₂|
 |---|---|
 |Molecular mass|120.91|
 |Boiling Point At one atmosphere|-29.75°C|
@@ -45,7 +45,7 @@ l

 |Temp|Pressure||Volume|||Density||Enthalpy|||Entropy|Temp|
 |---|---|---|---|---|---|---|---|---|---|---|---|---|
-|°C|[kPa]|[m3 Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m3] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C|
+|°C|[kPa]|[m³ Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m³] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C|

 |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100|
 |---|---|---|---|---|---|---|---|---|---|---|---|
Author	SHA1	Message	Date
Abimael MartellandClaude Fable 5	50dc4e5871	chore: source Python package version from Cargo.toml via maturin Address PR review (cubic P2): pyproject.toml pinned version = "0.1.0", which overrides Cargo.toml, so a maturin build produced a 0.1.0 Python artifact regardless of the crate version (it had drifted since the PyO3 bindings were added). Switch to dynamic = ["version"] so maturin sources the version from Cargo.toml [package] version and the two can no longer diverge. No workflow auto-publishes the Python package, so this is metadata hygiene rather than a release-path fix. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>	2026-07-09 12:36:59 -07:00
Abimael MartellandClaude Fable 5	974c2cea23	fix(lib): make cipher detection case-agnostic via sorted-histogram shape Address PR review follow-up: the mixed-case guard from the previous commit returned before the vowel/frequency checks, creating a blind spot — a uniform-case (all-lower or all-upper) substitution cipher is a plausible broken-CMap output and would bypass OCR entirely. Replace the case proxy with the actual invariant. A substitution cipher is a bijection over a real language's alphabet, so it preserves the frequency SHAPE (the sorted histogram) while scrambling letter POSITIONS (the unsorted histogram). Signal 2 now flags when english_cosine < 0.60 (positions unlike English) AND english_shape_cosine >= 0.90 (profile is still English-shaped). This is independent of case, so it catches all-lower, all-upper, and case-straddling shifts alike. The exempted structured content fails one half: DNA/hex dumps have too steep a profile (shape cosine 0.74 / 0.81 < 0.90), while protein sequences, ticker symbols and base64 are not sufficiently unlike English in position (unsorted cosine 0.74 / 0.75 / 0.77 >= 0.60). All stay out of OCR. Still strictly corpus-safe: every real Latin document scores unsorted cosine >= 0.70 (min 0.80), far above the 0.60 gate, so none can reach Signal 2. Re-verified byte-identical to a baseline main binary across all 185 eval PDFs; att10k remains flagged. Drops the now-unused case counters and adds all-lowercase / all-uppercase shifted-prose regression tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>	2026-07-09 12:35:24 -07:00
Abimael MartellandClaude Fable 5	cc602ab92e	fix(lib): exempt uniform-case structured content from cipher detection Address PR review (cubic P2): the frequency branch (english_cosine < 0.60) fired on any Latin-dominant, low-vowel letter distribution unlike English, so non-linguistic ASCII — DNA/protein sequences, ticker symbols, hex dumps — could be suppressed and routed to OCR despite not being garbled. Measured: DNA cosine 0.428 / vowel ratio 0.260, protein 0.738, tickers 0.747, hex 0.549 — all would have flagged. Add a mixed-case guard to looks_garbled: garbled English is a permutation of natural language and carries sentence capitalization (block-straddling shifts invert the ratio — att10k is 60% uppercase; in-case Caesar shifts preserve it at ~3%), so both keep some of each case. The exempted structured content is uniform case (all upper or all lower). Requiring the minority case to be >=1% of ASCII letters exempts single-case sequences while preserving both garble signals, including the in-case-shift scenario the frequency branch exists for. Strictly tightens the detector: it can only remove flags, so the eval corpus stays at zero false positives (verified byte-identical to a baseline main binary across all 185 PDFs) and att10k remains flagged. Adds regression tests for DNA, protein, tickers, and an in-case Caesar shift. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>	2026-07-09 12:25:10 -07:00
Abimael MartellandClaude Fable 5	f4541598ee	chore: bump pdf-inspector to 0.1.4, npm package to 1.9.11 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>	2026-07-09 12:16:35 -07:00
Abimael MartellandClaude Fable 5	f50cbd8044	fix(lib): detect substitution-cipher garbled text from broken ToUnicode CMaps ParseBench text_simple__att10k.pdf (issue #118) ships Type0/Identity-H fonts whose ToUnicode CMaps are authored garbled: every bfrange maps with a wrong constant delta, so text extracts as pure-ASCII ciphertext ("Certificate" -> "8VceZWZTReV"). The embedded subset font has no cmap table and no glyph names, so no decode source can recover the real text (poppler and mupdf emit the same ciphertext). The only correct behavior is to flag the page for OCR instead of serving the garbage silently -- but the text is 100% printable ASCII with word-like tokens, so it slipped past is_garbage_text and detect_encoding_issues. Add CipherGarbleStats, a letter-statistics discriminator that flags a Latin-dominant sample (>=200 ASCII letters) when vowels are starved (<=30% of letters) AND either: - lowercase->uppercase transitions inside words exceed 10% of letter bigrams (a shifted lowercase alphabet straddles the ASCII uppercase block), or - the letter histogram's cosine similarity against English letter frequencies drops below 0.60 (catches shifts that stay within case blocks). Wired into analyze_text_quality (per-page, item-level) and detect_encoding_issues (markdown-level), so extract_pages_markdown reports needs_ocr + suspected_garbled_text and suppresses the garbage. Thresholds validated against the 380-document pdf-evals snapshot corpus (Swedish, Finnish, Turkish, German, romaji, schematics, all-caps and camelCase-heavy docs): zero false positives, and byte-identical eval output vs main. Garbled page measures vowel ratio 0.245 / case-shift rate 0.225 / cosine 0.532; closest legitimate document on each axis is 0.264 / 0.021 / 0.801. Fixes #118 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>	2026-07-09 12:12:20 -07:00
Abimael MartellandCursor	b375d6f102	feat(markdown): underline emission, Unicode scripts, style-preserving merges (#117 ) * feat(markdown): underline emission, Unicode scripts, style-preserving merges (ENG-5015 2b) Three formatting losses in the direct-extraction markdown path: 1. text_with_formatting gains <u> run emission (detect_underline option, default on) using the geometric is_underline flag from 1.9.9. Underline runs stay free of nested bold/italic markers — consumers match tag content literally. Heading lines keep plain text for bold/italic but preserve <u>: the tag carries meaning `#` doesn't. 2. merge_subscript_items now maps absorbed digit scripts to Unicode sub/superscript forms with direction from the baseline offset ("H"+"2" -> "H₂", "word"+raised "2" -> "word²", "m"+"3" -> "m³"). NFKC/NFKD folds these back to plain digits so text matching downstream is unaffected; renderers keep the script semantics. 3. merge_text_items no longer merges across bold/italic boundaries — absorbing a styled run into a plain neighbor erased the styling before markdown emission ever saw it. On eval docs this recovers 20-82 italic runs per document that previously emitted as plain. Snapshots regenerated (diffs are the features: CCl₂F₂, m³, underlined legal section headings, finer bold runs). pdf-evals regression suite: 202/202 real PDFs pass. napi 1.9.9 -> 1.9.10. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(extractor): break merges at underline boundaries too (review) OR-merging underline stretched the eventual <u> span over neighboring plain fragments. Merge runs now break on any style-flag change, the redundant accumulator is gone, and format_list_item learned to move bullet markers outside <u> wrappers so fully-underlined bullet lines still render as markdown lists. td9264 snapshot regenerated — spans are tighter (trailing periods correctly outside the tag). Co-authored-by: Cursor <cursoragent@cursor.com> * fix(markdown): strip stray spaces before sentence punctuation (review) Style-boundary item splits can strand a trailing period in its own fragment, and multiple assembly paths join fragments with spaces, yielding "word ." artifacts. Rather than chasing every join site, a postprocess pass removes a space before `.`/`,`/`;` when the mark ends its token (whitespace, cell boundary `\|`, or end of text follows). Dot leaders/ellipses and mid-token periods are untouched. Fixes the td9264 "companies ." artifacts and two pre-existing "armoring ," artifacts in the 2013-app2 snapshot. pdf-evals: zero markdown diffs across all 203 corpus PDFs vs committed baselines. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(tables): trim spaces inside parenthetical cell fragments * fix(tables): reject sparse prose row-stripe tables --------- Co-authored-by: Cursor <cursoragent@cursor.com>	2026-07-08 23:40:08 -07:00
Abimael MartellandCursor	422a2ff118	feat(extractor): geometric underline detection on TextItem (#116 ) * feat(extractor): geometric underline detection on TextItem (ENG-5015) PDFs carry no underline font flag — underlines are stroked horizontal lines or thin filled rects drawn under the baseline. Correlate those graphics (already parsed from the content stream) with text items in a post-pass: a rule within ~0.35em below the baseline covering >=60% of an item's width marks is_underline. Exposed through the napi and python bindings. Verified on real docs: 4/4 underlined sentences flagged on a Japanese report, links/headings flagged on 8 of 10 underline-bearing eval docs, zero flags on docs without underlines. Known FP source (table cell borders) documented — downstream applies inline styling only to plain-text regions. napi 1.9.8 -> 1.9.9. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(extractor): underline rules only from painted rects, normalized extents (review) Two review fixes: (1) normalize rect extents before the thickness/width checks — `re` operands pass through the CTM so width/height can be negative, which missed negative-width rules and let negative-height bands pass as thin; (2) only feed painted rects to underline detection — `re` rects now wait in a pending list until a paint operator (S/s, f/F/ f, B/B/b/b) confirms them, and `re W n` clip-only paths are discarded at `n`, so invisible clip boundaries no longer underline nearby text. Marking moved into content_stream where paint state lives (pre-rotation, consistent device space). Co-authored-by: Cursor <cursoragent@cursor.com> fix(extractor): harden underline detection * feat(cli): export positioned text item json --------- Co-authored-by: Cursor <cursoragent@cursor.com>	2026-07-08 10:20:18 -07:00