fix(markdown): strip garbled running footers

* fix(extractor): reduce garbled OCR false positives

* fix(extractor): tighten garbled text OCR routing

* fix(extractor): decode UTF-16 ToUnicode destinations

* fix(extractor): narrow ToUnicode destination cleanup

* fix(extractor): decode Aptos private ff ligature

.github/workflows/publish-crate.yml

@@ -0,0 +1,87 @@
+name: Publish Rust crate
+
+on:
+  push:
+    branches: [main]
+    paths: ['Cargo.toml']
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check-version:
+    name: Check version change
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      published: ${{ steps.check.outputs.published }}
+      version: ${{ steps.check.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Check if version changed
+        id: check
+        run: |
+          NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
+          OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
+          echo "old=$OLD_VERSION new=$NEW_VERSION"
+          echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
+
+          if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "published=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          echo "changed=true" >> "$GITHUB_OUTPUT"
+
+          HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
+            -H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
+            "https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
+
+          case "$HTTP_STATUS" in
+            200)
+              echo "published=true" >> "$GITHUB_OUTPUT"
+              echo "pdf-inspector v$NEW_VERSION is already published"
+              ;;
+            404)
+              echo "published=false" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              cat /tmp/crate-version.json
+              echo "Unexpected crates.io response: $HTTP_STATUS" >&2
+              exit 1
+              ;;
+          esac
+
+  publish:
+    name: Publish to crates.io
+    needs: check-version
+    if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
+    runs-on: ubuntu-latest
+    environment: crates-io
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Verify package
+        run: cargo publish --dry-run
+
+      - name: Authenticate with crates.io
+        id: auth
+        uses: rust-lang/crates-io-auth-action@v1
+
+      - name: Publish crate
+        run: cargo publish
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector"
-version = "0.1.0"
+version = "0.1.3"
 edition = "2021"
 autobins = false
 authors = ["Firecrawl Team"]

docs/publishing.md

@@ -0,0 +1,21 @@
+# Publishing
+
+The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
+
+## crates.io Trusted Publisher
+
+Configure the trusted publisher for the `pdf-inspector` crate with:
+
+- Repository: `firecrawl/pdf-inspector`
+- Workflow: `publish-crate.yml`
+- Environment: `crates-io`
+
+The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
+
+## Release Steps
+
+1. Update `version` in `Cargo.toml`.
+2. Merge the version bump to `main`.
+3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
+
+If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.

napi/Cargo.lock

@@ -672,8 +672,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "lopdf"
-version = "0.40.0"
-source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d"
 dependencies = [
  "aes",
  "bitflags",
@@ -829,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
 [[package]]
 name = "pdf-inspector"
-version = "0.1.0"
+version = "0.1.3"
 dependencies = [
  "env_logger",
  "log",
@@ -844,7 +845,7 @@ dependencies = [
 
 [[package]]
 name = "pdf-inspector-napi"
-version = "0.2.0"
+version = "0.2.2"
 dependencies = [
  "napi",
  "napi-build",

napi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector-napi"
-version = "0.2.0"
+version = "0.2.2"
 edition = "2021"
 
 [lib]

napi/README.md

@@ -37,7 +37,7 @@ console.log(result.confidence)     // 0.875
 
 Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pipelines where a layout model detects regions in rendered page images, and this function extracts text from the PDF structure for text-based pages — skipping GPU OCR.
 
-Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues).
+Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). When the cause is a suspected garbled text layer, `ocrReason` is set to `"suspected_garbled_text"`.
 
 ```typescript
 import { extractTextInRegions } from '@firecrawl/pdf-inspector'
@@ -84,6 +84,7 @@ interface PageRegionTexts {
 interface RegionText {
   text: string
   needsOcr: boolean         // true when text is unreliable
+  ocrReason?: string        // "suspected_garbled_text" when known
 }
 ```
 

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.5",
+  "version": "1.9.8",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -40,6 +40,8 @@ pub struct PdfResult {
     pub processing_time_ms: u32,
     /// 1-indexed page numbers that need OCR.
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    pub ocr_reasons_by_page: Vec<PageOcrReasons>,
     pub title: Option<String>,
     pub confidence: f64,
     pub is_complex_layout: bool,
@@ -48,6 +50,13 @@ pub struct PdfResult {
     pub has_encoding_issues: bool,
 }
 
+/// OCR reasons for a single 1-indexed page.
+#[napi(object)]
+pub struct PageOcrReasons {
+    pub page: u32,
+    pub reasons: Vec<String>,
+}
+
 /// Lightweight PDF classification result.
 #[napi(object)]
 pub struct PdfClassification {
@@ -90,6 +99,8 @@ pub struct RegionText {
     pub text: String,
     /// `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues).
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    pub ocr_reason: Option<String>,
 }
 
 /// Extracted text for one page's regions.
@@ -126,6 +137,7 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
         page_count: r.page_count,
         processing_time_ms: r.processing_time_ms as u32,
         pages_needing_ocr: r.pages_needing_ocr,
+        ocr_reasons_by_page: to_napi_page_ocr_reasons(r.ocr_reasons_by_page),
         title: r.title,
         confidence: r.confidence as f64,
         is_complex_layout: r.layout.is_complex,
@@ -135,6 +147,18 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
     }
 }
 
+fn to_napi_page_ocr_reasons(
+    reasons: Vec<pdf_inspector::PageOcrReasons>,
+) -> Vec<PageOcrReasons> {
+    reasons
+        .into_iter()
+        .map(|reason| PageOcrReasons {
+            page: reason.page,
+            reasons: reason.reasons,
+        })
+        .collect()
+}
+
 fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
     match t {
         pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
@@ -563,6 +587,8 @@ pub struct PageMarkdownResult {
     pub markdown: String,
     /// `true` when text on this page is unreliable.
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    pub ocr_reason: Option<String>,
 }
 
 /// Combined per-page markdown extraction and layout classification result.
@@ -576,6 +602,8 @@ pub struct PagesExtractionResult {
     pub pages_with_columns: Vec<u32>,
     /// 1-indexed pages that need OCR (scanned/image-based).
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    pub ocr_reasons_by_page: Vec<PageOcrReasons>,
     /// True if any page has tables or columns.
     pub is_complex: bool,
 }
@@ -607,11 +635,13 @@ pub fn extract_pages_markdown(
                     page: r.page,
                     markdown: r.markdown,
                     needs_ocr: r.needs_ocr,
+                    ocr_reason: r.ocr_reason,
                 })
                 .collect(),
             pages_with_tables: result.pages_with_tables,
             pages_with_columns: result.pages_with_columns,
             pages_needing_ocr: result.pages_needing_ocr,
+            ocr_reasons_by_page: to_napi_page_ocr_reasons(result.ocr_reasons_by_page),
             is_complex: result.is_complex,
         })
     })
@@ -648,6 +678,7 @@ fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<Pa
                 .map(|r| RegionText {
                     text: r.text,
                     needs_ocr: r.needs_ocr,
+                    ocr_reason: r.ocr_reason,
                 })
                 .collect(),
         })

src/bin/pdf2md.rs

@@ -31,6 +31,22 @@ fn json_escape(s: &str) -> String {
     out
 }
 
+fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> String {
+    reasons
+        .iter()
+        .map(|entry| {
+            let reasons_json = entry
+                .reasons
+                .iter()
+                .map(|reason| format!(r#""{}""#, json_escape(reason)))
+                .collect::<Vec<_>>()
+                .join(",");
+            format!(r#"{{"page":{},"reasons":[{}]}}"#, entry.page, reasons_json)
+        })
+        .collect::<Vec<_>>()
+        .join(",")
+}
+
 /// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
 fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
     let mut pages = HashSet::new();
@@ -177,12 +193,14 @@ fn main() {
                         .iter()
                         .map(|p| p.to_string())
                         .collect();
+                    let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page);
                     println!(
-                        r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{}}}"#,
+                        r#"{{"pdf_type":"{}","page_count":{},"processing_time_ms":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{}}}"#,
                         pdf_type_str,
                         result.page_count,
                         result.processing_time_ms,
                         ocr_pages.join(","),
+                        ocr_reasons,
                         result.layout.is_complex,
                         table_pages.join(","),
                         col_pages.join(","),
@@ -223,8 +241,9 @@ fn main() {
                     .iter()
                     .map(|p| p.to_string())
                     .collect();
+                let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page);
                 println!(
-                    r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{},"markdown":"{}"}}"#,
+                    r#"{{"pdf_type":"{}","page_count":{},"has_text":{},"processing_time_ms":{},"markdown_length":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"has_encoding_issues":{},"markdown":"{}"}}"#,
                     match result.pdf_type {
                         PdfType::TextBased => "text_based",
                         PdfType::Scanned => "scanned",
@@ -236,6 +255,7 @@ fn main() {
                     result.processing_time_ms,
                     result.markdown.as_ref().map(|m| m.len()).unwrap_or(0),
                     ocr_pages.join(","),
+                    ocr_reasons,
                     result.layout.is_complex,
                     table_pages.join(","),
                     col_pages.join(","),

src/extractor/content_stream.rs

@@ -188,7 +188,17 @@ pub(crate) fn extract_page_text_items(
     // Graphics state tracking
     let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
     let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
-    let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();
+    #[derive(Clone)]
+    struct SavedGraphicsState {
+        ctm: [f32; 6],
+        text_rendering_mode: i32,
+        char_spacing: f32,
+        word_spacing: f32,
+        text_leading: f32,
+        current_font: String,
+        current_font_size: f32,
+    }
+    let mut gstate_stack: Vec<SavedGraphicsState> = Vec::new();
 
     // Text state tracking
     let mut current_font = String::new();
@@ -227,15 +237,26 @@ pub(crate) fn extract_page_text_items(
         match op.operator.as_str() {
             "q" => {
                 // Save graphics state
-                gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
+                gstate_stack.push(SavedGraphicsState {
+                    ctm,
+                    text_rendering_mode,
+                    char_spacing,
+                    word_spacing,
+                    text_leading,
+                    current_font: current_font.clone(),
+                    current_font_size,
+                });
             }
             "Q" => {
                 // Restore graphics state
-                if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
-                    ctm = saved_ctm;
-                    text_rendering_mode = saved_tr;
-                    char_spacing = saved_tc;
-                    word_spacing = saved_tw;
+                if let Some(saved) = gstate_stack.pop() {
+                    ctm = saved.ctm;
+                    text_rendering_mode = saved.text_rendering_mode;
+                    char_spacing = saved.char_spacing;
+                    word_spacing = saved.word_spacing;
+                    text_leading = saved.text_leading;
+                    current_font = saved.current_font;
+                    current_font_size = saved.current_font_size;
                 }
             }
             "cm" => {
@@ -1286,6 +1307,91 @@ mod tests {
         assert!(lines.is_empty());
     }
 
+    #[test]
+    fn test_q_restores_current_font_for_text_decoding() {
+        use crate::tounicode::FontCMaps;
+        use lopdf::{dictionary, Object, Stream};
+
+        fn cmap_stream(dst_hex: &str) -> Stream {
+            let cmap = format!(
+                r#"/CIDInit /ProcSet findresource begin
+12 dict begin
+begincmap
+/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
+/CMapName /Test-UCS def
+/CMapType 2 def
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+1 beginbfchar
+<41> <{dst_hex}>
+endbfchar
+endcmap
+CMapName currentdict /CMap defineresource pop
+end
+end"#
+            );
+            Stream::new(dictionary! {}, cmap.into_bytes())
+        }
+
+        let mut doc = lopdf::Document::new();
+        let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X
+        let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y
+        let f1 = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+            "ToUnicode" => Object::Reference(f1_cmap),
+        });
+        let f2 = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+            "ToUnicode" => Object::Reference(f2_cmap),
+        });
+
+        let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET
+q
+BT /F2 12 Tf 20 700 Tm <41> Tj ET
+Q
+BT 30 700 Tm <41> Tj ET";
+        let content_id = doc.add_object(Object::Stream(Stream::new(
+            dictionary! {},
+            content.to_vec(),
+        )));
+        let page_id = doc.add_object(dictionary! {
+            "Type" => "Page",
+            "Contents" => Object::Reference(content_id),
+            "Resources" => dictionary! {
+                "Font" => dictionary! {
+                    "F1" => Object::Reference(f1),
+                    "F2" => Object::Reference(f2),
+                },
+            },
+            "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
+        });
+        let pages_id = doc.add_object(dictionary! {
+            "Type" => "Pages",
+            "Count" => Object::Integer(1),
+            "Kids" => vec![Object::Reference(page_id)],
+        });
+        let catalog_id = doc.add_object(dictionary! {
+            "Type" => "Catalog",
+            "Pages" => Object::Reference(pages_id),
+        });
+        doc.trailer.set("Root", Object::Reference(catalog_id));
+
+        let font_cmaps = FontCMaps::from_doc(&doc);
+        let ((items, _, _), _, _) =
+            extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
+        let text = items
+            .iter()
+            .map(|item| item.text.as_str())
+            .collect::<String>();
+
+        assert_eq!(text, "XYX");
+    }
+
     #[test]
     fn test_strip_pdf_comments() {
         // Basic comment stripping

src/extractor/fonts.rs

@@ -526,6 +526,11 @@ pub(crate) fn parse_font_encoding(
     font_dict: &lopdf::Dictionary,
 ) -> Option<EncodingResult> {
     let encoding_obj = font_dict.get(b"Encoding").ok()?;
+    let base_font_name = font_dict
+        .get(b"BaseFont")
+        .ok()
+        .and_then(|o| o.as_name().ok())
+        .map(|n| String::from_utf8_lossy(n).to_string());
 
     // Encoding can be a name or a dictionary
     match encoding_obj {
@@ -538,12 +543,14 @@ pub(crate) fn parse_font_encoding(
         Object::Reference(obj_ref) => {
             // Reference to encoding dictionary
             if let Ok(enc_dict) = doc.get_dictionary(*obj_ref) {
-                parse_encoding_dictionary(doc, enc_dict)
+                parse_encoding_dictionary(doc, enc_dict, base_font_name.as_deref())
             } else {
                 None
             }
         }
-        Object::Dictionary(enc_dict) => parse_encoding_dictionary(doc, enc_dict),
+        Object::Dictionary(enc_dict) => {
+            parse_encoding_dictionary(doc, enc_dict, base_font_name.as_deref())
+        }
         _ => None,
     }
 }
@@ -562,6 +569,7 @@ pub(crate) struct EncodingResult {
 pub(crate) fn parse_encoding_dictionary(
     doc: &Document,
     enc_dict: &lopdf::Dictionary,
+    base_font_name: Option<&str>,
 ) -> Option<EncodingResult> {
     let differences = enc_dict.get(b"Differences").ok()?;
 
@@ -591,11 +599,9 @@ pub(crate) fn parse_encoding_dictionary(
             Object::Name(name) => {
                 // Map current code to glyph name -> Unicode
                 let glyph_name = String::from_utf8_lossy(&name).to_string();
-                if glyph_name == "fi"
-                    || glyph_name == "fl"
-                    || glyph_name == "ffi"
-                    || glyph_name == "ffl"
-                {
+                let mapped_char = glyph_to_char(&glyph_name)
+                    .or_else(|| private_glyph_to_char(&glyph_name, base_font_name));
+                if mapped_char.is_some_and(is_ligature_char) {
                     debug!(
                         "  Differences: code=0x{:02X} glyph={:?} (ligature)",
                         current_code, glyph_name
@@ -610,7 +616,7 @@ pub(crate) fn parse_encoding_dictionary(
                 {
                     gid_glyph_count += 1;
                 }
-                if let Some(ch) = glyph_to_char(&glyph_name) {
+                if let Some(ch) = mapped_char {
                     encoding_map.insert(current_code, ch);
                 } else {
                     debug!(
@@ -645,6 +651,31 @@ pub(crate) fn parse_encoding_dictionary(
     })
 }
 
+fn private_glyph_to_char(glyph_name: &str, base_font_name: Option<&str>) -> Option<char> {
+    let base_font_name = strip_subset_prefix(base_font_name?);
+
+    // Aptos CFF subsets from Office PDFs can expose the ff ligature as /g431
+    // without a ToUnicode map. Keep this font-scoped because /gNNN names are private.
+    if base_font_name.eq_ignore_ascii_case("Aptos") && glyph_name == "g431" {
+        Some('\u{FB00}')
+    } else {
+        None
+    }
+}
+
+fn strip_subset_prefix(font_name: &str) -> &str {
+    font_name
+        .split_once('+')
+        .map_or(font_name, |(_, stripped)| stripped)
+}
+
+fn is_ligature_char(ch: char) -> bool {
+    matches!(
+        ch,
+        '\u{FB00}' | '\u{FB01}' | '\u{FB02}' | '\u{FB03}' | '\u{FB04}'
+    )
+}
+
 /// Get the CMap lookup key for an Identity-H/V CID font without ToUnicode.
 /// Returns the object number used by `collect_cmaps_from_fonts` to store the CMap:
 /// - FontFile2 or FontFile3 obj_num (for embedded font cmap)
@@ -725,6 +756,8 @@ pub(crate) fn extract_text_from_operand(
     let is_type0_cid_font = font_widths
         .get(current_font)
         .is_some_and(|info| info.is_cid);
+    let use_cp1252_fallback =
+        should_use_cp1252_single_byte_fallback(base_font_name, is_type0_cid_font);
     let result = (|| -> Option<String> {
         if let Object::String(bytes, _) = obj {
             let mut decode_with_entry = |entry: &crate::tounicode::CMapEntry| -> Option<String> {
@@ -755,9 +788,12 @@ pub(crate) fn extract_text_from_operand(
                                     return Some(ch.to_string());
                                 }
                             }
-                            // 4. Printable ASCII/Latin-1 fallback
+                            // 4. Printable single-byte fallback
                             if b >= 0x20 {
-                                return Some((b as char).to_string());
+                                return Some(
+                                    decode_single_byte_fallback_char(b, use_cp1252_fallback)
+                                        .to_string(),
+                                );
                             }
                             None
                         })
@@ -880,8 +916,9 @@ pub(crate) fn extract_text_from_operand(
                                 Some(ch)
                             } else if b >= 0x20 {
                                 // Base encoding fallback for printable bytes.
-                                // For codes 0x20-0x7E this matches all standard PDF encodings.
-                                Some(b as char)
+                                // Most PDFs with simple fonts use WinAnsi/PDFDocEncoding
+                                // semantics, not ISO-8859-1 C1 controls.
+                                Some(decode_single_byte_fallback_char(b, use_cp1252_fallback))
                             } else {
                                 None // Skip unmapped control characters
                             }
@@ -937,6 +974,7 @@ pub(crate) fn extract_text_from_operand(
             // Try to decode using cached font encoding from lopdf
             if let Some(encoding) = encoding_cache.get(current_font) {
                 if let Ok(text) = Document::decode_text(encoding, bytes) {
+                    let text = normalize_cp1252_controls(text, use_cp1252_fallback);
                     if text.contains('\u{FFFD}') {
                         debug!(
                             "decode_text produced replacement for font={} bytes_len={}",
@@ -973,16 +1011,119 @@ pub(crate) fn extract_text_from_operand(
                 return Some(symbol_text);
             }
 
-            // Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
-            // 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
-            // use single-byte encodings where Latin-1 fallback is the
-            // canonical interpretation.
-            Some(bytes.iter().map(|&b| b as char).collect())
+            // Non-CID (Type1 / TrueType / Type3) fonts use single-byte
+            // encodings. In practice the fallback should follow WinAnsi for
+            // 0x80..=0x9F so bytes like 0x92 become smart punctuation instead
+            // of C1 controls that look like CID mojibake.
+            Some(decode_single_byte_fallback(bytes, use_cp1252_fallback))
         } else {
             None
         }
     })();
-    result.map(clean_symbol_pua)
+    result.map(|text| {
+        let text = clean_symbol_pua(text);
+        normalize_cp1252_controls(text, use_cp1252_fallback)
+    })
+}
+
+fn decode_single_byte_fallback(bytes: &[u8], use_cp1252_fallback: bool) -> String {
+    bytes
+        .iter()
+        .map(|&b| decode_single_byte_fallback_char(b, use_cp1252_fallback))
+        .collect()
+}
+
+fn decode_single_byte_fallback_char(byte: u8, use_cp1252_fallback: bool) -> char {
+    if !use_cp1252_fallback {
+        return byte as char;
+    }
+
+    match byte {
+        0x80 => '\u{20AC}',
+        0x82 => '\u{201A}',
+        0x83 => '\u{0192}',
+        0x84 => '\u{201E}',
+        0x85 => '\u{2026}',
+        0x86 => '\u{2020}',
+        0x87 => '\u{2021}',
+        0x88 => '\u{02C6}',
+        0x89 => '\u{2030}',
+        0x8A => '\u{0160}',
+        0x8B => '\u{2039}',
+        0x8C => '\u{0152}',
+        0x8E => '\u{017D}',
+        0x91 => '\u{2018}',
+        0x92 => '\u{2019}',
+        0x93 => '\u{201C}',
+        0x94 => '\u{201D}',
+        0x95 => '\u{2022}',
+        0x96 => '\u{2013}',
+        0x97 => '\u{2014}',
+        0x98 => '\u{02DC}',
+        0x99 => '\u{2122}',
+        0x9A => '\u{0161}',
+        0x9B => '\u{203A}',
+        0x9C => '\u{0153}',
+        0x9E => '\u{017E}',
+        0x9F => '\u{0178}',
+        _ => byte as char,
+    }
+}
+
+fn normalize_cp1252_controls(text: String, use_cp1252_fallback: bool) -> String {
+    if !use_cp1252_fallback {
+        return text;
+    }
+    if !text
+        .chars()
+        .any(|ch| ('\u{0080}'..='\u{009F}').contains(&ch))
+    {
+        return text;
+    }
+
+    text.chars()
+        .map(|ch| {
+            if ('\u{0080}'..='\u{009F}').contains(&ch) {
+                decode_single_byte_fallback_char(ch as u8, true)
+            } else {
+                ch
+            }
+        })
+        .collect()
+}
+
+fn should_use_cp1252_single_byte_fallback(
+    base_font_name: Option<&str>,
+    is_type0_cid_font: bool,
+) -> bool {
+    if is_type0_cid_font {
+        return false;
+    }
+
+    let Some(base_font_name) = base_font_name else {
+        return true;
+    };
+    let font_name = base_font_name
+        .rsplit_once('+')
+        .map_or(base_font_name, |(_, stripped)| stripped)
+        .to_ascii_lowercase();
+
+    // TeX/Computer Modern and math/symbol fonts often place ligatures or
+    // symbols in the C1 byte range. Treating those bytes as Windows-1252 makes
+    // words like "deficiente" become "de…ciente" and "fluid" become "‡uid".
+    let non_cp1252_prefixes = [
+        "cmr", "cmb", "cmmi", "cmsy", "cmex", "cmtt", "cmss", "cmti", "ecrm", "ecbx", "ecti",
+        "tcrm", "tctt", "msam", "msbm", "ttdc",
+    ];
+    if non_cp1252_prefixes
+        .iter()
+        .any(|prefix| font_name.starts_with(prefix))
+    {
+        return false;
+    }
+
+    let non_cp1252_names = ["math", "symbol", "dingbat", "emoji"];
+    !non_cp1252_names.iter().any(|name| font_name.contains(name))
 }
 
 /// Replace PUA characters in the F000-F0FF range with standard Unicode equivalents.
@@ -1104,6 +1245,7 @@ fn score_text(text: &str) -> i32 {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use lopdf::dictionary;
 
     fn make_font_info(widths: &[(u16, u16)], default_width: u16, is_cid: bool) -> FontWidthInfo {
         FontWidthInfo {
@@ -1228,6 +1370,51 @@ mod tests {
         assert!(score_text(good) > score_text(bad));
     }
 
+    fn doc_with_private_differences() -> (Document, lopdf::ObjectId) {
+        let mut doc = Document::with_version("1.7");
+        let encoding_id = doc.add_object(dictionary! {
+            "Differences" => Object::Array(vec![
+                Object::Integer(0x88),
+                Object::Name(b"g431".to_vec()),
+                Object::Name(b"fi".to_vec()),
+                Object::Integer(0xAD),
+                Object::Name(b"fl".to_vec()),
+            ]),
+        });
+
+        (doc, encoding_id)
+    }
+
+    #[test]
+    fn aptos_private_g431_maps_to_ff_ligature() {
+        let (doc, encoding_id) = doc_with_private_differences();
+        let font_dict = dictionary! {
+            "BaseFont" => Object::Name(b"NJEQOD+Aptos".to_vec()),
+            "Encoding" => Object::Reference(encoding_id),
+        };
+
+        let result = parse_font_encoding(&doc, &font_dict).expect("encoding should parse");
+
+        assert_eq!(result.map.get(&0x88u8), Some(&'\u{FB00}'));
+        assert_eq!(result.map.get(&0x89u8), Some(&'\u{FB01}'));
+        assert_eq!(result.map.get(&0xADu8), Some(&'\u{FB02}'));
+    }
+
+    #[test]
+    fn private_g431_does_not_map_for_unrelated_fonts() {
+        let (doc, encoding_id) = doc_with_private_differences();
+        let font_dict = dictionary! {
+            "BaseFont" => Object::Name(b"ABCDEF+OtherFont".to_vec()),
+            "Encoding" => Object::Reference(encoding_id),
+        };
+
+        let result = parse_font_encoding(&doc, &font_dict).expect("encoding should parse");
+
+        assert!(!result.map.contains_key(&0x88u8));
+        assert_eq!(result.map.get(&0x89u8), Some(&'\u{FB01}'));
+        assert_eq!(result.map.get(&0xADu8), Some(&'\u{FB02}'));
+    }
+
     #[test]
     fn cid_font_with_unparseable_cmap_does_not_emit_latin1_mojibake() {
         // Type0/CID font (font_widths reports `is_cid=true`) where the
@@ -1279,15 +1466,15 @@ mod tests {
     }
 
     #[test]
-    fn simple_font_latin1_fallback_passes_high_bytes_through() {
+    fn simple_font_single_byte_fallback_passes_high_bytes_through() {
         // A Type1/TrueType simple font (is_cid=false) with a `/ToUnicode`
         // reference but no usable CMap and no `/Differences` map.
-        // Per-byte Latin-1 IS the canonical interpretation here — these
-        // bytes are character codes, not CIDs. The CID guard must NOT
-        // strip them. Reproduces the false positive that an earlier
-        // version of the guard introduced for fonts in PDFs like
-        // pdf-evals/Navigating-Artificial-Intelligence-..., where bytes
-        // like 0xB6 are legitimate Latin-1 character codes.
+        // Per-byte fallback is the canonical interpretation here — these
+        // bytes are character codes, not CIDs. The CID guard must NOT strip
+        // them. Reproduces the false positive that an earlier version of the
+        // guard introduced for fonts in PDFs like pdf-evals/Navigating-
+        // Artificial-Intelligence-..., where bytes like 0xB6 are legitimate
+        // single-byte character codes.
         let bytes = vec![0x24_u8, 0x47, 0xB6, 0x56]; // "$G¶V"
         let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
 
@@ -1320,4 +1507,62 @@ mod tests {
             "simple font fallback must not stamp FFFD over legitimate bytes: {text:?}"
         );
     }
+
+    #[test]
+    fn simple_font_single_byte_fallback_maps_cp1252_punctuation() {
+        let bytes = vec![b'l', 0x92_u8, b'a', b'c', b'a', b'd'];
+        let obj = Object::String(bytes, lopdf::StringFormat::Hexadecimal);
+
+        let font_cmaps = FontCMaps::default();
+        let font_tounicode_refs: HashMap<String, u32> = HashMap::new();
+        let inline_cmaps = HashMap::new();
+        let font_encodings: PageFontEncodings = HashMap::new();
+        let encoding_cache: HashMap<String, Encoding<'_>> = HashMap::new();
+        let mut decisions = CMapDecisionCache::new();
+        let font_widths: PageFontWidths = HashMap::new();
+
+        let text = extract_text_from_operand(
+            &obj,
+            "F1",
+            None,
+            &font_cmaps,
+            &font_tounicode_refs,
+            &inline_cmaps,
+            &font_encodings,
+            &encoding_cache,
+            &mut decisions,
+            &font_widths,
+        )
+        .expect("simple font should decode CP1252 punctuation");
+
+        assert_eq!(text, "l’acad");
+    }
+
+    #[test]
+    fn cached_encoding_decode_normalizes_cp1252_controls() {
+        let text = normalize_cp1252_controls("d\u{92}un \u{96} test".to_string(), true);
+        assert_eq!(text, "d’un – test");
+    }
+
+    #[test]
+    fn tex_font_decode_keeps_c1_ligature_bytes_unmodified() {
+        let text = normalize_cp1252_controls("de\u{85}ciente \u{87}uid".to_string(), false);
+        assert_eq!(text, "de\u{85}ciente \u{87}uid");
+        assert!(!should_use_cp1252_single_byte_fallback(
+            Some("TTdcr10"),
+            false
+        ));
+        assert!(!should_use_cp1252_single_byte_fallback(
+            Some("cmr10"),
+            false
+        ));
+    }
+
+    #[test]
+    fn winansi_text_font_uses_cp1252_fallback() {
+        assert!(should_use_cp1252_single_byte_fallback(
+            Some("BJPQNQ+Times-Roman"),
+            false
+        ));
+    }
 }

src/extractor/layout.rs

@@ -1234,11 +1234,7 @@ pub(crate) fn group_into_lines_with_thresholds(
                             ci,
                             item.x,
                             item.y,
-                            if item.text.len() > 60 {
-                                &item.text[..60]
-                            } else {
-                                &item.text
-                            }
+                            super::trace_text_preview(&item.text, 60)
                         );
                     }
                 }

src/extractor/mod.rs

@@ -33,6 +33,13 @@ pub(crate) use layout::ColumnRegion;
 // Public API
 // ---------------------------------------------------------------------------
 
+pub(crate) fn trace_text_preview(text: &str, max_chars: usize) -> &str {
+    match text.char_indices().nth(max_chars) {
+        Some((idx, _)) => &text[..idx],
+        None => text,
+    }
+}
+
 /// Extract text from PDF file as plain string
 pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
     crate::validate_pdf_file(&path)?;
@@ -195,11 +202,7 @@ fn extract_positioned_text_impl(
                     item.width,
                     item.font_size,
                     item.font,
-                    if item.text.len() > 80 {
-                        &item.text[..80]
-                    } else {
-                        &item.text
-                    }
+                    trace_text_preview(&item.text, 80)
                 );
             }
         }
@@ -349,6 +352,133 @@ fn effective_merge_width(item: &TextItem) -> f32 {
     }
 }
 
+fn is_standalone_bullet_text(text: &str) -> bool {
+    matches!(text.trim(), "•" | "○" | "●" | "◦")
+}
+
+fn first_text_char(text: &str) -> Option<char> {
+    text.trim_start().chars().next()
+}
+
+fn is_short_alpha_fragment(text: &str) -> bool {
+    let trimmed = text.trim();
+    let char_count = trimmed.chars().count();
+    (1..=4).contains(&char_count) && trimmed.chars().all(char::is_alphabetic)
+}
+
+fn has_phrase_continuation_shape(text: &str) -> bool {
+    let trimmed = text.trim_start();
+    trimmed
+        .chars()
+        .take(24)
+        .any(|ch| ch.is_whitespace() || matches!(ch, '-'))
+}
+
+fn should_preserve_overlapping_stream_order(group: &[&TextItem]) -> bool {
+    if group.len() < 3 {
+        return false;
+    }
+
+    let Some(first) = group.iter().find(|item| !item.text.trim().is_empty()) else {
+        return false;
+    };
+    if group.iter().all(|item| item.mcid.is_none()) {
+        return false;
+    }
+
+    let mut nonempty_count = 0;
+    let mut saw_backtrack = false;
+    let mut nonspace_chars = 0;
+    let mut math_symbol_chars = 0;
+    let mut max_font_size = first.font_size;
+
+    for item in group {
+        if !item.text.trim().is_empty() {
+            nonempty_count += 1;
+        }
+        if (item.font_size - first.font_size).abs() > first.font_size * 0.25 {
+            return false;
+        }
+        max_font_size = max_font_size.max(item.font_size);
+        for ch in item.text.chars().filter(|ch| !ch.is_whitespace()) {
+            nonspace_chars += 1;
+            if matches!(
+                ch,
+                '*' | 'ˆ' | '^' | '=' | '+' | '_' | '[' | ']' | '{' | '}' | '|' | '<' | '>'
+            ) {
+                math_symbol_chars += 1;
+            }
+        }
+    }
+
+    if nonempty_count < 2 {
+        return false;
+    }
+    if nonspace_chars > 0 && math_symbol_chars * 4 > nonspace_chars {
+        return false;
+    }
+
+    let mut sorted_by_x = group.to_vec();
+    sorted_by_x.sort_by(|a, b| a.x.total_cmp(&b.x));
+    let cluster_start = sorted_by_x[0].x;
+    let mut cluster_end = cluster_start + effective_merge_width(sorted_by_x[0]);
+    for item in sorted_by_x.iter().skip(1) {
+        let gap = item.x - cluster_end;
+        if gap > max_font_size * 2.5 {
+            return false;
+        }
+        cluster_end = cluster_end.max(item.x + effective_merge_width(item));
+    }
+    if cluster_end - cluster_start > max_font_size * 36.0 {
+        return false;
+    }
+
+    for index in 0..group.len() - 1 {
+        let previous = group[index];
+        let next = group[index + 1];
+        let font_size = previous.font_size.max(next.font_size);
+        let backtrack_threshold = font_size * 0.25;
+        let previous_start = previous.x;
+        let next_start = next.x;
+        let next_end = next.x + effective_merge_width(next);
+        if next_start < previous_start - backtrack_threshold
+            && next_end > previous_start + backtrack_threshold
+        {
+            let has_near_prefix = group[..=index].iter().rev().take(4).any(|item| {
+                is_short_alpha_fragment(&item.text)
+                    && item.x >= next_start - font_size * 0.5
+                    && item.x <= next_start + font_size * 4.0
+            });
+            let starts_lowercase = first_text_char(&next.text).is_some_and(char::is_lowercase);
+            let phrase_continuation = has_phrase_continuation_shape(&next.text);
+            let has_near_bullet = group[..=index]
+                .iter()
+                .position(|item| {
+                    is_standalone_bullet_text(&item.text) && next_start <= item.x + font_size * 3.0
+                })
+                .is_some_and(|bullet_index| {
+                    if bullet_index >= index {
+                        return false;
+                    }
+                    group[bullet_index + 1..=index]
+                        .iter()
+                        .rev()
+                        .find(|item| !item.text.trim().is_empty())
+                        .is_some_and(|item| {
+                            item.text.trim().chars().count() <= 8
+                                && has_phrase_continuation_shape(&next.text)
+                        })
+                });
+            if (has_near_prefix && starts_lowercase && phrase_continuation) || has_near_bullet {
+                saw_backtrack = true;
+                break;
+            }
+        }
+    }
+
+    saw_backtrack
+}
+
 pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
     if items.is_empty() {
         return items;
@@ -369,28 +499,32 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
         }
     }
 
-    // Sort each group by X position (direction-aware)
-    for (_, _, group) in &mut line_groups {
+    let mut ordered_line_groups: Vec<(u32, f32, Vec<&TextItem>, bool)> = Vec::new();
+
+    // Sort each group by X position (direction-aware), except for lines whose
+    // content stream intentionally backtracks to overlay ActualText fragments.
+    for (page, y, mut group) in line_groups {
         let rtl = is_rtl_text(group.iter().map(|i| &i.text));
+        let preserve_stream_order = !rtl && should_preserve_overlapping_stream_order(&group);
         if rtl {
             group.sort_by(|a, b| b.x.total_cmp(&a.x));
-        } else {
+        } else if !preserve_stream_order {
             group.sort_by(|a, b| a.x.total_cmp(&b.x));
         }
+        ordered_line_groups.push((page, y, group, preserve_stream_order));
     }
 
     // Sort groups by page then Y descending (top of page first)
-    line_groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.total_cmp(&a.1)));
+    ordered_line_groups.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.total_cmp(&a.1)));
 
     let mut merged = Vec::new();
 
-    for (_, _, group) in &line_groups {
+    for (_, _, group, preserve_stream_order) in &ordered_line_groups {
         let mut i = 0;
         while i < group.len() {
             let first = group[i];
             let mut text = first.text.clone();
             let mut end_x = first.x + effective_merge_width(first);
-            let x_gap_max = first.font_size * 0.5;
 
             let mut j = i + 1;
             while j < group.len() {
@@ -400,10 +534,15 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                     break;
                 }
                 let gap = next.x - end_x;
+                let x_gap_max = if *preserve_stream_order && is_standalone_bullet_text(&text) {
+                    first.font_size * 1.2
+                } else {
+                    first.font_size * 0.5
+                };
                 if gap > x_gap_max {
                     break;
                 }
-                if gap < -first.font_size * 0.5 {
+                if gap < -first.font_size * 0.5 && !preserve_stream_order {
                     break;
                 }
                 // Insert space at word boundaries.
@@ -425,11 +564,19 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
                         first.font_size * 0.08
                     }
                 };
-                if gap > threshold {
+                let needs_bullet_space = *preserve_stream_order
+                    && is_standalone_bullet_text(&text)
+                    && !next.text.trim().is_empty();
+                if needs_bullet_space || gap > threshold {
                     text.push(' ');
                 }
                 text.push_str(&next.text);
-                end_x = next.x + effective_merge_width(next);
+                let next_end = next.x + effective_merge_width(next);
+                end_x = if *preserve_stream_order {
+                    end_x.max(next_end)
+                } else {
+                    next_end
+                };
                 j += 1;
             }
 
@@ -574,6 +721,21 @@ mod tests {
         }
     }
 
+    fn with_mcid(mut item: TextItem) -> TextItem {
+        item.mcid = Some(1);
+        item
+    }
+
+    #[test]
+    fn trace_text_preview_truncates_on_char_boundary() {
+        let text = format!("{}{}tail", "a".repeat(79), '\u{FFFD}');
+        let preview = trace_text_preview(&text, 80);
+
+        assert_eq!(preview.chars().count(), 80);
+        assert!(text.is_char_boundary(preview.len()));
+        assert!(preview.ends_with('\u{FFFD}'));
+    }
+
     #[test]
     fn merge_items_no_space_before_period() {
         // Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap
@@ -612,6 +774,83 @@ mod tests {
         assert_eq!(merged[0].text, "hello world");
     }
 
+    #[test]
+    fn merge_items_preserves_stream_order_for_backtracking_heading() {
+        // Some tagged PDFs emit first-letter ActualText fragments, then reset
+        // the text matrix and draw the rest of the word from the line start.
+        let items = vec![
+            with_mcid(make_merge_item("F", 79.4, 4.5)),
+            with_mcid(make_merge_item("r", 83.9, 3.3)),
+            with_mcid(make_merge_item("om tables to data-", 79.4, 89.7)),
+            with_mcid(make_merge_item("", 168.9, 33.9)),
+            with_mcid(make_merge_item("analytics-", 168.9, 75.5)),
+            with_mcid(make_merge_item("ready content", 210.5, 60.8)),
+        ];
+
+        let merged = merge_text_items(items);
+
+        assert_eq!(merged.len(), 1);
+        assert_eq!(
+            merged[0].text,
+            "From tables to data-analytics-ready content"
+        );
+    }
+
+    #[test]
+    fn merge_items_preserves_stream_order_for_reset_word_prefix() {
+        let items = vec![
+            with_mcid(make_merge_item("N", 68.0, 7.0)),
+            with_mcid(make_merge_item("e", 75.1, 4.0)),
+            with_mcid(make_merge_item("w fields created", 68.0, 82.0)),
+        ];
+
+        let merged = merge_text_items(items);
+
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "New fields created");
+    }
+
+    #[test]
+    fn merge_items_uses_x_order_for_untagged_backtracking_text() {
+        let items = vec![
+            make_merge_item("N", 68.0, 7.0),
+            make_merge_item("e", 75.1, 4.0),
+            make_merge_item("w fields created", 68.2, 82.0),
+        ];
+
+        let merged = merge_text_items(items);
+
+        let texts: Vec<_> = merged.iter().map(|item| item.text.as_str()).collect();
+        assert_eq!(texts, vec!["N", "w fields created", "e"]);
+    }
+
+    #[test]
+    fn merge_items_preserves_bullet_stream_order_with_backtracking() {
+        let items = vec![
+            with_mcid(make_merge_item("•", 79.4, 5.0)),
+            with_mcid(make_merge_item("The MS", 91.0, 32.6)),
+            with_mcid(make_merge_item("A LoS project", 84.4, 70.0)),
+        ];
+
+        let merged = merge_text_items(items);
+
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].text, "• The MSA LoS project");
+    }
+
+    #[test]
+    fn merge_items_keeps_normal_bullet_gap_limit_without_stream_order() {
+        let items = vec![
+            make_merge_item("•", 79.4, 5.0),
+            make_merge_item("Distant item", 91.0, 60.0),
+        ];
+
+        let merged = merge_text_items(items);
+
+        let texts: Vec<_> = merged.iter().map(|item| item.text.as_str()).collect();
+        assert_eq!(texts, vec!["•", "Distant item"]);
+    }
+
     #[test]
     fn test_group_into_lines() {
         let items = vec![

src/lib.rs

@@ -62,10 +62,23 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::path::Path;
 use tounicode::FontCMaps;
 
+/// OCR reason emitted when the extracted text layer appears garbled due to
+/// broken font decoding or mojibake.
+pub const OCR_REASON_SUSPECTED_GARBLED_TEXT: &str = "suspected_garbled_text";
+
 // =========================================================================
 // Result type
 // =========================================================================
 
+/// OCR reasons for a single 1-indexed page.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PageOcrReasons {
+    /// 1-indexed page number.
+    pub page: u32,
+    /// Machine-readable OCR reason identifiers.
+    pub reasons: Vec<String>,
+}
+
 /// High-level PDF processing result.
 #[derive(Debug)]
 pub struct PdfProcessResult {
@@ -79,6 +92,8 @@ pub struct PdfProcessResult {
     pub processing_time_ms: u64,
     /// 1-indexed page numbers that need OCR.
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    pub ocr_reasons_by_page: Vec<PageOcrReasons>,
     /// Title from PDF metadata (if available).
     pub title: Option<String>,
     /// Detection confidence score (0.0–1.0).
@@ -322,6 +337,8 @@ pub struct PageMarkdown {
     /// `true` when text on this page is unreliable (GID-encoded fonts,
     /// encoding issues, garbage text, or empty extraction).
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    pub ocr_reason: Option<String>,
 }
 
 /// Combined per-page markdown extraction and layout classification result.
@@ -335,6 +352,8 @@ pub struct PagesExtractionResult {
     pub pages_with_columns: Vec<u32>,
     /// 1-indexed pages that need OCR (scanned/image-based).
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    pub ocr_reasons_by_page: Vec<PageOcrReasons>,
     /// True if any page has tables or columns.
     pub is_complex: bool,
 }
@@ -368,6 +387,7 @@ pub fn extract_pages_markdown_mem(
     // Extract ALL pages to get accurate, document-wide font stats.
     let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
         extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
+    let text_quality = analyze_text_quality(&all_items);
 
     // Compute layout complexity from full document (near-zero cost).
     let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
@@ -387,6 +407,7 @@ pub fn extract_pages_markdown_mem(
 
     let mut results = Vec::with_capacity(pages_slice.len());
     let mut pages_needing_ocr = Vec::new();
+    let mut ocr_reasons_by_page = BTreeMap::new();
 
     for &page_0idx in pages_slice {
         // Out-of-range pages → empty + needs_ocr
@@ -396,6 +417,7 @@ pub fn extract_pages_markdown_mem(
                 page: page_0idx,
                 markdown: String::new(),
                 needs_ocr: true,
+                ocr_reason: None,
             });
             continue;
         }
@@ -416,6 +438,7 @@ pub fn extract_pages_markdown_mem(
             .collect();
 
         let has_gid = gid_pages.contains(&page_1idx);
+        let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx);
 
         // Build markdown with document-wide font stats
         let options = MarkdownOptions {
@@ -425,21 +448,33 @@ pub fn extract_pages_markdown_mem(
             ..MarkdownOptions::default()
         };
 
-        let md = markdown::to_markdown_from_items_with_rects_and_lines(
-            page_items,
-            options,
-            &page_rects,
-            &[],
-            &page_thresholds,
-            None,
-            &[],
-        );
+        let md = if has_text_quality_issue {
+            String::new()
+        } else {
+            markdown::to_markdown_from_items_with_rects_and_lines(
+                page_items,
+                options,
+                &page_rects,
+                &[],
+                &page_thresholds,
+                None,
+                &[],
+            )
+        };
 
-        let needs_ocr = md.trim().is_empty()
-            || has_gid
-            || is_garbage_text(&md)
-            || is_cid_garbage(&md)
-            || detect_encoding_issues(&md);
+        let has_decoding_issue = has_text_quality_issue
+            || (!md.is_empty() && (is_cid_garbage(&md) || detect_encoding_issues(&md)));
+        if has_decoding_issue {
+            add_ocr_reason(
+                &mut ocr_reasons_by_page,
+                page_1idx,
+                OCR_REASON_SUSPECTED_GARBLED_TEXT,
+            );
+        }
+        let ocr_reason = page_ocr_reason(&ocr_reasons_by_page, page_1idx);
+
+        let needs_ocr =
+            ocr_reason.is_some() || md.trim().is_empty() || has_gid || is_garbage_text(&md);
 
         if needs_ocr {
             pages_needing_ocr.push(page_1idx);
@@ -449,6 +484,7 @@ pub fn extract_pages_markdown_mem(
             page: page_0idx,
             markdown: if needs_ocr { String::new() } else { md },
             needs_ocr,
+            ocr_reason,
         });
     }
 
@@ -457,6 +493,7 @@ pub fn extract_pages_markdown_mem(
         pages_with_tables: complexity.pages_with_tables,
         pages_with_columns: complexity.pages_with_columns,
         pages_needing_ocr,
+        ocr_reasons_by_page: page_ocr_reasons_vec(ocr_reasons_by_page),
         is_complex: complexity.is_complex,
     })
 }
@@ -488,6 +525,8 @@ pub struct RegionText {
     /// Set when: the region is empty, the page uses GID-encoded fonts, or the
     /// extracted text fails garbage/encoding checks.
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    pub ocr_reason: Option<String>,
 }
 
 /// Result for a page's region extractions.
@@ -592,29 +631,36 @@ pub fn extract_text_in_regions_mem(
         for rect in regions {
             let [rx1, ry1, rx2, ry2] = *rect;
 
-            let text = match items {
-                Some(items) => collect_text_in_region_with_options(
-                    items,
-                    rx1,
-                    ry1,
-                    rx2,
-                    ry2,
-                    page_h,
-                    coords,
-                    adaptive_threshold,
-                ),
-                None => String::new(),
+            let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
+            let matched: Vec<TextItem> = match items {
+                Some(items) => items
+                    .iter()
+                    .filter(|item| region_overlaps_item(item, bounds))
+                    .cloned()
+                    .collect(),
+                None => Vec::new(),
+            };
+            let has_text_quality_issue = region_items_have_decoding_issue(&matched);
+            let text = collect_text_from_matched_items(matched, adaptive_threshold);
+            let has_cid_issue = is_cid_garbage(&text);
+            let has_encoding_issue = detect_encoding_issues(&text);
+            let ocr_reason = if has_text_quality_issue || has_cid_issue || has_encoding_issue {
+                Some(suspected_garbled_reason())
+            } else {
+                None
             };
 
             // Check per-region text quality instead of blanket page-level
             // GID rejection. A GID font in a logo elsewhere on the page
             // shouldn't force GPU OCR for clean text regions.
-            let needs_ocr = text.trim().is_empty()
-                || is_garbage_text(&text)
-                || is_cid_garbage(&text)
-                || detect_encoding_issues(&text);
+            let needs_ocr =
+                ocr_reason.is_some() || text.trim().is_empty() || is_garbage_text(&text);
 
-            page_results.push(RegionText { text, needs_ocr });
+            page_results.push(RegionText {
+                text,
+                needs_ocr,
+                ocr_reason,
+            });
         }
 
         results.push(PageRegionResult {
@@ -725,6 +771,16 @@ pub fn extract_tables_in_regions_mem(
                 page_results.push(RegionText {
                     text: String::new(),
                     needs_ocr: true,
+                    ocr_reason: None,
+                });
+                continue;
+            }
+
+            if region_items_have_decoding_issue(&matched) {
+                page_results.push(RegionText {
+                    text: String::new(),
+                    needs_ocr: true,
+                    ocr_reason: Some(suspected_garbled_reason()),
                 });
                 continue;
             }
@@ -899,10 +955,12 @@ pub fn extract_tables_in_regions_mem(
                 Some(candidate) => page_results.push(RegionText {
                     text: candidate.markdown.clone(),
                     needs_ocr: false,
+                    ocr_reason: None,
                 }),
                 None => page_results.push(RegionText {
                     text: String::new(),
                     needs_ocr: true,
+                    ocr_reason: None,
                 }),
             }
         }
@@ -3316,6 +3374,7 @@ fn process_document(
             page_count,
             processing_time_ms: start.elapsed().as_millis() as u64,
             pages_needing_ocr,
+            ocr_reasons_by_page: Vec::new(),
             title,
             confidence,
             layout: LayoutComplexity::default(),
@@ -3331,6 +3390,7 @@ fn process_document(
             page_count,
             processing_time_ms: start.elapsed().as_millis() as u64,
             pages_needing_ocr,
+            ocr_reasons_by_page: Vec::new(),
             title,
             confidence,
             layout: LayoutComplexity::default(),
@@ -3401,8 +3461,17 @@ fn process_document(
         })
         .unwrap_or((None, Vec::new()));
 
-    let (markdown, layout, has_encoding_issues, gid_pages) = match extracted {
+    let (
+        markdown,
+        layout,
+        has_encoding_issues,
+        gid_pages,
+        text_quality_pages,
+        text_quality_reasons_by_page,
+    ) = match extracted {
         Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => {
+            let mut ocr_reasons_by_page = BTreeMap::new();
+
             // For TextBased PDFs with pages flagged for OCR (Identity-H or
             // Type3 fonts without ToUnicode), check whether the CID-as-Unicode
             // passthrough actually produced readable text.  If a page's text
@@ -3435,6 +3504,13 @@ fn process_document(
                             "suppressing garbage text from OCR-flagged pages: {:?}",
                             garbage_pages
                         );
+                        for page in &garbage_pages {
+                            add_ocr_reason(
+                                &mut ocr_reasons_by_page,
+                                *page,
+                                OCR_REASON_SUSPECTED_GARBLED_TEXT,
+                            );
+                        }
                         let items: Vec<_> = items
                             .into_iter()
                             .filter(|i| !garbage_pages.contains(&i.page))
@@ -3451,6 +3527,8 @@ fn process_document(
                     }
                 };
 
+            let text_quality = analyze_text_quality(&items);
+            merge_ocr_reasons(&mut ocr_reasons_by_page, text_quality.reasons_by_page);
             let layout = compute_layout_complexity(&items, &rects, &lines);
 
             let md = if options.mode == ProcessMode::Analyze {
@@ -3467,14 +3545,25 @@ fn process_document(
                 ))
             };
 
-            let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m));
-            (md, layout, enc, gid_encoded_pages)
+            let enc = !ocr_reasons_by_page.is_empty()
+                || text_quality.has_encoding_issues
+                || md.as_ref().is_some_and(|m| detect_encoding_issues(m));
+            (
+                md,
+                layout,
+                enc,
+                gid_encoded_pages,
+                text_quality.pages_needing_ocr,
+                ocr_reasons_by_page,
+            )
         }
         None => (
             None,
             LayoutComplexity::default(),
             false,
             std::collections::HashSet::new(),
+            Vec::new(),
+            BTreeMap::new(),
         ),
     };
 
@@ -3516,6 +3605,19 @@ fn process_document(
         }
         pages_needing_ocr.sort_unstable();
     }
+    if !text_quality_pages.is_empty() {
+        log::debug!(
+            "pages with OCR reason {} (need OCR): {:?}",
+            OCR_REASON_SUSPECTED_GARBLED_TEXT,
+            text_quality_pages
+        );
+        for page in text_quality_pages {
+            if !pages_needing_ocr.contains(&page) {
+                pages_needing_ocr.push(page);
+            }
+        }
+        pages_needing_ocr.sort_unstable();
+    }
 
     // Detect sparse extraction: when a TEXT-BASED PDF produces very few
     // characters per page, the text is likely embedded in images/forms
@@ -3554,6 +3656,7 @@ fn process_document(
         page_count,
         processing_time_ms: start.elapsed().as_millis() as u64,
         pages_needing_ocr,
+        ocr_reasons_by_page: page_ocr_reasons_vec(text_quality_reasons_by_page),
         title,
         confidence,
         layout,
@@ -3581,6 +3684,10 @@ fn detect_encoding_issues(markdown: &str) -> bool {
     }
 
     // Heuristic 2: dollar-as-space pattern
+    has_dollar_as_space_pattern(markdown)
+}
+
+fn has_dollar_as_space_pattern(markdown: &str) -> bool {
     let total_dollars = markdown.matches('$').count();
     if total_dollars > 10 {
         let bytes = markdown.as_bytes();
@@ -3601,6 +3708,242 @@ fn detect_encoding_issues(markdown: &str) -> bool {
     false
 }
 
+#[derive(Debug, Default)]
+struct TextQualityReport {
+    pages_needing_ocr: Vec<u32>,
+    has_encoding_issues: bool,
+    reasons_by_page: BTreeMap<u32, Vec<String>>,
+}
+
+#[derive(Debug, Default)]
+struct PageTextQualityEvidence {
+    chars: usize,
+    replacement_chars: usize,
+    replacement_spans: usize,
+    longest_replacement_run: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum TextSpanIssueKind {
+    Replacement,
+    Strong,
+}
+
+fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
+    let mut reasons_by_page = BTreeMap::new();
+    let mut evidence_by_page = BTreeMap::<u32, PageTextQualityEvidence>::new();
+
+    for item in items {
+        if !matches!(item.item_type, crate::types::ItemType::Text) {
+            continue;
+        }
+
+        let evidence = evidence_by_page.entry(item.page).or_default();
+        evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
+
+        match text_span_decoding_issue_kind(&item.text) {
+            Some(TextSpanIssueKind::Strong) => {
+                add_ocr_reason(
+                    &mut reasons_by_page,
+                    item.page,
+                    OCR_REASON_SUSPECTED_GARBLED_TEXT,
+                );
+            }
+            Some(TextSpanIssueKind::Replacement) => {
+                let stats = replacement_text_stats(&item.text);
+                evidence.replacement_chars += stats.0;
+                evidence.replacement_spans += 1;
+                evidence.longest_replacement_run = evidence.longest_replacement_run.max(stats.1);
+            }
+            None => {}
+        }
+    }
+
+    for (page, evidence) in evidence_by_page {
+        if reasons_by_page.contains_key(&page) {
+            continue;
+        }
+        if page_replacement_evidence_needs_ocr(&evidence) {
+            add_ocr_reason(
+                &mut reasons_by_page,
+                page,
+                OCR_REASON_SUSPECTED_GARBLED_TEXT,
+            );
+        }
+    }
+
+    let pages_needing_ocr: Vec<u32> = reasons_by_page.keys().copied().collect();
+    TextQualityReport {
+        has_encoding_issues: !pages_needing_ocr.is_empty(),
+        pages_needing_ocr,
+        reasons_by_page,
+    }
+}
+
+fn suspected_garbled_reason() -> String {
+    OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()
+}
+
+fn add_ocr_reason(reasons_by_page: &mut BTreeMap<u32, Vec<String>>, page: u32, reason: &str) {
+    let reasons = reasons_by_page.entry(page).or_default();
+    if !reasons.iter().any(|existing| existing == reason) {
+        reasons.push(reason.to_string());
+    }
+}
+
+fn merge_ocr_reasons(
+    reasons_by_page: &mut BTreeMap<u32, Vec<String>>,
+    extra_reasons_by_page: BTreeMap<u32, Vec<String>>,
+) {
+    for (page, reasons) in extra_reasons_by_page {
+        for reason in reasons {
+            add_ocr_reason(reasons_by_page, page, &reason);
+        }
+    }
+}
+
+fn page_ocr_reason(reasons_by_page: &BTreeMap<u32, Vec<String>>, page: u32) -> Option<String> {
+    reasons_by_page
+        .get(&page)
+        .and_then(|reasons| reasons.first())
+        .cloned()
+}
+
+fn page_ocr_reasons_vec(reasons_by_page: BTreeMap<u32, Vec<String>>) -> Vec<PageOcrReasons> {
+    reasons_by_page
+        .into_iter()
+        .map(|(page, reasons)| PageOcrReasons { page, reasons })
+        .collect()
+}
+
+fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
+    items.iter().any(|item| {
+        matches!(item.item_type, crate::types::ItemType::Text)
+            && text_span_has_decoding_issue(&item.text)
+    })
+}
+
+fn text_span_has_decoding_issue(text: &str) -> bool {
+    text_span_decoding_issue_kind(text).is_some()
+}
+
+fn text_span_decoding_issue_kind(text: &str) -> Option<TextSpanIssueKind> {
+    let text = text.trim();
+    if text.is_empty() {
+        return None;
+    }
+
+    if has_dollar_as_space_pattern(text)
+        || has_private_use_text_run(text)
+        || is_cid_garbage(text)
+        || has_cid_control_token(text)
+    {
+        return Some(TextSpanIssueKind::Strong);
+    }
+
+    if has_replacement_text_run(text) {
+        return Some(TextSpanIssueKind::Replacement);
+    }
+
+    None
+}
+
+fn replacement_text_stats(text: &str) -> (usize, usize) {
+    let mut replacement = 0usize;
+    let mut current_run = 0usize;
+    let mut longest_run = 0usize;
+
+    for ch in text.chars() {
+        if ch == '\u{FFFD}' {
+            replacement += 1;
+            current_run += 1;
+            longest_run = longest_run.max(current_run);
+        } else {
+            current_run = 0;
+        }
+    }
+
+    (replacement, longest_run)
+}
+
+fn page_replacement_evidence_needs_ocr(evidence: &PageTextQualityEvidence) -> bool {
+    if evidence.replacement_chars == 0 || evidence.chars == 0 {
+        return false;
+    }
+
+    // If the entire page is only a short broken text layer, even a short
+    // replacement run is enough evidence. On otherwise text-heavy pages,
+    // require density so math formulas do not force full-page OCR.
+    if evidence.chars <= 80 && evidence.longest_replacement_run >= 2 {
+        return true;
+    }
+
+    let replacement_density_bps = evidence.replacement_chars * 10_000 / evidence.chars;
+    let enough_bad_text = evidence.replacement_chars >= 12 && replacement_density_bps >= 500;
+    let repeated_bad_spans = evidence.replacement_spans >= 3 && replacement_density_bps >= 250;
+    let long_bad_run = evidence.longest_replacement_run >= 8 && replacement_density_bps >= 250;
+
+    enough_bad_text || repeated_bad_spans || long_bad_run
+}
+
+fn has_replacement_text_run(text: &str) -> bool {
+    let (replacement, longest_run) = replacement_text_stats(text);
+    longest_run >= 2 || replacement >= 3
+}
+
+fn has_private_use_text_run(text: &str) -> bool {
+    let mut total = 0usize;
+    let mut private_use = 0usize;
+    let mut current_run = 0usize;
+    let mut longest_run = 0usize;
+
+    for ch in text.chars() {
+        if ch.is_whitespace() {
+            current_run = 0;
+            continue;
+        }
+        total += 1;
+        if is_private_use_char(ch) {
+            private_use += 1;
+            current_run += 1;
+            longest_run = longest_run.max(current_run);
+        } else {
+            current_run = 0;
+        }
+    }
+
+    if private_use == 0 {
+        return false;
+    }
+
+    longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
+}
+
+fn has_cid_control_token(text: &str) -> bool {
+    text.split_whitespace().any(token_has_cid_control)
+}
+
+fn token_has_cid_control(token: &str) -> bool {
+    let mut total = 0usize;
+    let mut c1_control = 0usize;
+
+    for ch in token.chars() {
+        total += 1;
+        if ('\u{0080}'..='\u{009F}').contains(&ch) {
+            c1_control += 1;
+        }
+    }
+
+    total >= 5 && c1_control >= 2 && c1_control * 20 >= total
+}
+
+fn is_private_use_char(ch: char) -> bool {
+    matches!(
+        ch as u32,
+        0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
+    )
+}
+
 /// Check if extracted text is predominantly garbage (non-alphanumeric).
 ///
 /// Broken font encodings produce text like "----1-.-.-.___  --.-. .._ I_---."
@@ -3609,20 +3952,36 @@ fn detect_encoding_issues(markdown: &str) -> bool {
 fn is_garbage_text(markdown: &str) -> bool {
     let mut alphanum = 0usize;
     let mut non_alphanum = 0usize;
-    for ch in markdown.chars() {
-        if ch.is_whitespace() {
-            continue;
+
+    let chars: Vec<char> = markdown.chars().collect();
+    let mut i = 0usize;
+    while i < chars.len() {
+        let ch = chars[i];
+        let mut run_end = i + 1;
+        while run_end < chars.len() && chars[run_end] == ch {
+            run_end += 1;
         }
-        // Skip markdown syntax chars that we add (not from the PDF)
-        if matches!(ch, '#' | '*' | '|' | '-' | '\n') {
-            continue;
-        }
-        if ch.is_alphanumeric() {
-            alphanum += 1;
-        } else {
-            non_alphanum += 1;
+
+        let is_decorative_leader = matches!(ch, '.' | '_' | '·') && run_end - i >= 3;
+        if !is_decorative_leader {
+            for &run_ch in &chars[i..run_end] {
+                if run_ch.is_whitespace() {
+                    continue;
+                }
+                // Skip markdown syntax chars that we add (not from the PDF)
+                if matches!(run_ch, '#' | '*' | '|' | '-' | '\n') {
+                    continue;
+                }
+                if run_ch.is_alphanumeric() {
+                    alphanum += 1;
+                } else {
+                    non_alphanum += 1;
+                }
+            }
         }
+        i = run_end;
     }
+
     let total = alphanum + non_alphanum;
     total >= 50 && alphanum * 2 < total
 }
@@ -3647,6 +4006,9 @@ fn is_cid_garbage(text: &str) -> bool {
         }
         total += 1;
         // C1 control characters (U+0080–U+009F) — almost never in real text
+        if ch == '·' {
+            continue;
+        }
         if ('\u{0080}'..='\u{009F}').contains(&ch) {
             c1_control += 1;
         }
@@ -3661,14 +4023,16 @@ fn is_cid_garbage(text: &str) -> bool {
         return false;
     }
     // If ≥5% of non-whitespace chars are C1 controls, it's garbage
-    if c1_control * 20 >= total {
+    if c1_control >= 2 && c1_control * 20 >= total {
         return true;
     }
     // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
     // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
-    // where CID values 0x80-0xFF become accented Latin characters).
+    // where CID values 0x80-0xFF become accented Latin characters).  Keep a
+    // minimum length so short math tokens like "2×()×" do not route a clean
+    // page to OCR.
     let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
-    high_latin * 5 >= total * 2 && ascii_letters * 3 < total
+    total >= 20 && high_latin * 5 >= total * 2 && ascii_letters * 3 < total
 }
 
 /// Detect markdown tables with suspicious structure that suggest the heuristic
@@ -5543,6 +5907,13 @@ mod tests {
         }
     }
 
+    fn test_text_item_on_page(page: u32, text: &str) -> TextItem {
+        TextItem {
+            page,
+            ..test_item(text, 10.0, 10.0, text.len() as f32 * 5.0, 12.0)
+        }
+    }
+
     #[test]
     fn test_detect_encoding_issues_fffd() {
         assert!(detect_encoding_issues(
@@ -5578,6 +5949,160 @@ mod tests {
         assert!(!detect_encoding_issues(text));
     }
 
+    #[test]
+    fn test_text_quality_flags_localized_cid_mojibake_span() {
+        let items = vec![
+            test_text_item_on_page(
+                1,
+                "Waiting Period 等待期 Maternity and newborn infant care benefit",
+            ),
+            test_text_item_on_page(
+                1,
+                "Inpatient and Day-care Benefits DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯",
+            ),
+            test_text_item_on_page(1, "Covered up to annual maximum. 赔付至年度最高保额。"),
+            test_text_item_on_page(2, "A clean second page should not be routed to OCR."),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(quality.has_encoding_issues);
+        assert_eq!(quality.pages_needing_ocr, vec![1]);
+        assert_eq!(
+            quality.reasons_by_page.get(&1).cloned(),
+            Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
+        );
+    }
+
+    #[test]
+    fn test_text_quality_flags_replacement_and_private_use_runs() {
+        let items = vec![
+            test_text_item_on_page(1, "broken \u{FFFD}\u{FFFD} text"),
+            test_text_item_on_page(3, "\u{E000}\u{E001}\u{E002}"),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert_eq!(quality.pages_needing_ocr, vec![1, 3]);
+        assert_eq!(
+            quality.reasons_by_page.get(&1).cloned(),
+            Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
+        );
+        assert_eq!(
+            quality.reasons_by_page.get(&3).cloned(),
+            Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
+        );
+    }
+
+    #[test]
+    fn test_text_quality_allows_clean_multilingual_and_latin1_text() {
+        let items = vec![
+            test_text_item_on_page(1, "你好世界，这是一段正常的中文文本。"),
+            test_text_item_on_page(1, "Résumé déjà vu: façade, São Paulo, año 2026."),
+            test_text_item_on_page(1, "A single icon \u{E000} should not force OCR."),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+        assert!(quality.reasons_by_page.is_empty());
+    }
+
+    #[test]
+    fn test_text_quality_allows_toc_leaders_form_rules_and_short_math() {
+        let items = vec![
+            test_text_item_on_page(
+                1,
+                "Feature Overview ........................................................................................................ 1-5",
+            ),
+            test_text_item_on_page(1, "Signature __________________________________________"),
+            test_text_item_on_page(1, "__________________________________________________"),
+            test_text_item_on_page(1, "2×()×"),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+    }
+
+    #[test]
+    fn test_text_quality_allows_isolated_replacement_character() {
+        let items = vec![
+            test_text_item_on_page(1, "\u{FFFD}2026 FINRA"),
+            test_text_item_on_page(1, "A mostly clean page should not be sent to OCR."),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+    }
+
+    #[test]
+    fn test_text_quality_allows_formula_replacement_on_clean_page() {
+        let items = vec![
+            test_text_item_on_page(
+                1,
+                "The LCOE of a power plant can be decomposed into three parts and described in prose.",
+            ),
+            test_text_item_on_page(
+                1,
+                "This page has enough normal text that a damaged equation should not force OCR.",
+            ),
+            test_text_item_on_page(1, "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} = x + y"),
+            test_text_item_on_page(1, "More normal explanatory text follows after the formula."),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+    }
+
+    #[test]
+    fn test_text_quality_flags_dense_replacement_text_page() {
+        let items = vec![
+            test_text_item_on_page(1, "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} broken layer"),
+            test_text_item_on_page(1, "more \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} broken text"),
+            test_text_item_on_page(1, "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}"),
+        ];
+
+        let quality = analyze_text_quality(&items);
+
+        assert_eq!(quality.pages_needing_ocr, vec![1]);
+        assert!(quality.has_encoding_issues);
+    }
+
+    #[test]
+    fn test_text_quality_allows_tex_ligature_c1_controls_in_words() {
+        let items = vec![test_text_item_on_page(
+            1,
+            "Especialmente de\u{85}ciente es nuestro conocimiento del control. The amniotic \u{87}uid is important.",
+        )];
+
+        let quality = analyze_text_quality(&items);
+
+        assert!(!quality.has_encoding_issues);
+        assert!(quality.pages_needing_ocr.is_empty());
+    }
+
+    #[test]
+    fn test_region_text_quality_is_scoped_to_matched_items() {
+        let clean_region = vec![
+            test_text_item_on_page(1, "Clean native text"),
+            test_text_item_on_page(1, "Résumé déjà vu"),
+        ];
+        let garbled_region = vec![
+            test_text_item_on_page(1, "Clean prefix"),
+            test_text_item_on_page(1, "DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯"),
+        ];
+
+        assert!(!region_items_have_decoding_issue(&clean_region));
+        assert!(region_items_have_decoding_issue(&garbled_region));
+    }
+
     #[test]
     fn test_garbage_text_detection() {
         // Simulates garbage output from Identity-H fonts without ToUnicode.
@@ -5621,6 +6146,18 @@ mod tests {
             !is_cid_garbage(japanese),
             "Valid Japanese text should not be flagged as garbage"
         );
+
+        let japanese_toc = "第１章 市政経営方針の位置づけ ···································· 1";
+        assert!(
+            !is_cid_garbage(japanese_toc),
+            "Japanese TOC dot leaders should not be flagged as garbage"
+        );
+
+        let tex_ligature = "amniotic \u{87}uid volume regulation";
+        assert!(
+            !is_cid_garbage(tex_ligature),
+            "A single TeX ligature byte inside a word should not be CID garbage"
+        );
     }
 
     #[test]

src/markdown/postprocess.rs

@@ -29,6 +29,7 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
     // text item, which combine with gap-based space insertion to produce
     // double spaces ("Vice  President" instead of "Vice President").
     collapse_consecutive_spaces(&mut text);
+    remove_spaces_before_closing_brackets(&mut text);
 
     // Remove excessive newlines (more than 2 in a row)
     while text.contains("\n\n\n") {
@@ -71,6 +72,20 @@ fn collapse_consecutive_spaces(text: &mut String) {
     *text = result;
 }
 
+/// Remove spaces before closing square brackets.
+/// Unit markers and markdown links occasionally pick up a gap-inserted space
+/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding.
+fn remove_spaces_before_closing_brackets(text: &mut String) {
+    let mut result = String::with_capacity(text.len());
+    for ch in text.chars() {
+        if ch == ']' && result.ends_with(' ') {
+            result.pop();
+        }
+        result.push(ch);
+    }
+    *text = result;
+}
+
 /// Collapse dot leaders (runs of 4+ dots) into " ... "
 /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
 fn collapse_dot_leaders(text: &str) -> String {
@@ -342,6 +357,18 @@ mod tests {
         assert!(result.contains("Chapter 2 ... 20"));
     }
 
+    // --- remove_spaces_before_closing_brackets ---
+
+    #[test]
+    fn test_remove_spaces_before_closing_brackets() {
+        let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string();
+        remove_spaces_before_closing_brackets(&mut input);
+        assert_eq!(
+            input,
+            "Density [kg/m3] and [linked text](https://example.com)"
+        );
+    }
+
     // --- fix_hyphenation ---
 
     #[test]

src/markdown/preprocess.rs

@@ -208,6 +208,157 @@ fn normalize_for_comparison(s: &str) -> String {
     trimmed.to_string()
 }
 
+/// Compact a comparison key for fuzzy matching of damaged running headers.
+///
+/// Some tagged PDFs emit running footer text with overlapping fragments, so one
+/// page may read "F rom ..." while later pages read "F om r ...". Exact
+/// normalized text still drives candidate discovery; this compact form is only
+/// used when deciding whether a one-off edge line is close enough to an already
+/// repeated candidate.
+fn compact_comparison_key(s: &str) -> String {
+    s.chars()
+        .filter(|c| c.is_ascii_alphanumeric())
+        .map(|c| c.to_ascii_lowercase())
+        .collect()
+}
+
+fn bounded_levenshtein(a: &str, b: &str, max_distance: usize) -> Option<usize> {
+    let a_chars: Vec<char> = a.chars().collect();
+    let b_chars: Vec<char> = b.chars().collect();
+
+    if a_chars.len().abs_diff(b_chars.len()) > max_distance {
+        return None;
+    }
+
+    let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
+    let mut curr = vec![0; b_chars.len() + 1];
+
+    for (i, a_ch) in a_chars.iter().enumerate() {
+        curr[0] = i + 1;
+        let mut row_min = curr[0];
+
+        for (j, b_ch) in b_chars.iter().enumerate() {
+            let cost = usize::from(a_ch != b_ch);
+            curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(prev[j] + cost);
+            row_min = row_min.min(curr[j + 1]);
+        }
+
+        if row_min > max_distance {
+            return None;
+        }
+
+        std::mem::swap(&mut prev, &mut curr);
+    }
+
+    let distance = prev[b_chars.len()];
+    (distance <= max_distance).then_some(distance)
+}
+
+fn matches_candidate(
+    normalized: &str,
+    candidates: &HashSet<String>,
+    compact_candidates: &[String],
+) -> bool {
+    if candidates.contains(normalized) {
+        return true;
+    }
+
+    if !has_broken_word_spacing(normalized) {
+        return false;
+    }
+
+    let compact = compact_comparison_key(normalized);
+    if compact.len() < 20 {
+        return false;
+    }
+
+    compact_candidates.iter().any(|candidate| {
+        candidate.len().abs_diff(compact.len()) <= 2
+            && bounded_levenshtein(&compact, candidate, 2).is_some()
+    })
+}
+
+fn ends_with_hyphen(raw: &str) -> bool {
+    matches!(
+        raw.chars().last(),
+        Some('-' | '\u{00ad}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}')
+    )
+}
+
+fn suspicious_short_token(raw: &str, alpha: &str, contains_equals: bool) -> bool {
+    let len = alpha.chars().count();
+    if len == 0 || len > 2 || ends_with_hyphen(raw) {
+        return false;
+    }
+
+    if contains_equals {
+        let raw_alpha: String = raw.chars().filter(|c| c.is_alphabetic()).collect();
+        if raw_alpha.chars().all(|c| c.is_uppercase()) {
+            return false;
+        }
+    }
+
+    true
+}
+
+fn is_uppercase_heavy(text: &str) -> bool {
+    let mut alpha = 0usize;
+    let mut uppercase = 0usize;
+    let mut lowercase = 0usize;
+
+    for ch in text.chars().filter(|ch| ch.is_alphabetic()) {
+        alpha += 1;
+        if ch.is_uppercase() {
+            uppercase += 1;
+        } else if ch.is_lowercase() {
+            lowercase += 1;
+        }
+    }
+
+    alpha >= 12 && lowercase == 0 && uppercase * 100 / alpha >= 80
+}
+
+fn has_broken_word_spacing(text: &str) -> bool {
+    if is_uppercase_heavy(text) {
+        return false;
+    }
+
+    let contains_equals = text.contains('=');
+    let tokens: Vec<(usize, bool)> = text
+        .split_whitespace()
+        .filter_map(|raw| {
+            let alpha: String = raw
+                .chars()
+                .filter(|c| c.is_alphabetic())
+                .flat_map(|c| c.to_lowercase())
+                .collect();
+            let len = alpha.chars().count();
+            (len > 0).then(|| (len, suspicious_short_token(raw, &alpha, contains_equals)))
+        })
+        .collect();
+
+    if tokens.len() < 4 {
+        return false;
+    }
+
+    let suspicious_tokens = tokens.iter().filter(|(_, suspicious)| *suspicious).count();
+    if suspicious_tokens < 3 {
+        return false;
+    }
+
+    let split_word_windows = tokens
+        .windows(3)
+        .filter(|window| window[0].0 >= 3 && window[1].1 && window[2].0 >= 3)
+        .count();
+    let adjacent_fragments = tokens
+        .windows(2)
+        .filter(|window| window[0].1 && window[1].1)
+        .count();
+
+    suspicious_tokens as f32 / tokens.len() as f32 >= 0.35
+        && (split_word_windows > 0 || adjacent_fragments > 0)
+}
+
 /// Returns true if the line looks like a list item or heading (should not be stripped).
 fn is_structural_line(text: &str) -> bool {
     let t = text.trim_start();
@@ -236,7 +387,9 @@ fn is_decorative_separator(text: &str) -> bool {
 /// Strip lines that repeat on many distinct pages (running headers/footers).
 ///
 /// A line is considered a repeated header/footer if:
-/// 1. Its normalized text appears on `>= max(3, page_count * 30%)` distinct pages
+/// 1. Its normalized text appears on enough distinct pages. The normal threshold
+///    is document-wide; visibly broken/letter-spaced running text can use a
+///    capped chapter-level threshold in long books.
 /// 2. It is at least 10 characters long
 /// 3. It doesn't look like a structural element (heading, list item)
 /// 4. It consistently appears in the top or bottom N distinct Y positions
@@ -253,13 +406,27 @@ fn is_decorative_separator(text: &str) -> bool {
 /// Page numbers are stripped from line text before comparison, so headers like
 /// "Chapter 3 — Page 5" and "Chapter 3 — Page 6" are treated as the same text.
 pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec<TextLine> {
-    if lines.is_empty() || page_count < 3 {
+    let removal_set = find_repeated_line_indices(&lines, page_count);
+    if removal_set.is_empty() {
         return lines;
     }
 
+    lines
+        .into_iter()
+        .enumerate()
+        .filter(|(idx, _)| !removal_set.contains(idx))
+        .map(|(_, line)| line)
+        .collect()
+}
+
+fn find_repeated_line_indices(lines: &[TextLine], page_count: u32) -> HashSet<usize> {
+    if lines.is_empty() || page_count < 3 {
+        return HashSet::new();
+    }
+
     // Compute Y range per page (min_y, max_y)
     let mut page_y_range: HashMap<u32, (f32, f32)> = HashMap::new();
-    for line in &lines {
+    for line in lines {
         let entry = page_y_range.entry(line.page).or_insert((line.y, line.y));
         if line.y < entry.0 {
             entry.0 = line.y;
@@ -271,7 +438,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
 
     // Build sorted Y values per page, so we can check line rank (position from edge)
     let mut page_sorted_ys: HashMap<u32, Vec<f32>> = HashMap::new();
-    for line in &lines {
+    for line in lines {
         page_sorted_ys.entry(line.page).or_default().push(line.y);
     }
     for ys in page_sorted_ys.values_mut() {
@@ -287,23 +454,39 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     // page margin.
     const EDGE_LINE_COUNT: usize = 5;
 
+    fn y_position_rank(
+        y: f32,
+        page: u32,
+        page_sorted_ys: &HashMap<u32, Vec<f32>>,
+    ) -> Option<(usize, usize)> {
+        let ys = page_sorted_ys.get(&page)?;
+        let pos = ys.iter().position(|&py| (py - y).abs() < 0.1)?;
+        Some((pos, ys.len()))
+    }
+
     /// Returns true if the given Y position is among the first or last N distinct
     /// Y positions on the specified page.
     fn is_y_at_edge(y: f32, page: u32, page_sorted_ys: &HashMap<u32, Vec<f32>>, n: usize) -> bool {
-        let ys = match page_sorted_ys.get(&page) {
-            Some(ys) => ys,
-            None => return false,
+        let Some((pos, len)) = y_position_rank(y, page, page_sorted_ys) else {
+            return false;
         };
-        if ys.len() <= n * 2 {
+        if len <= n * 2 {
             // Page has very few lines — everything is near the edge
             return true;
         }
-        // Check if this Y is among the first or last N
-        let pos = match ys.iter().position(|&py| (py - y).abs() < 0.1) {
-            Some(p) => p,
-            None => return false,
+        pos < n || pos >= len - n
+    }
+
+    fn is_y_at_strict_lower_edge(
+        y: f32,
+        page: u32,
+        page_sorted_ys: &HashMap<u32, Vec<f32>>,
+        n: usize,
+    ) -> bool {
+        let Some((pos, len)) = y_position_rank(y, page, page_sorted_ys) else {
+            return false;
         };
-        pos < n || pos >= ys.len() - n
+        len > n * 2 && pos < n
     }
 
     // Average page span for normalizing Y variance
@@ -327,8 +510,9 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     // Build frequency maps using normalize_for_comparison.
     // Individual line text -> distinct pages
     let mut freq: HashMap<String, HashSet<u32>> = HashMap::new();
+    let mut bottom_freq: HashMap<String, HashSet<u32>> = HashMap::new();
     let mut y_positions: HashMap<String, Vec<f32>> = HashMap::new();
-    for line in &lines {
+    for line in lines {
         if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
             continue;
         }
@@ -340,6 +524,12 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
         freq.entry(normalized.clone())
             .or_default()
             .insert(line.page);
+        if is_y_at_strict_lower_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            bottom_freq
+                .entry(normalized.clone())
+                .or_default()
+                .insert(line.page);
+        }
         y_positions.entry(normalized).or_default().push(line.y);
     }
 
@@ -347,6 +537,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     // This catches split column headers where individual fragments don't meet
     // the frequency threshold but the combined row does.
     let mut band_freq: HashMap<String, HashSet<u32>> = HashMap::new();
+    let mut band_bottom_freq: HashMap<String, HashSet<u32>> = HashMap::new();
     let mut band_y_positions: HashMap<String, Vec<f32>> = HashMap::new();
     for (&(page, _), indices) in &y_bands {
         if indices.len() < 2 {
@@ -371,11 +562,36 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
             .entry(normalized.clone())
             .or_default()
             .insert(page);
+        if is_y_at_strict_lower_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) {
+            band_bottom_freq
+                .entry(normalized.clone())
+                .or_default()
+                .insert(page);
+        }
         band_y_positions.entry(normalized).or_default().push(band_y);
     }
 
-    // Compute threshold
-    let threshold = 3u32.max(page_count * 30 / 100);
+    // Compute thresholds. Keep the conservative document-wide threshold for
+    // clean text, and allow a lower cap only for visibly broken/letter-spaced
+    // running headers in books where each chapter has its own footer/header.
+    let document_threshold = 3u32.max(page_count * 30 / 100);
+    let garbled_chapter_threshold = 3u32.max((page_count * 30 / 100).min(8));
+    let remove_all_bottom_threshold = document_threshold.min(garbled_chapter_threshold);
+    let meets_frequency_threshold =
+        |text: &str, pages: &HashSet<u32>, bottom_pages: &HashMap<String, HashSet<u32>>| -> bool {
+            pages.len() as u32 >= document_threshold
+                || (has_broken_word_spacing(text)
+                    && bottom_pages
+                        .get(text)
+                        .is_some_and(|pages| pages.len() as u32 >= garbled_chapter_threshold))
+        };
+    let should_remove_all_occurrences =
+        |text: &str, bottom_pages: &HashMap<String, HashSet<u32>>| -> bool {
+            has_broken_word_spacing(text)
+                && bottom_pages
+                    .get(text)
+                    .is_some_and(|pages| pages.len() as u32 >= remove_all_bottom_threshold)
+        };
 
     // Check Y-position consistency: headers/footers appear at the same position
     // on every page, table content varies. Require normalized stddev < 5% of
@@ -393,29 +609,61 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     };
 
     // Identify candidates from individual frequency map
-    let candidates: HashSet<String> = freq
-        .into_iter()
-        .filter(|(text, pages)| {
-            pages.len() as u32 >= threshold
-                && !is_structural_line(text)
-                && has_consistent_y(text, &y_positions)
-        })
-        .map(|(text, _)| text)
+    let mut remove_all_candidates: HashSet<String> = HashSet::new();
+    let mut candidates: HashSet<String> = HashSet::new();
+    for (text, pages) in freq {
+        if meets_frequency_threshold(&text, &pages, &bottom_freq)
+            && !is_structural_line(&text)
+            && has_consistent_y(&text, &y_positions)
+        {
+            if should_remove_all_occurrences(&text, &bottom_freq) {
+                remove_all_candidates.insert(text.clone());
+            }
+            candidates.insert(text);
+        }
+    }
+    let compact_candidates: Vec<String> = candidates
+        .iter()
+        .filter(|text| has_broken_word_spacing(text))
+        .map(|text| compact_comparison_key(text))
+        .filter(|text| text.len() >= 20)
+        .collect();
+    let compact_remove_all_candidates: Vec<String> = remove_all_candidates
+        .iter()
+        .filter(|text| has_broken_word_spacing(text))
+        .map(|text| compact_comparison_key(text))
+        .filter(|text| text.len() >= 20)
         .collect();
 
     // Identify candidates from coalesced band frequency map
-    let band_candidates: HashSet<String> = band_freq
-        .into_iter()
-        .filter(|(text, pages)| {
-            pages.len() as u32 >= threshold
-                && !is_structural_line(text)
-                && has_consistent_y(text, &band_y_positions)
-        })
-        .map(|(text, _)| text)
+    let mut remove_all_band_candidates: HashSet<String> = HashSet::new();
+    let mut band_candidates: HashSet<String> = HashSet::new();
+    for (text, pages) in band_freq {
+        if meets_frequency_threshold(&text, &pages, &band_bottom_freq)
+            && !is_structural_line(&text)
+            && has_consistent_y(&text, &band_y_positions)
+        {
+            if should_remove_all_occurrences(&text, &band_bottom_freq) {
+                remove_all_band_candidates.insert(text.clone());
+            }
+            band_candidates.insert(text);
+        }
+    }
+    let compact_band_candidates: Vec<String> = band_candidates
+        .iter()
+        .filter(|text| has_broken_word_spacing(text))
+        .map(|text| compact_comparison_key(text))
+        .filter(|text| text.len() >= 20)
+        .collect();
+    let compact_remove_all_band_candidates: Vec<String> = remove_all_band_candidates
+        .iter()
+        .filter(|text| has_broken_word_spacing(text))
+        .map(|text| compact_comparison_key(text))
+        .filter(|text| text.len() >= 20)
         .collect();
 
     if candidates.is_empty() && band_candidates.is_empty() {
-        return lines;
+        return HashSet::new();
     }
 
     // Build removal set.
@@ -424,8 +672,10 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     //   (b) its Y-band's coalesced text matches a band candidate, OR
     //   (c) any sibling in its Y-band was removed (propagation).
     //
-    // The first occurrence (lowest page number) of each repeated header/footer
-    // is kept so that document titles, column headers, etc. appear once.
+    // The first occurrence (lowest page number) of each repeated line is kept
+    // so that document titles, column headers, etc. appear once. Visibly broken
+    // footers proven by repeated lower-edge placement are removed from every
+    // matching edge occurrence, including sparse first pages.
     let mut removal_set: HashSet<usize> = HashSet::new();
 
     // Track which page first shows each candidate (to preserve first occurrence)
@@ -436,7 +686,15 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
         }
         let text = line.text();
         let normalized = normalize_for_comparison(&text);
-        if candidates.contains(&normalized) {
+        if matches_candidate(&normalized, &candidates, &compact_candidates) {
+            if matches_candidate(
+                &normalized,
+                &remove_all_candidates,
+                &compact_remove_all_candidates,
+            ) {
+                removal_set.insert(idx);
+                continue;
+            }
             let first = first_page_individual.entry(normalized).or_insert(line.page);
             if line.page > *first {
                 removal_set.insert(idx);
@@ -465,7 +723,7 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
             .collect::<Vec<_>>()
             .join(" ");
         let normalized = normalize_for_comparison(&coalesced);
-        if band_candidates.contains(&normalized) {
+        if matches_candidate(&normalized, &band_candidates, &compact_band_candidates) {
             let first = first_page_band.entry(normalized).or_insert(page);
             if page < *first {
                 *first = page;
@@ -489,7 +747,17 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
             .collect::<Vec<_>>()
             .join(" ");
         let normalized = normalize_for_comparison(&coalesced);
-        if band_candidates.contains(&normalized) {
+        if matches_candidate(&normalized, &band_candidates, &compact_band_candidates) {
+            if matches_candidate(
+                &normalized,
+                &remove_all_band_candidates,
+                &compact_remove_all_band_candidates,
+            ) {
+                for &idx in &sorted_indices {
+                    removal_set.insert(idx);
+                }
+                continue;
+            }
             let first = first_page_band.get(&normalized).copied().unwrap_or(0);
             if page > first {
                 for &idx in &sorted_indices {
@@ -514,15 +782,10 @@ pub(crate) fn strip_repeated_lines(lines: Vec<TextLine>, page_count: u32) -> Vec
     }
 
     if removal_set.is_empty() {
-        return lines;
+        return HashSet::new();
     }
 
-    lines
-        .into_iter()
-        .enumerate()
-        .filter(|(idx, _)| !removal_set.contains(idx))
-        .map(|(_, line)| line)
-        .collect()
+    removal_set
 }
 
 #[cfg(test)]
@@ -556,6 +819,29 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_has_broken_word_spacing_detects_split_words() {
+        assert!(has_broken_word_spacing(
+            "F rom p rese rva tion to access a nd be yond"
+        ));
+        assert!(has_broken_word_spacing("Conve rs ing w ith the pas t"));
+        assert!(has_broken_word_spacing("The Na tional Arch ives (U K)"));
+    }
+
+    #[test]
+    fn test_has_broken_word_spacing_ignores_normal_short_words() {
+        assert!(!has_broken_word_spacing(
+            "Reunir talento e empresas é um dos fatores po- sitivos para comunidades de sucesso"
+        ));
+        assert!(!has_broken_word_spacing("Witnessed on behalf of"));
+        assert!(!has_broken_word_spacing(
+            "V = Volume in m3/kg H = Enthalpy in kJ/kg S = Entropy in kJ/kg.K"
+        ));
+        assert!(!has_broken_word_spacing(
+            "TITULAR DEL PODER EJECUTIVO FEDERAL, A TRAVÉS DE LA SECRETARÍA DE ECONOMÍA, A HACER VALER EL PRINCIPIO DE"
+        ));
+    }
+
     #[test]
     fn test_merge_struct_tree_headings() {
         // Two consecutive lines tagged as H2 via struct tree, same font size as body
@@ -683,4 +969,222 @@ mod tests {
             .unwrap();
         assert_eq!(first_header.page, 1, "first occurrence should be on page 1");
     }
+
+    #[test]
+    fn test_strip_repeated_clean_bottom_footers_kept_below_document_threshold() {
+        let mut lines = Vec::new();
+        for page in 1..=8u32 {
+            for row in 0..12u32 {
+                lines.push(make_line(
+                    &format!("unique body content page {page} row {row}"),
+                    9.5,
+                    page,
+                    600.0 - row as f32 * 20.0,
+                    None,
+                ));
+            }
+            lines.push(make_line(
+                &format!("Chapter running footer {}", 90 + page),
+                7.5,
+                page,
+                39.5,
+                None,
+            ));
+        }
+
+        let result = strip_repeated_lines(lines, 200);
+
+        let footer_count = result
+            .iter()
+            .filter(|line| line.text().contains("Chapter running footer"))
+            .count();
+        assert_eq!(
+            footer_count, 8,
+            "clean repeated footer should not use the lower garbled-text threshold"
+        );
+    }
+
+    #[test]
+    fn test_strip_repeated_garbled_bottom_footers_removes_all_occurrences_in_long_doc() {
+        let mut lines = Vec::new();
+        for page in 1..=8u32 {
+            for row in 0..12u32 {
+                lines.push(make_line(
+                    &format!("unique body content page {page} row {row}"),
+                    9.5,
+                    page,
+                    600.0 - row as f32 * 20.0,
+                    None,
+                ));
+            }
+            lines.push(make_line(
+                &format!("M L a t the Na tional Libra ry of N orwa y {}", 90 + page),
+                7.5,
+                page,
+                39.5,
+                None,
+            ));
+        }
+
+        let result = strip_repeated_lines(lines, 200);
+
+        assert!(
+            result
+                .iter()
+                .all(|line| !line.text().contains("Na tional Libra")),
+            "garbled bottom running footer should be removed from every page"
+        );
+        assert!(
+            result
+                .iter()
+                .any(|line| line.text().contains("unique body content page 1 row 0")),
+            "body text should be preserved"
+        );
+    }
+
+    #[test]
+    fn test_strip_repeated_document_wide_garbled_footers_removes_all_occurrences() {
+        let mut lines = Vec::new();
+        for page in 1..=8u32 {
+            for row in 0..12u32 {
+                lines.push(make_line(
+                    &format!("unique body content page {page} row {row}"),
+                    9.5,
+                    page,
+                    600.0 - row as f32 * 20.0,
+                    None,
+                ));
+            }
+            lines.push(make_line(
+                &format!("F rom p rese rva tion to access a nd be yond {}", 90 + page),
+                7.5,
+                page,
+                39.5,
+                None,
+            ));
+        }
+
+        let result = strip_repeated_lines(lines, 8);
+
+        let footer_count = result
+            .iter()
+            .filter(|line| line.text().contains("be yond"))
+            .count();
+        assert_eq!(
+            footer_count, 0,
+            "document-wide garbled footers should be removed from every page"
+        );
+    }
+
+    #[test]
+    fn test_strip_repeated_sparse_uppercase_headers_keep_first_occurrence() {
+        let mut lines = Vec::new();
+        for page in 1..=5u32 {
+            lines.push(make_line(
+                "PROPOSICIÓN CON PUNTO DE ACUERDO POR EL QUE EL SENADO DE LA REPÚBLICA",
+                8.0,
+                page,
+                720.0,
+                None,
+            ));
+            lines.push(make_line(
+                "A TRAVÉS DE LA SECRETARÍA DE ECONOMÍA",
+                8.0,
+                page,
+                704.0,
+                None,
+            ));
+            lines.push(make_line(
+                &format!("unique sparse-page body text {page}"),
+                10.0,
+                page,
+                620.0,
+                None,
+            ));
+        }
+
+        let result = strip_repeated_lines(lines, 5);
+
+        let title_count = result
+            .iter()
+            .filter(|line| line.text().contains("PROPOSICIÓN CON PUNTO"))
+            .count();
+        assert_eq!(
+            title_count, 1,
+            "sparse repeated heading should keep the first occurrence"
+        );
+    }
+
+    #[test]
+    fn test_strip_repeated_bottom_footers_matches_minor_garbling() {
+        let mut lines = Vec::new();
+        for page in 1..=9u32 {
+            for row in 0..12u32 {
+                lines.push(make_line(
+                    &format!("distinct paragraph text page {page} row {row}"),
+                    9.5,
+                    page,
+                    600.0 - row as f32 * 20.0,
+                    None,
+                ));
+            }
+
+            let footer = if page == 1 {
+                "F rom p rese rva tion to access a nd be yond 95"
+            } else {
+                "F om r p rese rva tion to access a nd be yond 97"
+            };
+            lines.push(make_line(footer, 7.5, page, 39.5, None));
+        }
+
+        let result = strip_repeated_lines(lines, 200);
+
+        assert!(
+            result.iter().all(|line| !line.text().contains("be yond")),
+            "fuzzy footer variant should be removed once the repeated form is detected"
+        );
+        assert!(
+            result.iter().any(|line| line
+                .text()
+                .contains("distinct paragraph text page 9 row 11")),
+            "non-footer edge-adjacent body text should be preserved"
+        );
+    }
+
+    #[test]
+    fn test_strip_repeated_bottom_footers_matches_sparse_first_page_variant() {
+        let mut lines = Vec::new();
+        for page in 1..=9u32 {
+            let body_rows = if page == 1 { 3 } else { 12 };
+            for row in 0..body_rows {
+                lines.push(make_line(
+                    &format!("distinct paragraph text page {page} row {row}"),
+                    9.5,
+                    page,
+                    600.0 - row as f32 * 20.0,
+                    None,
+                ));
+            }
+
+            let footer = if page == 1 {
+                "F rom p rese rva tion to access a nd be yond 95"
+            } else {
+                "F om r p rese rva tion to access a nd be yond 97"
+            };
+            lines.push(make_line(footer, 7.5, page, 39.5, None));
+        }
+
+        let result = strip_repeated_lines(lines, 200);
+
+        assert!(
+            result.iter().all(|line| !line.text().contains("be yond")),
+            "sparse first page variant should be removed once later lower-edge footers prove the candidate"
+        );
+        assert!(
+            result
+                .iter()
+                .any(|line| line.text().contains("distinct paragraph text page 1 row 0")),
+            "sparse first page body text should be preserved"
+        );
+    }
 }

src/python.rs

@@ -30,6 +30,9 @@ pub struct PyPdfResult {
     /// 1-indexed page numbers that need OCR.
     #[pyo3(get)]
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    #[pyo3(get)]
+    pub ocr_reasons_by_page: Vec<PyPageOcrReasons>,
     /// Title from PDF metadata.
     #[pyo3(get)]
     pub title: Option<String>,
@@ -60,6 +63,28 @@ impl PyPdfResult {
     }
 }
 
+/// OCR reasons for a single 1-indexed page.
+#[pyclass(name = "PageOcrReasons")]
+#[derive(Clone)]
+pub struct PyPageOcrReasons {
+    /// 1-indexed page number.
+    #[pyo3(get)]
+    pub page: u32,
+    /// Machine-readable OCR reason identifiers.
+    #[pyo3(get)]
+    pub reasons: Vec<String>,
+}
+
+#[pymethods]
+impl PyPageOcrReasons {
+    fn __repr__(&self) -> String {
+        format!(
+            "PageOcrReasons(page={}, reasons={:?})",
+            self.page, self.reasons
+        )
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Classification wrapper (lightweight)
 // ---------------------------------------------------------------------------
@@ -106,6 +131,9 @@ pub struct PyRegionText {
     /// True when the text should not be trusted (empty, GID fonts, garbage, encoding issues).
     #[pyo3(get)]
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    #[pyo3(get)]
+    pub ocr_reason: Option<String>,
 }
 
 #[pymethods]
@@ -160,6 +188,9 @@ pub struct PyPageMarkdown {
     /// encoding issues, garbage text, or empty extraction).
     #[pyo3(get)]
     pub needs_ocr: bool,
+    /// Machine-readable OCR reason when the cause is known.
+    #[pyo3(get)]
+    pub ocr_reason: Option<String>,
 }
 
 #[pymethods]
@@ -190,6 +221,9 @@ pub struct PyPagesExtractionResult {
     /// 1-indexed pages that need OCR (scanned/image-based or unreliable text).
     #[pyo3(get)]
     pub pages_needing_ocr: Vec<u32>,
+    /// Machine-readable OCR reasons by 1-indexed page.
+    #[pyo3(get)]
+    pub ocr_reasons_by_page: Vec<PyPageOcrReasons>,
     /// True if any page has tables or columns.
     #[pyo3(get)]
     pub is_complex: bool,
@@ -268,6 +302,7 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult {
         page_count: r.page_count,
         processing_time_ms: r.processing_time_ms,
         pages_needing_ocr: r.pages_needing_ocr,
+        ocr_reasons_by_page: to_py_page_ocr_reasons(r.ocr_reasons_by_page),
         title: r.title,
         confidence: r.confidence,
         is_complex_layout: r.layout.is_complex,
@@ -277,6 +312,16 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult {
     }
 }
 
+fn to_py_page_ocr_reasons(reasons: Vec<crate::PageOcrReasons>) -> Vec<PyPageOcrReasons> {
+    reasons
+        .into_iter()
+        .map(|reason| PyPageOcrReasons {
+            page: reason.page,
+            reasons: reason.reasons,
+        })
+        .collect()
+}
+
 fn to_py_err(e: crate::PdfError) -> PyErr {
     PyValueError::new_err(e.to_string())
 }
@@ -350,11 +395,13 @@ fn to_py_pages_result(r: crate::PagesExtractionResult) -> PyPagesExtractionResul
                 page: p.page,
                 markdown: p.markdown,
                 needs_ocr: p.needs_ocr,
+                ocr_reason: p.ocr_reason,
             })
             .collect(),
         pages_with_tables: r.pages_with_tables,
         pages_with_columns: r.pages_with_columns,
         pages_needing_ocr: r.pages_needing_ocr,
+        ocr_reasons_by_page: to_py_page_ocr_reasons(r.ocr_reasons_by_page),
         is_complex: r.is_complex,
     }
 }
@@ -370,6 +417,7 @@ fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRe
                 .map(|r| PyRegionText {
                     text: r.text,
                     needs_ocr: r.needs_ocr,
+                    ocr_reason: r.ocr_reason,
                 })
                 .collect(),
         })
@@ -563,6 +611,7 @@ fn extract_pages_markdown_bytes(
 #[pymodule]
 fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyPdfResult>()?;
+    m.add_class::<PyPageOcrReasons>()?;
     m.add_class::<PyPdfClassification>()?;
     m.add_class::<PyTextItem>()?;
     m.add_class::<PyRegionText>()?;

src/tounicode.rs

@@ -329,7 +329,7 @@ impl ToUnicodeCMap {
                 if let (Some(start), Some(end), Some(base)) = (
                     parse_hex_u16(&start_hex),
                     parse_hex_u16(&end_hex),
-                    parse_hex_u32(&base_hex),
+                    hex_to_unicode_scalar(&base_hex),
                 ) {
                     self.ranges.push((start, end, base));
                 }
@@ -575,32 +575,86 @@ fn parse_hex_u16(hex: &str) -> Option<u16> {
     u16::from_str_radix(hex.trim(), 16).ok()
 }
 
-/// Parse a hex string to u32
-fn parse_hex_u32(hex: &str) -> Option<u32> {
-    u32::from_str_radix(hex.trim(), 16).ok()
-}
-
-/// Convert a hex string to a Unicode string
-/// Handles both 2-byte (BMP) and 4-byte (supplementary) codepoints
+/// Convert a ToUnicode destination hex string to Unicode.
+///
+/// PDF ToUnicode destinations are UTF-16BE strings. Supplementary-plane
+/// characters are encoded as surrogate pairs, so treating each 4-hex chunk as
+/// a scalar drops emoji like D83CDF1F.
 fn hex_to_unicode_string(hex: &str) -> Option<String> {
-    let hex = hex.trim();
-    let mut result = String::new();
-
-    // Process 4 hex digits at a time
-    let mut i = 0;
-    while i + 4 <= hex.len() {
-        if let Ok(cp) = u32::from_str_radix(&hex[i..i + 4], 16) {
-            if let Some(c) = char::from_u32(cp) {
-                result.push(c);
-            }
-        }
-        i += 4;
+    let hex: String = hex.chars().filter(|ch| !ch.is_ascii_whitespace()).collect();
+    if hex.is_empty() || !hex.len().is_multiple_of(2) {
+        return None;
     }
 
-    if result.is_empty() {
-        None
+    let bytes: Option<Vec<u8>> = (0..hex.len())
+        .step_by(2)
+        .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).ok())
+        .collect();
+    let bytes = bytes?;
+
+    if bytes.len().is_multiple_of(2) {
+        let units: Vec<u16> = bytes
+            .chunks_exact(2)
+            .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
+            .collect();
+        if let Ok(result) = String::from_utf16(&units) {
+            if !result.is_empty() {
+                return Some(normalize_tounicode_destination(result));
+            }
+        }
+    }
+
+    // Be permissive for non-standard one-byte destinations.
+    if bytes.len() == 1 {
+        let ch = bytes[0] as char;
+        if !ch.is_control() || ch == '\t' || ch == '\n' {
+            return Some(ch.to_string());
+        }
+    }
+
+    None
+}
+
+fn normalize_tounicode_destination(text: String) -> String {
+    let is_multi_char = text.chars().nth(1).is_some();
+
+    // Some malformed producer CMaps put a list of alternative whitespace or
+    // hyphen codepoints into one destination. Keep ordinary multi-character
+    // mappings intact unless that malformed signature is present.
+    if is_multi_char
+        && text.chars().all(char::is_whitespace)
+        && text.chars().any(|ch| matches!(ch, '\t' | '\n' | '\r'))
+    {
+        return if text.contains('\t') {
+            "\t".to_string()
+        } else {
+            " ".to_string()
+        };
+    }
+
+    if is_multi_char
+        && text.contains('\u{00ad}')
+        && text.chars().all(|ch| {
+            matches!(
+                ch,
+                '-' | '\u{00ad}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2212}'
+            )
+        })
+    {
+        return "-".to_string();
+    }
+
+    text
+}
+
+fn hex_to_unicode_scalar(hex: &str) -> Option<u32> {
+    let text = hex_to_unicode_string(hex)?;
+    let mut chars = text.chars();
+    let ch = chars.next()?;
+    if chars.next().is_none() {
+        Some(ch as u32)
     } else {
-        Some(result)
+        None
     }
 }
 
@@ -2607,6 +2661,97 @@ endbfrange
         assert_eq!(cmap.lookup(0x0005), Some("C".to_string()));
     }
 
+    #[test]
+    fn test_parse_bfchar_surrogate_pair_emoji() {
+        let cmap_content = r#"
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+2 beginbfchar
+<16> <D83CDF1F>
+<9D> <D83CDFAD>
+endbfchar
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        assert_eq!(cmap.code_byte_length, 1);
+        assert_eq!(cmap.lookup(0x16), Some("🌟".to_string()));
+        assert_eq!(cmap.lookup(0x9D), Some("🎭".to_string()));
+    }
+
+    #[test]
+    fn test_parse_bfrange_surrogate_pair_base() {
+        let cmap_content = r#"
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+1 beginbfrange
+<C8> <C9> <D83CDFD8>
+endbfrange
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        assert_eq!(cmap.code_byte_length, 1);
+        assert_eq!(cmap.lookup(0xC8), Some("🏘".to_string()));
+        assert_eq!(cmap.lookup(0xC9), Some("🏙".to_string()));
+    }
+
+    #[test]
+    fn test_parse_bfrange_preserves_single_hyphen_like_base() {
+        let cmap_content = r#"
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+1 beginbfrange
+<21> <22> <2013>
+endbfrange
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        assert_eq!(cmap.lookup(0x21), Some("–".to_string()));
+        assert_eq!(cmap.lookup(0x22), Some("—".to_string()));
+    }
+
+    #[test]
+    fn test_parse_spaced_destination_hex_without_control_noise() {
+        let cmap_content = r#"
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+3 beginbfchar
+<21> < 0009 000d 0020 00a0 >
+<22> < 002d 00ad 2010 >
+<23> <00a0>
+endbfchar
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        assert_eq!(cmap.lookup(0x21), Some("\t".to_string()));
+        assert_eq!(cmap.lookup(0x22), Some("-".to_string()));
+        assert_eq!(cmap.lookup(0x23), Some("\u{00a0}".to_string()));
+    }
+
+    #[test]
+    fn test_parse_preserves_valid_multi_character_destinations() {
+        let cmap_content = r#"
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+4 beginbfchar
+<21> <002d002d>
+<22> <20132013>
+<23> <002000a0>
+<24> <00660069>
+endbfchar
+"#;
+        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
+
+        assert_eq!(cmap.lookup(0x21), Some("--".to_string()));
+        assert_eq!(cmap.lookup(0x22), Some("––".to_string()));
+        assert_eq!(cmap.lookup(0x23), Some(" \u{00a0}".to_string()));
+        assert_eq!(cmap.lookup(0x24), Some("fi".to_string()));
+    }
+
     #[test]
     fn test_remap_to_sequential() {
         // Simulate a broken CMap where GIDs are from pre-subsetting:

tests/integration_tests.rs

@@ -1107,6 +1107,7 @@ fn test_pages_needing_ocr_field_accessible() {
         page_count: 1,
         processing_time_ms: 0,
         pages_needing_ocr: vec![1, 3],
+        ocr_reasons_by_page: Vec::new(),
         title: None,
         confidence: 1.0,
         layout: pdf_inspector::LayoutComplexity::default(),