chore: bump napi package version

fix(markdown): handle wrapped bold abstracts
2026-06-01 14:57:03 -07:00 · 2026-06-01 14:32:42 -07:00
17 changed files with 84 additions and 698 deletions
@@ -1,87 +0,0 @@
-name: Publish Rust crate
-
-on:
-  push:
-    branches: [main]
-    paths: ['Cargo.toml']
-
-permissions:
-  contents: read
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  check-version:
-    name: Check version change
-    runs-on: ubuntu-latest
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      published: ${{ steps.check.outputs.published }}
-      version: ${{ steps.check.outputs.version }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-
-      - name: Check if version changed
-        id: check
-        run: |
-          NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
-          OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
-          echo "old=$OLD_VERSION new=$NEW_VERSION"
-          echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
-
-          if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-            echo "published=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          echo "changed=true" >> "$GITHUB_OUTPUT"
-
-          HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
-            -H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
-            "https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
-
-          case "$HTTP_STATUS" in
-            200)
-              echo "published=true" >> "$GITHUB_OUTPUT"
-              echo "pdf-inspector v$NEW_VERSION is already published"
-              ;;
-            404)
-              echo "published=false" >> "$GITHUB_OUTPUT"
-              ;;
-            *)
-              cat /tmp/crate-version.json
-              echo "Unexpected crates.io response: $HTTP_STATUS" >&2
-              exit 1
-              ;;
-          esac
-
-  publish:
-    name: Publish to crates.io
-    needs: check-version
-    if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
-    runs-on: ubuntu-latest
-    environment: crates-io
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Verify package
-        run: cargo publish --dry-run
-
-      - name: Authenticate with crates.io
-        id: auth
-        uses: rust-lang/crates-io-auth-action@v1
-
-      - name: Publish crate
-        run: cargo publish
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
@@ -3,10 +3,7 @@ name: Publish npm package
 on:
  push:
    branches: [main]
-    paths:
-      - '.github/workflows/publish.yml'
-      - 'napi/package.json'
-      - 'napi/legacy/**'
+    paths: ['napi/package.json']

 permissions:
  contents: read
@@ -17,60 +14,29 @@ jobs:
    name: Check version change
    runs-on: ubuntu-latest
    outputs:
-      scoped_changed: ${{ steps.check.outputs.scoped_changed }}
-      scoped_version: ${{ steps.check.outputs.scoped_version }}
-      legacy_changed: ${{ steps.check.outputs.legacy_changed }}
-      legacy_version: ${{ steps.check.outputs.legacy_version }}
+      changed: ${{ steps.check.outputs.changed }}
+      version: ${{ steps.check.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

-      - name: Check if versions changed
+      - name: Check if version changed
        id: check
        run: |
-          set -euo pipefail
-
-          SCOPED_NEW=$(node -p "require('./napi/package.json').version")
-          if git show HEAD~1:napi/package.json >/tmp/scoped-package-old.json 2>/dev/null; then
-            SCOPED_OLD=$(node -p "JSON.parse(require('fs').readFileSync('/tmp/scoped-package-old.json','utf8')).version")
+          NEW_VERSION=$(node -p "require('./napi/package.json').version")
+          OLD_VERSION=$(git show HEAD~1:napi/package.json | node -p "JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')).version")
+          echo "old=$OLD_VERSION new=$NEW_VERSION"
+          if [ "$NEW_VERSION" != "$OLD_VERSION" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
          else
-            SCOPED_OLD=""
-          fi
-
-          LEGACY_NEW=$(node -p "require('./napi/legacy/package.json').version")
-          if git show HEAD~1:napi/legacy/package.json >/tmp/legacy-package-old.json 2>/dev/null; then
-            LEGACY_OLD=$(node -p "JSON.parse(require('fs').readFileSync('/tmp/legacy-package-old.json','utf8')).version")
-          else
-            LEGACY_OLD=""
-          fi
-
-          echo "scoped old=$SCOPED_OLD new=$SCOPED_NEW"
-          echo "legacy old=$LEGACY_OLD new=$LEGACY_NEW"
-
-          if [ "$LEGACY_NEW" != "$SCOPED_NEW" ]; then
-            echo "Legacy package version ($LEGACY_NEW) must match scoped package version ($SCOPED_NEW)." >&2
-            exit 1
-          fi
-
-          echo "scoped_version=$SCOPED_NEW" >> "$GITHUB_OUTPUT"
-          echo "legacy_version=$LEGACY_NEW" >> "$GITHUB_OUTPUT"
-
-          if [ "$SCOPED_NEW" != "$SCOPED_OLD" ]; then
-            echo "scoped_changed=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "scoped_changed=false" >> "$GITHUB_OUTPUT"
-          fi
-
-          if [ "$LEGACY_NEW" != "$LEGACY_OLD" ]; then
-            echo "legacy_changed=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "legacy_changed=false" >> "$GITHUB_OUTPUT"
+            echo "changed=false" >> "$GITHUB_OUTPUT"
          fi

  build:
    needs: check-version
-    if: needs.check-version.outputs.scoped_changed == 'true'
+    if: needs.check-version.outputs.changed == 'true'
    name: Build ${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    strategy:
@@ -133,7 +99,6 @@ jobs:
    name: Publish to npm
    needs: [check-version, build]
    runs-on: ubuntu-latest
-    if: ${{ always() && (needs.check-version.outputs.scoped_changed == 'true' || needs.check-version.outputs.legacy_changed == 'true') && (needs.check-version.outputs.scoped_changed != 'true' || needs.build.result == 'success') }}
    permissions:
      contents: read
      id-token: write
@@ -146,13 +111,11 @@ jobs:
          registry-url: 'https://registry.npmjs.org'

      - name: Download all artifacts
-        if: needs.check-version.outputs.scoped_changed == 'true'
        uses: actions/download-artifact@v4
        with:
          path: napi/artifacts

-      - name: Collect binaries and publish scoped package
-        if: needs.check-version.outputs.scoped_changed == 'true'
+      - name: Collect binaries and publish
        working-directory: napi
        run: |
          cp artifacts/bindings-*/*.node .
@@ -163,20 +126,3 @@ jobs:
          ls -la *.node index.js index.d.ts

          npm publish --provenance --access public
-
-      - name: Publish legacy package wrapper
-        if: needs.check-version.outputs.legacy_changed == 'true'
-        working-directory: napi/legacy
-        run: npm publish --provenance
-
-      - name: Deprecate legacy package
-        if: needs.check-version.outputs.legacy_changed == 'true'
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
-            echo "NPM_TOKEN is not configured; skipping npm deprecate."
-            exit 0
-          fi
-
-          npm deprecate firecrawl-pdf-inspector "Deprecated: this package has moved to @firecrawl/pdf-inspector. Please install @firecrawl/pdf-inspector instead."
@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector"
-version = "0.1.2"
+version = "0.1.0"
 edition = "2021"
 autobins = false
 authors = ["Firecrawl Team"]
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
 pyo3 = { version = "0.25", features = ["extension-module"], optional = true }

 # PDF parsing
-lopdf = { version = "0.41.0", features = ["rayon"] }
+lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }

 # Error handling
 thiserror = "2.0"
@@ -1,8 +1,5 @@
 # pdf-inspector

-[![Crates.io](https://img.shields.io/crates/v/pdf-inspector.svg)](https://crates.io/crates/pdf-inspector)
-[![npm](https://img.shields.io/npm/v/@firecrawl/pdf-inspector.svg)](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
-
 Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).

 Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
@@ -74,17 +71,9 @@ console.log(result.markdown);  // Markdown string or null

 ### Rust

-Install from [crates.io](https://crates.io/crates/pdf-inspector):
-
-```bash
-cargo add pdf-inspector
-```
-
-Or add it manually:
-
 ```toml
 [dependencies]
-pdf-inspector = "0.1"
+pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
 ```

 ```rust
@@ -102,34 +91,29 @@ if let Some(markdown) = &result.markdown {
 ### CLI

 ```bash
-# Install the CLI tools
-cargo install pdf-inspector
-
 # Convert PDF to Markdown
-pdf2md document.pdf
+cargo run --bin pdf2md -- document.pdf

 # JSON output (for piping)
-pdf2md document.pdf --json
+cargo run --bin pdf2md -- document.pdf --json

 # Raw markdown only (no headers)
-pdf2md document.pdf --raw
+cargo run --bin pdf2md -- document.pdf --raw

 # Insert page break markers (<!-- Page N -->)
-pdf2md document.pdf --pages
+cargo run --bin pdf2md -- document.pdf --pages

 # Process only specific pages
-pdf2md document.pdf --select-pages 1,3,5-10
+cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10

 # Detection only (no extraction)
-detect-pdf document.pdf
-detect-pdf document.pdf --json
+cargo run --bin detect-pdf -- document.pdf
+cargo run --bin detect-pdf -- document.pdf --json

 # Detection + layout analysis (tables, columns)
-detect-pdf document.pdf --analyze --json
+cargo run --bin detect-pdf -- document.pdf --analyze --json
 ```

-From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
-
 ## Architecture

 ```
@@ -1,44 +0,0 @@
-# Publishing
-
-The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
-
-## crates.io Trusted Publisher
-
-Configure the trusted publisher for the `pdf-inspector` crate with:
-
- Repository: `firecrawl/pdf-inspector`
- Workflow: `publish-crate.yml`
- Environment: `crates-io`
-
-The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
-
-## Release Steps
-
-1. Update `version` in `Cargo.toml`.
-2. Merge the version bump to `main`.
-3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
-
-If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
-
-## Legacy npm Package
-
-The unscoped `firecrawl-pdf-inspector` package is deprecated in favor of `@firecrawl/pdf-inspector`.
-
-The npm publish workflow publishes the compatibility wrapper in `napi/legacy` whenever `napi/legacy/package.json` has a version bump on `main`. The wrapper keeps older installs working while its README points users to the scoped package.
-
-Keep `napi/legacy/package.json` on the same version as `napi/package.json`, and pin its `@firecrawl/pdf-inspector` dependency to that same version. The publish workflow fails fast if the package versions drift.
-
-Configure npm trusted publishing for both packages against `.github/workflows/publish.yml`:
-
- `@firecrawl/pdf-inspector`
- `firecrawl-pdf-inspector`
-
-To mark the legacy package as deprecated from CI, configure a GitHub Actions `NPM_TOKEN` secret with permission to manage `firecrawl-pdf-inspector`. Without that secret, the workflow still publishes the wrapper but skips `npm deprecate`.
-
-You can also deprecate the legacy package manually from an npm account that owns it:
-
-```bash
-npm deprecate firecrawl-pdf-inspector "Deprecated: this package has moved to @firecrawl/pdf-inspector. Please install @firecrawl/pdf-inspector instead."
-```
-
-If the account has two-factor auth enabled, append `--otp=<code>`.
@@ -672,9 +672,8 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"

 [[package]]
 name = "lopdf"
-version = "0.41.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d"
+version = "0.40.0"
+source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
 dependencies = [
 "aes",
 "bitflags",
@@ -830,7 +829,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"

 [[package]]
 name = "pdf-inspector"
-version = "0.1.1"
+version = "0.1.0"
 dependencies = [
 "env_logger",
 "log",
@@ -845,7 +844,7 @@ dependencies = [

 [[package]]
 name = "pdf-inspector-napi"
-version = "0.2.1"
+version = "0.2.0"
 dependencies = [
 "napi",
 "napi-build",
@@ -1,6 +1,6 @@
 [package]
 name = "pdf-inspector-napi"
-version = "0.2.1"
+version = "0.2.0"
 edition = "2021"

 [lib]
@@ -1,30 +0,0 @@
-# firecrawl-pdf-inspector
-
-> Deprecated: this package has moved to [`@firecrawl/pdf-inspector`](https://www.npmjs.com/package/@firecrawl/pdf-inspector).
-> Please install and import the scoped Firecrawl package instead.
-
-```bash
-npm install @firecrawl/pdf-inspector
-# or
-bun add @firecrawl/pdf-inspector
-```
-
-```typescript
-import { processPdf, classifyPdf } from '@firecrawl/pdf-inspector'
-import { readFileSync } from 'fs'
-
-const pdf = readFileSync('document.pdf')
-const result = processPdf(pdf)
-
-console.log(result.pdfType)
-console.log(result.markdown)
-```
-
-This package remains only as a compatibility wrapper for older installs:
-
-```typescript
-import { processPdf } from 'firecrawl-pdf-inspector'
-```
-
-New projects should use `@firecrawl/pdf-inspector` directly.
-
@@ -1,4 +0,0 @@
-#!/usr/bin/env node
-
-import '@firecrawl/pdf-inspector/bin/pdf-inspector.mjs'
-
@@ -1,2 +0,0 @@
-export * from '@firecrawl/pdf-inspector'
-
@@ -1,20 +0,0 @@
-'use strict'
-
-const scoped = require('@firecrawl/pdf-inspector')
-
-module.exports = scoped
-module.exports.classifyPdf = scoped.classifyPdf
-module.exports.detectPdf = scoped.detectPdf
-module.exports.detectVectorGridInRegion = scoped.detectVectorGridInRegion
-module.exports.extractPagesMarkdown = scoped.extractPagesMarkdown
-module.exports.extractTablesInRegions = scoped.extractTablesInRegions
-module.exports.extractTablesWithStructure = scoped.extractTablesWithStructure
-module.exports.extractTablesWithStructureAuto = scoped.extractTablesWithStructureAuto
-module.exports.extractTablesWithStructureCells = scoped.extractTablesWithStructureCells
-module.exports.extractText = scoped.extractText
-module.exports.extractTextInRegions = scoped.extractTextInRegions
-module.exports.extractTextWithPositions = scoped.extractTextWithPositions
-module.exports.ItemType = scoped.ItemType
-module.exports.PdfType = scoped.PdfType
-module.exports.processPdf = scoped.processPdf
-
@@ -1,35 +0,0 @@
-{
-  "name": "firecrawl-pdf-inspector",
-  "version": "1.9.8",
-  "description": "Deprecated compatibility wrapper for @firecrawl/pdf-inspector.",
-  "main": "index.js",
-  "types": "index.d.ts",
-  "bin": {
-    "pdf-inspector": "bin/pdf-inspector.mjs"
-  },
-  "license": "MIT",
-  "keywords": [
-    "pdf",
-    "pdf-extraction",
-    "pdf-parser",
-    "text-extraction",
-    "ocr",
-    "pdf-classification",
-    "firecrawl",
-    "deprecated"
-  ],
-  "files": [
-    "index.js",
-    "index.d.ts",
-    "bin/",
-    "README.md"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/firecrawl/pdf-inspector"
-  },
-  "homepage": "https://github.com/firecrawl/pdf-inspector",
-  "dependencies": {
-    "@firecrawl/pdf-inspector": "1.9.8"
-  }
-}
@@ -1,6 +1,6 @@
 {
  "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.8",
+  "version": "1.9.5",
  "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
  "main": "index.js",
  "types": "index.d.ts",
@@ -188,17 +188,7 @@ pub(crate) fn extract_page_text_items(
    // Graphics state tracking
    let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
    let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
-    #[derive(Clone)]
-    struct SavedGraphicsState {
-        ctm: [f32; 6],
-        text_rendering_mode: i32,
-        char_spacing: f32,
-        word_spacing: f32,
-        text_leading: f32,
-        current_font: String,
-        current_font_size: f32,
-    }
-    let mut gstate_stack: Vec<SavedGraphicsState> = Vec::new();
+    let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();

    // Text state tracking
    let mut current_font = String::new();
@@ -237,26 +227,15 @@ pub(crate) fn extract_page_text_items(
        match op.operator.as_str() {
            "q" => {
                // Save graphics state
-                gstate_stack.push(SavedGraphicsState {
-                    ctm,
-                    text_rendering_mode,
-                    char_spacing,
-                    word_spacing,
-                    text_leading,
-                    current_font: current_font.clone(),
-                    current_font_size,
-                });
+                gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
            }
            "Q" => {
                // Restore graphics state
-                if let Some(saved) = gstate_stack.pop() {
-                    ctm = saved.ctm;
-                    text_rendering_mode = saved.text_rendering_mode;
-                    char_spacing = saved.char_spacing;
-                    word_spacing = saved.word_spacing;
-                    text_leading = saved.text_leading;
-                    current_font = saved.current_font;
-                    current_font_size = saved.current_font_size;
+                if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
+                    ctm = saved_ctm;
+                    text_rendering_mode = saved_tr;
+                    char_spacing = saved_tc;
+                    word_spacing = saved_tw;
                }
            }
            "cm" => {
@@ -1307,91 +1286,6 @@ mod tests {
        assert!(lines.is_empty());
    }

-    #[test]
-    fn test_q_restores_current_font_for_text_decoding() {
-        use crate::tounicode::FontCMaps;
-        use lopdf::{dictionary, Object, Stream};
-
-        fn cmap_stream(dst_hex: &str) -> Stream {
-            let cmap = format!(
-                r#"/CIDInit /ProcSet findresource begin
-12 dict begin
-begincmap
-/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
-/CMapName /Test-UCS def
-/CMapType 2 def
-1 begincodespacerange
-<00> <FF>
-endcodespacerange
-1 beginbfchar
-<41> <{dst_hex}>
-endbfchar
-endcmap
-CMapName currentdict /CMap defineresource pop
-end
-end"#
-            );
-            Stream::new(dictionary! {}, cmap.into_bytes())
-        }
-
-        let mut doc = lopdf::Document::new();
-        let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X
-        let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y
-        let f1 = doc.add_object(dictionary! {
-            "Type" => "Font",
-            "Subtype" => "Type1",
-            "BaseFont" => "Helvetica",
-            "ToUnicode" => Object::Reference(f1_cmap),
-        });
-        let f2 = doc.add_object(dictionary! {
-            "Type" => "Font",
-            "Subtype" => "Type1",
-            "BaseFont" => "Helvetica",
-            "ToUnicode" => Object::Reference(f2_cmap),
-        });
-
-        let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET
-q
-BT /F2 12 Tf 20 700 Tm <41> Tj ET
-Q
-BT 30 700 Tm <41> Tj ET";
-        let content_id = doc.add_object(Object::Stream(Stream::new(
-            dictionary! {},
-            content.to_vec(),
-        )));
-        let page_id = doc.add_object(dictionary! {
-            "Type" => "Page",
-            "Contents" => Object::Reference(content_id),
-            "Resources" => dictionary! {
-                "Font" => dictionary! {
-                    "F1" => Object::Reference(f1),
-                    "F2" => Object::Reference(f2),
-                },
-            },
-            "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
-        });
-        let pages_id = doc.add_object(dictionary! {
-            "Type" => "Pages",
-            "Count" => Object::Integer(1),
-            "Kids" => vec![Object::Reference(page_id)],
-        });
-        let catalog_id = doc.add_object(dictionary! {
-            "Type" => "Catalog",
-            "Pages" => Object::Reference(pages_id),
-        });
-        doc.trailer.set("Root", Object::Reference(catalog_id));
-
-        let font_cmaps = FontCMaps::from_doc(&doc);
-        let ((items, _, _), _, _) =
-            extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
-        let text = items
-            .iter()
-            .map(|item| item.text.as_str())
-            .collect::<String>();
-
-        assert_eq!(text, "XYX");
-    }
-
    #[test]
    fn test_strip_pdf_comments() {
        // Basic comment stripping
@@ -858,13 +858,6 @@ pub(crate) fn extract_text_from_operand(
            // unmapped. Don't fall through to text-interpretation fallbacks
            // (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
            // character codes (e.g. CID 0x01A9 → Latin-1 "©").
-            if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
-                // 2-byte CIDs (Identity-H) are by far the common case; for
-                // an odd byte count we still emit at least one marker so
-                // detection downstream fires.
-                let cid_count = (bytes.len() / 2).max(1);
-                return Some("\u{FFFD}".repeat(cid_count));
-            }

            // Try our custom encoding map from Differences arrays.
            // The Differences array overrides specific codes in a base encoding (typically
@@ -973,6 +966,27 @@ pub(crate) fn extract_text_from_operand(
                return Some(symbol_text);
            }

+            // Latin-1 fallback. Safe ONLY for fonts that use single-byte
+            // encodings — for these, an unmapped byte is a valid character
+            // code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
+            // emit multi-byte CIDs that aren't characters; per-byte Latin-1
+            // produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
+            // production scrape_id 019de78c-... samples).
+            //
+            // For a CID font (has_cmap is set OR a /ToUnicode reference
+            // exists) with any non-ASCII bytes, emit a single U+FFFD per
+            // CID instead. This both replaces the mojibake with a proper
+            // "decode failed" marker AND keeps `detect_encoding_issues`
+            // tripping so the page is flagged for OCR — the existing
+            // garbage-detection path that the high-Latin-1 mojibake used
+            // to satisfy by accident.
+            if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
+                // 2-byte CIDs (Identity-H) are by far the common case; for
+                // an odd byte count we still emit at least one marker so
+                // detection downstream fires.
+                let cid_count = (bytes.len() / 2).max(1);
+                return Some("\u{FFFD}".repeat(cid_count));
+            }
            // Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
            // 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
            // use single-byte encodings where Latin-1 fallback is the
@@ -368,7 +368,6 @@ pub fn extract_pages_markdown_mem(
    // Extract ALL pages to get accurate, document-wide font stats.
    let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
        extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
-    let text_quality = analyze_text_quality(&all_items);

    // Compute layout complexity from full document (near-zero cost).
    let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
@@ -417,7 +416,6 @@ pub fn extract_pages_markdown_mem(
            .collect();

        let has_gid = gid_pages.contains(&page_1idx);
-        let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx);

        // Build markdown with document-wide font stats
        let options = MarkdownOptions {
@@ -427,22 +425,17 @@ pub fn extract_pages_markdown_mem(
            ..MarkdownOptions::default()
        };

-        let md = if has_text_quality_issue {
-            String::new()
-        } else {
-            markdown::to_markdown_from_items_with_rects_and_lines(
-                page_items,
-                options,
-                &page_rects,
-                &[],
-                &page_thresholds,
-                None,
-                &[],
-            )
-        };
+        let md = markdown::to_markdown_from_items_with_rects_and_lines(
+            page_items,
+            options,
+            &page_rects,
+            &[],
+            &page_thresholds,
+            None,
+            &[],
+        );

-        let needs_ocr = has_text_quality_issue
-            || md.trim().is_empty()
+        let needs_ocr = md.trim().is_empty()
            || has_gid
            || is_garbage_text(&md)
            || is_cid_garbage(&md)
@@ -599,23 +592,24 @@ pub fn extract_text_in_regions_mem(
        for rect in regions {
            let [rx1, ry1, rx2, ry2] = *rect;

-            let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
-            let matched: Vec<TextItem> = match items {
-                Some(items) => items
-                    .iter()
-                    .filter(|item| region_overlaps_item(item, bounds))
-                    .cloned()
-                    .collect(),
-                None => Vec::new(),
+            let text = match items {
+                Some(items) => collect_text_in_region_with_options(
+                    items,
+                    rx1,
+                    ry1,
+                    rx2,
+                    ry2,
+                    page_h,
+                    coords,
+                    adaptive_threshold,
+                ),
+                None => String::new(),
            };
-            let has_text_quality_issue = region_items_have_decoding_issue(&matched);
-            let text = collect_text_from_matched_items(matched, adaptive_threshold);

            // Check per-region text quality instead of blanket page-level
            // GID rejection. A GID font in a logo elsewhere on the page
            // shouldn't force GPU OCR for clean text regions.
-            let needs_ocr = has_text_quality_issue
-                || text.trim().is_empty()
+            let needs_ocr = text.trim().is_empty()
                || is_garbage_text(&text)
                || is_cid_garbage(&text)
                || detect_encoding_issues(&text);
@@ -735,14 +729,6 @@ pub fn extract_tables_in_regions_mem(
                continue;
            }

-            if region_items_have_decoding_issue(&matched) {
-                page_results.push(RegionText {
-                    text: String::new(),
-                    needs_ocr: true,
-                });
-                continue;
-            }
-
            // Compute base_font_size as most common font size in the region
            let base_font_size = {
                let mut freq: HashMap<i32, usize> = HashMap::new();
@@ -3415,7 +3401,7 @@ fn process_document(
        })
        .unwrap_or((None, Vec::new()));

-    let (markdown, layout, has_encoding_issues, gid_pages, text_quality_pages) = match extracted {
+    let (markdown, layout, has_encoding_issues, gid_pages) = match extracted {
        Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => {
            // For TextBased PDFs with pages flagged for OCR (Identity-H or
            // Type3 fonts without ToUnicode), check whether the CID-as-Unicode
@@ -3465,7 +3451,6 @@ fn process_document(
                    }
                };

-            let text_quality = analyze_text_quality(&items);
            let layout = compute_layout_complexity(&items, &rects, &lines);

            let md = if options.mode == ProcessMode::Analyze {
@@ -3482,22 +3467,14 @@ fn process_document(
                ))
            };

-            let enc = text_quality.has_encoding_issues
-                || md.as_ref().is_some_and(|m| detect_encoding_issues(m));
-            (
-                md,
-                layout,
-                enc,
-                gid_encoded_pages,
-                text_quality.pages_needing_ocr,
-            )
+            let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m));
+            (md, layout, enc, gid_encoded_pages)
        }
        None => (
            None,
            LayoutComplexity::default(),
            false,
            std::collections::HashSet::new(),
-            Vec::new(),
        ),
    };

@@ -3539,18 +3516,6 @@ fn process_document(
        }
        pages_needing_ocr.sort_unstable();
    }
-    if !text_quality_pages.is_empty() {
-        log::debug!(
-            "pages with suspicious text-layer decoding (need OCR): {:?}",
-            text_quality_pages
-        );
-        for page in text_quality_pages {
-            if !pages_needing_ocr.contains(&page) {
-                pages_needing_ocr.push(page);
-            }
-        }
-        pages_needing_ocr.sort_unstable();
-    }

    // Detect sparse extraction: when a TEXT-BASED PDF produces very few
    // characters per page, the text is likely embedded in images/forms
@@ -3636,104 +3601,6 @@ fn detect_encoding_issues(markdown: &str) -> bool {
    false
 }

-#[derive(Debug, Default)]
-struct TextQualityReport {
-    pages_needing_ocr: Vec<u32>,
-    has_encoding_issues: bool,
-}
-
-fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
-    let mut pages = HashSet::new();
-
-    for item in items {
-        if !matches!(item.item_type, crate::types::ItemType::Text) {
-            continue;
-        }
-        if text_span_has_decoding_issue(&item.text) {
-            pages.insert(item.page);
-        }
-    }
-
-    let mut pages_needing_ocr: Vec<u32> = pages.into_iter().collect();
-    pages_needing_ocr.sort_unstable();
-    TextQualityReport {
-        has_encoding_issues: !pages_needing_ocr.is_empty(),
-        pages_needing_ocr,
-    }
-}
-
-fn region_items_have_decoding_issue(items: &[TextItem]) -> bool {
-    items.iter().any(|item| {
-        matches!(item.item_type, crate::types::ItemType::Text)
-            && text_span_has_decoding_issue(&item.text)
-    })
-}
-
-fn text_span_has_decoding_issue(text: &str) -> bool {
-    let text = text.trim();
-    if text.is_empty() {
-        return false;
-    }
-
-    detect_encoding_issues(text)
-        || has_private_use_text_run(text)
-        || is_cid_garbage(text)
-        || has_cid_control_token(text)
-}
-
-fn has_private_use_text_run(text: &str) -> bool {
-    let mut total = 0usize;
-    let mut private_use = 0usize;
-    let mut current_run = 0usize;
-    let mut longest_run = 0usize;
-
-    for ch in text.chars() {
-        if ch.is_whitespace() {
-            current_run = 0;
-            continue;
-        }
-        total += 1;
-        if is_private_use_char(ch) {
-            private_use += 1;
-            current_run += 1;
-            longest_run = longest_run.max(current_run);
-        } else {
-            current_run = 0;
-        }
-    }
-
-    if private_use == 0 {
-        return false;
-    }
-
-    longest_run >= 3 || (total >= 5 && private_use >= 2 && private_use * 2 >= total)
-}
-
-fn has_cid_control_token(text: &str) -> bool {
-    text.split_whitespace().any(token_has_cid_control)
-}
-
-fn token_has_cid_control(token: &str) -> bool {
-    let mut total = 0usize;
-    let mut c1_control = 0usize;
-
-    for ch in token.chars() {
-        total += 1;
-        if ('\u{0080}'..='\u{009F}').contains(&ch) {
-            c1_control += 1;
-        }
-    }
-
-    total >= 5 && c1_control > 0 && c1_control * 20 >= total
-}
-
-fn is_private_use_char(ch: char) -> bool {
-    matches!(
-        ch as u32,
-        0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD
-    )
-}
-
 /// Check if extracted text is predominantly garbage (non-alphanumeric).
 ///
 /// Broken font encodings produce text like "----1-.-.-.___  --.-. .._ I_---."
@@ -5676,13 +5543,6 @@ mod tests {
        }
    }

-    fn test_text_item_on_page(page: u32, text: &str) -> TextItem {
-        TextItem {
-            page,
-            ..test_item(text, 10.0, 10.0, text.len() as f32 * 5.0, 12.0)
-        }
-    }
-
    #[test]
    fn test_detect_encoding_issues_fffd() {
        assert!(detect_encoding_issues(
@@ -5718,68 +5578,6 @@ mod tests {
        assert!(!detect_encoding_issues(text));
    }

-    #[test]
-    fn test_text_quality_flags_localized_cid_mojibake_span() {
-        let items = vec![
-            test_text_item_on_page(
-                1,
-                "Waiting Period 等待期 Maternity and newborn infant care benefit",
-            ),
-            test_text_item_on_page(
-                1,
-                "Inpatient and Day-care Benefits DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯",
-            ),
-            test_text_item_on_page(1, "Covered up to annual maximum. 赔付至年度最高保额。"),
-            test_text_item_on_page(2, "A clean second page should not be routed to OCR."),
-        ];
-
-        let quality = analyze_text_quality(&items);
-
-        assert!(quality.has_encoding_issues);
-        assert_eq!(quality.pages_needing_ocr, vec![1]);
-    }
-
-    #[test]
-    fn test_text_quality_flags_replacement_and_private_use_runs() {
-        let items = vec![
-            test_text_item_on_page(1, "broken \u{FFFD} text"),
-            test_text_item_on_page(3, "\u{E000}\u{E001}\u{E002}"),
-        ];
-
-        let quality = analyze_text_quality(&items);
-
-        assert_eq!(quality.pages_needing_ocr, vec![1, 3]);
-    }
-
-    #[test]
-    fn test_text_quality_allows_clean_multilingual_and_latin1_text() {
-        let items = vec![
-            test_text_item_on_page(1, "你好世界，这是一段正常的中文文本。"),
-            test_text_item_on_page(1, "Résumé déjà vu: façade, São Paulo, año 2026."),
-            test_text_item_on_page(1, "A single icon \u{E000} should not force OCR."),
-        ];
-
-        let quality = analyze_text_quality(&items);
-
-        assert!(!quality.has_encoding_issues);
-        assert!(quality.pages_needing_ocr.is_empty());
-    }
-
-    #[test]
-    fn test_region_text_quality_is_scoped_to_matched_items() {
-        let clean_region = vec![
-            test_text_item_on_page(1, "Clean native text"),
-            test_text_item_on_page(1, "Résumé déjà vu"),
-        ];
-        let garbled_region = vec![
-            test_text_item_on_page(1, "Clean prefix"),
-            test_text_item_on_page(1, "DÂB\u{009B}A4gÉ9¶0ÅDÂB\u{009B}Ê(D>öBÑ9¯"),
-        ];
-
-        assert!(!region_items_have_decoding_issue(&clean_region));
-        assert!(region_items_have_decoding_issue(&garbled_region));
-    }
-
    #[test]
    fn test_garbage_text_detection() {
        // Simulates garbage output from Identity-H fonts without ToUnicode.
@@ -29,7 +29,6 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str
    // text item, which combine with gap-based space insertion to produce
    // double spaces ("Vice  President" instead of "Vice President").
    collapse_consecutive_spaces(&mut text);
-    remove_spaces_before_closing_brackets(&mut text);

    // Remove excessive newlines (more than 2 in a row)
    while text.contains("\n\n\n") {
@@ -72,20 +71,6 @@ fn collapse_consecutive_spaces(text: &mut String) {
    *text = result;
 }

-/// Remove spaces before closing square brackets.
-/// Unit markers and markdown links occasionally pick up a gap-inserted space
-/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding.
-fn remove_spaces_before_closing_brackets(text: &mut String) {
-    let mut result = String::with_capacity(text.len());
-    for ch in text.chars() {
-        if ch == ']' && result.ends_with(' ') {
-            result.pop();
-        }
-        result.push(ch);
-    }
-    *text = result;
-}
-
 /// Collapse dot leaders (runs of 4+ dots) into " ... "
 /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1"
 fn collapse_dot_leaders(text: &str) -> String {
@@ -357,18 +342,6 @@ mod tests {
        assert!(result.contains("Chapter 2 ... 20"));
    }

-    // --- remove_spaces_before_closing_brackets ---
-
-    #[test]
-    fn test_remove_spaces_before_closing_brackets() {
-        let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string();
-        remove_spaces_before_closing_brackets(&mut input);
-        assert_eq!(
-            input,
-            "Density [kg/m3] and [linked text](https://example.com)"
-        );
-    }
-
    // --- fix_hyphenation ---

    #[test]
Author	SHA1	Message	Date
Abimael Martell	12d670fe1e	chore: bump napi package version	2026-06-01 14:57:03 -07:00
Abimael Martell	ce742ca183	fix(markdown): handle wrapped bold abstracts	2026-06-01 14:32:42 -07:00