Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
062ede203e | ||
|
Abimael Martell
|
16fbe29b52 |
@@ -180,31 +180,18 @@ jobs:
|
||||
test -n "$ort_path"
|
||||
echo "ORT_DYLIB_PATH=$ort_path" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Configure isolated model cache
|
||||
shell: bash
|
||||
run: echo "PDF_INSPECTOR_MODEL_CACHE=$RUNNER_TEMP/pdf-inspector-models" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build OCR CLI
|
||||
run: cargo build --features ocr --bin pdf2md
|
||||
|
||||
- name: Test PDFium runtime
|
||||
run: cargo test --features ocr --test local_render_tests
|
||||
|
||||
- name: Provision OCR model cache
|
||||
shell: bash
|
||||
run: |
|
||||
target/debug/pdf2md \
|
||||
tests/fixtures/scan_with_native_header_text.pdf \
|
||||
--ocr force \
|
||||
--json > /dev/null
|
||||
|
||||
- name: Run OCR CLI
|
||||
shell: bash
|
||||
run: |
|
||||
target/debug/pdf2md \
|
||||
tests/fixtures/scan_with_native_header_text.pdf \
|
||||
--ocr auto \
|
||||
--ocr-offline \
|
||||
--json > "$RUNNER_TEMP/ocr-result.json"
|
||||
|
||||
- name: Validate OCR JSON contract
|
||||
@@ -226,12 +213,6 @@ jobs:
|
||||
assert "layout_ms" not in result["pages"][0]["timings"]
|
||||
PY
|
||||
|
||||
- name: Run OCR launch smoke set
|
||||
shell: bash
|
||||
run: |
|
||||
export PDF_INSPECTOR_OCR_TEST_MODELS="$PDF_INSPECTOR_MODEL_CACHE/pp-ocrv6-small/oar-ocr-v0.7.0"
|
||||
cargo test --features ocr --test ocr_tests -- --nocapture
|
||||
|
||||
- name: Build Node binding
|
||||
working-directory: napi
|
||||
run: |
|
||||
|
||||
@@ -86,14 +86,12 @@ jobs:
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
docker-options: -e CFLAGS_aarch64_unknown_linux_gnu=-D__ARM_ARCH=8
|
||||
# macos-13 was retired by GitHub; macos-15-intel is the remaining
|
||||
# Intel runner label (available through 2027).
|
||||
- os: macos-15-intel
|
||||
@@ -115,9 +113,6 @@ jobs:
|
||||
target: ${{ matrix.target }}
|
||||
args: --release --out dist
|
||||
manylinux: auto
|
||||
# The manylinux AArch64 GCC omits this macro while preprocessing
|
||||
# ring's assembly. AArch64 is ARMv8 by definition.
|
||||
docker-options: ${{ matrix.docker-options }}
|
||||
|
||||
- name: Upload wheel
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -60,7 +60,6 @@ jobs:
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
@@ -71,7 +70,6 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
build-flags: --use-napi-cross
|
||||
cflags: -D__ARM_ARCH=8
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-musl
|
||||
build-flags: -x
|
||||
@@ -128,10 +126,6 @@ jobs:
|
||||
|
||||
- name: Build native addon
|
||||
working-directory: napi
|
||||
env:
|
||||
# napi-cross's old AArch64 GCC omits this predefined macro while
|
||||
# preprocessing ring's assembly. AArch64 is ARMv8 by definition.
|
||||
CFLAGS_aarch64_unknown_linux_gnu: ${{ matrix.cflags }}
|
||||
run: bunx napi build --platform --release --target ${{ matrix.target }} ${{ matrix.build-flags }}
|
||||
|
||||
- name: Upload native binary
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
autobins = false
|
||||
authors = ["Firecrawl Team"]
|
||||
|
||||
@@ -48,7 +48,8 @@ Use the [paired benchmark harness](docs/benchmarking.md) to compare two local bu
|
||||
### Python
|
||||
|
||||
```bash
|
||||
pip install pdf-inspector
|
||||
pip install maturin
|
||||
maturin develop --release
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -181,9 +182,8 @@ and confidence, warnings, and pages recommended for the hosted document
|
||||
pipeline. Native Python and Node packages expose the same pipeline without a
|
||||
source-build feature. All native entry points still require separately
|
||||
installed PDFium and ONNX Runtime libraries only when OCR is routed. See the
|
||||
[OCR runtime setup guide](docs/ocr-runtime.md) for pinned downloads, platform
|
||||
support, model-cache behavior, and hosted-fallback integration. See the
|
||||
[Rust API guide](docs/rust-api.md#complete-ocr-api) for lower-level controls.
|
||||
[Rust API guide](docs/rust-api.md#complete-ocr-api) for model cache and offline
|
||||
configuration.
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# OCR runtime setup
|
||||
|
||||
Selective OCR is available from the Rust library and CLI, Python, and Node.js.
|
||||
Clean native-text documents do not load an OCR dependency or download a model.
|
||||
When `auto` routes at least one page, the process needs PDFium, ONNX Runtime,
|
||||
and the pinned PP-OCRv6 Small model set.
|
||||
|
||||
## Validated versions
|
||||
|
||||
The reproducible runtime path uses these builds:
|
||||
|
||||
- [Firecrawl PDFium `native-v7988`](https://github.com/firecrawl/pdfium-rs/releases/tag/native-v7988),
|
||||
containing PDFium `153.0.7988.0`
|
||||
- [ONNX Runtime `1.27.0`](https://github.com/microsoft/onnxruntime/releases/tag/v1.27.0)
|
||||
- PP-OCRv6 Small artifact revision `oar-ocr-v0.7.0`
|
||||
|
||||
Use these versions for the reproducible path. Other compatible shared-library
|
||||
builds may work, but are not part of the release smoke test.
|
||||
|
||||
## Install the shared libraries
|
||||
|
||||
Download and extract the matching archives:
|
||||
|
||||
| Platform | PDFium asset | ONNX Runtime asset |
|
||||
|---|---|---|
|
||||
| Linux x64 | `firecrawl-pdfium-linux-x64.tgz` | `onnxruntime-linux-x64-1.27.0.tgz` |
|
||||
| Linux ARM64 | `firecrawl-pdfium-linux-arm64.tgz` | `onnxruntime-linux-aarch64-1.27.0.tgz` |
|
||||
| macOS Apple Silicon | `firecrawl-pdfium-mac-arm64.tgz` | `onnxruntime-osx-arm64-1.27.0.tgz` |
|
||||
| Windows x64 | `firecrawl-pdfium-win-x64.tgz` | `onnxruntime-win-x64-1.27.0.zip` |
|
||||
|
||||
The PDFium release publishes `SHA256SUMS`, build provenance, license files,
|
||||
and an SPDX document for every platform archive. GitHub publishes a SHA-256
|
||||
digest with each ONNX Runtime asset.
|
||||
|
||||
Point pdf-inspector at the extracted shared libraries when they are not on the
|
||||
platform library search path:
|
||||
|
||||
```bash
|
||||
export PDFIUM_LIB_PATH=/absolute/path/to/libpdfium.so
|
||||
export ORT_DYLIB_PATH=/absolute/path/to/libonnxruntime.so
|
||||
pdf2md scan.pdf --ocr auto --json
|
||||
```
|
||||
|
||||
On macOS the filenames end in `.dylib`. On Windows, use PowerShell and point
|
||||
the variables at `pdfium.dll` and `onnxruntime.dll`:
|
||||
|
||||
```powershell
|
||||
$env:PDFIUM_LIB_PATH = "C:\absolute\path\to\pdfium.dll"
|
||||
$env:ORT_DYLIB_PATH = "C:\absolute\path\to\onnxruntime.dll"
|
||||
pdf2md scan.pdf --ocr auto --json
|
||||
```
|
||||
|
||||
The native extraction packages also support platforms without these exact
|
||||
runtime assets. In particular, the Python package has an Intel macOS wheel,
|
||||
but ONNX Runtime 1.27.0 does not publish an Intel macOS archive; local OCR on
|
||||
that target requires a compatible custom ONNX Runtime build.
|
||||
|
||||
The full OCR path is exercised end to end on Linux x64 in CI. macOS and
|
||||
Windows compile and run the feature's platform-independent tests, while their
|
||||
external-runtime paths should be treated as preview until equivalent smoke
|
||||
jobs are added.
|
||||
|
||||
## Model cache and offline mode
|
||||
|
||||
The first routed page downloads and SHA-256-verifies three pinned artifacts:
|
||||
the detection model, recognition model, and character dictionary. Together
|
||||
they are about 31 MB. They are stored below the platform cache directory.
|
||||
Set `PDF_INSPECTOR_MODEL_CACHE` to choose a managed cache root.
|
||||
|
||||
For hermetic deployments, populate the model directory ahead of time and use
|
||||
the language-specific offline option:
|
||||
|
||||
- CLI: `--ocr-offline --ocr-model-dir /models/pp-ocrv6-small`
|
||||
- Rust: `ModelDownloadPolicy::Offline` with `OcrOptions::model_directory`
|
||||
- Python: `offline=True, model_directory="/models/pp-ocrv6-small"`
|
||||
- Node.js: `offline: true, modelDirectory: "/models/pp-ocrv6-small"`
|
||||
|
||||
The model artifacts come from
|
||||
[`GreatV/oar-ocr`](https://github.com/GreatV/oar-ocr/releases/tag/v0.7.0),
|
||||
whose OCR implementation and upstream PaddleOCR project use Apache-2.0
|
||||
licensing. Models are downloaded at runtime and are not embedded in any
|
||||
pdf-inspector package.
|
||||
|
||||
## Hosted fallback boundary
|
||||
|
||||
`pages_recommending_hosted` is available after the local pipeline completes.
|
||||
It marks pages whose completed OCR result is empty, low-confidence, or still
|
||||
appears incomplete.
|
||||
|
||||
Setup and execution failures happen before that result exists. A missing or
|
||||
incompatible PDFium/ONNX Runtime library, failed model acquisition, or OCR
|
||||
execution error is returned as an error. A downstream integration that has a
|
||||
hosted parser should catch that error and route the document to the hosted
|
||||
path. This keeps deployment problems distinct from page-quality judgments.
|
||||
|
||||
In `auto`, documents with no routed pages return successfully without touching
|
||||
PDFium, ONNX Runtime, the model cache, or the network.
|
||||
+1
-3
@@ -44,9 +44,7 @@ OCR calls that route work require compatible PDFium and ONNX Runtime shared
|
||||
libraries. Set `PDFIUM_LIB_PATH` and `ORT_DYLIB_PATH` when they are not on the
|
||||
platform library search path. The pinned OCR model set is downloaded and
|
||||
checksum-verified on the first routed page; use `offline=True` with a warm
|
||||
cache or `model_directory` to prohibit network access. See the
|
||||
[OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
for pinned downloads, supported platforms, and hosted-fallback behavior.
|
||||
cache or `model_directory` to prohibit network access.
|
||||
|
||||
## Usage
|
||||
|
||||
|
||||
@@ -383,10 +383,6 @@ shape; `Force` renders every selected page. OCR uses the existing deterministic
|
||||
table, column, reading-order, and Markdown assembly path; no learned layout
|
||||
model is included.
|
||||
|
||||
The [OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
lists the pinned PDFium and ONNX Runtime builds, environment variables, model
|
||||
cache behavior, and the error boundary downstream hosted fallbacks should use.
|
||||
|
||||
For ambiguous mixed pages, `Auto` privately retains clean native fragments
|
||||
instead of discarding them when OCR is selected. After recognition it compares
|
||||
script-agnostic text quality, OCR confidence, character overlap, and material
|
||||
|
||||
Generated
+2
-2
@@ -2089,7 +2089,7 @@ checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"dirs",
|
||||
"env_logger",
|
||||
@@ -2114,7 +2114,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-napi"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
|
||||
+1
-3
@@ -41,9 +41,7 @@ OCR calls that route work require compatible PDFium and ONNX Runtime shared
|
||||
libraries. Set `PDFIUM_LIB_PATH` and `ORT_DYLIB_PATH` when they are not on the
|
||||
platform library search path. The pinned OCR model set is downloaded and
|
||||
checksum-verified on the first routed page; use `offline: true` with a warm
|
||||
cache or `modelDirectory` to prohibit network access. See the
|
||||
[OCR runtime setup guide](https://github.com/firecrawl/pdf-inspector/blob/main/docs/ocr-runtime.md)
|
||||
for pinned downloads, supported platforms, and hosted-fallback behavior.
|
||||
cache or `modelDirectory` to prohibit network access.
|
||||
|
||||
## API
|
||||
|
||||
|
||||
+6
-6
@@ -8,12 +8,12 @@
|
||||
"@napi-rs/cli": "^3.4.1",
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+7
-7
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.15.0",
|
||||
"version": "1.14.2",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
@@ -52,11 +52,11 @@
|
||||
"@napi-rs/cli": "^3.4.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.15.0",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.15.0"
|
||||
"@firecrawl/pdf-inspector-linux-x64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-x64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-gnu": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-linux-arm64-musl": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-darwin-arm64": "1.14.2",
|
||||
"@firecrawl/pdf-inspector-win32-x64-msvc": "1.14.2"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "maturin"
|
||||
name = "pdf-inspector"
|
||||
# Keep package versions in sync with `python3 scripts/version.py <version>`.
|
||||
# CI publishes automatically when the synchronized change lands on main.
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
readme = "docs/python.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
+1
-1
@@ -975,7 +975,7 @@ result = pdf_inspector.<span class="fn">process_pdf</span>(<span class="str">"do
|
||||
<script>
|
||||
(() => {
|
||||
const MAX_FILE_SIZE = 25 * 1024 * 1024;
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.15.0/pdf_inspector_wasm.js";
|
||||
const WASM_MODULE_URL = "https://cdn.jsdelivr.net/npm/@firecrawl/pdf-inspector-wasm@1.14.2/pdf_inspector_wasm.js";
|
||||
const input = document.querySelector("#pdf-input");
|
||||
const dropZone = document.querySelector("#drop-zone");
|
||||
const filePanel = document.querySelector("#demo-file");
|
||||
|
||||
@@ -406,9 +406,6 @@ fn main() {
|
||||
" --compact Collapse token-heavy source formatting such as dot leaders"
|
||||
);
|
||||
eprintln!(" --pages Insert page break markers (<!-- Page N -->)");
|
||||
eprintln!(
|
||||
" --formula-latex Rewrite native math glyph runs into $...$ LaTeX (experimental)"
|
||||
);
|
||||
eprintln!(" --select-pages N Only process specified pages (e.g. 1,3,5-10)");
|
||||
eprintln!(" --password PW Password for an encrypted PDF");
|
||||
eprintln!(" --detect-only Only detect PDF type (no extraction)");
|
||||
@@ -428,7 +425,6 @@ fn main() {
|
||||
let raw_output = args.iter().any(|a| a == "--raw");
|
||||
let compact_output = args.iter().any(|a| a == "--compact");
|
||||
let page_numbers = args.iter().any(|a| a == "--pages");
|
||||
let formula_latex = args.iter().any(|a| a == "--formula-latex");
|
||||
let detect_only = args.iter().any(|a| a == "--detect-only");
|
||||
let analyze = args.iter().any(|a| a == "--analyze");
|
||||
let ocr_mode_argument = argument_value(&args, "--ocr").unwrap_or_else(|error| {
|
||||
@@ -546,7 +542,6 @@ fn main() {
|
||||
markdown.profile = pdf_inspector::MarkdownProfile::Compact;
|
||||
}
|
||||
markdown.include_page_numbers = page_numbers;
|
||||
markdown.formula_latex = formula_latex;
|
||||
let mut pdf_options = OcrPdfOptions::new()
|
||||
.render(RenderOptions::new().dpi(dpi))
|
||||
.ocr(ocr)
|
||||
@@ -622,7 +617,6 @@ fn main() {
|
||||
options.markdown.profile = pdf_inspector::MarkdownProfile::Compact;
|
||||
}
|
||||
options.markdown.include_page_numbers = page_numbers;
|
||||
options.markdown.formula_latex = formula_latex;
|
||||
if let Some(pages) = page_filter {
|
||||
options.page_filter = Some(pages);
|
||||
}
|
||||
|
||||
-1120
File diff suppressed because it is too large
Load Diff
@@ -1,391 +0,0 @@
|
||||
//! Unicode to LaTeX character mapping.
|
||||
//!
|
||||
//! Maps Unicode math symbols, Greek letters, operators, and relations to their
|
||||
//! LaTeX command equivalents. Only includes characters that commonly appear in
|
||||
//! PDF text extraction of mathematical formulas.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Returns a reference to the global Unicode → LaTeX mapping table.
|
||||
pub fn unicode_to_latex_map() -> &'static HashMap<char, &'static str> {
|
||||
static MAP: OnceLock<HashMap<char, &'static str>> = OnceLock::new();
|
||||
MAP.get_or_init(build_map)
|
||||
}
|
||||
|
||||
fn build_map() -> HashMap<char, &'static str> {
|
||||
let entries: &[(char, &str)] = &[
|
||||
// ── Greek lowercase ─────────────────────────────────────────
|
||||
('\u{03B1}', r"\alpha"),
|
||||
('\u{03B2}', r"\beta"),
|
||||
('\u{03B3}', r"\gamma"),
|
||||
('\u{03B4}', r"\delta"),
|
||||
('\u{03B5}', r"\varepsilon"),
|
||||
('\u{03F5}', r"\epsilon"),
|
||||
('\u{03B6}', r"\zeta"),
|
||||
('\u{03B7}', r"\eta"),
|
||||
('\u{03B8}', r"\theta"),
|
||||
('\u{03D1}', r"\vartheta"),
|
||||
('\u{03B9}', r"\iota"),
|
||||
('\u{03BA}', r"\kappa"),
|
||||
('\u{03BB}', r"\lambda"),
|
||||
('\u{03BC}', r"\mu"),
|
||||
('\u{03BD}', r"\nu"),
|
||||
('\u{03BE}', r"\xi"),
|
||||
('\u{03C0}', r"\pi"),
|
||||
('\u{03D6}', r"\varpi"),
|
||||
('\u{03C1}', r"\rho"),
|
||||
('\u{03C2}', r"\varsigma"),
|
||||
('\u{03C3}', r"\sigma"),
|
||||
('\u{03C4}', r"\tau"),
|
||||
('\u{03C5}', r"\upsilon"),
|
||||
('\u{03C6}', r"\varphi"),
|
||||
('\u{03D5}', r"\phi"),
|
||||
('\u{03C7}', r"\chi"),
|
||||
('\u{03C8}', r"\psi"),
|
||||
('\u{03C9}', r"\omega"),
|
||||
// ── Greek uppercase ─────────────────────────────────────────
|
||||
('\u{0393}', r"\Gamma"),
|
||||
('\u{0394}', r"\Delta"),
|
||||
('\u{0398}', r"\Theta"),
|
||||
('\u{039B}', r"\Lambda"),
|
||||
('\u{039E}', r"\Xi"),
|
||||
('\u{03A0}', r"\Pi"),
|
||||
('\u{03A3}', r"\Sigma"),
|
||||
('\u{03A5}', r"\Upsilon"),
|
||||
('\u{03A6}', r"\Phi"),
|
||||
('\u{03A8}', r"\Psi"),
|
||||
('\u{03A9}', r"\Omega"),
|
||||
// ── Large operators ─────────────────────────────────────────
|
||||
('\u{222B}', r"\int"),
|
||||
('\u{222C}', r"\iint"),
|
||||
('\u{222D}', r"\iiint"),
|
||||
('\u{222E}', r"\oint"),
|
||||
('\u{2211}', r"\sum"),
|
||||
('\u{220F}', r"\prod"),
|
||||
('\u{2210}', r"\coprod"),
|
||||
// ── Roots / radicals ────────────────────────────────────────
|
||||
('\u{221A}', r"\sqrt"),
|
||||
// ── Calculus / differential ─────────────────────────────────
|
||||
('\u{2202}', r"\partial"),
|
||||
('\u{2207}', r"\nabla"),
|
||||
// ── Binary operators ────────────────────────────────────────
|
||||
('\u{00B1}', r"\pm"),
|
||||
('\u{2213}', r"\mp"),
|
||||
('\u{00D7}', r"\times"),
|
||||
('\u{00F7}', r"\div"),
|
||||
('\u{2217}', r"\ast"),
|
||||
('\u{22C6}', r"\star"),
|
||||
('\u{00B7}', r"\cdot"),
|
||||
('\u{2219}', r"\bullet"),
|
||||
('\u{2218}', r"\circ"),
|
||||
('\u{2020}', r"\dagger"),
|
||||
('\u{2021}', r"\ddagger"),
|
||||
('\u{2295}', r"\oplus"),
|
||||
('\u{2297}', r"\otimes"),
|
||||
('\u{2227}', r"\wedge"),
|
||||
('\u{2228}', r"\vee"),
|
||||
('\u{2229}', r"\cap"),
|
||||
('\u{222A}', r"\cup"),
|
||||
// ── Relations ───────────────────────────────────────────────
|
||||
('\u{2264}', r"\leq"),
|
||||
('\u{2265}', r"\geq"),
|
||||
('\u{2260}', r"\neq"),
|
||||
('\u{2248}', r"\approx"),
|
||||
('\u{223C}', r"\sim"),
|
||||
('\u{2243}', r"\simeq"),
|
||||
('\u{2261}', r"\equiv"),
|
||||
('\u{226A}', r"\ll"),
|
||||
('\u{226B}', r"\gg"),
|
||||
('\u{221D}', r"\propto"),
|
||||
('\u{2208}', r"\in"),
|
||||
('\u{2209}', r"\notin"),
|
||||
('\u{220B}', r"\ni"),
|
||||
('\u{2282}', r"\subset"),
|
||||
('\u{2283}', r"\supset"),
|
||||
('\u{2286}', r"\subseteq"),
|
||||
('\u{2287}', r"\supseteq"),
|
||||
('\u{22A2}', r"\vdash"),
|
||||
('\u{22A3}', r"\dashv"),
|
||||
('\u{22A4}', r"\top"),
|
||||
('\u{22A5}', r"\bot"),
|
||||
('\u{2225}', r"\parallel"),
|
||||
('\u{22A5}', r"\perp"),
|
||||
// ── Arrows ──────────────────────────────────────────────────
|
||||
('\u{2190}', r"\leftarrow"),
|
||||
('\u{2192}', r"\to"),
|
||||
('\u{2191}', r"\uparrow"),
|
||||
('\u{2193}', r"\downarrow"),
|
||||
('\u{2194}', r"\leftrightarrow"),
|
||||
('\u{21D0}', r"\Leftarrow"),
|
||||
('\u{21D2}', r"\Rightarrow"),
|
||||
('\u{21D4}', r"\Leftrightarrow"),
|
||||
('\u{21A6}', r"\mapsto"),
|
||||
('\u{2197}', r"\nearrow"),
|
||||
('\u{2198}', r"\searrow"),
|
||||
// ── Miscellaneous symbols ───────────────────────────────────
|
||||
('\u{221E}', r"\infty"),
|
||||
('\u{2200}', r"\forall"),
|
||||
('\u{2203}', r"\exists"),
|
||||
('\u{2204}', r"\nexists"),
|
||||
('\u{2205}', r"\emptyset"),
|
||||
('\u{00AC}', r"\neg"),
|
||||
('\u{00B0}', r"^\circ"),
|
||||
('\u{2032}', r"'"), // prime (common in physics: x')
|
||||
('\u{2033}', r"''"), // double prime
|
||||
('\u{210F}', r"\hbar"),
|
||||
('\u{2113}', r"\ell"),
|
||||
('\u{211C}', r"\Re"),
|
||||
('\u{2111}', r"\Im"),
|
||||
('\u{2118}', r"\wp"),
|
||||
('\u{2135}', r"\aleph"),
|
||||
// ── Dots ────────────────────────────────────────────────────
|
||||
('\u{22EF}', r"\cdots"),
|
||||
('\u{22EE}', r"\vdots"),
|
||||
('\u{22F1}', r"\ddots"),
|
||||
('\u{2026}', r"\ldots"),
|
||||
// ── Delimiters / brackets ───────────────────────────────────
|
||||
('\u{27E8}', r"\langle"),
|
||||
('\u{27E9}', r"\rangle"),
|
||||
('\u{2308}', r"\lceil"),
|
||||
('\u{2309}', r"\rceil"),
|
||||
('\u{230A}', r"\lfloor"),
|
||||
('\u{230B}', r"\rfloor"),
|
||||
('\u{2016}', r"\|"),
|
||||
// ── Accents / decorations (as standalone chars) ─────────────
|
||||
('\u{0302}', r"\hat{}"),
|
||||
('\u{0303}', r"\tilde{}"),
|
||||
('\u{0304}', r"\bar{}"),
|
||||
('\u{0307}', r"\dot{}"),
|
||||
('\u{0308}', r"\ddot{}"),
|
||||
('\u{20D7}', r"\vec{}"),
|
||||
// Hat/tilde as standalone characters (sometimes extracted separately)
|
||||
('\u{02C6}', r"\hat{}"),
|
||||
('\u{02DC}', r"\tilde{}"),
|
||||
// ── Subscript/superscript digits (Unicode) ──────────────────
|
||||
('\u{2070}', "^{0}"),
|
||||
('\u{00B9}', "^{1}"),
|
||||
('\u{00B2}', "^{2}"),
|
||||
('\u{00B3}', "^{3}"),
|
||||
('\u{2074}', "^{4}"),
|
||||
('\u{2075}', "^{5}"),
|
||||
('\u{2076}', "^{6}"),
|
||||
('\u{2077}', "^{7}"),
|
||||
('\u{2078}', "^{8}"),
|
||||
('\u{2079}', "^{9}"),
|
||||
('\u{207A}', "^{+}"),
|
||||
('\u{207B}', "^{-}"),
|
||||
('\u{2080}', "_{0}"),
|
||||
('\u{2081}', "_{1}"),
|
||||
('\u{2082}', "_{2}"),
|
||||
('\u{2083}', "_{3}"),
|
||||
('\u{2084}', "_{4}"),
|
||||
('\u{2085}', "_{5}"),
|
||||
('\u{2086}', "_{6}"),
|
||||
('\u{2087}', "_{7}"),
|
||||
('\u{2088}', "_{8}"),
|
||||
('\u{2089}', "_{9}"),
|
||||
('\u{208A}', "_{+}"),
|
||||
('\u{208B}', "_{-}"),
|
||||
// ── Math italic letters (sometimes used in PDF fonts) ───────
|
||||
// These map back to plain ASCII in LaTeX (math mode handles italics)
|
||||
// ── Codepoint variants of glyphs above ──────────────────────
|
||||
// TeX/legacy fonts frequently decode to the "wrong" sibling
|
||||
// codepoint: MICRO SIGN for mu, GREEK LUNATE EPSILON for epsilon,
|
||||
// RING OPERATOR vs MASCULINE ORDINAL. Map every sibling.
|
||||
('\u{00B5}', r"\mu"), // MICRO SIGN
|
||||
('\u{2212}', "-"), // MINUS SIGN → ASCII hyphen-minus
|
||||
('\u{2044}', "/"), // FRACTION SLASH
|
||||
('\u{2215}', "/"), // DIVISION SLASH
|
||||
// ── Blackboard bold (double-struck) letters ─────────────────
|
||||
('\u{2102}', r"\mathbb{C}"),
|
||||
('\u{210D}', r"\mathbb{H}"),
|
||||
('\u{2115}', r"\mathbb{N}"),
|
||||
('\u{2119}', r"\mathbb{P}"),
|
||||
('\u{211A}', r"\mathbb{Q}"),
|
||||
('\u{211D}', r"\mathbb{R}"),
|
||||
('\u{2124}', r"\mathbb{Z}"),
|
||||
('\u{1D53C}', r"\mathbb{E}"),
|
||||
('\u{1D53D}', r"\mathbb{F}"),
|
||||
// ── Script (calligraphic) letters with dedicated codepoints ──
|
||||
('\u{2110}', r"\mathcal{I}"),
|
||||
('\u{2112}', r"\mathcal{L}"),
|
||||
('\u{2130}', r"\mathcal{E}"),
|
||||
('\u{2131}', r"\mathcal{F}"),
|
||||
('\u{2133}', r"\mathcal{M}"),
|
||||
('\u{212C}', r"\mathcal{B}"),
|
||||
('\u{210B}', r"\mathcal{H}"),
|
||||
('\u{211B}', r"\mathcal{R}"),
|
||||
// ── Additional relations seen in TeX corpora ─────────────────
|
||||
('\u{2245}', r"\cong"),
|
||||
('\u{224A}', r"\approxeq"),
|
||||
('\u{227A}', r"\prec"),
|
||||
('\u{227B}', r"\succ"),
|
||||
('\u{2AAF}', r"\preceq"),
|
||||
('\u{2AB0}', r"\succeq"),
|
||||
('\u{22C0}', r"\bigwedge"),
|
||||
('\u{22C1}', r"\bigvee"),
|
||||
('\u{22C2}', r"\bigcap"),
|
||||
('\u{22C3}', r"\bigcup"),
|
||||
('\u{2294}', r"\sqcup"),
|
||||
('\u{2293}', r"\sqcap"),
|
||||
('\u{2296}', r"\ominus"),
|
||||
('\u{2298}', r"\oslash"),
|
||||
('\u{2299}', r"\odot"),
|
||||
('\u{22C4}', r"\diamond"),
|
||||
('\u{25CB}', r"\bigcirc"),
|
||||
('\u{2216}', r"\setminus"),
|
||||
('\u{2234}', r"\therefore"),
|
||||
('\u{2235}', r"\because"),
|
||||
('\u{221F}', r"\angle"),
|
||||
('\u{2220}', r"\angle"),
|
||||
('\u{22C8}', r"\bowtie"),
|
||||
('\u{228E}', r"\uplus"),
|
||||
('\u{2201}', r"\complement"),
|
||||
('\u{22D8}', r"\lll"),
|
||||
('\u{22D9}', r"\ggg"),
|
||||
('\u{227E}', r"\precsim"),
|
||||
('\u{2272}', r"\lesssim"),
|
||||
('\u{2273}', r"\gtrsim"),
|
||||
('\u{2266}', r"\leqq"),
|
||||
('\u{2267}', r"\geqq"),
|
||||
('\u{2268}', r"\lneq"),
|
||||
('\u{2269}', r"\gneq"),
|
||||
('\u{228A}', r"\subsetneq"),
|
||||
('\u{228B}', r"\supsetneq"),
|
||||
('\u{21C0}', r"\rightharpoonup"),
|
||||
('\u{21CC}', r"\rightleftharpoons"),
|
||||
('\u{27F6}', r"\longrightarrow"),
|
||||
('\u{27F5}', r"\longleftarrow"),
|
||||
('\u{27F7}', r"\longleftrightarrow"),
|
||||
('\u{27F9}', r"\Longrightarrow"),
|
||||
('\u{21AA}', r"\hookrightarrow"),
|
||||
('\u{21A9}', r"\hookleftarrow"),
|
||||
('\u{21D1}', r"\Uparrow"),
|
||||
('\u{21D3}', r"\Downarrow"),
|
||||
('\u{22A8}', r"\models"),
|
||||
('\u{22A9}', r"\Vdash"),
|
||||
('\u{225C}', r"\triangleq"),
|
||||
('\u{2250}', r"\doteq"),
|
||||
('\u{226E}', r"\nless"),
|
||||
('\u{226F}', r"\ngtr"),
|
||||
('\u{2270}', r"\nleq"),
|
||||
('\u{2271}', r"\ngeq"),
|
||||
('\u{2288}', r"\nsubseteq"),
|
||||
('\u{2289}', r"\nsupseteq"),
|
||||
('\u{22E6}', r"\lnsim"),
|
||||
('\u{25B3}', r"\triangle"),
|
||||
('\u{25A1}', r"\square"),
|
||||
('\u{25A0}', r"\blacksquare"),
|
||||
('\u{2662}', r"\diamondsuit"),
|
||||
('\u{22D5}', r"\#"),
|
||||
];
|
||||
|
||||
let mut map = HashMap::with_capacity(entries.len());
|
||||
for &(ch, latex) in entries {
|
||||
map.insert(ch, latex);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Convert a single character to its LaTeX representation.
|
||||
///
|
||||
/// Returns `Some(latex_str)` if the character has a known mapping,
|
||||
/// or `None` if it should be kept as-is.
|
||||
pub fn char_to_latex(ch: char) -> Option<&'static str> {
|
||||
unicode_to_latex_map().get(&ch).copied()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_greek_lowercase() {
|
||||
assert_eq!(char_to_latex('\u{03B1}'), Some(r"\alpha"));
|
||||
assert_eq!(char_to_latex('\u{03B2}'), Some(r"\beta"));
|
||||
assert_eq!(char_to_latex('\u{03B3}'), Some(r"\gamma"));
|
||||
assert_eq!(char_to_latex('\u{03B4}'), Some(r"\delta"));
|
||||
assert_eq!(char_to_latex('\u{03B5}'), Some(r"\varepsilon"));
|
||||
assert_eq!(char_to_latex('\u{03B6}'), Some(r"\zeta"));
|
||||
assert_eq!(char_to_latex('\u{03B7}'), Some(r"\eta"));
|
||||
assert_eq!(char_to_latex('\u{03B8}'), Some(r"\theta"));
|
||||
assert_eq!(char_to_latex('\u{03D1}'), Some(r"\vartheta"));
|
||||
assert_eq!(char_to_latex('\u{03B9}'), Some(r"\iota"));
|
||||
assert_eq!(char_to_latex('\u{03BA}'), Some(r"\kappa"));
|
||||
assert_eq!(char_to_latex('\u{03BB}'), Some(r"\lambda"));
|
||||
assert_eq!(char_to_latex('\u{03BC}'), Some(r"\mu"));
|
||||
assert_eq!(char_to_latex('\u{03BD}'), Some(r"\nu"));
|
||||
assert_eq!(char_to_latex('\u{03BE}'), Some(r"\xi"));
|
||||
assert_eq!(char_to_latex('\u{03C0}'), Some(r"\pi"));
|
||||
assert_eq!(char_to_latex('\u{03C1}'), Some(r"\rho"));
|
||||
assert_eq!(char_to_latex('\u{03C3}'), Some(r"\sigma"));
|
||||
assert_eq!(char_to_latex('\u{03C4}'), Some(r"\tau"));
|
||||
assert_eq!(char_to_latex('\u{03C9}'), Some(r"\omega"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_greek_uppercase() {
|
||||
assert_eq!(char_to_latex('\u{0393}'), Some(r"\Gamma"));
|
||||
assert_eq!(char_to_latex('\u{0394}'), Some(r"\Delta"));
|
||||
assert_eq!(char_to_latex('\u{0398}'), Some(r"\Theta"));
|
||||
assert_eq!(char_to_latex('\u{039B}'), Some(r"\Lambda"));
|
||||
assert_eq!(char_to_latex('\u{03A3}'), Some(r"\Sigma"));
|
||||
assert_eq!(char_to_latex('\u{03A6}'), Some(r"\Phi"));
|
||||
assert_eq!(char_to_latex('\u{03A9}'), Some(r"\Omega"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operators() {
|
||||
assert_eq!(char_to_latex('\u{222B}'), Some(r"\int"));
|
||||
assert_eq!(char_to_latex('\u{2211}'), Some(r"\sum"));
|
||||
assert_eq!(char_to_latex('\u{220F}'), Some(r"\prod"));
|
||||
assert_eq!(char_to_latex('\u{221A}'), Some(r"\sqrt"));
|
||||
assert_eq!(char_to_latex('\u{2202}'), Some(r"\partial"));
|
||||
assert_eq!(char_to_latex('\u{2207}'), Some(r"\nabla"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_relations() {
|
||||
assert_eq!(char_to_latex('\u{2264}'), Some(r"\leq"));
|
||||
assert_eq!(char_to_latex('\u{2265}'), Some(r"\geq"));
|
||||
assert_eq!(char_to_latex('\u{2260}'), Some(r"\neq"));
|
||||
assert_eq!(char_to_latex('\u{2248}'), Some(r"\approx"));
|
||||
assert_eq!(char_to_latex('\u{223C}'), Some(r"\sim"));
|
||||
assert_eq!(char_to_latex('\u{226A}'), Some(r"\ll"));
|
||||
assert_eq!(char_to_latex('\u{226B}'), Some(r"\gg"));
|
||||
assert_eq!(char_to_latex('\u{221E}'), Some(r"\infty"));
|
||||
assert_eq!(char_to_latex('\u{2208}'), Some(r"\in"));
|
||||
assert_eq!(char_to_latex('\u{2209}'), Some(r"\notin"));
|
||||
assert_eq!(char_to_latex('\u{2282}'), Some(r"\subset"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_misc_symbols() {
|
||||
assert_eq!(char_to_latex('\u{00B1}'), Some(r"\pm"));
|
||||
assert_eq!(char_to_latex('\u{00D7}'), Some(r"\times"));
|
||||
assert_eq!(char_to_latex('\u{00B7}'), Some(r"\cdot"));
|
||||
assert_eq!(char_to_latex('\u{00B0}'), Some(r"^\circ"));
|
||||
assert_eq!(char_to_latex('\u{2192}'), Some(r"\to"));
|
||||
assert_eq!(char_to_latex('\u{21D2}'), Some(r"\Rightarrow"));
|
||||
assert_eq!(char_to_latex('\u{210F}'), Some(r"\hbar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unicode_super_sub_digits() {
|
||||
assert_eq!(char_to_latex('\u{00B2}'), Some("^{2}"));
|
||||
assert_eq!(char_to_latex('\u{00B3}'), Some("^{3}"));
|
||||
assert_eq!(char_to_latex('\u{2082}'), Some("_{2}"));
|
||||
assert_eq!(char_to_latex('\u{2083}'), Some("_{3}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_codepoint_variants_map_to_same_commands() {
|
||||
// MICRO SIGN and GREEK SMALL MU both mean \mu in extracted PDFs.
|
||||
assert_eq!(char_to_latex('\u{00B5}'), Some(r"\mu"));
|
||||
assert_eq!(char_to_latex('\u{03BC}'), Some(r"\mu"));
|
||||
assert_eq!(char_to_latex('\u{2212}'), Some("-"));
|
||||
assert_eq!(char_to_latex('\u{211D}'), Some(r"\mathbb{R}"));
|
||||
assert_eq!(char_to_latex('\u{211A}'), Some(r"\mathbb{Q}"));
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,6 @@ pub mod python;
|
||||
pub mod adobe_korea1;
|
||||
pub mod detector;
|
||||
pub mod extractor;
|
||||
mod formula;
|
||||
pub mod glyph_names;
|
||||
pub mod markdown;
|
||||
pub mod process_mode;
|
||||
|
||||
@@ -702,13 +702,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// Merge drop caps with following text
|
||||
let lines = merge_drop_caps(lines, base_size);
|
||||
|
||||
// Rewrite confident native math glyph runs into $...$ LaTeX
|
||||
let lines = if options.formula_latex {
|
||||
crate::formula::rewrite_math_runs(lines)
|
||||
} else {
|
||||
lines
|
||||
};
|
||||
|
||||
// Discover heading tiers for this document
|
||||
let heading_tiers = compute_heading_tiers(&lines, base_size);
|
||||
|
||||
@@ -1267,13 +1260,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
// Merge drop caps with following text
|
||||
let lines = merge_drop_caps(lines, base_size);
|
||||
|
||||
// Rewrite confident native math glyph runs into $...$ LaTeX
|
||||
let lines = if options.formula_latex {
|
||||
crate::formula::rewrite_math_runs(lines)
|
||||
} else {
|
||||
lines
|
||||
};
|
||||
|
||||
// Discover heading tiers for this document
|
||||
let heading_tiers = compute_heading_tiers(&lines, base_size);
|
||||
|
||||
|
||||
@@ -1055,11 +1055,6 @@ pub struct MarkdownOptions {
|
||||
pub include_page_numbers: bool,
|
||||
/// Strip repeated headers/footers that appear on many pages
|
||||
pub strip_headers_footers: bool,
|
||||
/// Rewrite native math glyph runs into `$...$` LaTeX (experimental).
|
||||
/// Off by default: reconstruction is confidence-gated and structurally
|
||||
/// flat, and downstream consumers matching on the original Unicode text
|
||||
/// would see it replaced by LaTeX.
|
||||
pub formula_latex: bool,
|
||||
}
|
||||
|
||||
impl Default for MarkdownOptions {
|
||||
@@ -1089,7 +1084,6 @@ impl Default for MarkdownOptions {
|
||||
include_links: true,
|
||||
include_page_numbers: false,
|
||||
strip_headers_footers: true,
|
||||
formula_latex: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -724,7 +724,7 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"env_logger",
|
||||
"include_dir",
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "pdf-inspector-wasm"
|
||||
version = "1.15.0"
|
||||
version = "1.14.2"
|
||||
edition = "2021"
|
||||
authors = ["Firecrawl Team"]
|
||||
description = "Browser WebAssembly bindings for pdf-inspector"
|
||||
|
||||
Reference in New Issue
Block a user