diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..efab531 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,99 @@ +name: Publish npm package + +on: + push: + tags: ['v*'] + +permissions: + contents: read + packages: write + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: macos-14 + target: aarch64-apple-darwin + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + napi/target/ + key: ${{ runner.os }}-cargo-napi-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-napi- + + - name: Install dependencies + working-directory: napi + run: bun install + + - name: Build native addon + working-directory: napi + run: bunx napi build --platform --release + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.target }} + path: napi/*.node + if-no-files-found: error + + publish: + name: Publish to GitHub Packages + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + working-directory: napi + run: bun install + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: napi/artifacts + + - name: Move artifacts + run: bunx napi artifacts --dir napi/artifacts --napi-dir napi + working-directory: napi + + - name: List packages + run: ls -R npm/ + working-directory: napi + + - name: Publish + working-directory: napi + run: | + echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > .npmrc + echo "@firecrawl:registry=https://npm.pkg.github.com" >> .npmrc + bunx napi prepublish -t npm + npm publish --access public + for dir in npm/*/; do + if [ -f "$dir/package.json" ]; then + echo "Publishing $dir" + (cd "$dir" && npm publish --access public) + fi + done diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..05ccaa0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,79 @@ +# pdf-inspector + +Fast PDF text extraction to structured Markdown. CLI binary: `pdf2md`. Detection binary: `detect-pdf`. + +## Build & Test + +```bash +cargo fmt # format +cargo clippy -- -D warnings # lint (enforced, zero warnings) +cargo test # unit + integration tests (267+ unit, 73+ integration) +cargo build --release # release binary for benchmarks +``` + +All three must pass before committing. + +## Binaries + +- `pdf2md` — extract PDF → Markdown. Supports `--json` for structured output. +- `detect-pdf` — classify PDF type (TextBased/Scanned/Mixed/ImageBased). Supports `--analyze --json`. + +## Architecture + +``` +src/ + lib.rs – public API, process_pdf_with_options, encoding issue detection + detector.rs – PDF type classification, tiled-scan detection, page sampling + types.rs – TextItem, TextLine, PdfRect, PdfLine + tounicode.rs – CMap/ToUnicode parsing, CID decoding + text_utils.rs – CJK/RTL handling, Otsu threshold, ligature expansion, NFKC + extractor/ + mod.rs – top-level extraction orchestrator + content_stream.rs – PDF operator state machine (Tj/TJ/Td/Tm/q/Q) + fonts.rs – font width/encoding, CMapDecisionCache, TrueType cmap fallback + layout.rs – column detection (histogram), newspaper/tabular classification, + spanning-line pre-masking, sidebar detection + tables/ + detect_rects.rs – rect-based table detection (union-find clustering) + detect_heuristic.rs – heuristic table detection (gap-histogram, body-font tables) + detect_lines.rs – line-based table detection (H/V line grids) + grid.rs – column/row boundaries, cell assignment + format.rs – table→Markdown formatting, continuation row merging + markdown/ + convert.rs – core line→Markdown loop, struct-tree role support + analysis.rs – font stats, heading tiers, paragraph thresholds + classify.rs – line classification (header, list, code, caption) + preprocess.rs – drop cap merging, heading line merging + postprocess.rs – dot leaders, hyphenation, page numbers, URL formatting +``` + +## Key design decisions + +- **Primary audience is AI agents.** Output optimized for token efficiency and semantic quality, not visual formatting. No cosmetic padding. +- **Three table detection strategies** run in priority order: rect-based → line-based → heuristic. First valid result wins. +- **Column detection** uses horizontal projection histograms with valley detection. Multi-item spanning lines (titles, headers) are pre-masked using column-aware thresholds before column assignment. +- **Newspaper vs tabular** classification determines reading order: newspaper reads columns sequentially, tabular Y-interleaves them. +- **Tiled-scan detection** catches scanned PDFs with JBIG2/strip images where no single tile exceeds the template threshold but aggregate area does (≥2M pixels). +- **Garbage text upgrade** reclassifies Mixed PDFs as Scanned when extracted text is <50% alphanumeric. +- **Tagged PDF support** uses structure tree roles (H1-H6, P, L, Code, BlockQuote) when available, falling back to font-size heuristics. + +## Testing + +- **Unit tests**: inline `#[cfg(test)] mod tests` in each module with synthetic data. +- **Integration tests**: `tests/integration_tests.rs` with fixture PDFs in `tests/fixtures/`. +- **Regression suite**: sibling repo `pdf-evals` with 179+ snapshot PDFs. Run `cargo build --release` then `bench.py test` in that repo before committing. + +## Debugging + +```bash +RUST_LOG=pdf_inspector::extractor::layout=debug cargo run --bin pdf2md -- file.pdf +RUST_LOG=pdf_inspector::tables=debug cargo run --bin pdf2md -- file.pdf +RUST_LOG=pdf_inspector::detector=debug cargo run --release --bin detect-pdf -- file.pdf +``` + +## Conventions + +- Clippy: use `is_some_and(...)` not `map_or(false, ...)` +- lopdf quirk: `ParseError` is private — match by string for `InvalidFileHeader` +- Column limit for tables: 25 (wide statistical tables) +- `propagate_merged_cells` skipped for >10 columns (spanning rects = background fills) diff --git a/Cargo.toml b/Cargo.toml index f534f95..f1d7a8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"] pyo3 = { version = "0.22", features = ["extension-module"], optional = true } # PDF parsing -lopdf = { git = "https://github.com/firecrawl/lopdf", branch = "firecrawl/zlib-checksum-encrypted", features = ["rayon"] } +lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "052674053814a9f4897af94f0b8e46a545c9b329", features = ["rayon"] } # Error handling thiserror = "2.0" @@ -32,6 +32,7 @@ env_logger = "0.11" # Text processing regex = "1.10" once_cell = "1.19" +unicode-normalization = "0.1" # TrueType font parsing (for Identity-H CID font cmap extraction) ttf-parser = "0.25" diff --git a/napi/.gitignore b/napi/.gitignore new file mode 100644 index 0000000..01254fc --- /dev/null +++ b/napi/.gitignore @@ -0,0 +1,6 @@ +target/ +node_modules/ +*.node + +# Override root .gitignore — lock file needed for reproducible napi builds +!Cargo.lock diff --git a/napi/Cargo.lock b/napi/Cargo.lock new file mode 100644 index 0000000..8a11ccd --- /dev/null +++ b/napi/Cargo.lock @@ -0,0 +1,1509 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "352d39c2f7bef1d6ad73db6f5160efcaed66d94ef8c6c573a8410c00bf909a98" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dtor" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1057d6c64987086ff8ed0fd3fbf377a6b7d205cc7715868cd401705f715cbe4" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lopdf" +version = "0.40.0" +source = "git+https://github.com/J-F-Liu/lopdf?rev=052674053814a9f4897af94f0b8e46a545c9b329#052674053814a9f4897af94f0b8e46a545c9b329" +dependencies = [ + "aes", + "bitflags", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "nom_locate", + "rand", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "napi" +version = "3.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb7848c221fb7bb789e02f01875287ebb1e078b92a6566a34de01ef8806e7c2b" +dependencies = [ + "bitflags", + "ctor", + "futures", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", + "serde", + "serde_json", +] + +[[package]] +name = "napi-build" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1" + +[[package]] +name = "napi-derive" +version = "3.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60867ff9a6f76e82350e0c3420cb0736f5866091b61d7d8a024baa54b0ec17dd" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "5.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0864cf6a82e2cfb69067374b64c9253d7e910e5b34db833ed7495dda56ccb18" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb602b84d7c1edae45e50bbf1374696548f36ae179dfa667f577e384bb90c2b" +dependencies = [ + "libloading", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pdf-inspector" +version = "0.1.0" +dependencies = [ + "env_logger", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + +[[package]] +name = "pdf-inspector-napi" +version = "0.2.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "pdf-inspector", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/napi/Cargo.toml b/napi/Cargo.toml new file mode 100644 index 0000000..55a8730 --- /dev/null +++ b/napi/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pdf-inspector-napi" +version = "0.2.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +pdf-inspector = { path = ".." } +napi = { version = "3.0.0", features = ["serde-json"] } +napi-derive = "3.0.0" + +[build-dependencies] +napi-build = "2" + +[profile.release] +lto = true +strip = "debuginfo" diff --git a/napi/build.rs b/napi/build.rs new file mode 100644 index 0000000..bbfc9e4 --- /dev/null +++ b/napi/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/napi/bun.lock b/napi/bun.lock new file mode 100644 index 0000000..cd7f371 --- /dev/null +++ b/napi/bun.lock @@ -0,0 +1,231 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@firecrawl/pdf-inspector-js", + "devDependencies": { + "@napi-rs/cli": "^3.4.1", + }, + }, + }, + "packages": { + "@emnapi/core": ["@emnapi/core@1.9.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.9.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw=="], + + "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], + + "@inquirer/ansi": ["@inquirer/ansi@2.0.4", "", {}, "sha512-DpcZrQObd7S0R/U3bFdkcT5ebRwbTTC4D3tCc1vsJizmgPLxNJBo+AAFmrZwe8zk30P2QzgzGWZ3Q9uJwWuhIg=="], + + "@inquirer/checkbox": ["@inquirer/checkbox@5.1.2", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-PubpMPO2nJgMufkoB3P2wwxNXEMUXnBIKi/ACzDUYfaoPuM7gSTmuxJeMscoLVEsR4qqrCMf5p0SiYGWnVJ8kw=="], + + "@inquirer/confirm": ["@inquirer/confirm@6.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-tiNyA73pgpQ0FQ7axqtoLUe4GDYjNCDcVsbgcA5anvwg2z6i+suEngLKKJrWKJolT//GFPZHwN30binDIHgSgQ=="], + + "@inquirer/core": ["@inquirer/core@11.1.7", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4", "cli-width": "^4.1.0", "fast-wrap-ansi": "^0.2.0", "mute-stream": "^3.0.0", "signal-exit": "^4.1.0" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-1BiBNDk9btIwYIzNZpkikIHXWeNzNncJePPqwDyVMhXhD1ebqbpn1mKGctpoqAbzywZfdG0O4tvmsGIcOevAPQ=="], + + "@inquirer/editor": ["@inquirer/editor@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/external-editor": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-VJx4XyaKea7t8hEApTw5dxeIyMtWXre2OiyJcICCRZI4hkoHsMoCnl/KbUnJJExLbH9csLLHMVR144ZhFE1CwA=="], + + "@inquirer/expand": ["@inquirer/expand@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-fC0UHJPXsTRvY2fObiwuQYaAnHrp3aDqfwKUJSdfpgv18QUG054ezGbaRNStk/BKD5IPijeMKWej8VV8O5Q/eQ=="], + + "@inquirer/external-editor": ["@inquirer/external-editor@2.0.4", "", { "dependencies": { "chardet": "^2.1.1", "iconv-lite": "^0.7.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-Prenuv9C1PHj2Itx0BcAOVBTonz02Hc2Nd2DbU67PdGUaqn0nPCnV34oDyyoaZHnmfRxkpuhh/u51ThkrO+RdA=="], + + "@inquirer/figures": ["@inquirer/figures@2.0.4", "", {}, "sha512-eLBsjlS7rPS3WEhmOmh1znQ5IsQrxWzxWDxO51e4urv+iVrSnIHbq4zqJIOiyNdYLa+BVjwOtdetcQx1lWPpiQ=="], + + "@inquirer/input": ["@inquirer/input@5.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-nvZ6qEVeX/zVtZ1dY2hTGDQpVGD3R7MYPLODPgKO8Y+RAqxkrP3i/3NwF3fZpLdaMiNuK0z2NaYIx9tPwiSegQ=="], + + "@inquirer/number": ["@inquirer/number@4.0.10", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-Ht8OQstxiS3APMGjHV0aYAjRAysidWdwurWEo2i8yI5xbhOBWqizT0+MU1S2GCcuhIBg+3SgWVjEoXgfhY+XaA=="], + + "@inquirer/password": ["@inquirer/password@5.0.10", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-QbNyvIE8q2GTqKLYSsA8ATG+eETo+m31DSR0+AU7x3d2FhaTWzqQek80dj3JGTo743kQc6mhBR0erMjYw5jQ0A=="], + + "@inquirer/prompts": ["@inquirer/prompts@8.3.2", "", { "dependencies": { "@inquirer/checkbox": "^5.1.2", "@inquirer/confirm": "^6.0.10", "@inquirer/editor": "^5.0.10", "@inquirer/expand": "^5.0.10", "@inquirer/input": "^5.0.10", "@inquirer/number": "^4.0.10", "@inquirer/password": "^5.0.10", "@inquirer/rawlist": "^5.2.6", "@inquirer/search": "^4.1.6", "@inquirer/select": "^5.1.2" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-yFroiSj2iiBFlm59amdTvAcQFvWS6ph5oKESls/uqPBect7rTU2GbjyZO2DqxMGuIwVA8z0P4K6ViPcd/cp+0w=="], + + "@inquirer/rawlist": ["@inquirer/rawlist@5.2.6", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-jfw0MLJ5TilNsa9zlJ6nmRM0ZFVZhhTICt4/6CU2Dv1ndY7l3sqqo1gIYZyMMDw0LvE1u1nzJNisfHEhJIxq5w=="], + + "@inquirer/search": ["@inquirer/search@4.1.6", "", { "dependencies": { "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-3/6kTRae98hhDevENScy7cdFEuURnSpM3JbBNg8yfXLw88HgTOl+neUuy/l9W0No5NzGsLVydhBzTIxZP7yChQ=="], + + "@inquirer/select": ["@inquirer/select@5.1.2", "", { "dependencies": { "@inquirer/ansi": "^2.0.4", "@inquirer/core": "^11.1.7", "@inquirer/figures": "^2.0.4", "@inquirer/type": "^4.0.4" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-kTK8YIkHV+f02y7bWCh7E0u2/11lul5WepVTclr3UMBtBr05PgcZNWfMa7FY57ihpQFQH/spLMHTcr0rXy50tA=="], + + "@inquirer/type": ["@inquirer/type@4.0.4", "", { "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-PamArxO3cFJZoOzspzo6cxVlLeIftyBsZw/S9bKY5DzxqJVZgjoj1oP8d0rskKtp7sZxBycsoer1g6UeJV1BBA=="], + + "@napi-rs/cli": ["@napi-rs/cli@3.6.0", "", { "dependencies": { "@inquirer/prompts": "^8.0.0", "@napi-rs/cross-toolchain": "^1.0.3", "@napi-rs/wasm-tools": "^1.0.1", "@octokit/rest": "^22.0.1", "clipanion": "^4.0.0-rc.4", "colorette": "^2.0.20", "emnapi": "^1.9.1", "es-toolkit": "^1.41.0", "js-yaml": "^4.1.0", "obug": "^2.0.0", "semver": "^7.7.3", "typanion": "^3.14.0" }, "peerDependencies": { "@emnapi/runtime": "^1.7.1" }, "optionalPeers": ["@emnapi/runtime"], "bin": { "napi": "dist/cli.js", "napi-raw": "cli.mjs" } }, "sha512-aA8m4+9XxnK1+0sr4GplZP0Ze90gkzO8sMKaplOK0zXbLnzsLl6O2BQQt6rTCcTRzIN24wrrByakr/imM+CxhA=="], + + "@napi-rs/cross-toolchain": ["@napi-rs/cross-toolchain@1.0.3", "", { "dependencies": { "@napi-rs/lzma": "^1.4.5", "@napi-rs/tar": "^1.1.0", "debug": "^4.4.1" }, "peerDependencies": { "@napi-rs/cross-toolchain-arm64-target-aarch64": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-armv7": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-ppc64le": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-s390x": "^1.0.3", "@napi-rs/cross-toolchain-arm64-target-x86_64": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-aarch64": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-armv7": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-ppc64le": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-s390x": "^1.0.3", "@napi-rs/cross-toolchain-x64-target-x86_64": "^1.0.3" }, "optionalPeers": ["@napi-rs/cross-toolchain-arm64-target-aarch64", "@napi-rs/cross-toolchain-arm64-target-armv7", "@napi-rs/cross-toolchain-arm64-target-ppc64le", "@napi-rs/cross-toolchain-arm64-target-s390x", "@napi-rs/cross-toolchain-arm64-target-x86_64", "@napi-rs/cross-toolchain-x64-target-aarch64", "@napi-rs/cross-toolchain-x64-target-armv7", "@napi-rs/cross-toolchain-x64-target-ppc64le", "@napi-rs/cross-toolchain-x64-target-s390x", "@napi-rs/cross-toolchain-x64-target-x86_64"] }, "sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg=="], + + "@napi-rs/lzma": ["@napi-rs/lzma@1.4.5", "", { "optionalDependencies": { "@napi-rs/lzma-android-arm-eabi": "1.4.5", "@napi-rs/lzma-android-arm64": "1.4.5", "@napi-rs/lzma-darwin-arm64": "1.4.5", "@napi-rs/lzma-darwin-x64": "1.4.5", "@napi-rs/lzma-freebsd-x64": "1.4.5", "@napi-rs/lzma-linux-arm-gnueabihf": "1.4.5", "@napi-rs/lzma-linux-arm64-gnu": "1.4.5", "@napi-rs/lzma-linux-arm64-musl": "1.4.5", "@napi-rs/lzma-linux-ppc64-gnu": "1.4.5", "@napi-rs/lzma-linux-riscv64-gnu": "1.4.5", "@napi-rs/lzma-linux-s390x-gnu": "1.4.5", "@napi-rs/lzma-linux-x64-gnu": "1.4.5", "@napi-rs/lzma-linux-x64-musl": "1.4.5", "@napi-rs/lzma-wasm32-wasi": "1.4.5", "@napi-rs/lzma-win32-arm64-msvc": "1.4.5", "@napi-rs/lzma-win32-ia32-msvc": "1.4.5", "@napi-rs/lzma-win32-x64-msvc": "1.4.5" } }, "sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg=="], + + "@napi-rs/lzma-android-arm-eabi": ["@napi-rs/lzma-android-arm-eabi@1.4.5", "", { "os": "android", "cpu": "arm" }, "sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q=="], + + "@napi-rs/lzma-android-arm64": ["@napi-rs/lzma-android-arm64@1.4.5", "", { "os": "android", "cpu": "arm64" }, "sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ=="], + + "@napi-rs/lzma-darwin-arm64": ["@napi-rs/lzma-darwin-arm64@1.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA=="], + + "@napi-rs/lzma-darwin-x64": ["@napi-rs/lzma-darwin-x64@1.4.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw=="], + + "@napi-rs/lzma-freebsd-x64": ["@napi-rs/lzma-freebsd-x64@1.4.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw=="], + + "@napi-rs/lzma-linux-arm-gnueabihf": ["@napi-rs/lzma-linux-arm-gnueabihf@1.4.5", "", { "os": "linux", "cpu": "arm" }, "sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw=="], + + "@napi-rs/lzma-linux-arm64-gnu": ["@napi-rs/lzma-linux-arm64-gnu@1.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg=="], + + "@napi-rs/lzma-linux-arm64-musl": ["@napi-rs/lzma-linux-arm64-musl@1.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w=="], + + "@napi-rs/lzma-linux-ppc64-gnu": ["@napi-rs/lzma-linux-ppc64-gnu@1.4.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ=="], + + "@napi-rs/lzma-linux-riscv64-gnu": ["@napi-rs/lzma-linux-riscv64-gnu@1.4.5", "", { "os": "linux", "cpu": "none" }, "sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA=="], + + "@napi-rs/lzma-linux-s390x-gnu": ["@napi-rs/lzma-linux-s390x-gnu@1.4.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA=="], + + "@napi-rs/lzma-linux-x64-gnu": ["@napi-rs/lzma-linux-x64-gnu@1.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw=="], + + "@napi-rs/lzma-linux-x64-musl": ["@napi-rs/lzma-linux-x64-musl@1.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ=="], + + "@napi-rs/lzma-wasm32-wasi": ["@napi-rs/lzma-wasm32-wasi@1.4.5", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA=="], + + "@napi-rs/lzma-win32-arm64-msvc": ["@napi-rs/lzma-win32-arm64-msvc@1.4.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg=="], + + "@napi-rs/lzma-win32-ia32-msvc": ["@napi-rs/lzma-win32-ia32-msvc@1.4.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg=="], + + "@napi-rs/lzma-win32-x64-msvc": ["@napi-rs/lzma-win32-x64-msvc@1.4.5", "", { "os": "win32", "cpu": "x64" }, "sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q=="], + + "@napi-rs/tar": ["@napi-rs/tar@1.1.0", "", { "optionalDependencies": { "@napi-rs/tar-android-arm-eabi": "1.1.0", "@napi-rs/tar-android-arm64": "1.1.0", "@napi-rs/tar-darwin-arm64": "1.1.0", "@napi-rs/tar-darwin-x64": "1.1.0", "@napi-rs/tar-freebsd-x64": "1.1.0", "@napi-rs/tar-linux-arm-gnueabihf": "1.1.0", "@napi-rs/tar-linux-arm64-gnu": "1.1.0", "@napi-rs/tar-linux-arm64-musl": "1.1.0", "@napi-rs/tar-linux-ppc64-gnu": "1.1.0", "@napi-rs/tar-linux-s390x-gnu": "1.1.0", "@napi-rs/tar-linux-x64-gnu": "1.1.0", "@napi-rs/tar-linux-x64-musl": "1.1.0", "@napi-rs/tar-wasm32-wasi": "1.1.0", "@napi-rs/tar-win32-arm64-msvc": "1.1.0", "@napi-rs/tar-win32-ia32-msvc": "1.1.0", "@napi-rs/tar-win32-x64-msvc": "1.1.0" } }, "sha512-7cmzIu+Vbupriudo7UudoMRH2OA3cTw67vva8MxeoAe5S7vPFI7z0vp0pMXiA25S8IUJefImQ90FeJjl8fjEaQ=="], + + "@napi-rs/tar-android-arm-eabi": ["@napi-rs/tar-android-arm-eabi@1.1.0", "", { "os": "android", "cpu": "arm" }, "sha512-h2Ryndraj/YiKgMV/r5by1cDusluYIRT0CaE0/PekQ4u+Wpy2iUVqvzVU98ZPnhXaNeYxEvVJHNGafpOfaD0TA=="], + + "@napi-rs/tar-android-arm64": ["@napi-rs/tar-android-arm64@1.1.0", "", { "os": "android", "cpu": "arm64" }, "sha512-DJFyQHr1ZxNZorm/gzc1qBNLF/FcKzcH0V0Vwan5P+o0aE2keQIGEjJ09FudkF9v6uOuJjHCVDdK6S6uHtShAw=="], + + "@napi-rs/tar-darwin-arm64": ["@napi-rs/tar-darwin-arm64@1.1.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Zz2sXRzjIX4e532zD6xm2SjXEym6MkvfCvL2RMpG2+UwNVDVscHNcz3d47Pf3sysP2e2af7fBB3TIoK2f6trPw=="], + + "@napi-rs/tar-darwin-x64": ["@napi-rs/tar-darwin-x64@1.1.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-EI+CptIMNweT0ms9S3mkP/q+J6FNZ1Q6pvpJOEcWglRfyfQpLqjlC0O+dptruTPE8VamKYuqdjxfqD8hifZDOA=="], + + "@napi-rs/tar-freebsd-x64": ["@napi-rs/tar-freebsd-x64@1.1.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J0PIqX+pl6lBIAckL/c87gpodLbjZB1OtIK+RDscKC9NLdpVv6VGOxzUV/fYev/hctcE8EfkLbgFOfpmVQPg2g=="], + + "@napi-rs/tar-linux-arm-gnueabihf": ["@napi-rs/tar-linux-arm-gnueabihf@1.1.0", "", { "os": "linux", "cpu": "arm" }, "sha512-SLgIQo3f3EjkZ82ZwvrEgFvMdDAhsxCYjyoSuWfHCz0U16qx3SuGCp8+FYOPYCECHN3ZlGjXnoAIt9ERd0dEUg=="], + + "@napi-rs/tar-linux-arm64-gnu": ["@napi-rs/tar-linux-arm64-gnu@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-d014cdle52EGaH6GpYTQOP9Py7glMO1zz/+ynJPjjzYFSxvdYx0byrjumZk2UQdIyGZiJO2MEFpCkEEKFSgPYA=="], + + "@napi-rs/tar-linux-arm64-musl": ["@napi-rs/tar-linux-arm64-musl@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ=="], + + "@napi-rs/tar-linux-ppc64-gnu": ["@napi-rs/tar-linux-ppc64-gnu@1.1.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ=="], + + "@napi-rs/tar-linux-s390x-gnu": ["@napi-rs/tar-linux-s390x-gnu@1.1.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ=="], + + "@napi-rs/tar-linux-x64-gnu": ["@napi-rs/tar-linux-x64-gnu@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww=="], + + "@napi-rs/tar-linux-x64-musl": ["@napi-rs/tar-linux-x64-musl@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ=="], + + "@napi-rs/tar-wasm32-wasi": ["@napi-rs/tar-wasm32-wasi@1.1.0", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw=="], + + "@napi-rs/tar-win32-arm64-msvc": ["@napi-rs/tar-win32-arm64-msvc@1.1.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-vfpG71OB0ijtjemp3WTdmBKJm9R70KM8vsSExMsIQtV0lVzP07oM1CW6JbNRPXNLhRoue9ofYLiUDk8bE0Hckg=="], + + "@napi-rs/tar-win32-ia32-msvc": ["@napi-rs/tar-win32-ia32-msvc@1.1.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-hGPyPW60YSpOSgzfy68DLBHgi6HxkAM+L59ZZZPMQ0TOXjQg+p2EW87+TjZfJOkSpbYiEkULwa/f4a2hcVjsqQ=="], + + "@napi-rs/tar-win32-x64-msvc": ["@napi-rs/tar-win32-x64-msvc@1.1.0", "", { "os": "win32", "cpu": "x64" }, "sha512-L6Ed1DxXK9YSCMyvpR8MiNAyKNkQLjsHsHK9E0qnHa8NzLFqzDKhvs5LfnWxM2kJ+F7m/e5n9zPm24kHb3LsVw=="], + + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.2", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-sNXv5oLJ7ob93xkZ1XnxisYhGYXfaG9f65/ZgYuAu3qt7b3NadcOEhLvx28hv31PgX8SZJRYrAIPQilQmFpLVw=="], + + "@napi-rs/wasm-tools": ["@napi-rs/wasm-tools@1.0.1", "", { "optionalDependencies": { "@napi-rs/wasm-tools-android-arm-eabi": "1.0.1", "@napi-rs/wasm-tools-android-arm64": "1.0.1", "@napi-rs/wasm-tools-darwin-arm64": "1.0.1", "@napi-rs/wasm-tools-darwin-x64": "1.0.1", "@napi-rs/wasm-tools-freebsd-x64": "1.0.1", "@napi-rs/wasm-tools-linux-arm64-gnu": "1.0.1", "@napi-rs/wasm-tools-linux-arm64-musl": "1.0.1", "@napi-rs/wasm-tools-linux-x64-gnu": "1.0.1", "@napi-rs/wasm-tools-linux-x64-musl": "1.0.1", "@napi-rs/wasm-tools-wasm32-wasi": "1.0.1", "@napi-rs/wasm-tools-win32-arm64-msvc": "1.0.1", "@napi-rs/wasm-tools-win32-ia32-msvc": "1.0.1", "@napi-rs/wasm-tools-win32-x64-msvc": "1.0.1" } }, "sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ=="], + + "@napi-rs/wasm-tools-android-arm-eabi": ["@napi-rs/wasm-tools-android-arm-eabi@1.0.1", "", { "os": "android", "cpu": "arm" }, "sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw=="], + + "@napi-rs/wasm-tools-android-arm64": ["@napi-rs/wasm-tools-android-arm64@1.0.1", "", { "os": "android", "cpu": "arm64" }, "sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg=="], + + "@napi-rs/wasm-tools-darwin-arm64": ["@napi-rs/wasm-tools-darwin-arm64@1.0.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g=="], + + "@napi-rs/wasm-tools-darwin-x64": ["@napi-rs/wasm-tools-darwin-x64@1.0.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A=="], + + "@napi-rs/wasm-tools-freebsd-x64": ["@napi-rs/wasm-tools-freebsd-x64@1.0.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg=="], + + "@napi-rs/wasm-tools-linux-arm64-gnu": ["@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q=="], + + "@napi-rs/wasm-tools-linux-arm64-musl": ["@napi-rs/wasm-tools-linux-arm64-musl@1.0.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ=="], + + "@napi-rs/wasm-tools-linux-x64-gnu": ["@napi-rs/wasm-tools-linux-x64-gnu@1.0.1", "", { "os": "linux", "cpu": "x64" }, "sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw=="], + + "@napi-rs/wasm-tools-linux-x64-musl": ["@napi-rs/wasm-tools-linux-x64-musl@1.0.1", "", { "os": "linux", "cpu": "x64" }, "sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ=="], + + "@napi-rs/wasm-tools-wasm32-wasi": ["@napi-rs/wasm-tools-wasm32-wasi@1.0.1", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.0.3" }, "cpu": "none" }, "sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA=="], + + "@napi-rs/wasm-tools-win32-arm64-msvc": ["@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ=="], + + "@napi-rs/wasm-tools-win32-ia32-msvc": ["@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw=="], + + "@napi-rs/wasm-tools-win32-x64-msvc": ["@napi-rs/wasm-tools-win32-x64-msvc@1.0.1", "", { "os": "win32", "cpu": "x64" }, "sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ=="], + + "@octokit/auth-token": ["@octokit/auth-token@6.0.0", "", {}, "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w=="], + + "@octokit/core": ["@octokit/core@7.0.6", "", { "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "before-after-hook": "^4.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q=="], + + "@octokit/endpoint": ["@octokit/endpoint@11.0.3", "", { "dependencies": { "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag=="], + + "@octokit/graphql": ["@octokit/graphql@9.0.3", "", { "dependencies": { "@octokit/request": "^10.0.6", "@octokit/types": "^16.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA=="], + + "@octokit/openapi-types": ["@octokit/openapi-types@27.0.0", "", {}, "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA=="], + + "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@14.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw=="], + + "@octokit/plugin-request-log": ["@octokit/plugin-request-log@6.0.0", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q=="], + + "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@17.0.0", "", { "dependencies": { "@octokit/types": "^16.0.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw=="], + + "@octokit/request": ["@octokit/request@10.0.8", "", { "dependencies": { "@octokit/endpoint": "^11.0.3", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "fast-content-type-parse": "^3.0.0", "json-with-bigint": "^3.5.3", "universal-user-agent": "^7.0.2" } }, "sha512-SJZNwY9pur9Agf7l87ywFi14W+Hd9Jg6Ifivsd33+/bGUQIjNujdFiXII2/qSlN2ybqUHfp5xpekMEjIBTjlSw=="], + + "@octokit/request-error": ["@octokit/request-error@7.1.0", "", { "dependencies": { "@octokit/types": "^16.0.0" } }, "sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw=="], + + "@octokit/rest": ["@octokit/rest@22.0.1", "", { "dependencies": { "@octokit/core": "^7.0.6", "@octokit/plugin-paginate-rest": "^14.0.0", "@octokit/plugin-request-log": "^6.0.0", "@octokit/plugin-rest-endpoint-methods": "^17.0.0" } }, "sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw=="], + + "@octokit/types": ["@octokit/types@16.0.0", "", { "dependencies": { "@octokit/openapi-types": "^27.0.0" } }, "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg=="], + + "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + + "before-after-hook": ["before-after-hook@4.0.0", "", {}, "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ=="], + + "chardet": ["chardet@2.1.1", "", {}, "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ=="], + + "cli-width": ["cli-width@4.1.0", "", {}, "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ=="], + + "clipanion": ["clipanion@4.0.0-rc.4", "", { "dependencies": { "typanion": "^3.8.0" } }, "sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q=="], + + "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "emnapi": ["emnapi@1.9.2", "", { "peerDependencies": { "node-addon-api": ">= 6.1.0" }, "optionalPeers": ["node-addon-api"] }, "sha512-OdUoQe/8so7FvubnE/DNV9sNNSFwDYQiK4ZCAz4agMnD1s6faLuDn2gzxfJrmMoKfxZhhsckqGNwqPnS5K140A=="], + + "es-toolkit": ["es-toolkit@1.45.1", "", {}, "sha512-/jhoOj/Fx+A+IIyDNOvO3TItGmlMKhtX8ISAHKE90c4b/k1tqaqEZ+uUqfpU8DMnW5cgNJv606zS55jGvza0Xw=="], + + "fast-content-type-parse": ["fast-content-type-parse@3.0.0", "", {}, "sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg=="], + + "fast-string-truncated-width": ["fast-string-truncated-width@3.0.3", "", {}, "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g=="], + + "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], + + "fast-wrap-ansi": ["fast-wrap-ansi@0.2.0", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w=="], + + "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + + "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + + "json-with-bigint": ["json-with-bigint@3.5.8", "", {}, "sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "mute-stream": ["mute-stream@3.0.0", "", {}, "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw=="], + + "obug": ["obug@2.1.1", "", {}, "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "typanion": ["typanion@3.14.0", "", {}, "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug=="], + + "universal-user-agent": ["universal-user-agent@7.0.3", "", {}, "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A=="], + } +} diff --git a/napi/index.d.ts b/napi/index.d.ts new file mode 100644 index 0000000..8254452 --- /dev/null +++ b/napi/index.d.ts @@ -0,0 +1,49 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ +/** + * Classify a PDF: detect type (TextBased/Scanned/Mixed/ImageBased), + * page count, and which pages need OCR. Takes PDF bytes as Buffer. + */ +export declare function classifyPdf(buffer: Buffer): PdfClassification + +/** + * Extract text within bounding-box regions from a PDF. + * + * For hybrid OCR: layout model detects regions in rendered images, + * this extracts PDF text within those regions — skipping GPU OCR + * for text-based pages. + * + * Each region result includes `needs_ocr` — set when the extracted text + * is unreliable (empty, GID-encoded fonts, garbage, encoding issues). + * + * Coordinates are PDF points with top-left origin. + */ +export declare function extractTextInRegions(buffer: Buffer, pageRegions: Array): Array + +/** A page's regions for text extraction: (page_index_0based, bboxes). */ +export interface PageRegions { + page: number + /** Each bbox is [x1, y1, x2, y2] in PDF points, top-left origin. */ + regions: Array> +} + +/** Extracted text for one page's regions. */ +export interface PageRegionTexts { + page: number + regions: Array +} + +/** Lightweight PDF classification result. */ +export interface PdfClassification { + pdfType: string + pageCount: number + pagesNeedingOcr: Array + confidence: number +} + +/** Extracted text for a single region. */ +export interface RegionText { + text: string + /** `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues). */ + needsOcr: boolean +} diff --git a/napi/index.js b/napi/index.js new file mode 100644 index 0000000..23f7a6e --- /dev/null +++ b/napi/index.js @@ -0,0 +1,580 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* auto-generated by NAPI-RS */ + +const { readFileSync } = require('node:fs') +let nativeBinding = null +const loadErrors = [] + +const isMusl = () => { + let musl = false + if (process.platform === 'linux') { + musl = isMuslFromFilesystem() + if (musl === null) { + musl = isMuslFromReport() + } + if (musl === null) { + musl = isMuslFromChildProcess() + } + } + return musl +} + +const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-') + +const isMuslFromFilesystem = () => { + try { + return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl') + } catch { + return null + } +} + +const isMuslFromReport = () => { + let report = null + if (typeof process.report?.getReport === 'function') { + process.report.excludeNetwork = true + report = process.report.getReport() + } + if (!report) { + return null + } + if (report.header && report.header.glibcVersionRuntime) { + return false + } + if (Array.isArray(report.sharedObjects)) { + if (report.sharedObjects.some(isFileMusl)) { + return true + } + } + return false +} + +const isMuslFromChildProcess = () => { + try { + return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl') + } catch (e) { + // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false + return false + } +} + +function requireNative() { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + try { + return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH); + } catch (err) { + loadErrors.push(err) + } + } else if (process.platform === 'android') { + if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.android-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-android-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-android-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./pdf-inspector.android-arm-eabi.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-android-arm-eabi') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-android-arm-eabi/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`)) + } + } else if (process.platform === 'win32') { + if (process.arch === 'x64') { + if (process.config?.variables?.shlib_suffix === 'dll.a' || process.config?.variables?.node_target_type === 'shared_library') { + try { + return require('./pdf-inspector.win32-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-x64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.win32-x64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-x64-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-x64-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ia32') { + try { + return require('./pdf-inspector.win32-ia32-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-ia32-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-ia32-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.win32-arm64-msvc.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-win32-arm64-msvc') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-win32-arm64-msvc/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`)) + } + } else if (process.platform === 'darwin') { + try { + return require('./pdf-inspector.darwin-universal.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-universal') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-universal/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + if (process.arch === 'x64') { + try { + return require('./pdf-inspector.darwin-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.darwin-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-darwin-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-darwin-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`)) + } + } else if (process.platform === 'freebsd') { + if (process.arch === 'x64') { + try { + return require('./pdf-inspector.freebsd-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-freebsd-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-freebsd-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.freebsd-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-freebsd-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-freebsd-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`)) + } + } else if (process.platform === 'linux') { + if (process.arch === 'x64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-x64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-x64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-x64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-x64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-x64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-x64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-arm64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-arm64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'arm') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-arm-musleabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm-musleabihf') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm-musleabihf/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-arm-gnueabihf.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-arm-gnueabihf') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-arm-gnueabihf/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'loong64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-loong64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-loong64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-loong64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-loong64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-loong64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-loong64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'riscv64') { + if (isMusl()) { + try { + return require('./pdf-inspector.linux-riscv64-musl.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-riscv64-musl') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-riscv64-musl/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + try { + return require('./pdf-inspector.linux-riscv64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-riscv64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-riscv64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } + } else if (process.arch === 'ppc64') { + try { + return require('./pdf-inspector.linux-ppc64-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-ppc64-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-ppc64-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 's390x') { + try { + return require('./pdf-inspector.linux-s390x-gnu.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-linux-s390x-gnu') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-linux-s390x-gnu/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`)) + } + } else if (process.platform === 'openharmony') { + if (process.arch === 'arm64') { + try { + return require('./pdf-inspector.openharmony-arm64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-arm64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-arm64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'x64') { + try { + return require('./pdf-inspector.openharmony-x64.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-x64') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-x64/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else if (process.arch === 'arm') { + try { + return require('./pdf-inspector.openharmony-arm.node') + } catch (e) { + loadErrors.push(e) + } + try { + const binding = require('@firecrawl/pdf-inspector-js-openharmony-arm') + const bindingPackageVersion = require('@firecrawl/pdf-inspector-js-openharmony-arm/package.json').version + if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + } + return binding + } catch (e) { + loadErrors.push(e) + } + } else { + loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`)) + } + } else { + loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`)) + } +} + +nativeBinding = requireNative() + +if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { + let wasiBinding = null + let wasiBindingError = null + try { + wasiBinding = require('./pdf-inspector.wasi.cjs') + nativeBinding = wasiBinding + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + wasiBindingError = err + } + } + if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { + try { + wasiBinding = require('@firecrawl/pdf-inspector-js-wasm32-wasi') + nativeBinding = wasiBinding + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + if (!wasiBindingError) { + wasiBindingError = err + } else { + wasiBindingError.cause = err + } + loadErrors.push(err) + } + } + } + if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) { + const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error') + error.cause = wasiBindingError + throw error + } +} + +if (!nativeBinding) { + if (loadErrors.length > 0) { + throw new Error( + `Cannot find native binding. ` + + `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` + + 'Please try `npm i` again after removing both package-lock.json and node_modules directory.', + { + cause: loadErrors.reduce((err, cur) => { + cur.cause = err + return cur + }), + }, + ) + } + throw new Error(`Failed to load native binding`) +} + +module.exports = nativeBinding +module.exports.classifyPdf = nativeBinding.classifyPdf +module.exports.extractTextInRegions = nativeBinding.extractTextInRegions diff --git a/napi/package.json b/napi/package.json new file mode 100644 index 0000000..86b28eb --- /dev/null +++ b/napi/package.json @@ -0,0 +1,34 @@ +{ + "name": "@firecrawl/pdf-inspector-js", + "version": "0.2.0", + "main": "index.js", + "types": "index.d.ts", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/firecrawl/pdf-inspector" + }, + "publishConfig": { + "registry": "https://npm.pkg.github.com", + "access": "public" + }, + "napi": { + "binaryName": "pdf-inspector", + "targets": [ + "x86_64-unknown-linux-gnu", + "aarch64-apple-darwin" + ], + "package": { + "name": "@firecrawl/pdf-inspector-js" + } + }, + "scripts": { + "build": "napi build --platform --release", + "build:debug": "napi build --platform", + "prepublishOnly": "napi prepublish -t npm", + "artifacts": "napi artifacts" + }, + "devDependencies": { + "@napi-rs/cli": "^3.4.1" + } +} diff --git a/napi/src/lib.rs b/napi/src/lib.rs new file mode 100644 index 0000000..c13ffdf --- /dev/null +++ b/napi/src/lib.rs @@ -0,0 +1,116 @@ +#![deny(clippy::all)] + +use napi::bindgen_prelude::*; +use napi_derive::napi; + +/// Lightweight PDF classification result. +#[napi(object)] +pub struct PdfClassification { + pub pdf_type: String, + pub page_count: u32, + pub pages_needing_ocr: Vec, + pub confidence: f64, +} + +/// A page's regions for text extraction: (page_index_0based, bboxes). +#[napi(object)] +pub struct PageRegions { + pub page: u32, + /// Each bbox is [x1, y1, x2, y2] in PDF points, top-left origin. + pub regions: Vec>, +} + +/// Extracted text for a single region. +#[napi(object)] +pub struct RegionText { + pub text: String, + /// `true` when the text should not be trusted (empty, GID fonts, garbage, encoding issues). + pub needs_ocr: bool, +} + +/// Extracted text for one page's regions. +#[napi(object)] +pub struct PageRegionTexts { + pub page: u32, + pub regions: Vec, +} + +/// Classify a PDF: detect type (TextBased/Scanned/Mixed/ImageBased), +/// page count, and which pages need OCR. Takes PDF bytes as Buffer. +#[napi] +pub fn classify_pdf(buffer: Buffer) -> Result { + let result = pdf_inspector::classify_pdf_mem(&buffer).map_err(|e| { + Error::new(Status::GenericFailure, format!("classify_pdf failed: {e}")) + })?; + + Ok(PdfClassification { + pdf_type: match result.pdf_type { + pdf_inspector::PdfType::TextBased => "TextBased".to_string(), + pdf_inspector::PdfType::Scanned => "Scanned".to_string(), + pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(), + pdf_inspector::PdfType::Mixed => "Mixed".to_string(), + }, + page_count: result.page_count, + pages_needing_ocr: result.pages_needing_ocr, + confidence: result.confidence as f64, + }) +} + +/// Extract text within bounding-box regions from a PDF. +/// +/// For hybrid OCR: layout model detects regions in rendered images, +/// this extracts PDF text within those regions — skipping GPU OCR +/// for text-based pages. +/// +/// Each region result includes `needs_ocr` — set when the extracted text +/// is unreliable (empty, GID-encoded fonts, garbage, encoding issues). +/// +/// Coordinates are PDF points with top-left origin. +#[napi] +pub fn extract_text_in_regions( + buffer: Buffer, + page_regions: Vec, +) -> Result> { + // Convert from napi types to the Rust API's expected format + let regions: Vec<(u32, Vec<[f32; 4]>)> = page_regions + .iter() + .map(|pr| { + let bboxes: Vec<[f32; 4]> = pr + .regions + .iter() + .map(|r| { + if r.len() != 4 { + [0.0, 0.0, 0.0, 0.0] + } else { + [r[0] as f32, r[1] as f32, r[2] as f32, r[3] as f32] + } + }) + .collect(); + (pr.page, bboxes) + }) + .collect(); + + let results = pdf_inspector::extract_text_in_regions_mem(&buffer, ®ions).map_err(|e| { + Error::new( + Status::GenericFailure, + format!("extract_text_in_regions failed: {e}"), + ) + })?; + + Ok( + results + .into_iter() + .map(|page_result| PageRegionTexts { + page: page_result.page, + regions: page_result + .regions + .into_iter() + .map(|r| RegionText { + text: r.text, + needs_ocr: r.needs_ocr, + }) + .collect(), + }) + .collect(), + ) +} diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index c930076..77baf31 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -268,6 +268,9 @@ fn main() { eprintln!("Pages: {}", result.page_count); eprintln!("Processing time: {}ms", result.processing_time_ms); print_layout_info(&result.layout); + if !result.pages_needing_ocr.is_empty() { + eprintln!("Pages needing OCR: {:?}", result.pages_needing_ocr); + } if let Some(markdown) = &result.markdown { if let Some(output) = output_file { diff --git a/src/detector.rs b/src/detector.rs index ab5f9d8..4e84516 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -144,7 +144,7 @@ pub fn detect_pdf_type_mem_with_config( let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; @@ -197,11 +197,13 @@ pub(crate) fn detect_from_document( let analysis = analyze_page_content(doc, page_id); pages_actually_sampled += 1; log::debug!( - "page {}: text_ops={} images={} image_count={} template={} unique_chars={} path_ops={} vector_text={} image_area={}", + "page {}: text_ops={} images={} image_count={} template={} unique_chars={} alphanum={} path_ops={} vector_text={} image_area={} identity_h_no_tounicode={} type3_only={} font_changes={}", page_num, analysis.text_operator_count, analysis.has_images, analysis.image_count, analysis.has_template_image, - analysis.unique_text_chars, analysis.path_op_count, analysis.has_vector_text, - analysis.total_image_area + analysis.unique_text_chars, analysis.unique_alphanum_chars, + analysis.path_op_count, analysis.has_vector_text, + analysis.total_image_area, analysis.has_identity_h_no_tounicode, + analysis.has_only_type3_fonts, analysis.font_change_count ); let is_image_dominated = analysis.image_count > 10 && analysis.image_count > analysis.text_operator_count * 3; @@ -214,6 +216,7 @@ pub(crate) fn detect_from_document( && !is_image_dominated && analysis.unique_text_chars >= 5 && !analysis.has_vector_text + && !analysis.has_only_type3_fonts { pages_with_text += 1; } @@ -265,9 +268,8 @@ pub(crate) fn detect_from_document( // Classification logic let (pdf_type, confidence) = if has_template_images && pages_with_text > 0 { - // Template-based PDF: has text but images provide essential context - // Classify as Mixed with lower confidence ocr_recommended = true; + // Template-based PDF: has text but images provide essential context (PdfType::Mixed, 0.5 + (0.3 * (1.0 - template_ratio))) } else if text_ratio >= config.text_page_ratio_threshold { ocr_recommended = false; @@ -291,8 +293,51 @@ pub(crate) fn detect_from_document( (PdfType::TextBased, text_ratio.max(0.5)) }; + // Phase 1b: Newspaper-style layout detection. + // Dense multi-column newspapers (WSJ, NYT) have extractable text but produce + // poor output due to complex interleaved article layouts. Detect via consistently + // high text density combined with moderate font switches and a low Tf/Tj ratio. + // + // The Tf/Tj ratio distinguishes newspapers from styled legal/business documents: + // - Newspapers: ratio 0.02-0.06 (dense prose with occasional font switches) + // - Rich-styled docs (DPA, contracts): ratio 0.25-0.35 (per-character styling) + // + // Thresholds calibrated against: + // - WSJ 50-page newspaper: text_ops 1500-3800, font_changes 50-194, ratio 0.02-0.06 + // - DPA/contracts: text_ops 1300-2260, font_changes 327-630, ratio 0.25-0.32 + // - SEC filings: text_ops 1-1800, font_changes 1-65 (only 1-2 dense pages) + // - Normal docs: text_ops < 700, font_changes < 55 + let ocr_recommended = if pdf_type == PdfType::TextBased && pages_sampled >= 3 { + let mut newspaper_pages = 0u32; + for analysis in analysis_cache.values() { + let ratio = if analysis.text_operator_count > 0 { + analysis.font_change_count as f32 / analysis.text_operator_count as f32 + } else { + 1.0 + }; + if analysis.text_operator_count >= 1500 + && analysis.font_change_count >= 50 + && ratio < 0.15 + { + newspaper_pages += 1; + } + } + let newspaper_ratio = newspaper_pages as f32 / pages_sampled as f32; + if newspaper_ratio >= 0.5 { + log::debug!( + "newspaper layout detected: {}/{} pages with high text_ops + font_changes → OCR recommended", + newspaper_pages, pages_sampled + ); + true + } else { + ocr_recommended + } + } else { + ocr_recommended + }; + // Phase 2: Build per-page OCR list - let pages_needing_ocr = match pdf_type { + let mut pages_needing_ocr = match pdf_type { PdfType::TextBased => Vec::new(), PdfType::Scanned | PdfType::ImageBased => (1..=total_pages).collect(), PdfType::Mixed => { @@ -319,6 +364,34 @@ pub(crate) fn detect_from_document( } }; + // Phase 3: Flag pages with undecodable fonts for OCR. + // - Identity-H/V without ToUnicode: raw CID values can't map to Unicode + // - Type3-only without ToUnicode: glyph bitmaps can't map to Unicode + for (&page_num, analysis) in &analysis_cache { + if (analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts) + && !pages_needing_ocr.contains(&page_num) + { + pages_needing_ocr.push(page_num); + } + } + // Check uncached pages too (when not all pages were sampled) + if pages_needing_ocr.len() < total_pages as usize { + for page_num in 1..=total_pages { + if analysis_cache.contains_key(&page_num) || pages_needing_ocr.contains(&page_num) { + continue; + } + if let Some(&page_id) = pages.get(&page_num) { + if page_has_identity_h_no_tounicode(doc, page_id) + || page_has_only_type3_fonts(doc, page_id) + { + pages_needing_ocr.push(page_num); + } + } + } + } + pages_needing_ocr.sort(); + pages_needing_ocr.dedup(); + // Try to get title from metadata let title = get_document_title(doc); @@ -383,11 +456,22 @@ struct PageAnalysis { image_count: u32, /// Number of unique non-whitespace text characters found in string operands unique_text_chars: u32, + /// Number of unique ASCII alphanumeric bytes (letters + digits) in string operands + unique_alphanum_chars: u32, /// Number of path construction/painting ops (m, l, c, h, f, re, etc.) #[allow(dead_code)] path_op_count: u32, /// Whether the page has vector-outlined text (massive path ops, minimal text ops) has_vector_text: bool, + /// Whether the page has Type0 fonts with Identity-H/V encoding but no ToUnicode CMap. + /// These fonts produce garbage text because CID values can't be mapped to Unicode. + has_identity_h_no_tounicode: bool, + /// Whether the page uses only Type3 fonts (no normal text fonts). + /// Type3 fonts render each glyph as a custom drawing/bitmap — without a + /// ToUnicode CMap, the character codes can't be mapped to Unicode. + has_only_type3_fonts: bool, + /// Number of Tf (set font) operators — high count indicates many font switches + font_change_count: u32, } /// Analyze a page's content stream for text operators and images @@ -396,6 +480,7 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { let mut has_images = false; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; let mut all_unique_chars: HashSet = HashSet::new(); // Get content streams for this page @@ -409,12 +494,13 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { Err(_) => stream.content.clone(), }; - // Scan for text operators (Tj, TJ), image operators (Do), and path ops - let (ops, imgs, paths) = + // Scan for text operators (Tj, TJ), font changes (Tf), image operators (Do), and path ops + let (ops, imgs, paths, fonts) = scan_content_for_text_operators(&content, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } } @@ -423,20 +509,22 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { if let Ok((resource_dict, resource_ids)) = doc.get_page_resources(page_id) { let mut visited = HashSet::new(); if let Some(resources) = resource_dict { - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_xobjects_in_resources(doc, resources, &mut visited, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } for resource_id in resource_ids { if let Ok(resources) = doc.get_dictionary(resource_id) { - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_xobjects_in_resources(doc, resources, &mut visited, &mut all_unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; has_images = has_images || imgs > 0; } } @@ -454,6 +542,18 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { // outlined text produces thousands of path ops. let has_vector_text = path_ops >= 1000 && path_ops > text_ops.saturating_mul(200); + let unique_alphanum_chars = all_unique_chars + .iter() + .filter(|b| b.is_ascii_alphanumeric()) + .count() as u32; + + // Check for Identity-H/V fonts without ToUnicode — these produce garbage text + let has_identity_h_no_tounicode = + text_ops > 0 && page_has_identity_h_no_tounicode(doc, page_id); + + // Check for Type3-only fonts — glyph bitmaps without Unicode mapping + let has_only_type3_fonts = text_ops > 0 && page_has_only_type3_fonts(doc, page_id); + PageAnalysis { text_operator_count: text_ops, has_images, @@ -461,20 +561,101 @@ fn analyze_page_content(doc: &Document, page_id: ObjectId) -> PageAnalysis { total_image_area, image_count, unique_text_chars: all_unique_chars.len() as u32, + unique_alphanum_chars, path_op_count: path_ops, has_vector_text, + has_identity_h_no_tounicode, + has_only_type3_fonts, + font_change_count: font_changes, } } +/// Check if a page has Type0 fonts with Identity-H/V encoding and no ToUnicode CMap. +/// These fonts encode text as raw CID values that can't be mapped to Unicode without +/// a ToUnicode CMap, producing garbage output for non-Latin scripts (e.g. Cyrillic). +fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool { + let fonts = match doc.get_page_fonts(page_id) { + Ok(f) => f, + Err(_) => return false, + }; + for font_dict in fonts.values() { + let subtype = font_dict + .get(b"Subtype") + .ok() + .and_then(|o| o.as_name().ok()); + if subtype != Some(b"Type0") { + continue; + } + let encoding = font_dict + .get(b"Encoding") + .ok() + .and_then(|o| o.as_name().ok()); + let is_identity = matches!(encoding, Some(b"Identity-H") | Some(b"Identity-V")); + if !is_identity { + continue; + } + // Has ToUnicode? Then the font is decodable. + if font_dict.get(b"ToUnicode").is_ok() { + continue; + } + // Identity-H/V without ToUnicode — flag it + log::debug!( + "page has Identity-H/V font without ToUnicode: {:?}", + font_dict + .get(b"BaseFont") + .ok() + .and_then(|o| o.as_name().ok()) + .map(|n| String::from_utf8_lossy(n).to_string()) + ); + return true; + } + false +} + +/// Returns true if every font on the page is Type3 (no normal text fonts). +/// Type3 fonts render glyphs as custom drawings/bitmaps. Without a ToUnicode +/// CMap, character codes can't be mapped to Unicode — the page needs OCR. +fn page_has_only_type3_fonts(doc: &Document, page_id: ObjectId) -> bool { + let fonts = match doc.get_page_fonts(page_id) { + Ok(f) => f, + Err(_) => return false, + }; + if fonts.is_empty() { + return false; + } + let mut has_type3 = false; + for font_dict in fonts.values() { + let subtype = font_dict + .get(b"Subtype") + .ok() + .and_then(|o| o.as_name().ok()); + if subtype == Some(b"Type3") { + // Type3 with a ToUnicode CMap can still produce usable text + if font_dict.get(b"ToUnicode").is_ok() { + return false; + } + has_type3 = true; + } else { + // Has a non-Type3 font — page has real text fonts + return false; + } + } + if has_type3 { + log::debug!("page has only Type3 fonts without ToUnicode — text is undecodable"); + } + has_type3 +} + fn scan_xobjects_in_resources( doc: &Document, resources: &lopdf::Dictionary, visited: &mut HashSet, unique_chars: &mut HashSet, -) -> (u32, u32, u32) { +) -> (u32, u32, u32, u32) { let mut text_ops = 0u32; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; let xobjects = match resources.get(b"XObject").ok() { Some(Object::Dictionary(d)) => Some(d.clone()), @@ -503,22 +684,24 @@ fn scan_xobjects_in_resources( let content = stream .decompressed_content() .unwrap_or_else(|_| stream.content.clone()); - let (ops, imgs, paths) = + let (ops, imgs, paths, fonts) = scan_content_for_text_operators(&content, unique_chars); text_ops += ops; image_count += imgs; path_ops += paths; + font_changes += fonts; if let Some(res) = stream .dict .get(b"Resources") .ok() .and_then(|o| o.as_dict().ok()) { - let (ops2, imgs2, paths2) = + let (ops2, imgs2, paths2, fonts2) = scan_xobjects_in_resources(doc, res, visited, unique_chars); text_ops += ops2; image_count += imgs2; path_ops += paths2; + font_changes += fonts2; } } Some(b"Image") => { @@ -529,7 +712,7 @@ fn scan_xobjects_in_resources( } } - (text_ops, image_count, path_ops) + (text_ops, image_count, path_ops, font_changes) } /// Fast scan of content stream bytes for text operators @@ -540,15 +723,16 @@ fn scan_xobjects_in_resources( /// - "'" - move to next line and show text /// - "\"" - set word/char spacing, move to next line, show text /// -/// Returns (text_op_count, image_count, path_op_count). +/// Returns (text_op_count, image_count, path_op_count, font_change_count). /// Unique non-whitespace text characters are collected into `unique_chars`. fn scan_content_for_text_operators( content: &[u8], unique_chars: &mut HashSet, -) -> (u32, u32, u32) { +) -> (u32, u32, u32, u32) { let mut text_ops = 0u32; let mut image_count = 0u32; let mut path_ops = 0u32; + let mut font_changes = 0u32; // Helper: check if position is a word boundary (start of content or preceded by whitespace) let is_word_start = |pos: usize| -> bool { pos == 0 || content[pos - 1].is_ascii_whitespace() }; @@ -561,7 +745,7 @@ fn scan_content_for_text_operators( while i < content.len() { let b = content[i]; - // Look for 'T' followed by 'j' or 'J' + // Look for 'T' followed by 'j', 'J', or 'f' if b == b'T' && i + 1 < content.len() { let next = content[i + 1]; if next == b'j' || next == b'J' { @@ -575,6 +759,15 @@ fn scan_content_for_text_operators( // Scan backward for text string operand to collect unique chars collect_text_chars_before(content, i, unique_chars); } + } else if next == b'f' { + // Tf = set font operator + if i + 2 >= content.len() + || content[i + 2].is_ascii_whitespace() + || content[i + 2] == b'\n' + || content[i + 2] == b'\r' + { + font_changes += 1; + } } } @@ -619,7 +812,7 @@ fn scan_content_for_text_operators( i += 1; } - (text_ops, image_count, path_ops) + (text_ops, image_count, path_ops, font_changes) } /// Scan backward from a Tj/TJ operator to find the preceding string operand @@ -798,9 +991,58 @@ fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { TEMPLATE_IMAGE_THRESHOLD, &mut visited, ); + + // Also check Pattern resources: tiling patterns can contain + // XObject images (e.g., screenshots pasted into PDFs via + // Chrome "Save as PDF"). + if let Ok(pattern_obj) = resources.get(b"Pattern") { + let pattern_dict = match pattern_obj { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(dict) => Some(dict), + _ => None, + }; + if let Some(pattern_dict) = pattern_dict { + for (_, value) in pattern_dict.iter() { + let pat_ref = match value.as_reference() { + Ok(r) => r, + _ => continue, + }; + if !visited.insert(pat_ref) { + continue; + } + if let Ok(Object::Stream(stream)) = doc.get_object(pat_ref) { + if let Ok(pat_resources) = stream.dict.get(b"Resources") { + let pat_res_dict = match pat_resources { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(dict) => Some(dict), + _ => None, + }; + if let Some(pat_res) = pat_res_dict { + collect_images_from_resources( + doc, + pat_res, + &mut has_images, + &mut total_area, + &mut has_template_image, + TEMPLATE_IMAGE_THRESHOLD, + &mut visited, + ); + } + } + } + } + } + } } } + // Tiled scans: many small image tiles (e.g., JBIG2 strips) that together + // cover the full page. No individual tile triggers the template threshold, + // but the aggregate area clearly indicates a scanned/image-backed page. + if !has_template_image && total_area >= TEMPLATE_IMAGE_THRESHOLD * 4 { + has_template_image = true; + } + (has_images, total_area, has_template_image) } @@ -929,7 +1171,7 @@ mod tests { // Sample PDF content stream with text operators let content = b"BT /F1 12 Tf 100 700 Td (Hello World) Tj ET"; - let (ops, imgs, _) = scan_content_for_text_operators(content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars); assert_eq!(ops, 1); assert_eq!(imgs, 0); // "Hello World" without space: H, e, l, o, W, r, d = 7 unique @@ -938,7 +1180,7 @@ mod tests { // Content with TJ array uchars.clear(); let content2 = b"BT /F1 12 Tf 100 700 Td [(H) 10 (ello)] TJ ET"; - let (ops2, _, _) = scan_content_for_text_operators(content2, &mut uchars); + let (ops2, _, _, _) = scan_content_for_text_operators(content2, &mut uchars); assert_eq!(ops2, 1); // H, e, l, o = 4 unique assert!(uchars.len() >= 4); @@ -946,7 +1188,7 @@ mod tests { // Content with Do (image) uchars.clear(); let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q"; - let (ops3, imgs3, _) = scan_content_for_text_operators(content3, &mut uchars); + let (ops3, imgs3, _, _) = scan_content_for_text_operators(content3, &mut uchars); assert_eq!(ops3, 0); assert_eq!(imgs3, 1); } @@ -965,7 +1207,7 @@ mod tests { content.extend_from_slice(b"BT (x) Tj ET\n"); let mut uchars = HashSet::new(); - let (ops, imgs, _) = scan_content_for_text_operators(&content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(ops, 3); assert_eq!(imgs, 50); // Only 'x' unique char @@ -983,7 +1225,7 @@ mod tests { let content = b"BT /F1 12 Tf (The quick brown fox jumps over the lazy dog) Tj ET\n\ /Img1 Do\n/Img2 Do\n"; let mut uchars = HashSet::new(); - let (ops, imgs, _) = scan_content_for_text_operators(content, &mut uchars); + let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars); assert_eq!(ops, 1); assert_eq!(imgs, 2); // Many unique chars from the sentence @@ -1006,7 +1248,7 @@ mod tests { content.extend_from_slice(b"f\n"); let mut uchars = HashSet::new(); - let (text, imgs, paths) = scan_content_for_text_operators(&content, &mut uchars); + let (text, imgs, paths, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(text, 1); assert_eq!(imgs, 0); // 500 * (m + l + c + h) + 1 f = 2001 @@ -1031,7 +1273,7 @@ mod tests { } let mut uchars = HashSet::new(); - let (text, _, paths) = scan_content_for_text_operators(&content, &mut uchars); + let (text, _, paths, _) = scan_content_for_text_operators(&content, &mut uchars); assert_eq!(text, 20); assert!(paths >= 40, "expected >= 40 path ops, got {paths}"); @@ -1074,4 +1316,116 @@ mod tests { ); assert!(result.ocr_recommended); } + + #[test] + fn test_page_has_identity_h_no_tounicode_positive() { + // Build a minimal PDF with a Type0 Identity-H font and no ToUnicode. + use lopdf::dictionary; + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let page_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => Object::Name(b"Type0".to_vec()), + "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()), + "Encoding" => Object::Name(b"Identity-H".to_vec()), + }); + let resources = dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(font_id), + }, + }; + doc.objects.insert( + page_id, + Object::Dictionary(dictionary! { + "Type" => "Page", + "Parent" => Object::Reference(pages_id), + "Resources" => resources, + }), + ); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => Object::Integer(1), + }), + ); + assert!(page_has_identity_h_no_tounicode(&doc, page_id)); + } + + #[test] + fn test_page_has_identity_h_with_tounicode_negative() { + // Type0 Identity-H font WITH ToUnicode — should NOT flag. + use lopdf::dictionary; + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let page_id = doc.new_object_id(); + let cmap_id = doc.add_object(Object::Stream(lopdf::Stream::new( + dictionary! {}, + b"fake cmap".to_vec(), + ))); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => Object::Name(b"Type0".to_vec()), + "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()), + "Encoding" => Object::Name(b"Identity-H".to_vec()), + "ToUnicode" => Object::Reference(cmap_id), + }); + let resources = dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(font_id), + }, + }; + doc.objects.insert( + page_id, + Object::Dictionary(dictionary! { + "Type" => "Page", + "Parent" => Object::Reference(pages_id), + "Resources" => resources, + }), + ); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => Object::Integer(1), + }), + ); + assert!(!page_has_identity_h_no_tounicode(&doc, page_id)); + } + + #[test] + fn test_scan_content_counts_tf_operators() { + let mut uchars = HashSet::new(); + let content = b"BT /F1 12 Tf (Hello) Tj /F2 10 Tf (World) Tj ET"; + let (ops, _, _, fonts) = scan_content_for_text_operators(content, &mut uchars); + assert_eq!(ops, 2); + assert_eq!(fonts, 2); + } + + #[test] + fn test_newspaper_heuristic_thresholds() { + // Newspaper page: high text ops, moderate font changes, low ratio + let text_ops = 3500u32; + let font_changes = 150u32; + let ratio = font_changes as f32 / text_ops as f32; + assert!(text_ops >= 1500); + assert!(font_changes >= 50); + assert!(ratio < 0.15); // 0.043 + + // Dense styled doc (DPA/contract): high text ops, very high font changes, high ratio + let text_ops = 1800u32; + let font_changes = 540u32; + let ratio = font_changes as f32 / text_ops as f32; + assert!(text_ops >= 1500); + assert!(font_changes >= 50); + assert!(ratio >= 0.15); // 0.30 — should NOT trigger newspaper heuristic + + // Normal doc: low text ops — doesn't qualify at all + let text_ops = 300u32; + let font_changes = 50u32; + assert!(text_ops < 1500); + } } diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 2ce06e6..7696563 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -20,12 +20,69 @@ use super::fonts::{ use super::xobjects::{extract_form_xobject_text, get_page_xobjects, XObjectType}; use super::{get_number, multiply_matrices}; +/// Strip PDF comments (% to end of line) from content stream bytes. +/// +/// Some PDF generators (e.g. PD4ML) embed comments in content streams that +/// confuse lopdf's `Content::decode` parser. Comments inside string literals +/// (parentheses) are NOT stripped — only top-level comments. +fn strip_pdf_comments(data: &[u8]) -> Vec { + // Quick check: if no '%' present, return as-is (common case) + if !data.contains(&b'%') { + return data.to_vec(); + } + + let mut result = Vec::with_capacity(data.len()); + let mut i = 0; + let mut in_string = 0i32; // parenthesis nesting depth + let mut in_hex_string = false; + + while i < data.len() { + let b = data[i]; + match b { + b'(' if !in_hex_string => { + in_string += 1; + result.push(b); + } + b')' if !in_hex_string && in_string > 0 => { + in_string -= 1; + result.push(b); + } + b'<' if in_string == 0 && !in_hex_string => { + in_hex_string = true; + result.push(b); + } + b'>' if in_hex_string => { + in_hex_string = false; + result.push(b); + } + b'%' if in_string == 0 && !in_hex_string => { + // Skip until end of line + while i < data.len() && data[i] != b'\n' && data[i] != b'\r' { + i += 1; + } + // Replace comment with a space to preserve token separation + result.push(b' '); + continue; // Don't increment i again + } + _ => { + result.push(b); + } + } + i += 1; + } + + result +} + +/// Returns `(page_extraction, has_gid_fonts)` where `has_gid_fonts` indicates +/// the page uses fonts with unresolvable gid-encoded glyphs. pub(crate) fn extract_page_text_items( doc: &Document, page_id: ObjectId, page_num: u32, font_cmaps: &FontCMaps, -) -> Result { + include_invisible: bool, +) -> Result<(PageExtraction, bool), PdfError> { use lopdf::content::Content; let mut items = Vec::new(); @@ -45,7 +102,7 @@ pub(crate) fn extract_page_text_items( let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); // Build font encoding maps from Differences arrays - let font_encodings = build_font_encodings(doc, &fonts); + let (font_encodings, has_gid_fonts) = build_font_encodings(doc, &fonts); // Build font width info for accurate text positioning let font_widths = build_font_widths(doc, &fonts); @@ -72,12 +129,13 @@ pub(crate) fn extract_page_text_items( if let Ok(obj_ref) = tounicode.as_reference() { font_tounicode_refs.insert(resource_name, obj_ref.0); } else if let Object::Stream(s) = tounicode { - if let Ok(data) = s.decompressed_content() { - if let Some(entry) = - crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) - { - inline_cmaps.insert(resource_name, entry); - } + let data = s + .decompressed_content() + .unwrap_or_else(|_| s.content.clone()); + if let Some(entry) = + crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) + { + inline_cmaps.insert(resource_name, entry); } } } @@ -109,38 +167,74 @@ pub(crate) fn extract_page_text_items( .get_page_content(page_id) .map_err(|e| PdfError::Parse(e.to_string()))?; + // Strip PDF comments (% to end of line) from the content stream. + // Some PDF generators (e.g. PD4ML) embed comments that confuse lopdf's + // Content::decode parser, causing it to skip operators like ET and Q. + let content_data = strip_pdf_comments(&content_data); + let content = Content::decode(&content_data).map_err(|e| PdfError::Parse(e.to_string()))?; + const MAX_OPERATIONS: usize = 1_000_000; + if content.operations.len() > MAX_OPERATIONS { + log::warn!( + "page {}: skipping extraction — {} operations exceeds limit ({})", + page_num, + content.operations.len(), + MAX_OPERATIONS + ); + return Ok(((Vec::new(), Vec::new(), Vec::new()), false)); + } + // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible - let mut gstate_stack: Vec<([f32; 6], i32)> = Vec::new(); + let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new(); // Text state tracking let mut current_font = String::new(); let mut current_font_size: f32 = 12.0; let mut text_leading: f32 = 0.0; // TL parameter (in text-space units) + let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled) + let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled) let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; let mut in_text_block = false; - // Marked content (ActualText) tracking - let mut marked_content_stack: Vec> = Vec::new(); + // Track text direction votes: (horizontal_count, rotated_count). + // For each text item, if |combined[0]| > |combined[1]| the text runs + // horizontally (normal); otherwise it's rotated ~90°. + let mut rotation_votes = RotationVotes { + horizontal: 0, + rotated: 0, + }; + + // Marked content tracking: (ActualText, MCID) per nesting level + struct MarkedContentEntry { + actual_text: Option, + mcid: Option, + } + let mut marked_content_stack: Vec = Vec::new(); let mut suppress_glyph_extraction = false; let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry + /// Get the innermost MCID from the marked content stack. + fn current_mcid(stack: &[MarkedContentEntry]) -> Option { + stack.iter().rev().find_map(|e| e.mcid) + } for op in &content.operations { trace!("{} {:?}", op.operator, op.operands); match op.operator.as_str() { "q" => { // Save graphics state - gstate_stack.push((ctm, text_rendering_mode)); + gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing)); } "Q" => { // Restore graphics state - if let Some((saved_ctm, saved_tr)) = gstate_stack.pop() { + if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() { ctm = saved_ctm; text_rendering_mode = saved_tr; + char_spacing = saved_tc; + word_spacing = saved_tw; } } "cm" => { @@ -193,6 +287,18 @@ pub(crate) fn extract_page_text_items( text_rendering_mode = mode as i32; } } + "Tc" => { + // Set character spacing (extra space added after each character) + if let Some(tc) = op.operands.first().and_then(get_number) { + char_spacing = tc; + } + } + "Tw" => { + // Set word spacing (extra space added for each space character) + if let Some(tw) = op.operands.first().and_then(get_number) { + word_spacing = tw; + } + } "Td" | "TD" => { // Move text position: TLM = T(tx,ty) × TLM; Tm = TLM // tx,ty are in text space — must be scaled by the text line matrix @@ -233,8 +339,15 @@ pub(crate) fn extract_page_text_items( if in_text_block && !op.operands.is_empty() { // Advance text matrix regardless of visibility let w_ts_opt = font_widths.get(¤t_font).and_then(|fi| { - get_operand_bytes(&op.operands[0]) - .map(|raw| compute_string_width_ts(raw, fi, current_font_size)) + get_operand_bytes(&op.operands[0]).map(|raw| { + compute_string_width_ts( + raw, + fi, + current_font_size, + char_spacing, + word_spacing, + ) + }) }); // ActualText: suppress glyph extraction, just advance text matrix if suppress_glyph_extraction { @@ -244,8 +357,10 @@ pub(crate) fn extract_page_text_items( } continue; } - // Skip invisible (Tr=3) text but still advance text matrix - if text_rendering_mode == 3 { + // Skip invisible (Tr=3) text but still advance text matrix. + // For Mixed/template PDFs, include_invisible=true extracts + // the OCR text layer that sits behind scanned images. + if text_rendering_mode == 3 && !include_invisible { if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -266,6 +381,11 @@ pub(crate) fn extract_page_text_items( let combined = multiply_matrices(&text_matrix, &ctm); let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let width = if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -292,6 +412,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -302,7 +423,8 @@ pub(crate) fn extract_page_text_items( if in_text_block && !op.operands.is_empty() { if let Ok(array) = op.operands[0].as_array() { let font_info = font_widths.get(¤t_font); - let is_invisible = text_rendering_mode == 3 || suppress_glyph_extraction; + let is_invisible = (text_rendering_mode == 3 && !include_invisible) + || suppress_glyph_extraction; // Compute space threshold based on font metrics when available let space_threshold = if let Some(font_info) = font_info { @@ -379,8 +501,13 @@ pub(crate) fn extract_page_text_items( } if let Some(fi) = font_info { if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); + total_width_ts += compute_string_width_ts( + raw_bytes, + fi, + current_font_size, + char_spacing, + word_spacing, + ); } } if !is_invisible { @@ -406,6 +533,11 @@ pub(crate) fn extract_page_text_items( // Emit one TextItem per sub-item if !sub_items.is_empty() { let combined = multiply_matrices(&text_matrix, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let rendered_size = effective_font_size(current_font_size, &combined); let base_font = font_base_names .get(¤t_font) @@ -440,6 +572,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -461,7 +594,7 @@ pub(crate) fn extract_page_text_items( line_matrix[4] += (-tl) * line_matrix[2]; line_matrix[5] += (-tl) * line_matrix[3]; text_matrix = line_matrix; - if !(text_rendering_mode == 3 + if !((text_rendering_mode == 3 && !include_invisible) || suppress_glyph_extraction || op.operands.is_empty()) { @@ -478,6 +611,11 @@ pub(crate) fn extract_page_text_items( ) { if !text.trim().is_empty() { let combined = multiply_matrices(&text_matrix, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); let base_font = font_base_names @@ -496,6 +634,7 @@ pub(crate) fn extract_page_text_items( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: current_mcid(&marked_content_stack), }); } } @@ -531,11 +670,15 @@ pub(crate) fn extract_page_text_items( } "BMC" => { // Begin Marked Content (no properties) - marked_content_stack.push(None); + marked_content_stack.push(MarkedContentEntry { + actual_text: None, + mcid: None, + }); } "BDC" => { - // Begin Marked Content with properties — extract ActualText + // Begin Marked Content with properties — extract ActualText and MCID let mut actual_text: Option = None; + let mut mcid: Option = None; if op.operands.len() >= 2 { let dict = match &op.operands[1] { Object::Dictionary(d) => Some(d.clone()), @@ -549,47 +692,61 @@ pub(crate) fn extract_page_text_items( _ => None, }; } + if let Ok(Object::Integer(id)) = d.get(b"MCID") { + mcid = Some(*id); + } } } if actual_text.is_some() { suppress_glyph_extraction = true; actual_text_start_tm = Some(text_matrix); } - marked_content_stack.push(actual_text); + marked_content_stack.push(MarkedContentEntry { actual_text, mcid }); } "EMC" => { // End Marked Content — emit ActualText item with correct width - if let Some(Some(at)) = marked_content_stack.pop() { - // Compute width from text matrix advancement during BDC..EMC - if let Some(start_tm) = actual_text_start_tm.take() { - let combined = multiply_matrices(&start_tm, &ctm); - let rendered_size = effective_font_size(current_font_size, &combined); - let (x, y) = (combined[4], combined[5]); - // Width in device space from text matrix delta - let delta_ts = text_matrix[4] - start_tm[4]; - let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; - let width = (delta_ts * scale_x).abs(); - if !at.trim().is_empty() { - let base_font = font_base_names - .get(¤t_font) - .map(|s| s.as_str()) - .unwrap_or(¤t_font); - items.push(TextItem { - text: expand_ligatures(&at), - x, - y, - width, - height: rendered_size, - font: current_font.clone(), - font_size: rendered_size, - page: page_num, - is_bold: is_bold_font(base_font), - is_italic: is_italic_font(base_font), - item_type: ItemType::Text, - }); + if let Some(entry) = marked_content_stack.pop() { + if let Some(at) = entry.actual_text { + // Compute width from text matrix advancement during BDC..EMC + if let Some(start_tm) = actual_text_start_tm.take() { + let combined = multiply_matrices(&start_tm, &ctm); + if combined[0].abs() >= combined[1].abs() { + rotation_votes.horizontal += 1; + } else { + rotation_votes.rotated += 1; + } + let rendered_size = effective_font_size(current_font_size, &combined); + let (x, y) = (combined[4], combined[5]); + // Width in device space from text matrix delta + let delta_ts = text_matrix[4] - start_tm[4]; + let scale_x = start_tm[0] * ctm[0] + start_tm[1] * ctm[2]; + let width = (delta_ts * scale_x).abs(); + if !at.trim().is_empty() { + let base_font = font_base_names + .get(¤t_font) + .map(|s| s.as_str()) + .unwrap_or(¤t_font); + items.push(TextItem { + text: expand_ligatures(&at), + x, + y, + width, + height: rendered_size, + font: current_font.clone(), + font_size: rendered_size, + page: page_num, + is_bold: is_bold_font(base_font), + is_italic: is_italic_font(base_font), + item_type: ItemType::Text, + mcid: entry + .mcid + .or_else(|| current_mcid(&marked_content_stack)), + }); + } } + suppress_glyph_extraction = + marked_content_stack.iter().any(|e| e.actual_text.is_some()); } - suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); } } "re" => { @@ -848,8 +1005,94 @@ pub(crate) fn extract_page_text_items( } } + // Detect dominant text rotation and transform coordinates if needed. + // Some PDFs embed landscape content in portrait pages using a rotated text + // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW). The layout engine + // assumes x=horizontal, y=vertical — so we swap coordinates to match. + let (items, rects, lines) = correct_rotated_page(items, rects, lines, &rotation_votes); + let items = super::merge_text_items(items); - Ok((items, rects, lines)) + let items = super::merge_subscript_items(items); + Ok(((items, rects, lines), has_gid_fonts)) +} + +/// Counts of text operators with horizontal vs rotated combined matrices. +struct RotationVotes { + horizontal: u32, + rotated: u32, +} + +/// Detect if most text items on a page are rotated 90° or 270°, and if so, +/// swap x↔y coordinates (plus widths/heights) so the layout engine sees +/// them as horizontal text on a landscape page. +fn correct_rotated_page( + mut items: Vec, + mut rects: Vec, + mut lines: Vec, + votes: &RotationVotes, +) -> (Vec, Vec, Vec) { + if items.len() < 2 { + return (items, rects, lines); + } + + // Use the combined-matrix direction votes collected during extraction. + // For normal text, combined[0] (the x-component of the text x-axis) is + // large; for 90° rotated text, combined[1] dominates instead. + let total_votes = votes.horizontal + votes.rotated; + if total_votes == 0 || votes.rotated * 3 < total_votes * 2 { + // Less than ~67% of text operators are rotated → not a rotated page + return (items, rects, lines); + } + + log::debug!( + "detected rotated page text: {}/{} text ops are rotated — swapping coordinates", + votes.rotated, + total_votes + ); + + // For 90° CCW rotation (the common case: Tm = [0, b, -b, 0, tx, ty]): + // device x increases = visual "down" → negate when mapping to y + // device y increases = visual "right" → use directly as x + // The layout engine sorts by y descending (highest = top of page), so + // we negate old_x so that visual-top (low device x) gets high new_y. + for item in &mut items { + let new_x = item.y; + let new_y = -item.x; + item.x = new_x; + item.y = new_y; + // For rotated text, the "width" along the reading direction was + // lost (computed as 0 due to scale_x ≈ 0). Estimate from text + // length × approximate char width. font_size is the rendered + // height in device space, which for 90° rotation corresponds to + // the horizontal extent of one em. + if item.width < 0.5 { + let char_count = item.text.chars().count() as f32; + item.width = char_count * item.font_size * 0.5; + } + } + + // Transform rectangles + for rect in &mut rects { + let new_x = rect.y; + let new_y = -(rect.x + rect.width.abs()); + rect.x = new_x; + rect.y = new_y; + std::mem::swap(&mut rect.width, &mut rect.height); + } + + // Transform lines + for line in &mut lines { + let new_x1 = line.y1; + let new_y1 = -line.x1; + let new_x2 = line.y2; + let new_y2 = -line.x2; + line.x1 = new_x1; + line.y1 = new_y1; + line.x2 = new_x2; + line.y2 = new_y2; + } + + (items, rects, lines) } /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance). @@ -949,4 +1192,77 @@ mod tests { dedup_rects(&mut single); assert_eq!(single.len(), 1); } + + #[test] + fn test_skip_excessive_operations() { + use crate::tounicode::FontCMaps; + use lopdf::{dictionary, Object, Stream}; + + let mut doc = lopdf::Document::new(); + + // "0 0 m\n" = 6 bytes per op, 1_100_000 ops → ~6.6 MB content stream + let ops_bytes = "0 0 m\n".repeat(1_100_000).into_bytes(); + let stream = Stream::new(dictionary! {}, ops_bytes); + let content_id = doc.add_object(Object::Stream(stream)); + + let page_dict = dictionary! { + "Type" => "Page", + "Contents" => Object::Reference(content_id), + "Resources" => dictionary! {}, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }; + let page_id = doc.add_object(page_dict); + + // Register the page so get_page_content can find it + let pages_dict = dictionary! { + "Type" => "Pages", + "Count" => Object::Integer(1), + "Kids" => vec![Object::Reference(page_id)], + }; + let pages_id = doc.add_object(pages_dict); + let catalog = dictionary! { + "Type" => "Catalog", + "Pages" => Object::Reference(pages_id), + }; + doc.add_object(catalog); + + let font_cmaps = FontCMaps::from_doc(&doc); + let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let ((items, rects, lines), _has_gid) = result; + assert!(items.is_empty()); + assert!(rects.is_empty()); + assert!(lines.is_empty()); + } + + #[test] + fn test_strip_pdf_comments() { + // Basic comment stripping + let input = b"BT\n% comment\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"BT\n \nTj\nET\n"); + + // No comments = unchanged + let input = b"BT\nTj\nET\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, input.to_vec()); + + // Don't strip inside string literals + let input = b"(text with % not a comment)\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"(text with % not a comment)\n \n"); + + // Don't strip inside hex strings + let input = b"<0033% not a comment>\n% real comment\n"; + let output = strip_pdf_comments(input); + assert_eq!(output, b"<0033% not a comment>\n \n"); + + // PD4ML style: comment between Tj and ET + let input = b"<0033> Tj\n\t% Mission Statement\n\tET\n"; + let output = strip_pdf_comments(input); + let output_str = String::from_utf8_lossy(&output); + assert!( + output_str.contains("ET"), + "ET should be preserved after comment stripping" + ); + } } diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index e6f3b78..e4f6b84 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -429,15 +429,23 @@ pub(crate) fn parse_cid_w_array( /// Compute the width of a string in text space units, /// given raw bytes and font width info. /// Returns width in text space units (font_units * units_scale * font_size). +/// +/// `char_spacing` (Tc) is added per character and `word_spacing` (Tw) is added +/// per space character (byte 0x20), both in unscaled text-space units. +/// Per the PDF spec: tx = (w0 × Tfs + Tc + Tw_if_space) per glyph. pub(crate) fn compute_string_width_ts( bytes: &[u8], font_info: &FontWidthInfo, font_size: f32, + char_spacing: f32, + word_spacing: f32, ) -> f32 { let mut total: f32 = 0.0; - if font_info.is_cid { + let mut num_spaces: usize = 0; + let num_chars = if font_info.is_cid { // 2-byte (big-endian) character codes let mut j = 0; + let mut count = 0usize; while j + 1 < bytes.len() { let cid = u16::from_be_bytes([bytes[j], bytes[j + 1]]); let w = font_info @@ -446,8 +454,14 @@ pub(crate) fn compute_string_width_ts( .copied() .unwrap_or(font_info.default_width); total += w as f32; + // CID 32 = space in most CID fonts + if cid == 32 { + num_spaces += 1; + } + count += 1; j += 2; } + count } else { // 1-byte character codes for &b in bytes { @@ -458,10 +472,17 @@ pub(crate) fn compute_string_width_ts( .copied() .unwrap_or(font_info.default_width); total += w as f32; + if b == 0x20 { + num_spaces += 1; + } } - } + bytes.len() + }; // Convert from font units to text space using the font's scale factor + // Then add Tc per character and Tw per space character total * font_info.units_scale * font_size + + num_chars as f32 * char_spacing + + num_spaces as f32 * word_spacing } /// Extract raw bytes from a PDF operand (String object) @@ -473,29 +494,37 @@ pub(crate) fn get_operand_bytes(obj: &Object) -> Option<&[u8]> { } } -/// Build encoding maps for all fonts on a page +/// Build encoding maps for all fonts on a page. +/// Returns `(encodings, has_gid_fonts)` where `has_gid_fonts` is true when +/// any font uses raw glyph ID names (gidNNNNN) that can't be decoded. pub(crate) fn build_font_encodings( doc: &Document, fonts: &std::collections::BTreeMap, &lopdf::Dictionary>, -) -> PageFontEncodings { +) -> (PageFontEncodings, bool) { let mut encodings = PageFontEncodings::new(); + let mut has_gid_fonts = false; for (font_name, font_dict) in fonts { let resource_name = String::from_utf8_lossy(font_name).to_string(); - if let Some(encoding_map) = parse_font_encoding(doc, font_dict) { - encodings.insert(resource_name, encoding_map); + if let Some(result) = parse_font_encoding(doc, font_dict) { + if result.gid_glyph_count > 0 { + has_gid_fonts = true; + } + if !result.map.is_empty() { + encodings.insert(resource_name, result.map); + } } } - encodings + (encodings, has_gid_fonts) } /// Parse font encoding from a font dictionary pub(crate) fn parse_font_encoding( doc: &Document, font_dict: &lopdf::Dictionary, -) -> Option { +) -> Option { let encoding_obj = font_dict.get(b"Encoding").ok()?; // Encoding can be a name or a dictionary @@ -519,11 +548,21 @@ pub(crate) fn parse_font_encoding( } } +/// Result of parsing an encoding dictionary's Differences array. +pub(crate) struct EncodingResult { + pub map: FontEncodingMap, + /// Number of glyph names matching the `gidNNNNN` pattern (raw glyph IDs). + /// These indicate a font with unresolvable encoding — the glyph IDs + /// reference the original font's glyph table, but without the original + /// font's cmap there is no way to map them to Unicode. + pub gid_glyph_count: u32, +} + /// Parse an encoding dictionary with Differences array pub(crate) fn parse_encoding_dictionary( doc: &Document, enc_dict: &lopdf::Dictionary, -) -> Option { +) -> Option { let differences = enc_dict.get(b"Differences").ok()?; let diff_array = match differences { @@ -541,6 +580,7 @@ pub(crate) fn parse_encoding_dictionary( let mut encoding_map = FontEncodingMap::new(); let mut current_code: u8 = 0; let mut ligature_count = 0u32; + let mut gid_glyph_count = 0u32; for item in diff_array { match item { @@ -562,6 +602,14 @@ pub(crate) fn parse_encoding_dictionary( ); ligature_count += 1; } + // Detect raw glyph ID names (e.g. "gid00053") that can't be + // mapped to Unicode without the original font's cmap table. + if glyph_name.starts_with("gid") + && glyph_name.len() >= 4 + && glyph_name[3..].chars().all(|c| c.is_ascii_digit()) + { + gid_glyph_count += 1; + } if let Some(ch) = glyph_to_char(&glyph_name) { encoding_map.insert(current_code, ch); } else { @@ -584,11 +632,17 @@ pub(crate) fn parse_encoding_dictionary( ); } - if encoding_map.is_empty() { - None - } else { - Some(encoding_map) + if gid_glyph_count > 0 { + debug!( + " Differences: {} gid-encoded glyphs (unresolvable without original font)", + gid_glyph_count + ); } + + Some(EncodingResult { + map: encoding_map, + gid_glyph_count, + }) } /// Get the CMap lookup key for an Identity-H/V CID font without ToUnicode. @@ -1037,6 +1091,92 @@ fn score_text(text: &str) -> i32 { mod tests { use super::*; + fn make_font_info(widths: &[(u16, u16)], default_width: u16, is_cid: bool) -> FontWidthInfo { + FontWidthInfo { + widths: widths.iter().copied().collect(), + default_width, + space_width: widths + .iter() + .find(|(k, _)| *k == 32) + .map(|(_, v)| *v) + .unwrap_or(default_width), + is_cid, + units_scale: 0.001, + wmode: 0, + } + } + + #[test] + fn compute_string_width_ts_no_tc_tw() { + // Without Tc/Tw (both 0), width = glyph widths only + let fi = make_font_info(&[(72, 500), (101, 400), (108, 300)], 600, false); + let bytes = b"Hello"; // H=500, e=400, l=300, l=300, o=600(default) + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.0); + // (500+400+300+300+600) * 0.001 * 10 = 21.0 + assert!((w - 21.0).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_positive_tc() { + // Positive Tc adds char_spacing per character + let fi = make_font_info(&[], 500, false); + let bytes = b"ab"; // 2 chars, each 500 default + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.5, 0.0); + // glyph: (500+500)*0.001*10 = 10.0, Tc: 2*0.5 = 1.0, total = 11.0 + assert!((w - 11.0).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_negative_tc() { + // Negative Tc (tight tracking) reduces width + let fi = make_font_info(&[], 500, false); + let bytes = b"ab"; + let w = compute_string_width_ts(bytes, &fi, 10.0, -0.3, 0.0); + // glyph: 10.0, Tc: 2*(-0.3) = -0.6, total = 9.4 + assert!((w - 9.4).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_tw() { + // Tw applies only to space characters (byte 0x20) + let fi = make_font_info(&[(32, 250)], 500, false); + let bytes = b"a b"; // 'a'=500, ' '=250, 'b'=500 + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.8); + // glyph: (500+250+500)*0.001*10 = 12.5, Tw: 1*0.8 = 0.8, total = 13.3 + assert!((w - 13.3).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_with_tc_and_tw() { + // Both Tc and Tw + let fi = make_font_info(&[(32, 250)], 500, false); + let bytes = b"a b"; // 3 chars, 1 space + let w = compute_string_width_ts(bytes, &fi, 10.0, 0.1, 0.5); + // glyph: 12.5, Tc: 3*0.1 = 0.3, Tw: 1*0.5 = 0.5, total = 13.3 + assert!((w - 13.3).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_cid_font() { + // CID font: 2-byte codes, space is CID 32 + let fi = make_font_info(&[(65, 500), (32, 250)], 600, true); + // "A " in CID: [0,65, 0,32] + let bytes = &[0u8, 65, 0, 32]; + let w = compute_string_width_ts(bytes, &fi, 12.0, 0.2, 0.3); + // glyph: (500+250)*0.001*12 = 9.0, Tc: 2*0.2 = 0.4, Tw: 1*0.3 = 0.3 + assert!((w - 9.7).abs() < 0.01); + } + + #[test] + fn compute_string_width_ts_large_tc() { + // Large Tc (character-spreading) is applied in full + let fi = make_font_info(&[], 500, false); + let bytes = b"abc"; // 3 chars + let w = compute_string_width_ts(bytes, &fi, 10.0, 5.0, 0.0); + // glyph: (500*3)*0.001*10 = 15.0, Tc: 3*5.0 = 15.0, total = 30.0 + assert!((w - 30.0).abs() < 0.01); + } + #[test] fn score_text_cjk() { // Correct Japanese text should score well diff --git a/src/extractor/layout.rs b/src/extractor/layout.rs index ee582a8..60cf114 100644 --- a/src/extractor/layout.rs +++ b/src/extractor/layout.rs @@ -1,5 +1,7 @@ //! Column detection, line grouping, and reading-order layout. +use std::collections::{HashMap, HashSet}; + use crate::text_utils::{effective_width, sort_line_items}; use crate::types::{TextItem, TextLine}; use log::debug; @@ -16,7 +18,11 @@ pub(crate) struct ColumnRegion { /// Builds an occupancy histogram across the page width and finds empty valleys /// (gutters) where no text exists. Validates valleys with vertical consistency /// checks to avoid false positives. -pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec { +pub(crate) fn detect_columns( + items: &[TextItem], + page: u32, + page_has_table: bool, +) -> Vec { const BIN_WIDTH: f32 = 2.0; const MIN_GUTTER_WIDTH: f32 = 8.0; const MIN_VERTICAL_SPAN_RATIO: f32 = 0.30; @@ -108,10 +114,358 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec }) .collect(); - if valleys.is_empty() { + // Fallback: if no absolute valleys found, try relative valley detection. + // Justified text can leave gutter bins non-empty because item widths extend + // to the column edge. Look for local minima that are significantly lower + // than the peaks on either side. + // Only attempt this for dense pages (>=100 items) — sparse pages with shallow + // histogram dips are likely not multi-column. + // Skip on pages with detected tables — table column gaps look like gutters + // in the histogram but the table pipeline already handles reading order. + if valleys.is_empty() && page_items.len() >= 100 && !page_has_table { + let rel_valleys = find_relative_valleys( + &histogram, + num_bins, + x_min, + BIN_WIDTH, + page_width, + margin_threshold, + ); + if !rel_valleys.is_empty() { + let result = validate_and_build_columns( + &rel_valleys, + &page_items, + x_min, + BIN_WIDTH, + x_max, + MIN_ITEMS_PER_COLUMN, + MIN_VERTICAL_SPAN_RATIO, + page, + true, // center-based assignment for relative valleys + ); + if result.len() > 1 { + // Validate that both sides contain paragraph-like content. + // Tables, forms, and checklists have short scattered items + // that create false gutter signals. Only commit to relative + // valley columns when both sides look like flowing prose. + if columns_have_prose(&result, &page_items) { + debug!( + "page {}: relative valley detection found {} columns", + page, + result.len() + ); + return result; + } else { + debug!( + "page {}: relative valley rejected — columns lack prose density", + page, + ); + } + } + } return vec![ColumnRegion { x_min, x_max }]; } + return validate_and_build_columns( + &valleys, + &page_items, + x_min, + BIN_WIDTH, + x_max, + MIN_ITEMS_PER_COLUMN, + MIN_VERTICAL_SPAN_RATIO, + page, + false, // edge-based assignment for absolute valleys + ); +} + +/// Check whether each proposed column contains paragraph-like content. +/// +/// Groups items per column into rough lines by Y-proximity, then measures +/// what fraction of those lines span a significant portion of the column +/// width. Two-column prose (justified or ragged-right) produces lines that +/// fill most of the column width. Tables, forms, and checklists produce +/// short scattered items that don't. +/// +/// Returns true only when *every* column passes a minimum prose density. +fn columns_have_prose(columns: &[ColumnRegion], items: &[&TextItem]) -> bool { + const Y_TOL: f32 = 3.0; // y-proximity to group items into the same line + const LINE_FILL_THRESHOLD: f32 = 0.45; // line must span ≥45% of column width + const MIN_PROSE_RATIO: f32 = 0.40; // ≥40% of lines must be "full" + const MIN_LINES: usize = 8; // need enough lines to judge + const MIN_COL_WIDTH: f32 = 120.0; // columns must be ≥120pt (not narrow sidebars/fragments) + const MAX_AVG_ITEMS_PER_LINE: f32 = 3.5; // prose has 1-3 items/line; tables/forms have 4+ + + for col in columns { + let col_width = col.x_max - col.x_min; + if col_width < MIN_COL_WIDTH { + return false; + } + + // Collect items whose center falls within this column + let col_items: Vec<&TextItem> = items + .iter() + .filter(|i| { + let center = i.x + effective_width(i) / 2.0; + center >= col.x_min && center <= col.x_max + }) + .copied() + .collect(); + + if col_items.len() < MIN_LINES { + return false; + } + + // Sort by Y descending (top of page = higher Y in PDF coords) + let mut sorted: Vec<&TextItem> = col_items; + sorted.sort_by(|a, b| b.y.partial_cmp(&a.y).unwrap_or(std::cmp::Ordering::Equal)); + + // Group into lines by Y-proximity and measure fill + item count + let mut full_lines = 0usize; + let mut total_lines = 0usize; + let mut total_items_in_lines = 0usize; + let mut line_items: Vec<&TextItem> = Vec::new(); + let mut line_y = f32::NAN; + + let flush_line = |line_items: &[&TextItem], + full: &mut usize, + total: &mut usize, + total_items: &mut usize| { + if line_items.is_empty() { + return; + } + *total += 1; + *total_items += line_items.len(); + // Compute the span of text on this line within the column + let left = line_items + .iter() + .map(|i| i.x.max(col.x_min)) + .fold(f32::INFINITY, f32::min); + let right = line_items + .iter() + .map(|i| (i.x + effective_width(i)).min(col.x_max)) + .fold(f32::NEG_INFINITY, f32::max); + let span = (right - left).max(0.0); + if span >= col_width * LINE_FILL_THRESHOLD { + *full += 1; + } + }; + + for item in &sorted { + if line_items.is_empty() || (line_y - item.y).abs() < Y_TOL { + if line_items.is_empty() { + line_y = item.y; + } + line_items.push(item); + } else { + flush_line( + &line_items, + &mut full_lines, + &mut total_lines, + &mut total_items_in_lines, + ); + line_items.clear(); + line_y = item.y; + line_items.push(item); + } + } + flush_line( + &line_items, + &mut full_lines, + &mut total_lines, + &mut total_items_in_lines, + ); + + if total_lines < MIN_LINES { + return false; + } + + let ratio = full_lines as f32 / total_lines as f32; + let avg_items = total_items_in_lines as f32 / total_lines as f32; + debug!( + "columns_have_prose: col [{:.0}..{:.0}] lines={} full={} ratio={:.2} avg_items={:.1}", + col.x_min, col.x_max, total_lines, full_lines, ratio, avg_items + ); + if ratio < MIN_PROSE_RATIO { + return false; + } + // Tables and forms tend to have many small items per line (one per cell), + // while prose has few items per line (one per word-run or phrase). + if avg_items > MAX_AVG_ITEMS_PER_LINE { + return false; + } + } + + true +} + +/// Find relative valleys (local minima) in the histogram. +/// +/// When justified text fills gutters, the absolute noise threshold fails. +/// This finds local minima where the count drops significantly below +/// the peaks on either side — indicating a gutter even when not empty. +fn find_relative_valleys( + histogram: &[u32], + num_bins: usize, + _x_min: f32, + bin_width: f32, + page_width: f32, + margin_threshold: f32, +) -> Vec<(usize, usize)> { + const MIN_GUTTER_BINS: usize = 2; // minimum 4pt gutter + const CONTRAST_THRESHOLD: f32 = 0.60; // valley must be < 60% of surrounding peaks + const PEAK_WINDOW: usize = 25; // look 50pt on each side for peaks + const MIN_PEAK_HEIGHT: f32 = 20.0; // peaks must be ≥20 (dense text columns) + + if num_bins < 10 { + return vec![]; + } + + // Smooth histogram with a 5-bin moving average to reduce noise + let mut smoothed = vec![0.0f32; num_bins]; + let half_win = 2usize; + for (i, s) in smoothed.iter_mut().enumerate().take(num_bins) { + let lo = i.saturating_sub(half_win); + let hi = (i + half_win + 1).min(num_bins); + let sum: u32 = histogram[lo..hi].iter().sum(); + *s = sum as f32 / (hi - lo) as f32; + } + + // Find local minima: positions where smoothed value is lower than + // both sides within a search window + let mut candidates: Vec<(usize, f32, f32)> = Vec::new(); // (bin, valley_val, contrast) + + for i in PEAK_WINDOW..num_bins.saturating_sub(PEAK_WINDOW) { + let val = smoothed[i]; + if val < 1.0 { + continue; // skip empty margins + } + + // Check this is a local minimum within a small window + let local_lo = i.saturating_sub(3); + let local_hi = (i + 4).min(num_bins); + let is_local_min = (local_lo..local_hi).all(|j| smoothed[j] >= val - 0.5); + if !is_local_min { + continue; + } + + // Find peak values on each side + let left_peak = smoothed[i.saturating_sub(PEAK_WINDOW)..i] + .iter() + .cloned() + .fold(0.0f32, f32::max); + let right_peak = smoothed[(i + 1)..(i + 1 + PEAK_WINDOW).min(num_bins)] + .iter() + .cloned() + .fold(0.0f32, f32::max); + + if left_peak < MIN_PEAK_HEIGHT || right_peak < MIN_PEAK_HEIGHT { + continue; + } + + // Both peaks must be substantial — prevents detecting margin drop-offs + // as gutters in single-column layouts with ragged text. + let peak_balance = left_peak.min(right_peak) / left_peak.max(right_peak); + if peak_balance < 0.40 { + continue; + } + + // Contrast: ratio of valley to the smaller of the two peaks + let ref_peak = left_peak.min(right_peak); + let contrast = val / ref_peak; + + if contrast < CONTRAST_THRESHOLD { + // Check margin constraint + let center_pts = i as f32 * bin_width; + if center_pts > margin_threshold && center_pts < (page_width - margin_threshold) { + candidates.push((i, val, contrast)); + } + } + } + + if candidates.is_empty() { + return vec![]; + } + + // Group adjacent candidates into valley ranges and pick the deepest point + let mut valleys: Vec<(usize, usize)> = Vec::new(); + let mut best_bin = candidates[0].0; + let mut best_contrast = candidates[0].2; + + for window in candidates.windows(2) { + let (prev_bin, _, _) = window[0]; + let (next_bin, _, next_contrast) = window[1]; + + if next_bin - prev_bin <= 5 { + // Same group + if next_contrast < best_contrast { + best_bin = next_bin; + best_contrast = next_contrast; + } + } else { + // End current group + let half = MIN_GUTTER_BINS; + valleys.push(( + best_bin.saturating_sub(half), + (best_bin + half + 1).min(num_bins), + )); + best_bin = next_bin; + best_contrast = next_contrast; + } + } + // Close last group + let half = MIN_GUTTER_BINS; + valleys.push(( + best_bin.saturating_sub(half), + (best_bin + half + 1).min(num_bins), + )); + + // Limit to the single best valley (deepest contrast). + // Multi-column layouts with 3+ columns typically have clear gutters that + // the absolute valley detection handles. The relative fallback is designed + // for 2-column layouts where justified text fills the gutter. + if valleys.len() > 1 { + // Keep only the valley with the best (lowest) contrast in the candidates + let mut best_idx = 0; + let mut best_c = f32::MAX; + for (vi, v) in valleys.iter().enumerate() { + let mid = (v.0 + v.1) / 2; + // Find the candidate closest to this valley's midpoint + if let Some(c) = candidates + .iter() + .filter(|(b, _, _)| (*b as isize - mid as isize).unsigned_abs() <= 5) + .map(|(_, _, c)| *c) + .reduce(f32::min) + { + if c < best_c { + best_c = c; + best_idx = vi; + } + } + } + return vec![valleys[best_idx]]; + } + + valleys +} + +/// Validate valley candidates with vertical consistency checks and build column regions. +/// +/// When `center_assign` is true, items are assigned to columns based on their +/// center point rather than their right edge. This helps when justified text +/// items extend past the gutter. +#[allow(clippy::too_many_arguments)] +fn validate_and_build_columns( + valleys: &[(usize, usize)], + page_items: &[&TextItem], + x_min: f32, + bin_width: f32, + x_max: f32, + min_items: usize, + min_vertical_span: f32, + page: u32, + center_assign: bool, +) -> Vec { // Compute Y range of the page let y_min = page_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min); let y_max = page_items @@ -121,22 +475,37 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let y_range = y_max - y_min; // Validate each valley with vertical consistency - // Each entry: (start_bin, end_bin, left_count, right_count) let mut valid_valleys: Vec<(usize, usize, usize, usize)> = Vec::new(); - for &(start, end) in &valleys { - let gutter_left = x_min + start as f32 * BIN_WIDTH; - let gutter_right = x_min + end as f32 * BIN_WIDTH; + for &(start, end) in valleys { + let gutter_left = x_min + start as f32 * bin_width; + let gutter_right = x_min + end as f32 * bin_width; let gutter_center = (gutter_left + gutter_right) / 2.0; - // Collect items on each side of the gutter + // Collect items on each side of the gutter. + // Center-based: use item midpoint (better for justified text). + // Edge-based: use item right edge (original behavior). let left_items: Vec<&&TextItem> = page_items .iter() - .filter(|i| i.x + effective_width(i) <= gutter_center) + .filter(|i| { + if center_assign { + i.x + effective_width(i) / 2.0 <= gutter_center + } else { + i.x + effective_width(i) <= gutter_center + } + }) + .collect(); + let right_items: Vec<&&TextItem> = page_items + .iter() + .filter(|i| { + if center_assign { + i.x + effective_width(i) / 2.0 > gutter_center + } else { + i.x >= gutter_center + } + }) .collect(); - let right_items: Vec<&&TextItem> = - page_items.iter().filter(|i| i.x >= gutter_center).collect(); - if left_items.len() < MIN_ITEMS_PER_COLUMN || right_items.len() < MIN_ITEMS_PER_COLUMN { + if left_items.len() < min_items || right_items.len() < min_items { continue; } @@ -160,7 +529,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let overlap_max = left_y_max.min(right_y_max); let overlap = (overlap_max - overlap_min).max(0.0); - if overlap / y_range < MIN_VERTICAL_SPAN_RATIO { + if overlap / y_range < min_vertical_span { continue; } } @@ -169,6 +538,11 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec } if valid_valleys.is_empty() { + debug!( + "page {}: {} valleys found but none passed validation", + page, + valleys.len() + ); return vec![ColumnRegion { x_min, x_max }]; } @@ -178,14 +552,12 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec valid_valleys.len() + 1, valid_valleys .iter() - .map(|(s, e, _, _)| x_min + ((*s + *e) as f32 / 2.0) * BIN_WIDTH) + .map(|(s, e, _, _)| x_min + ((*s + *e) as f32 / 2.0) * bin_width) .collect::>() ); // Limit to at most 3 gutters (4 columns). // Score = width_in_bins * min(left_count, right_count) - // This prefers gutters that separate substantial content on both sides, - // rather than just the physically widest gaps (which may be intra-column). if valid_valleys.len() > 3 { valid_valleys.sort_by(|a, b| { let score_a = (a.1 - a.0) as f32 * (a.2.min(a.3) as f32); @@ -195,7 +567,6 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec .unwrap_or(std::cmp::Ordering::Equal) }); valid_valleys.truncate(3); - // Re-sort by position (left to right) valid_valleys.sort_by_key(|v| v.0); } @@ -203,7 +574,7 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec let mut columns = Vec::new(); let mut col_start = x_min; for &(start, end, _, _) in &valid_valleys { - let gutter_center = x_min + ((start + end) as f32 / 2.0) * BIN_WIDTH; + let gutter_center = x_min + ((start + end) as f32 / 2.0) * bin_width; columns.push(ColumnRegion { x_min: col_start, x_max: gutter_center, @@ -218,6 +589,105 @@ pub(crate) fn detect_columns(items: &[TextItem], page: u32) -> Vec columns } +/// Identify items that belong to lines spanning across detected columns. +/// +/// Groups items into rough lines by Y-proximity and marks items whose line's +/// combined X-span exceeds 1.3× the widest column AND has no gap located at +/// a detected gutter boundary. Returns a boolean mask parallel to `items`. +fn identify_spanning_lines(items: &[TextItem], columns: &[ColumnRegion]) -> Vec { + let n = items.len(); + let mut mask = vec![false; n]; + + if n < 3 || columns.len() < 2 { + return mask; + } + + let max_col_width = columns + .iter() + .map(|c| c.x_max - c.x_min) + .fold(0.0_f32, f32::max); + let span_threshold = max_col_width * 1.3; + + // Gutter centers: boundaries between adjacent columns + let gutters: Vec = columns.windows(2).map(|c| c[0].x_max).collect(); + let gutter_tol = 15.0; + let y_tol = 5.0; + + // Build (original_index, y) pairs sorted by Y descending for grouping + let mut indexed: Vec<(usize, f32)> = + items.iter().enumerate().map(|(i, it)| (i, it.y)).collect(); + indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + // Group by Y-proximity into rough lines (as index sets) + let mut groups: Vec> = Vec::new(); + let mut current_group: Vec = Vec::new(); + let mut current_y = f32::NAN; + + for (idx, y) in indexed { + if current_group.is_empty() || (current_y - y).abs() < y_tol { + if current_group.is_empty() { + current_y = y; + } + current_group.push(idx); + } else { + groups.push(std::mem::take(&mut current_group)); + current_y = y; + current_group.push(idx); + } + } + if !current_group.is_empty() { + groups.push(current_group); + } + + for group in groups { + if group.len() < 2 { + continue; + } + + // Sort group indices by X to compute span + let mut sorted_by_x: Vec = group; + sorted_by_x.sort_by(|&a, &b| { + items[a] + .x + .partial_cmp(&items[b].x) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let line_left = items[sorted_by_x[0]].x; + let last = *sorted_by_x.last().unwrap(); + let line_right = items[last].x + effective_width(&items[last]); + let span = line_right - line_left; + + if span <= span_threshold { + continue; + } + + // Check if any inter-item gap falls at a detected gutter boundary. + // If so, this is items from different columns at the same Y, not a + // true spanning line (like a title or section header). + let has_gutter_gap = sorted_by_x.windows(2).any(|pair| { + let left_end = items[pair[0]].x + effective_width(&items[pair[0]]); + let right_start = items[pair[1]].x; + let gap = right_start - left_end; + if gap < 5.0 { + return false; + } + // Check if any gutter falls within the gap interval (with tolerance) + gutters + .iter() + .any(|&g| g > left_end - gutter_tol && g < right_start + gutter_tol) + }); + + if !has_gutter_gap { + for &idx in &sorted_by_x { + mask[idx] = true; + } + } + } + + mask +} + /// Determines if a text item spans across multiple column regions (e.g. full-width headers/titles). fn spans_multiple_columns(item: &TextItem, columns: &[ColumnRegion]) -> bool { let w = effective_width(item); @@ -255,21 +725,78 @@ fn is_page_number(item: &TextItem) -> bool { /// Group text items into lines, with multi-column support /// Detect newspaper-style columns: independent text flows that should be read /// sequentially (all of col1, then col2) rather than Y-interleaved. -pub(crate) fn is_newspaper_layout(per_column_lines: &[Vec]) -> bool { +pub(crate) fn is_newspaper_layout( + per_column_lines: &[Vec], + columns: &[ColumnRegion], +) -> bool { if per_column_lines.len() < 2 { return false; } // Each column must independently have substantial content let min_lines = per_column_lines.iter().map(|c| c.len()).min().unwrap_or(0); + let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0); + + if min_lines < 5 { + return false; + } + if min_lines < 15 { + // Sidebar detection: a narrow annotation column beside a wide body column. + // Guards: + // - Only 2 columns (sidebars are body+sidebar, not 3+ columns) + // - width_ratio < 0.50: sidebar is much narrower than body + // - line_balance < 0.35: sidebar has significantly fewer lines + // - max_lines >= 20: body column has substantial prose content + // - narrower column has fewer lines (not a dense reference column) + if columns.len() == 2 && per_column_lines.len() == 2 { + let w0 = columns[0].x_max - columns[0].x_min; + let w1 = columns[1].x_max - columns[1].x_min; + let width_ratio = w0.min(w1) / w0.max(w1); + let line_balance = if max_lines > 0 { + min_lines as f32 / max_lines as f32 + } else { + 1.0 + }; + let narrow_width = w0.min(w1); + if width_ratio < 0.50 && line_balance < 0.35 && max_lines >= 20 && narrow_width >= 160.0 + { + let narrower_idx = if w0 < w1 { 0 } else { 1 }; + let fewest_idx = if per_column_lines[0].len() <= per_column_lines[1].len() { + 0 + } else { + 1 + }; + if narrower_idx == fewest_idx { + // Sparse density check: sidebar annotations are spread thinly + // across the page height while regular two-column text is dense. + // Compare average Y-gap between successive lines in each column. + let narrow = &per_column_lines[narrower_idx]; + let wide = &per_column_lines[1 - narrower_idx]; + let avg_gap = |lines: &[TextLine]| -> f32 { + if lines.len() < 2 { + return 0.0; + } + let mut ys: Vec = lines.iter().map(|l| l.y).collect(); + ys.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let span = ys.last().unwrap() - ys.first().unwrap(); + span / (lines.len() as f32 - 1.0) + }; + let narrow_gap = avg_gap(narrow); + let wide_gap = avg_gap(wide); + // Sidebar annotations have >2.5x the average gap of body text + if wide_gap > 0.0 && narrow_gap / wide_gap >= 2.5 { + return true; + } + } + } + } return false; } // Dense balanced columns (similar line counts) are newspaper regardless of Y-alignment. // By this point table items are already removed, so two dense balanced columns // of remaining text are independent prose flows. - let max_lines = per_column_lines.iter().map(|c| c.len()).max().unwrap_or(0); let balance_ratio = min_lines as f32 / max_lines as f32; if balance_ratio > 0.7 { return true; @@ -367,6 +894,17 @@ fn split_column_stragglers(lines: Vec) -> (Vec, Vec) -> Vec { + group_into_lines_with_thresholds(items, &HashMap::new(), &HashSet::new()) +} + +/// Group text items into lines, using pre-computed per-page adaptive thresholds +/// from Canva-style letter-spacing detection. Falls back to computing the +/// threshold from item gaps when no pre-computed value is available. +pub(crate) fn group_into_lines_with_thresholds( + items: Vec, + page_thresholds: &HashMap, + table_pages: &HashSet, +) -> Vec { if items.is_empty() { return Vec::new(); } @@ -387,23 +925,41 @@ pub fn group_into_lines(items: Vec) -> Vec { for page in pages { let page_items: Vec = items.iter().filter(|i| i.page == page).cloned().collect(); + // Use pre-computed threshold from fix_letterspaced_items if available + // (computed before embedded-space removal, with full signal). + // Non-Canva pages use the default 0.10 threshold. + let adaptive_threshold = page_thresholds.get(&page).copied().unwrap_or(0.10); + // Detect columns for this page - let columns = detect_columns(&page_items, page); + let columns = detect_columns(&page_items, page, table_pages.contains(&page)); if columns.len() <= 1 { // Single column - use simple sorting - let lines = group_single_column(page_items); + let lines = group_single_column(page_items, adaptive_threshold); all_lines.extend(lines); } else { - // Multi-column - separate spanning items from column items + // Multi-column detected. Pre-mask lines that span the full page + // width (titles, section headers, footers). These multi-item lines + // would otherwise be split across column buckets, corrupting + // newspaper detection and reading order. + let spanning_mask = identify_spanning_lines(&page_items, &columns); + let premasked_count = spanning_mask.iter().filter(|&&m| m).count(); + if premasked_count > 0 { + debug!( + "page {}: pre-masked {} spanning-line items", + page, premasked_count + ); + } + + // Partition items preserving original order let mut spanning_items: Vec = Vec::new(); let mut column_items: Vec = Vec::new(); - for item in &page_items { - if spans_multiple_columns(item, &columns) { - spanning_items.push(item.clone()); + for (i, item) in page_items.into_iter().enumerate() { + if spanning_mask[i] || spans_multiple_columns(&item, &columns) { + spanning_items.push(item); } else { - column_items.push(item.clone()); + column_items.push(item); } } @@ -461,14 +1017,14 @@ pub fn group_into_lines(items: Vec) -> Vec { let mut per_column_lines: Vec> = Vec::new(); for col_items in col_buckets { - let lines = group_single_column(col_items); + let lines = group_single_column(col_items, adaptive_threshold); per_column_lines.push(lines); } // Process spanning items as their own group - let spanning_lines = group_single_column(spanning_items); + let spanning_lines = group_single_column(spanning_items, adaptive_threshold); - let is_newspaper = is_newspaper_layout(&per_column_lines); + let is_newspaper = is_newspaper_layout(&per_column_lines, &columns); debug!( "page {}: layout={}", page, @@ -618,7 +1174,7 @@ fn should_use_y_sorting(items: &[TextItem]) -> bool { /// Group items from a single column into lines /// Uses heuristics to decide between PDF stream order and Y-position sorting. -fn group_single_column(items: Vec) -> Vec { +fn group_single_column(items: Vec, adaptive_threshold: f32) -> Vec { if items.is_empty() { return Vec::new(); } @@ -687,6 +1243,7 @@ fn group_single_column(items: Vec) -> Vec { items: vec![item], y, page, + adaptive_threshold, }); } } @@ -721,6 +1278,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -751,7 +1309,7 @@ mod tests { items.extend(fill_zone(1, 345.0, 660.0, 750.0, 50.0)); items.extend(fill_zone(1, 675.0, 800.0, 750.0, 50.0)); - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 3, "Expected 3 columns, got {}", cols.len()); // Gutter 1 should be in the gap between left and middle zones @@ -776,7 +1334,7 @@ mod tests { items.extend(fill_zone(1, 30.0, 280.0, 750.0, 50.0)); items.extend(fill_zone(1, 320.0, 570.0, 750.0, 50.0)); - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 2, "Expected 2 columns, got {}", cols.len()); let gutter = cols[0].x_max; @@ -807,7 +1365,7 @@ mod tests { )); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); // Should detect the gutters between the 3 dense zones, not the wide gap // before the sparse zone assert!( @@ -816,4 +1374,269 @@ mod tests { cols.len() ); } + + /// Helper: create items that fill a zone but with widths that extend past + /// the zone boundary (simulating justified text). Items start within the zone + /// but their reported width extends `overshoot` points past the zone end. + fn fill_zone_justified( + page: u32, + x_start: f32, + x_end: f32, + overshoot: f32, + y_start: f32, + y_end: f32, + ) -> Vec { + let mut items = Vec::new(); + let mut y = y_start; + while y >= y_end { + // Each line: 3-4 items that together span x_start to x_end+overshoot + let item_width = (x_end - x_start + overshoot) / 3.0; + for i in 0..3 { + let x = x_start + i as f32 * (x_end - x_start) / 3.0; + let text_len = (item_width / 6.0).ceil() as usize; + let text: String = "W".repeat(text_len); + items.push(TextItem { + text, + x, + y, + width: item_width, + height: 12.0, + font_size: 12.0, + font: String::new(), + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + }); + } + y -= 14.0; + } + items + } + + #[test] + fn relative_valley_detects_justified_text_columns() { + // Two columns of justified text where item widths overshoot the gutter + // by a few points, preventing absolute valley detection from finding + // an empty gutter. + let mut items = Vec::new(); + // Left column: x=40..290, items extend to ~297 (7pt overshoot) + items.extend(fill_zone_justified(1, 40.0, 290.0, 7.0, 750.0, 50.0)); + // Right column: x=300..550, items extend to ~557 + items.extend(fill_zone_justified(1, 300.0, 550.0, 7.0, 750.0, 50.0)); + + let cols = detect_columns(&items, 1, false); + assert_eq!( + cols.len(), + 2, + "Expected 2 columns for justified text, got {}", + cols.len() + ); + + let gutter = cols[0].x_max; + assert!( + (280.0..=310.0).contains(&gutter), + "Gutter at {gutter}, expected ~295" + ); + } + + #[test] + fn relative_valley_rejects_single_column_margin() { + // Single column of text — the right margin drop-off should NOT be + // detected as a column gutter. + let items = fill_zone_justified(1, 40.0, 350.0, 0.0, 750.0, 50.0); + + let cols = detect_columns(&items, 1, false); + assert_eq!( + cols.len(), + 1, + "Expected 1 column for single-column text, got {}", + cols.len() + ); + } + + /// Helper: build a Vec with `n` lines at given X, starting at Y=700. + fn make_lines(n: usize, x: f32) -> Vec { + (0..n) + .map(|i| { + let y = 700.0 - i as f32 * 14.0; + let item = make_item(1, x, y, "SomeText__"); + TextLine { + y, + page: 1, + adaptive_threshold: 0.10, + items: vec![item], + } + }) + .collect() + } + + #[test] + fn sidebar_layout_detected_as_newspaper() { + // Wide body column (x 0..400) with 40 lines, + // narrow sidebar (x 420..590, width 170) with 12 lines. + // width_ratio = 170/400 = 0.425, line_balance = 12/40 = 0.30 → sidebar → newspaper + // Sidebar lines have ~3x gap of body lines (sparse annotations). + let body = make_lines(40, 50.0); + let sidebar: Vec = (0..12) + .map(|i| { + let y = 693.0 - i as f32 * 45.0; // sparse annotations: ~3x body gap + let item = make_item(1, 440.0, y, "SomeText__"); + TextLine { + y, + page: 1, + adaptive_threshold: 0.10, + items: vec![item], + } + }) + .collect(); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 400.0, + }, + ColumnRegion { + x_min: 420.0, + x_max: 590.0, + }, + ]; + assert!( + is_newspaper_layout(&[body, sidebar], &cols), + "Wide body + narrow sidebar should be detected as newspaper" + ); + } + + #[test] + fn borderless_table_not_misclassified() { + // Two columns of similar width and equal line counts → borderless table, not newspaper. + // width_ratio = 250/300 = 0.83 (> 0.50), so sidebar guard fails → false. + let col1 = make_lines(10, 50.0); + let col2 = make_lines(10, 350.0); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 550.0, + }, + ]; + assert!( + !is_newspaper_layout(&[col1, col2], &cols), + "Equal-width equal-row columns should NOT be newspaper (borderless table)" + ); + } + + #[test] + fn premask_spanning_title_removed_from_columns() { + // Title spans x=30..550 as 5 adjacent items (no gap near gutter at x=300) + // Two columns: left (x=0..300), right (x=300..600) + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Spanning title: 5 items at Y=750, each ~100pt wide, gaps ~4pt + // No item gap falls near the gutter at x=300 + for i in 0..5 { + items.push(make_item( + 1, + 30.0 + i as f32 * 104.0, + 750.0, + "TitleWord_________", + )); + } + + // Left column body: 20 lines + for i in 0..20 { + items.push(make_item(1, 30.0, 700.0 - i as f32 * 14.0, "LeftText__")); + } + + // Right column body: 20 lines + for i in 0..20 { + items.push(make_item(1, 320.0, 700.0 - i as f32 * 14.0, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + let non_spanning_count = mask.iter().filter(|&&m| !m).count(); + assert_eq!(spanning_count, 5, "Title items should be pre-masked"); + assert_eq!(non_spanning_count, 40, "Column items should remain"); + } + + #[test] + fn premask_does_not_mask_column_items_at_same_y() { + // Two items at same Y with gap at gutter → NOT masked + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Items in two columns at same Y — gap center ~305 is near gutter at 300 + for i in 0..15 { + let y = 700.0 - i as f32 * 14.0; + items.push(make_item(1, 30.0, y, "LeftText__")); + items.push(make_item(1, 320.0, y, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + assert_eq!( + spanning_count, 0, + "Column items with gap at gutter should NOT be pre-masked" + ); + } + + #[test] + fn premask_narrow_line_not_masked() { + // Items that form a line spanning only ~40% of column width → not masked + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + let mut items = Vec::new(); + + // Narrow header at top (spans ~240pt, max col width = 300, threshold = 390) + for i in 0..3 { + items.push(make_item( + 1, + 180.0 + i as f32 * 84.0, + 750.0, + "SmallHeader___", + )); + } + + // Two columns below + for i in 0..15 { + let y = 700.0 - i as f32 * 14.0; + items.push(make_item(1, 30.0, y, "LeftText__")); + items.push(make_item(1, 400.0, y, "RightText_")); + } + + let mask = identify_spanning_lines(&items, &cols); + let spanning_count = mask.iter().filter(|&&m| m).count(); + assert_eq!(spanning_count, 0, "Narrow header should NOT be pre-masked"); + } } diff --git a/src/extractor/links.rs b/src/extractor/links.rs index f7e9090..74a1fae 100644 --- a/src/extractor/links.rs +++ b/src/extractor/links.rs @@ -79,6 +79,7 @@ pub fn extract_page_links(doc: &Document, page_id: ObjectId, page_num: u32) -> V is_bold: false, is_italic: false, item_type: ItemType::Link(url), + mcid: None, }); } } @@ -316,5 +317,6 @@ pub(crate) fn walk_form_fields( is_bold: false, is_italic: false, item_type: ItemType::FormField, + mcid: None, }); } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index e0ea518..93b465e 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -2,7 +2,7 @@ //! //! This module extracts text with position information for structure detection. -mod content_stream; +pub(crate) mod content_stream; mod fonts; mod layout; mod links; @@ -25,6 +25,9 @@ pub use crate::text_utils::{is_bold_font, is_italic_font}; pub use crate::types::{ItemType, TextLine}; pub(crate) use layout::detect_columns; pub use layout::group_into_lines; +pub(crate) use layout::group_into_lines_with_thresholds; +pub(crate) use layout::is_newspaper_layout; +pub(crate) use layout::ColumnRegion; // --------------------------------------------------------------------------- // Public API @@ -49,7 +52,7 @@ pub fn extract_text_mem(buffer: &[u8]) -> Result { let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; @@ -96,7 +99,9 @@ pub(crate) fn extract_text_with_positions_and_rects>( Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) + let (extraction, _thresholds, _gid_pages) = + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; + Ok(extraction) } /// Extract text with positions from memory buffer @@ -122,28 +127,56 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( let doc = match Document::load_mem(buffer) { Ok(d) => d, Err(ref e) if crate::is_encrypted_lopdf_error(e) => { - Document::load_mem_with_password(buffer, "")? + Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))? } Err(e) => return Err(e.into()), }; let font_cmaps = FontCMaps::from_doc(&doc); - extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) + let (extraction, _thresholds, _gid_pages) = + extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; + Ok(extraction) } // --------------------------------------------------------------------------- // Orchestration // --------------------------------------------------------------------------- +/// Per-page adaptive join thresholds from Canva-style letter-spacing detection. +pub(crate) type PageThresholds = HashMap; + /// Extract positioned text, rectangles, and line segments from a pre-loaded document. +/// +/// Also returns per-page adaptive join thresholds for Canva-style pages. pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result { +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, false) +} + +/// Extract with option to include invisible (Tr=3) text. +/// Used for Mixed/template PDFs where the OCR text layer is invisible. +pub(crate) fn extract_positioned_text_include_invisible( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { + extract_positioned_text_impl(doc, font_cmaps, page_filter, true) +} + +fn extract_positioned_text_impl( + doc: &Document, + font_cmaps: &FontCMaps, + page_filter: Option<&HashSet>, + include_invisible: bool, +) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); let mut all_rects = Vec::new(); let mut all_lines = Vec::new(); + let mut page_thresholds: PageThresholds = HashMap::new(); + let mut gid_encoded_pages: HashSet = HashSet::new(); // Build page ObjectId → page number map for form field extraction let page_id_to_num: HashMap = @@ -155,13 +188,26 @@ pub(crate) fn extract_positioned_text_from_doc( continue; } } - let (items, rects, lines) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + let ((mut items, rects, lines), has_gid_fonts) = + extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?; + if has_gid_fonts { + gid_encoded_pages.insert(*page_num); + } + let threshold = crate::text_utils::fix_letterspaced_items(&mut items); + if threshold > 0.10 { + page_thresholds.insert(*page_num, threshold); + } debug!( - "page {}: {} text items, {} rects, {} lines", + "page {}: {} text items, {} rects, {} lines{}", page_num, items.len(), rects.len(), - lines.len() + lines.len(), + if has_gid_fonts { + " [gid-encoded fonts]" + } else { + "" + } ); if log::log_enabled!(log::Level::Trace) { for item in &items { @@ -194,7 +240,11 @@ pub(crate) fn extract_positioned_text_from_doc( let form_items = extract_form_fields(doc, &page_id_to_num); all_items.extend(form_items); - Ok((all_items, all_rects, all_lines)) + Ok(( + (all_items, all_rects, all_lines), + page_thresholds, + gid_encoded_pages, + )) } // --------------------------------------------------------------------------- @@ -222,6 +272,44 @@ pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] { /// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each /// group by X, then merges consecutive items that share a similar font size /// and are close horizontally. +/// Cap item width for merge-gap computation to guard against Tw inflation. +/// +/// When PDF word-spacing (Tw) is large (used for text justification), the +/// advance width of strings containing spaces extends far past the visible +/// glyph extent. This inflated width collapses inter-column gaps, making +/// `merge_text_items` incorrectly merge items from different table columns. +/// +/// Only applies to non-CJK items whose text contains spaces (where Tw +/// contributes) and whose average width-per-character is abnormally high. +fn effective_merge_width(item: &TextItem) -> f32 { + use crate::text_utils::is_cjk_char; + + if item.width <= 0.0 || item.font_size <= 0.0 { + return item.width; + } + // Tw only inflates strings that contain space characters. + if !item.text.contains(' ') { + return item.width; + } + // CJK characters are naturally ~1.0× font_size wide; skip the cap. + if item.text.chars().any(is_cjk_char) { + return item.width; + } + let char_count = item.text.chars().count(); + if char_count == 0 { + return item.width; + } + let avg = item.width / char_count as f32; + // Normal proportional text: ~0.5× font_size per char. + // Monospace: ~0.6×. Threshold at 0.85× catches Tw inflation. + if avg > item.font_size * 0.85 { + let capped = char_count as f32 * item.font_size * 0.6; + capped.min(item.width) + } else { + item.width + } +} + pub(crate) fn merge_text_items(items: Vec) -> Vec { if items.is_empty() { return items; @@ -265,7 +353,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { while i < group.len() { let first = group[i]; let mut text = first.text.clone(); - let mut end_x = first.x + first.width; + let mut end_x = first.x + effective_merge_width(first); let x_gap_max = first.font_size * 0.5; let mut j = i + 1; @@ -282,12 +370,30 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { if gap < -first.font_size * 0.5 { break; } - // Insert space at word boundaries - if gap > first.font_size * 0.08 { + // Insert space at word boundaries. + // Base threshold 0.08; raised to 0.13 for lowercase→lowercase + // junctions to accommodate Tc/Tw character-spacing adjustments + // that shift advance widths relative to Td positioning. + let threshold = { + let prev_last = text.trim_end().chars().last(); + let next_first = next.text.trim_start().chars().next(); + // Never insert space before joining punctuation + if next_first.is_some_and(|c| matches!(c, '.' | ',' | ';' | ')' | ']' | '}')) { + first.font_size * 0.25 + } else if prev_last.is_some_and(|c| c.is_lowercase()) + && next_first.is_some_and(|c| c.is_lowercase()) + { + // Lowercase→lowercase: likely mid-word, use wider threshold + first.font_size * 0.13 + } else { + first.font_size * 0.08 + } + }; + if gap > threshold { text.push(' '); } text.push_str(&next.text); - end_x = next.x + next.width; + end_x = next.x + effective_merge_width(next); j += 1; } @@ -303,6 +409,7 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { is_bold: first.is_bold, is_italic: first.is_italic, item_type: first.item_type.clone(), + mcid: first.mcid, }); i = j; @@ -312,6 +419,92 @@ pub(crate) fn merge_text_items(items: Vec) -> Vec { merged } +/// Merge subscript/superscript items into their adjacent parent items. +/// +/// Subscripts (e.g. "2" in H₂O) are rendered as separate text items with a +/// much smaller font size and a slight Y offset. This pass finds such items +/// and absorbs them into the preceding normal-sized item so that downstream +/// table detection and line grouping see complete text (e.g. "H2O" not "H"+"2"+"O"). +pub(crate) fn merge_subscript_items(items: Vec) -> Vec { + if items.len() < 2 { + return items; + } + + // Group items by (page, approximate Y) with generous tolerance to capture + // both the parent line and the subscript/superscript offset. + let y_tolerance = 5.0; + let mut line_groups: Vec<(u32, f32, Vec)> = Vec::new(); + + for item in items { + let found = line_groups + .iter_mut() + .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance); + if let Some((_, _, group)) = found { + group.push(item); + } else { + let page = item.page; + let y = item.y; + line_groups.push((page, y, vec![item])); + } + } + + let mut result = Vec::new(); + + for (_, _, mut group) in line_groups { + // Sort by X position + group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)); + + // Find the dominant (most common) font size in this group + let max_fs = group.iter().map(|i| i.font_size).fold(0.0_f32, f32::max); + + if max_fs < 1.0 { + result.extend(group); + continue; + } + + let sub_threshold = max_fs * 0.75; + + // Walk through items and merge subscripts into their preceding parent + let mut merged: Vec = Vec::new(); + for item in group { + if item.font_size < sub_threshold + && item.font_size > 0.0 + && item.text.len() <= 4 + && item.text.chars().all(|c| c.is_ascii_digit()) + { + // This is a candidate numeric subscript/superscript (e.g. "2" in H₂O). + // Only merge purely numeric text to avoid false positives with small + // bullets, ordinal indicators, or letter-based labels. + if let Some(parent) = merged.last_mut() { + // Only merge into a parent that is normal-sized, not another subscript, + // and whose text ends with a letter. This prevents merging into numbers + // (e.g. "33" + "1" in "33 1/3%") or punctuation, while preserving + // chemical formulas (NH + "3") and footnote refs (word + "2"). + let ends_with_letter = parent + .text + .chars() + .last() + .is_some_and(|c| c.is_alphabetic()); + if parent.font_size >= sub_threshold && ends_with_letter { + let parent_right = parent.x + parent.width; + let gap = item.x - parent_right; + // Subscripts must be tightly adjacent (within ~1pt) + if gap < parent.font_size * 0.2 && gap > -parent.font_size * 0.3 { + parent.text.push_str(&item.text); + parent.width = (item.x + item.width) - parent.x; + continue; + } + } + } + } + merged.push(item); + } + result.extend(merged); + } + + result +} + /// Helper to get f32 from Object pub(crate) fn get_number(obj: &Object) -> Option { match obj { @@ -326,7 +519,62 @@ mod tests { use super::*; use crate::text_utils::{is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items}; use crate::types::{ItemType, TextLine}; - use layout::{detect_columns, is_newspaper_layout}; + use layout::{detect_columns, is_newspaper_layout, ColumnRegion}; + + fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y: 700.0, + width, + height: 12.0, + font: "F1".into(), + font_size: 12.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn merge_items_no_space_before_period() { + // Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap + // to "." due to negative Tc, but period should still join without space. + let items = vec![ + make_merge_item("date", 227.25, 89.25), // end = 316.50 + make_merge_item(".", 318.00, 3.0), // gap = 1.50 (0.125 × fs) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "date."); + } + + #[test] + fn merge_items_lowercase_join_with_tc() { + // Lowercase→lowercase junction: "deve" + "lopers" with Tc-affected gap + // Gap of 0.12 × font_size should merge without space + let items = vec![ + make_merge_item("deve", 100.0, 30.0), // end = 130.0 + make_merge_item("lopers", 131.44, 40.0), // gap = 1.44 (0.12 × 12) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "developers"); + } + + #[test] + fn merge_items_space_at_word_boundary() { + // Word boundary gap (> 0.13 × font_size) should insert space + let items = vec![ + make_merge_item("hello", 100.0, 30.0), + make_merge_item("world", 132.0, 30.0), // gap = 2.0 (0.167 × 12) + ]; + let merged = merge_text_items(items); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].text, "hello world"); + } #[test] fn test_group_into_lines() { @@ -343,6 +591,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -356,6 +605,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Next line".into(), @@ -369,6 +619,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -422,6 +673,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Prague".into(), @@ -435,6 +687,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "Rules".into(), @@ -448,6 +701,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -472,6 +726,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "A".into(), @@ -485,6 +740,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "V".into(), @@ -498,6 +754,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -524,6 +781,7 @@ mod tests { is_bold: true, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -557,6 +815,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -591,6 +850,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "履行義務".into(), @@ -604,6 +864,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "を識別す".into(), @@ -617,6 +878,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; @@ -638,6 +900,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -650,7 +913,7 @@ mod tests { items.push(make_item("Left text here", 72.0, y, 200.0)); items.push(make_item("Right text here", 350.0, y, 200.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols); assert!(cols[0].x_min < cols[1].x_min); } @@ -665,7 +928,7 @@ mod tests { items.push(make_item("Col two", 220.0, y, 140.0)); items.push(make_item("Col three", 390.0, y, 140.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols); } @@ -683,7 +946,7 @@ mod tests { let y = 700.0 - (i as f32) * 14.0; items.push(make_item("wide", 72.0, y, 320.0)); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert!( cols.len() >= 2, "Width bleed should not prevent column detection, got {:?}", @@ -704,7 +967,7 @@ mod tests { 468.0, )); } - let cols = detect_columns(&items, 1); + let cols = detect_columns(&items, 1, false); assert!( cols.len() <= 1, "Full-width text should not be split into columns, got {:?}", @@ -749,6 +1012,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "\u{05D1}".into(), // bet at x=200 (rightmost) @@ -762,6 +1026,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -785,6 +1050,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, TextItem { text: "World".into(), @@ -798,6 +1064,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }, ]; sort_line_items(&mut items); @@ -824,6 +1091,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, @@ -836,6 +1104,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -846,7 +1115,17 @@ mod tests { .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(is_newspaper_layout(&[col1, col2], &cols)); } #[test] @@ -856,6 +1135,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, @@ -868,6 +1148,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -879,7 +1160,17 @@ mod tests { .map(|i| make_line(685.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(is_newspaper_layout(&[col1, col2], &cols)); } #[test] @@ -888,6 +1179,7 @@ mod tests { let make_line = |y: f32, x: f32, page: u32| TextLine { y, page, + adaptive_threshold: 0.10, items: vec![TextItem { text: "text".into(), x, @@ -900,6 +1192,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, }], }; @@ -910,6 +1203,133 @@ mod tests { .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1)) .collect(); - assert!(!is_newspaper_layout(&[col1, col2])); + let cols = vec![ + ColumnRegion { + x_min: 0.0, + x_max: 300.0, + }, + ColumnRegion { + x_min: 300.0, + x_max: 600.0, + }, + ]; + assert!(!is_newspaper_layout(&[col1, col2], &cols)); + } + + fn make_item_fs(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.into(), + x, + y, + width, + height: font_size, + font: "F1".into(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn test_merge_subscript_items_chemical_formula() { + // NH₃: "NH" at fs=8 followed by subscript "3" at fs=4.7 + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 4.7), + make_item_fs("Cl", 100.0, 499.0, 7.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "NH3"); + assert_eq!(merged[1].text, "Cl"); + } + + #[test] + fn test_merge_subscript_items_h2o() { + // H₂O: "H" then subscript "2" then "O" + let items = vec![ + make_item_fs("H", 250.0, 499.0, 5.0, 8.0), + make_item_fs("2", 255.0, 496.0, 2.3, 4.7), + make_item_fs("O", 257.5, 499.0, 6.0, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "H2"); + assert_eq!(merged[1].text, "O"); + } + + #[test] + fn test_merge_subscript_items_no_merge_far_gap() { + // Subscript-sized item that's far from the parent should NOT merge + let items = vec![ + make_item_fs("Text", 78.0, 499.0, 20.0, 8.0), + make_item_fs("▶", 120.0, 498.0, 3.0, 3.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "Text"); + assert_eq!(merged[1].text, "▶"); + } + + #[test] + fn test_merge_subscript_items_no_merge_long_text() { + // Long subscript-sized text should NOT merge (not a true subscript) + let items = vec![ + make_item_fs("Title", 78.0, 499.0, 30.0, 8.0), + make_item_fs("footnote", 108.0, 496.0, 20.0, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_same_font_size() { + // Same font size items should NOT be treated as subscripts + let items = vec![ + make_item_fs("NH", 78.0, 499.0, 12.0, 8.0), + make_item_fs("3", 90.0, 496.0, 2.3, 8.0), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + } + + #[test] + fn test_merge_subscript_items_no_merge_non_numeric() { + // Non-numeric subscript text (e.g. "sol", "º", "vf") should NOT merge + let items = vec![ + make_item_fs("∆", 200.0, 639.0, 5.5, 8.0), + make_item_fs("sol", 205.8, 636.9, 5.7, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "∆"); + assert_eq!(merged[1].text, "sol"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_digit() { + // "33" + "1" in "33 1/3%" — parent ends with digit, should NOT merge + let items = vec![ + make_item_fs("33", 78.0, 499.0, 10.0, 8.0), + make_item_fs("1", 88.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].text, "33"); + assert_eq!(merged[1].text, "1"); + } + + #[test] + fn test_merge_subscript_items_no_merge_parent_ends_with_space() { + // "Health " + "1" — parent ends with space (table credit), should NOT merge + let items = vec![ + make_item_fs("Health ", 78.0, 499.0, 30.0, 8.0), + make_item_fs("1", 108.0, 496.0, 2.3, 4.7), + ]; + let merged = merge_subscript_items(items); + assert_eq!(merged.len(), 2); } } diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index c9e5da1..2cab02f 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -144,9 +144,10 @@ fn extract_form_xobject_text_inner( return items; }; - // Decompress the content stream - let Ok(content_data) = stream.decompressed_content() else { - return items; + // Decompress the content stream (fall back to raw bytes for uncompressed streams) + let content_data = match stream.decompressed_content() { + Ok(data) => data, + Err(_) => stream.content.clone(), }; // Decode the content stream @@ -156,7 +157,7 @@ fn extract_form_xobject_text_inner( // Get fonts from the Form's Resources let form_fonts = get_form_fonts(doc, &stream.dict); - let font_encodings = build_font_encodings(doc, &form_fonts); + let (font_encodings, _has_gid_fonts) = build_font_encodings(doc, &form_fonts); // Build font width info for the form let font_widths = build_font_widths(doc, &form_fonts); @@ -179,12 +180,13 @@ fn extract_form_xobject_text_inner( if let Ok(obj_ref) = tounicode.as_reference() { font_tounicode_refs.insert(resource_name, obj_ref.0); } else if let Object::Stream(s) = tounicode { - if let Ok(data) = s.decompressed_content() { - if let Some(entry) = - crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) - { - inline_cmaps.insert(resource_name, entry); - } + let data = s + .decompressed_content() + .unwrap_or_else(|_| s.content.clone()); + if let Some(entry) = + crate::tounicode::build_cmap_entry_from_stream(&data, font_dict, doc, 0) + { + inline_cmaps.insert(resource_name, entry); } } } @@ -352,6 +354,8 @@ fn extract_form_xobject_text_inner( raw_bytes, font_info, current_font_size, + 0.0, + 0.0, ); text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -379,6 +383,8 @@ fn extract_form_xobject_text_inner( raw_bytes, font_info, current_font_size, + 0.0, + 0.0, ); text_matrix[4] += w_ts * text_matrix[0]; text_matrix[5] += w_ts * text_matrix[1]; @@ -408,6 +414,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } @@ -490,8 +497,13 @@ fn extract_form_xobject_text_inner( } if let Some(fi) = font_info { if let Some(raw_bytes) = get_operand_bytes(element) { - total_width_ts += - compute_string_width_ts(raw_bytes, fi, current_font_size); + total_width_ts += compute_string_width_ts( + raw_bytes, + fi, + current_font_size, + 0.0, + 0.0, + ); } } if !fill_is_white { @@ -549,6 +561,7 @@ fn extract_form_xobject_text_inner( is_bold: is_bold_font(base_font), is_italic: is_italic_font(base_font), item_type: ItemType::Text, + mcid: None, }); } } diff --git a/src/lib.rs b/src/lib.rs index f80718c..3c61add 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,7 @@ pub mod extractor; pub mod glyph_names; pub mod markdown; pub mod process_mode; +pub mod structure_tree; pub mod tables; pub mod text_utils; pub mod tounicode; @@ -48,7 +49,7 @@ pub use process_mode::ProcessMode; pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem}; use lopdf::Document; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::Path; use tounicode::FontCMaps; @@ -266,6 +267,264 @@ pub fn process_pdf_mem_with_config( ) } +// ========================================================================= +// Region-based text extraction (for hybrid OCR pipelines) +// ========================================================================= + +/// Lightweight classification result for routing decisions. +#[derive(Debug)] +pub struct PdfClassification { + /// The detected PDF type. + pub pdf_type: PdfType, + /// Total page count. + pub page_count: u32, + /// 0-indexed page numbers that need OCR (scanned/image pages). + pub pages_needing_ocr: Vec, + /// Detection confidence score (0.0–1.0). + pub confidence: f32, +} + +/// Classify a PDF from a memory buffer without extracting text. +/// Returns the PDF type and which pages need OCR (~10-50ms). +pub fn classify_pdf_mem(buffer: &[u8]) -> Result { + validate_pdf_bytes(buffer)?; + let (doc, page_count) = load_document_from_mem(buffer)?; + let detection = detector::detect_from_document(&doc, page_count, &DetectionConfig::default())?; + Ok(PdfClassification { + pdf_type: detection.pdf_type, + page_count, + // Convert from 1-indexed to 0-indexed for caller convenience + pages_needing_ocr: detection.pages_needing_ocr.iter().map(|&p| p - 1).collect(), + confidence: detection.confidence, + }) +} + +/// Result for a single region's text extraction. +#[derive(Debug)] +pub struct RegionText { + /// Extracted text (may be empty if region has no text items). + pub text: String, + /// `true` when the text should not be trusted and OCR should be used instead. + /// Set when: the region is empty, the page uses GID-encoded fonts, or the + /// extracted text fails garbage/encoding checks. + pub needs_ocr: bool, +} + +/// Result for a page's region extractions. +#[derive(Debug)] +pub struct PageRegionResult { + /// 0-indexed page number. + pub page: u32, + /// Per-region results, parallel to the input regions. + pub regions: Vec, +} + +/// Extract text within bounding-box regions from a PDF in memory. +/// +/// This is designed for hybrid OCR pipelines: a layout model detects regions +/// in a rendered page image, and this function extracts the PDF text that +/// falls within each region — avoiding GPU OCR for text-based pages. +/// +/// Each region result includes a `needs_ocr` flag that is set when extraction +/// quality is suspect (empty text, GID-encoded fonts, garbage/encoding issues). +/// +/// # Arguments +/// +/// * `buffer` — PDF file bytes +/// * `page_regions` — list of `(page_number_0indexed, Vec<[x1, y1, x2, y2]>)`. +/// Coordinates are in **PDF points** with **top-left origin** (matching typical +/// layout model output after coordinate conversion). +/// +/// # Returns +/// +/// A `Vec` parallel to `page_regions`. +pub fn extract_text_in_regions_mem( + buffer: &[u8], + page_regions: &[(u32, Vec<[f32; 4]>)], +) -> Result, PdfError> { + validate_pdf_bytes(buffer)?; + let (doc, _page_count) = load_document_from_mem(buffer)?; + let font_cmaps = FontCMaps::from_doc(&doc); + let pages = doc.get_pages(); + + // Build a set of pages we need to extract + let needed_pages: HashSet = page_regions.iter().map(|(p, _)| p + 1).collect(); // to 1-indexed + + // Extract text items for needed pages only + let mut items_by_page: HashMap> = HashMap::new(); + let mut page_heights: HashMap = HashMap::new(); + let mut gid_pages: HashSet = HashSet::new(); + + for (page_num, &page_id) in pages.iter() { + if !needed_pages.contains(page_num) { + continue; + } + + // Get page height from MediaBox for coordinate flip + let height = get_page_height(&doc, page_id).unwrap_or(792.0); + page_heights.insert(*page_num, height); + + // Extract text items for this page + let ((mut items, _rects, _lines), has_gid) = + extractor::content_stream::extract_page_text_items( + &doc, + page_id, + *page_num, + &font_cmaps, + false, + )?; + text_utils::fix_letterspaced_items(&mut items); + if has_gid { + gid_pages.insert(*page_num); + } + items_by_page.insert(*page_num, items); + } + + // For each page's regions, filter and assemble text + let mut results = Vec::with_capacity(page_regions.len()); + + for (page_0idx, regions) in page_regions { + let page_1idx = page_0idx + 1; + let items = items_by_page.get(&page_1idx); + let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0); + let page_has_gid = gid_pages.contains(&page_1idx); + + let mut page_results = Vec::with_capacity(regions.len()); + + for rect in regions { + let [rx1, ry1, rx2, ry2] = *rect; + + let text = match items { + Some(items) => collect_text_in_region(items, rx1, ry1, rx2, ry2, page_h), + None => String::new(), + }; + + let needs_ocr = text.trim().is_empty() + || page_has_gid + || is_garbage_text(&text) + || detect_encoding_issues(&text); + + page_results.push(RegionText { text, needs_ocr }); + } + + results.push(PageRegionResult { + page: *page_0idx, + regions: page_results, + }); + } + + Ok(results) +} + +/// Get page height in points from MediaBox. +fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option { + let page_dict = doc.get_dictionary(page_id).ok()?; + // Try MediaBox directly, then follow reference + let media_box = page_dict.get(b"MediaBox").ok()?; + let arr = match media_box { + lopdf::Object::Array(a) => a, + lopdf::Object::Reference(r) => { + if let Ok(lopdf::Object::Array(a)) = doc.get_object(*r) { + a + } else { + return None; + } + } + _ => return None, + }; + if arr.len() >= 4 { + let y1 = obj_to_f32(&arr[1])?; + let y2 = obj_to_f32(&arr[3])?; + Some((y2 - y1).abs()) + } else { + None + } +} + +fn obj_to_f32(obj: &lopdf::Object) -> Option { + match obj { + lopdf::Object::Integer(i) => Some(*i as f32), + lopdf::Object::Real(f) => Some(*f), + _ => None, + } +} + +/// Collect text items that fall within a region bbox (top-left origin, PDF points) +/// and return them as a single string in reading order. +fn collect_text_in_region( + items: &[TextItem], + rx1: f32, + ry1: f32, + rx2: f32, + ry2: f32, + page_height: f32, +) -> String { + // Convert region from top-left to bottom-left origin + let by1 = page_height - ry2; // top-left y2 → bottom-left y1 + let by2 = page_height - ry1; // top-left y1 → bottom-left y2 + + // Collect items whose center falls within the region + let mut matched: Vec<&TextItem> = items + .iter() + .filter(|item| { + let cx = item.x + item.width / 2.0; + let cy = item.y + item.height / 2.0; + cx >= rx1 && cx <= rx2 && cy >= by1 && cy <= by2 + }) + .collect(); + + if matched.is_empty() { + return String::new(); + } + + // Sort top→bottom (descending Y in bottom-left coords), then left→right + matched.sort_by(|a, b| { + let line_threshold = a.font_size.max(b.font_size) * 0.5; + let y_diff = b.y - a.y; // descending Y = top to bottom + if y_diff.abs() < line_threshold { + a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal) + } else { + y_diff + .partial_cmp(&0.0_f32) + .unwrap_or(std::cmp::Ordering::Equal) + } + }); + + // Group into lines and join + let mut lines: Vec = Vec::new(); + let mut current_line = String::new(); + let mut last_y = f32::NAN; + let mut last_x_end = 0.0_f32; + + for item in &matched { + let line_threshold = item.font_size * 0.5; + let same_line = (item.y - last_y).abs() < line_threshold; + + if !same_line && !current_line.is_empty() { + lines.push(current_line.clone()); + current_line.clear(); + } + + if !current_line.is_empty() { + // Insert space if there's a gap between items on the same line + let gap = item.x - last_x_end; + if gap > item.font_size * 0.15 { + current_line.push(' '); + } + } + + current_line.push_str(&item.text); + last_y = item.y; + last_x_end = item.x + item.width; + } + + if !current_line.is_empty() { + lines.push(current_line); + } + + lines.join("\n") +} + // ========================================================================= // Internal: single-load document pipeline // ========================================================================= @@ -276,20 +535,23 @@ pub fn process_pdf_mem_with_config( /// are combined here, but lopdf loads the full doc in `load()` so we extract /// page count from it directly to avoid the metadata-only round-trip. fn load_document_from_path>(path: P) -> Result<(Document, u32), PdfError> { - let doc = match Document::load(&path) { - Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_with_password(&path, "")?, - Err(e) => return Err(e.into()), - }; - let page_count = doc.get_pages().len() as u32; - Ok((doc, page_count)) + let buffer = std::fs::read(&path)?; + load_document_from_mem(&buffer) } /// Load a PDF from a memory buffer. fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> { - let doc = match Document::load_mem(buffer) { + // Fix malformed struct element names before parsing. Some PDF generators + // write bare names (/S Code) instead of proper PDF names (/S /Code), which + // causes lopdf to silently drop the entire object. + let fixed = structure_tree::fix_bare_struct_names(buffer); + let buf = fixed.as_ref(); + + let doc = match Document::load_mem(buf) { Ok(d) => d, - Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buffer, "")?, + Err(ref e) if is_encrypted_lopdf_error(e) => { + Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))? + } Err(e) => return Err(e.into()), }; let page_count = doc.get_pages().len() as u32; @@ -343,7 +605,38 @@ fn process_document( // Step 2 — Extraction (reuses the already-loaded document) let extracted = { let font_cmaps = FontCMaps::from_doc(&doc); - extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, options.page_filter.as_ref()) + let result = extractor::extract_positioned_text_from_doc( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ); + + // For Mixed/template PDFs: if normal extraction produces garbage text + // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. + // This unlocks OCR text layers behind scanned images. + if pdf_type == PdfType::Mixed { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _)| e) { + let sample: String = items.iter().take(200).map(|i| i.text.as_str()).collect(); + if is_garbage_text(&sample) || sample.trim().is_empty() { + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } else { + result + } + } else { + // Normal extraction failed — try invisible as fallback + extractor::extract_positioned_text_include_invisible( + &doc, + &font_cmaps, + options.page_filter.as_ref(), + ) + } + } else { + result + } }; // For Mixed PDFs, extraction failure is non-fatal @@ -353,8 +646,75 @@ fn process_document( Some(extracted?) }; - let (markdown, layout, has_encoding_issues) = match extracted { - Some((items, rects, lines)) => { + // Parse structure tree for tagged PDFs (reuses the loaded document) + let (struct_roles, struct_tables) = structure_tree::StructTree::from_doc(&doc) + .map(|tree| { + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + let tables = tree.extract_tables(&page_ids); + if !roles.is_empty() { + log::debug!( + "structure tree: {} pages with MCID roles, {} total MCIDs, {} tagged tables", + roles.len(), + tree.mcid_count(), + tables.len() + ); + } + let roles = if roles.is_empty() { None } else { Some(roles) }; + (roles, tables) + }) + .unwrap_or((None, Vec::new())); + + let (markdown, layout, has_encoding_issues, gid_pages) = match extracted { + Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { + // For TextBased PDFs with pages flagged for OCR (Identity-H or + // Type3 fonts without ToUnicode), check whether the CID-as-Unicode + // passthrough actually produced readable text. If a page's text + // is garbage, strip its items so we don't emit mojibake. + // Only applies to TextBased — for Mixed PDFs, OCR flags come from + // template images rather than font encoding issues. + let (items, rects, lines) = + if pages_needing_ocr.is_empty() || pdf_type != PdfType::TextBased { + (items, rects, lines) + } else { + let ocr_set: std::collections::HashSet = + pages_needing_ocr.iter().copied().collect(); + // Collect text per OCR-flagged page and check quality + let mut garbage_pages: std::collections::HashSet = + std::collections::HashSet::new(); + for &pg in &ocr_set { + let page_text: String = items + .iter() + .filter(|i| i.page == pg) + .map(|i| i.text.as_str()) + .collect(); + if is_cid_garbage(&page_text) { + garbage_pages.insert(pg); + } + } + if garbage_pages.is_empty() { + (items, rects, lines) + } else { + log::debug!( + "suppressing garbage text from OCR-flagged pages: {:?}", + garbage_pages + ); + let items: Vec<_> = items + .into_iter() + .filter(|i| !garbage_pages.contains(&i.page)) + .collect(); + let rects: Vec<_> = rects + .into_iter() + .filter(|r| !garbage_pages.contains(&r.page)) + .collect(); + let lines: Vec<_> = lines + .into_iter() + .filter(|l| !garbage_pages.contains(&l.page)) + .collect(); + (items, rects, lines) + } + }; + let layout = compute_layout_complexity(&items, &rects, &lines); let md = if options.mode == ProcessMode::Analyze { @@ -365,13 +725,91 @@ fn process_document( options.markdown, &rects, &lines, + &page_thresholds, + struct_roles.as_ref(), + &struct_tables, )) }; let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m)); - (md, layout, enc) + (md, layout, enc, gid_encoded_pages) } - None => (None, LayoutComplexity::default(), false), + None => ( + None, + LayoutComplexity::default(), + false, + std::collections::HashSet::new(), + ), + }; + + // If the extracted text is predominantly garbage (non-alphanumeric) and + // the PDF is image-backed (Mixed/template), upgrade to Scanned — the text + // layer comes from a bad OCR pass, and callers should use proper OCR. + let (pdf_type, markdown, confidence) = + if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { + (PdfType::Scanned, None, 0.95) + } else { + (pdf_type, markdown, confidence) + }; + + // If a TextBased PDF produces garbage text, the fonts are undecodable + // (e.g. Identity-H without ToUnicode for non-Latin scripts like Cyrillic). + // Drop the useless markdown and flag all pages for OCR. + let (markdown, has_encoding_issues, force_ocr_all) = if pdf_type == PdfType::TextBased + && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) + { + log::debug!("TextBased PDF has garbage text — flagging all pages for OCR"); + (None, true, true) + } else { + (markdown, has_encoding_issues, false) + }; + + // Add pages with gid-encoded fonts (unresolvable encoding) to OCR list. + // When ALL pages have gid-encoded fonts, suppress unreliable markdown. + let all_gid = !gid_pages.is_empty() && gid_pages.len() as u32 >= page_count; + let mut pages_needing_ocr = pages_needing_ocr; + if force_ocr_all { + pages_needing_ocr = (1..=page_count).collect(); + } + if !gid_pages.is_empty() { + log::debug!("pages with gid-encoded fonts (need OCR): {:?}", gid_pages); + for page in gid_pages { + if !pages_needing_ocr.contains(&page) { + pages_needing_ocr.push(page); + } + } + pages_needing_ocr.sort_unstable(); + } + + // Detect sparse extraction: when a TEXT-BASED PDF produces very few + // characters per page, the text is likely embedded in images/forms + // that need OCR. Flag all pages for OCR in this case. + // Only check when markdown was actually generated (not in Analyze mode). + if pdf_type == PdfType::TextBased + && page_count > 0 + && pages_needing_ocr.is_empty() + && markdown.is_some() + { + let md_len = markdown.as_ref().map_or(0, |m| m.len()); + let chars_per_page = md_len as f32 / page_count as f32; + if chars_per_page < 50.0 && md_len < 500 { + log::debug!( + "sparse extraction: {:.0} chars/page — recommending OCR for all {} pages", + chars_per_page, + page_count + ); + pages_needing_ocr = (1..=page_count).collect(); + } + } + + let markdown = if all_gid { + log::debug!( + "all {} pages have gid-encoded fonts — suppressing markdown output", + page_count + ); + None + } else { + markdown }; Ok(PdfProcessResult { @@ -427,6 +865,76 @@ fn detect_encoding_issues(markdown: &str) -> bool { false } +/// Check if extracted text is predominantly garbage (non-alphanumeric). +/// +/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---." +/// where most characters are punctuation/symbols. Real text in any language +/// has >50% alphanumeric characters. +fn is_garbage_text(markdown: &str) -> bool { + let mut alphanum = 0usize; + let mut non_alphanum = 0usize; + for ch in markdown.chars() { + if ch.is_whitespace() { + continue; + } + // Skip markdown syntax chars that we add (not from the PDF) + if matches!(ch, '#' | '*' | '|' | '-' | '\n') { + continue; + } + if ch.is_alphanumeric() { + alphanum += 1; + } else { + non_alphanum += 1; + } + } + let total = alphanum + non_alphanum; + total >= 50 && alphanum * 2 < total +} + +/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts. +/// +/// When CID values don't correspond to Unicode codepoints, the raw bytes often +/// produce characters in the C1 control range (U+0080–U+009F) or Private Use +/// Area, mixed with random Latin Extended characters. Valid text in any +/// language almost never contains C1 controls. We also fall back to the +/// general `is_garbage_text` check for non-alphanumeric-heavy patterns. +fn is_cid_garbage(text: &str) -> bool { + if is_garbage_text(text) { + return true; + } + let mut total = 0usize; + let mut c1_control = 0usize; + let mut high_latin = 0usize; + for ch in text.chars() { + if ch.is_whitespace() { + continue; + } + total += 1; + // C1 control characters (U+0080–U+009F) — almost never in real text + if ('\u{0080}'..='\u{009F}').contains(&ch) { + c1_control += 1; + } + // High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text + // but when combined with ASCII in CID passthrough, indicates mojibake + // from CID values being misinterpreted as Latin-1 characters. + if ('\u{00A0}'..='\u{00FF}').contains(&ch) { + high_latin += 1; + } + } + if total < 5 { + return false; + } + // If ≥5% of non-whitespace chars are C1 controls, it's garbage + if c1_control * 20 >= total { + return true; + } + // If ≥40% of non-whitespace chars are high Latin-1 AND the text has few + // ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs + // where CID values 0x80-0xFF become accented Latin characters). + let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count(); + high_latin * 5 >= total * 2 && ascii_letters * 3 < total +} + /// Analyse extracted items and rects for layout complexity. fn compute_layout_complexity( items: &[types::TextItem], @@ -507,7 +1015,7 @@ fn compute_layout_complexity( let mut pages_with_columns: Vec = Vec::new(); for page in seen_pages { - let cols = extractor::detect_columns(items, page); + let cols = extractor::detect_columns(items, page, pages_with_tables.contains(&page)); if cols.len() >= 2 { pages_with_columns.push(page); } @@ -726,4 +1234,49 @@ mod tests { let text = "a$b c$d e$f"; assert!(!detect_encoding_issues(text)); } + + #[test] + fn test_garbage_text_detection() { + // Simulates garbage output from Identity-H fonts without ToUnicode. + // Needs >= 50 non-whitespace chars and < 50% alphanumeric. + let garbage = ",&/5 /5&(#(8-!5 *,(6--( *,%@/-A W"; + assert!(is_garbage_text(garbage)); + + // Normal text should not be garbage + let normal = "This is a normal paragraph with words and sentences that contains enough characters to pass the threshold."; + assert!(!is_garbage_text(normal)); + + // Cyrillic text should not be garbage + let cyrillic = + "Роботизированные технологии комплексы для производства металлургических предприятий"; + assert!(!is_garbage_text(cyrillic)); + } + + #[test] + fn test_cid_garbage_detection() { + // Simulates CID garbage from Identity-H fonts: Latin Extended chars + // mixed with C1 control characters (U+0080–U+009F). + let cid_garbage = "Ë>íÓ\tý\r\u{0088}æ&Ït\u{0094}äí;\ný;wAL¢©èåD\rü£\ + qq\u{0096}¶Í Æ\réá; Ô 7G\u{008B}ý;èÕç¢ £ ý;C"; + assert!( + is_cid_garbage(cid_garbage), + "CID garbage with C1 controls should be detected" + ); + + // Valid Korean text (CID-as-Unicode passthrough) should NOT be garbage + let korean = "본 가격표는 국내 거주 중인 외국인을 위한 한국어 가격표의 비공식 번역본입니다"; + assert!( + !is_cid_garbage(korean), + "Valid Korean text should not be flagged as garbage" + ); + + // Valid Japanese text should NOT be garbage + let japanese = "羽田空港新飛行経路に係る航空機騒音の測定結果"; + assert!( + !is_cid_garbage(japanese), + "Valid Japanese text should not be flagged as garbage" + ); + } } diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 42fb107..6d256e3 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; +use crate::structure_tree::StructRole; use crate::types::TextLine; use super::analysis::{ @@ -13,6 +14,50 @@ use super::postprocess::clean_markdown; use super::preprocess::{merge_drop_caps, merge_heading_lines}; use super::MarkdownOptions; +/// Resolve the dominant structure role for a text line by looking up its items' MCIDs. +/// +/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span). +/// These wrapper roles don't carry useful semantic info for markdown generation. +fn resolve_line_struct_role( + line: &TextLine, + struct_roles: &std::collections::HashMap>, +) -> Option { + let page_roles = struct_roles.get(&line.page)?; + for item in &line.items { + if let Some(mcid) = item.mcid { + if let Some(role) = page_roles.get(&mcid) { + match role { + // Skip container/wrapper roles — not useful for line classification + StructRole::Document + | StructRole::Part + | StructRole::Art + | StructRole::Sect + | StructRole::Div + | StructRole::NonStruct + | StructRole::Span + | StructRole::Private => continue, + _ => return Some(role.clone()), + } + } + } + } + None +} + +/// Map a StructRole heading variant to a markdown heading level (1–6). +fn struct_role_heading_level(role: &StructRole) -> Option { + match role { + StructRole::H => Some(1), // Generic heading → H1 + StructRole::H1 => Some(1), + StructRole::H2 => Some(2), + StructRole::H3 => Some(3), + StructRole::H4 => Some(4), + StructRole::H5 => Some(5), + StructRole::H6 => Some(6), + _ => None, + } +} + /// Merge continuation tables that span across page breaks. /// /// When consecutive pages each have exactly one table with the same number of columns @@ -182,6 +227,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( page_tables: std::collections::HashMap>, page_images: std::collections::HashMap>, band_split_pages: &HashSet, + struct_roles: Option< + &std::collections::HashMap>, + >, ) -> String { if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { return String::new(); @@ -200,7 +248,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let heading_tiers = compute_heading_tiers(&lines, base_size); // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); + let lines = merge_heading_lines(lines, base_size, &heading_tiers, struct_roles); // Compute the typical line spacing for paragraph break detection. // For double-spaced documents (like legal/government PDFs), the normal @@ -215,6 +263,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let mut in_list = false; let mut in_paragraph = false; let mut last_list_x: Option = None; + let mut in_code_block = false; let mut prev_had_dot_leaders = false; let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new(); let mut inserted_images: HashSet<(u32, usize)> = HashSet::new(); @@ -233,6 +282,10 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( if line.page != current_page { // Flush current page's remaining tables and images if current_page > 0 { + if in_code_block { + output.push_str("```\n"); + in_code_block = false; + } flush_page_tables_and_images( current_page, &page_tables, @@ -352,7 +405,25 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Detect figure/table captions and source citations // These should be on their own line followed by a paragraph break - if is_caption_line(plain_trimmed) { + let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(&line, roles)); + + // Determine if this line is code (struct-tree or font-based) for block accumulation + let is_code_line = struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Code)) + || (options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))); + + // Close code block when transitioning to non-code + if in_code_block && !is_code_line { + output.push_str("```\n"); + in_code_block = false; + } + + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::Caption)) + || is_caption_line(plain_trimmed) + { if in_paragraph { output.push_str("\n\n"); in_paragraph = false; @@ -362,27 +433,48 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( continue; } - // Detect headers by font size - // Note: Headers typically shouldn't have bold markers since they're already emphasized - // Skip very short text (drop caps/labels) and very long text (body paragraphs) - if options.detect_headers + // Detect headers: structure-tree headings win, then font-size heuristics. + // Structure roles ADD headings (e.g. same-size text tagged H2) but do NOT + // suppress headings that the font heuristic would detect (some tagged PDFs + // mark obvious headings as P or Span). + let struct_heading = struct_role.as_ref().and_then(struct_role_heading_level); + let heuristic_heading = if options.detect_headers && plain_trimmed.len() > 3 && plain_trimmed.split_whitespace().count() <= 15 { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - if let Some(header_level) = - detect_header_level(line_font_size, base_size, &heading_tiers) - { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - let prefix = "#".repeat(header_level); - // Use plain text for headers to avoid redundant formatting - output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); - in_list = false; - continue; + detect_header_level(line_font_size, base_size, &heading_tiers) + } else { + None + }; + + if let Some(level) = struct_heading.or(heuristic_heading) { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + let prefix = "#".repeat(level); + // Use plain text for headers to avoid redundant formatting + output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed)); + in_list = false; + continue; + } + + // Structure-tree list item (LI only — LBody is a continuation, not a new item) + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::LI)) + && !is_list_item(plain_trimmed) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + output.push_str(&format!("- {}", trimmed)); + output.push('\n'); + in_list = true; + last_list_x = line.items.first().map(|i| i.x); + continue; } // Detect list items @@ -428,18 +520,32 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( } } - // Detect code blocks by font - if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; + // Structure-tree block quote + if struct_role + .as_ref() + .is_some_and(|r| matches!(r, StructRole::BlockQuote)) + { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; } + output.push_str(&format!("> {}\n", trimmed)); + continue; + } + + // Code block accumulation (struct-tree Code role or monospace font) + if is_code_line { + if in_paragraph { + output.push_str("\n\n"); + in_paragraph = false; + } + if !in_code_block { + output.push_str("```\n"); + in_code_block = true; + } + output.push_str(plain_trimmed); + output.push('\n'); + continue; } // Regular text - join lines within same paragraph with space @@ -456,6 +562,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( prev_had_dot_leaders = cur_dot_leaders; } + // Close any trailing code block + if in_code_block { + output.push_str("```\n"); + } + // Flush current page and any remaining pages with tables/images // (handles table-only pages after the last text line, and trailing image-only pages) flush_page_tables_and_images( @@ -510,7 +621,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> let heading_tiers = compute_heading_tiers(&lines, base_size); // Merge consecutive heading lines at the same level (e.g., wrapped titles) - let lines = merge_heading_lines(lines, base_size, &heading_tiers); + let lines = merge_heading_lines(lines, base_size, &heading_tiers, None); // Compute the typical line spacing for paragraph break detection let para_threshold = compute_paragraph_threshold(&lines, base_size); @@ -680,3 +791,271 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Clean up and post-process clean_markdown(output, &options) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::structure_tree::StructRole; + use crate::types::TextItem; + use std::collections::HashMap; + + fn make_item(text: &str, page: u32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x: 72.0, + y: 700.0, + width: 100.0, + height: 12.0, + font: "Helvetica".to_string(), + font_size: 12.0, + page, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid, + } + } + + fn make_line(items: Vec) -> TextLine { + let y = items.first().map(|i| i.y).unwrap_or(0.0); + let page = items.first().map(|i| i.page).unwrap_or(1); + TextLine { + items, + y, + page, + adaptive_threshold: 0.10, + } + } + + #[test] + fn test_struct_role_heading() { + let lines = vec![ + make_line(vec![make_item("Introduction", 1, Some(0))]), + make_line(vec![{ + let mut item = make_item("Body text here.", 1, Some(1)); + item.y = 680.0; + item + }]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::H1); + page_roles.insert(1i64, StructRole::P); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("# Introduction"), + "Should have H1 heading: {md}" + ); + assert!( + md.contains("Body text here."), + "Should have body text: {md}" + ); + } + + #[test] + fn test_struct_role_list_item() { + let lines = vec![make_line(vec![make_item("First item", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::LI); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("- First item"), + "Should format as list item: {md}" + ); + } + + #[test] + fn test_struct_role_blockquote() { + let lines = vec![make_line(vec![make_item("Quoted text", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::BlockQuote); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("> Quoted text"), + "Should format as blockquote: {md}" + ); + } + + #[test] + fn test_struct_role_heading_levels() { + let mcids = vec![ + (StructRole::H1, "Title"), + (StructRole::H2, "Section"), + (StructRole::H3, "Subsection"), + ]; + + let mut lines = Vec::new(); + let mut page_roles = HashMap::new(); + for (i, (role, text)) in mcids.iter().enumerate() { + let mut item = make_item(text, 1, Some(i as i64)); + item.y = 700.0 - (i as f32 * 30.0); + lines.push(make_line(vec![item])); + page_roles.insert(i as i64, role.clone()); + } + + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!(md.contains("# Title"), "H1 → #: {md}"); + assert!(md.contains("## Section"), "H2 → ##: {md}"); + assert!(md.contains("### Subsection"), "H3 → ###: {md}"); + } + + #[test] + fn test_no_struct_roles_falls_back_to_heuristics() { + let mut item = make_item("Big Title", 1, None); + item.font_size = 24.0; + item.height = 24.0; + + let lines = vec![ + make_line(vec![item]), + make_line(vec![{ + let mut body = make_item("Normal body text.", 1, None); + body.y = 660.0; + body + }]), + ]; + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + None, + ); + + assert!( + md.contains("# Big Title"), + "Font heuristic should detect heading: {md}" + ); + } + + #[test] + fn test_resolve_line_struct_role_skips_containers() { + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Div); + page_roles.insert(1i64, StructRole::H2); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let line = make_line(vec![ + make_item("Part ", 1, Some(0)), + make_item("Title", 1, Some(1)), + ]); + + let role = resolve_line_struct_role(&line, &roles); + assert_eq!(role, Some(StructRole::H2)); + } + + #[test] + fn test_struct_role_code() { + let lines = vec![make_line(vec![make_item("fn main() {}", 1, Some(0))])]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + assert!( + md.contains("```\nfn main() {}\n```"), + "Should format as code block: {md}" + ); + } + + #[test] + fn test_struct_role_code_multiline_accumulation() { + let mut line1 = make_item("fn main() {", 1, Some(0)); + line1.y = 700.0; + let mut line2 = make_item(" println!(\"hello\");", 1, Some(1)); + line2.y = 688.0; + let mut line3 = make_item("}", 1, Some(2)); + line3.y = 676.0; + + let lines = vec![ + make_line(vec![line1]), + make_line(vec![line2]), + make_line(vec![line3]), + ]; + + let mut page_roles = HashMap::new(); + page_roles.insert(0i64, StructRole::Code); + page_roles.insert(1i64, StructRole::Code); + page_roles.insert(2i64, StructRole::Code); + let mut roles = HashMap::new(); + roles.insert(1u32, page_roles); + + let md = to_markdown_from_lines_with_tables_and_images( + lines, + MarkdownOptions::default(), + HashMap::new(), + HashMap::new(), + &std::collections::HashSet::new(), + Some(&roles), + ); + + // Should produce a single fenced block, not three separate ones + assert!( + md.contains("```\nfn main() {\nprintln!(\"hello\");\n}\n```"), + "Should accumulate consecutive code lines into one block: {md}" + ); + // Should NOT have adjacent fences + assert!( + !md.contains("```\n```"), + "Should not have adjacent close/open fences: {md}" + ); + } +} diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 5a8184b..84a1471 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -16,7 +16,7 @@ pub use convert::to_markdown_from_lines; use std::collections::{HashMap, HashSet}; -use crate::extractor::group_into_lines; +use crate::extractor::group_into_lines_with_thresholds; use crate::types::{PdfLine, PdfRect, TextItem}; use analysis::calculate_font_stats_from_items; @@ -124,6 +124,51 @@ pub(crate) fn split_side_by_side(items: &[TextItem]) -> Vec<(f32, f32)> { return vec![]; } + // Don't split when the left side is text labels and the right side is numeric + // data at matching Y positions — this is a single table (labels + numbers), + // not two independent side-by-side regions. + // Requires ALL THREE: left side is mostly non-numeric, right side is mostly + // numeric, AND high Y-correlation between the two sides. + let is_numeric_item = |item: &&&TextItem| -> bool { + let text = item.text.trim(); + if text.is_empty() { + return false; + } + let data_chars = text + .chars() + .filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c)) + .count(); + data_chars as f32 / text.chars().count() as f32 >= 0.6 + }; + + let left_items: Vec<&TextItem> = items + .iter() + .filter(|i| i.x + i.width / 2.0 < best_split) + .collect(); + let right_items: Vec<&TextItem> = items + .iter() + .filter(|i| i.x + i.width / 2.0 >= best_split) + .collect(); + + if !left_items.is_empty() && !right_items.is_empty() { + let left_numeric_ratio = + left_items.iter().filter(is_numeric_item).count() as f32 / left_items.len() as f32; + let right_numeric_ratio = + right_items.iter().filter(is_numeric_item).count() as f32 / right_items.len() as f32; + + // Left side is mostly text (< 30% numeric) AND right side is mostly numbers (≥ 70%) + if left_numeric_ratio < 0.30 && right_numeric_ratio >= 0.70 { + let y_tol = 5.0; + let y_matches = right_items + .iter() + .filter(|ri| left_items.iter().any(|li| (li.y - ri.y).abs() < y_tol)) + .count(); + if y_matches as f32 / right_items.len() as f32 >= 0.5 { + return vec![]; + } + } + } + vec![(x_min, best_split), (best_split, x_max)] } @@ -452,7 +497,15 @@ pub fn to_markdown_from_items_with_rects( options: MarkdownOptions, rects: &[crate::types::PdfRect], ) -> String { - to_markdown_from_items_with_rects_and_lines(items, options, rects, &[]) + to_markdown_from_items_with_rects_and_lines( + items, + options, + rects, + &[], + &HashMap::new(), + None, + &[], + ) } /// Convert positioned text items to markdown, using rectangles and line segments for table detection. @@ -464,10 +517,13 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( options: MarkdownOptions, rects: &[crate::types::PdfRect], pdf_lines: &[crate::types::PdfLine], + page_thresholds: &HashMap, + struct_roles: Option<&HashMap>>, + struct_tables: &[crate::structure_tree::StructTable], ) -> String { use crate::tables::{ - detect_tables, detect_tables_from_lines, detect_tables_from_rects, table_to_markdown, - try_build_rect_guided_table, + detect_tables, detect_tables_from_lines, detect_tables_from_rects, + detect_tables_from_struct_tree, table_to_markdown, try_build_rect_guided_table, }; use crate::types::ItemType; @@ -593,6 +649,33 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .collect() }; + // When the page is split into bands but no band produces a table, + // retry with all items merged as a single band. This handles + // borderless tables whose column alignment is misclassified as + // page-layout columns by split_side_by_side. + let was_split = band_specs.len() > 1; + log::debug!( + "page {}: {} bands (was_split={})", + page, + band_specs.len(), + was_split + ); + let merged_band: BandSpec = if was_split { + let identity: Vec = (0..page_items.len()).collect(); + ( + page_items.clone(), + identity, + rects.iter().filter(|r| r.page == page).cloned().collect(), + pdf_lines + .iter() + .filter(|l| l.page == page) + .cloned() + .collect(), + ) + } else { + (Vec::new(), Vec::new(), Vec::new(), Vec::new()) + }; + for (band_items, band_index_map, band_rects, band_lines) in &band_specs { if band_items.is_empty() { continue; @@ -601,10 +684,46 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // Track which band-local indices are claimed by structural detection let mut rect_claimed: HashSet = HashSet::new(); - // 1. Rect-based detection first (well-tested, high precision) + // 0. Structure-tree detection (highest priority — semantic PDF tagging) + // Only use struct-tree tables when they capture a majority (≥50%) of + // band items. Incomplete struct trees (partial tagging) should fall + // through to geometry detection which sees all items. + if !struct_tables.is_empty() { + let st_tables = detect_tables_from_struct_tree(band_items, struct_tables, page); + for table in &st_tables { + let coverage = table.item_indices.len() as f32 / band_items.len().max(1) as f32; + if coverage < 0.5 { + continue; + } + for &idx in &table.item_indices { + rect_claimed.insert(idx); + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } + + // 1. Rect-based detection (skips tables overlapping struct-tree claims) let (rect_tables, hint_regions) = detect_tables_from_rects(band_items, band_rects, page); for table in &rect_tables { + if !rect_claimed.is_empty() + && table + .item_indices + .iter() + .any(|idx| rect_claimed.contains(idx)) + { + continue; + } for &idx in &table.item_indices { rect_claimed.insert(idx); if let Some(&page_idx) = band_index_map.get(idx) { @@ -753,9 +872,102 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .unzip(); run_heuristic(&unclaimed_items, &unclaimed_map, 6); } + + // 4. Column-based table detection: last resort for borderless tabular + // layouts (e.g. exam/reference grids) when ALL structural methods + // found nothing. Only runs when no rects/lines exist (truly borderless) + // and no other detection method found tables in this band. + let band_has_tables = band_items.iter().enumerate().any(|(idx, _)| { + band_index_map + .get(idx) + .and_then(|&page_idx| group.get(page_idx)) + .is_some_and(|&(global_idx, _)| table_items.contains(&global_idx)) + }); + let has_structural_elements = band_rects.len() >= 6 || band_lines.len() >= 4; + if !band_has_tables && !has_structural_elements { + if let Some(table) = crate::tables::try_build_table_from_columns(band_items, page) { + for &idx in &table.item_indices { + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } + } + + // Merged-band retry: if we split into bands but found no tables in + // any band, retry heuristic detection with all items as a single band. + // This catches borderless tables whose text-column alignment was + // misclassified as page-layout columns. + if was_split && !page_tables.contains_key(&page) && !merged_band.0.is_empty() { + let (ref band_items, ref band_index_map, _, _) = merged_band; + log::debug!( + "page {}: merged-band retry ({} items, was_split={})", + page, + band_items.len(), + was_split + ); + let heuristic_tables = detect_tables(band_items, base_size, false); + for table in &heuristic_tables { + for &idx in &table.item_indices { + if let Some(&page_idx) = band_index_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } } } + // Check structure tree coverage on ALL text items (before table filtering) + // to decide whether to use structure-aware markdown generation. + let struct_roles_coverage_ok = struct_roles.is_some_and(|roles| { + let total = text_items.len(); + if total == 0 { + return false; + } + let tagged = text_items + .iter() + .filter(|item| { + item.mcid + .and_then(|mcid| { + roles + .get(&item.page) + .and_then(|page_roles| page_roles.get(&mcid)) + }) + .is_some() + }) + .count(); + let coverage = tagged as f32 / total as f32; + log::debug!( + "structure tree coverage: {}/{} items ({:.0}%)", + tagged, + total, + coverage * 100.0 + ); + coverage >= 0.5 + }); + let effective_struct_roles = if struct_roles_coverage_ok { + struct_roles + } else { + None + }; + // Filter out table items and process the rest let non_table_items: Vec = text_items .into_iter() @@ -777,11 +989,15 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // Merge continuation tables across page breaks, but only for table-only pages merge_continuation_tables(&mut page_tables, &table_only_pages); + // Collect pages that have detected tables — used to suppress relative valley + // column detection on pages where table column gaps would be misidentified. + let table_page_set: HashSet = page_tables.keys().copied().collect(); + // Split non-table items by band boundaries before line grouping so that // items from different side-by-side zones (e.g. left/right month columns // in a calendar) don't merge into the same line. let lines = if page_band_splits.is_empty() { - group_into_lines(non_table_items) + group_into_lines_with_thresholds(non_table_items, page_thresholds, &table_page_set) } else { // Separate items into band-split pages and non-split pages let mut split_page_items: HashMap> = HashMap::new(); @@ -794,7 +1010,8 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( } } // Process unsplit pages normally - let mut all_lines = group_into_lines(unsplit_items); + let mut all_lines = + group_into_lines_with_thresholds(unsplit_items, page_thresholds, &table_page_set); // Process each split page's bands independently, then interleave // by Y position so paired zones (e.g. left/right months) appear together. let mut split_pages: Vec = split_page_items.keys().copied().collect(); @@ -811,7 +1028,11 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( .cloned() .collect(); if !band_items.is_empty() { - page_lines.extend(group_into_lines(band_items)); + page_lines.extend(group_into_lines_with_thresholds( + band_items, + page_thresholds, + &table_page_set, + )); } } // Sort by Y descending (top to bottom) so left and right @@ -837,6 +1058,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( page_tables, page_images, &band_split_page_set, + effective_struct_roles, ) } @@ -922,6 +1144,7 @@ mod tests { is_bold: false, is_italic: false, item_type: crate::types::ItemType::Text, + mcid: None, } } @@ -957,4 +1180,33 @@ mod tests { .collect(); assert!(split_from_hint_regions(&items, &rects, 1).is_empty()); } + + #[test] + fn no_split_label_plus_number_table() { + // Balance sheet layout: text labels on left, numbers on right. + // Should NOT split because it's one table, not side-by-side regions. + let mut items = Vec::new(); + for row in 0..30 { + // Label at x=50 + let mut label = make_item(50.0, 700.0 - row as f32 * 15.0, 1); + label.text = format!("Row label {}", row); + label.width = 100.0; + items.push(label); + // Number at x=400 + let mut num1 = make_item(400.0, 700.0 - row as f32 * 15.0, 1); + num1.text = format!("{},000.0", 100 + row); + num1.width = 50.0; + items.push(num1); + // Number at x=470 + let mut num2 = make_item(470.0, 700.0 - row as f32 * 15.0, 1); + num2.text = format!("{},500.0", 200 + row); + num2.width = 50.0; + items.push(num2); + } + let split = split_side_by_side(&items); + assert!( + split.is_empty(), + "label+number table should not be split side-by-side" + ); + } } diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs index 2bd9970..6da6090 100644 --- a/src/markdown/postprocess.rs +++ b/src/markdown/postprocess.rs @@ -24,6 +24,12 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str text = format_urls(&text); } + // Collapse consecutive spaces within text lines. + // OCR text layers and some PDF producers emit trailing spaces on each + // text item, which combine with gap-based space insertion to produce + // double spaces ("Vice President" instead of "Vice President"). + collapse_consecutive_spaces(&mut text); + // Remove excessive newlines (more than 2 in a row) while text.contains("\n\n\n") { text = text.replace("\n\n\n", "\n\n"); @@ -36,6 +42,35 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str text } +/// Collapse runs of 2+ spaces to a single space within each line. +/// Preserves leading indentation and markdown table pipe alignment. +fn collapse_consecutive_spaces(text: &mut String) { + let mut result = String::with_capacity(text.len()); + for line in text.split('\n') { + if !result.is_empty() { + result.push('\n'); + } + // Preserve leading whitespace + let trimmed = line.trim_start(); + let leading = &line[..line.len() - trimmed.len()]; + result.push_str(leading); + // Collapse inner runs of spaces to single space + let mut prev_space = false; + for ch in trimmed.chars() { + if ch == ' ' { + if !prev_space { + result.push(' '); + } + prev_space = true; + } else { + prev_space = false; + result.push(ch); + } + } + } + *text = result; +} + /// Collapse dot leaders (runs of 4+ dots) into " ... " /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" fn collapse_dot_leaders(text: &str) -> String { diff --git a/src/markdown/preprocess.rs b/src/markdown/preprocess.rs index fa89efa..657440b 100644 --- a/src/markdown/preprocess.rs +++ b/src/markdown/preprocess.rs @@ -2,20 +2,62 @@ use std::collections::{HashMap, HashSet}; +use crate::structure_tree::StructRole; use crate::types::TextLine; use super::analysis::detect_header_level; +/// Resolve a heading level for a line, considering both struct-tree roles and font heuristics. +/// Struct-tree headings take priority. +fn effective_heading_level( + line: &TextLine, + base_size: f32, + heading_tiers: &[f32], + struct_roles: Option<&HashMap>>, +) -> Option { + // Check struct-tree role first + if let Some(roles) = struct_roles { + if let Some(page_roles) = roles.get(&line.page) { + for item in &line.items { + if let Some(mcid) = item.mcid { + if let Some(role) = page_roles.get(&mcid) { + let level = match role { + StructRole::H => Some(1), + StructRole::H1 => Some(1), + StructRole::H2 => Some(2), + StructRole::H3 => Some(3), + StructRole::H4 => Some(4), + StructRole::H5 => Some(5), + StructRole::H6 => Some(6), + _ => None, + }; + if level.is_some() { + return level; + } + } + } + } + } + } + + // Fall back to font-size heuristic + let font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); + detect_header_level(font, base_size, heading_tiers) +} + /// Merge consecutive heading lines at the same level into a single line. /// /// When a heading wraps across multiple text lines (e.g., "About Glenair, the Mission-Critical" /// and "Interconnect Company"), each fragment becomes a separate `# Header` in the output. /// This function detects consecutive lines at the same heading tier on the same page /// with a small Y gap and merges them into one line. +/// +/// Both font-size heuristic headings and struct-tree tagged headings are considered. pub(crate) fn merge_heading_lines( lines: Vec, base_size: f32, heading_tiers: &[f32], + struct_roles: Option<&HashMap>>, ) -> Vec { if lines.is_empty() { return lines; @@ -24,19 +66,23 @@ pub(crate) fn merge_heading_lines( let mut result: Vec = Vec::with_capacity(lines.len()); for line in lines { + let line_level = effective_heading_level(&line, base_size, heading_tiers, struct_roles); let line_font = line.items.first().map(|i| i.font_size).unwrap_or(base_size); - let line_level = detect_header_level(line_font, base_size, heading_tiers); // Check if the previous line is a heading at the same level on the same page let should_merge = if let (Some(prev), Some(curr_level)) = (result.last(), line_level) { - let prev_font = prev.items.first().map(|i| i.font_size).unwrap_or(base_size); - let prev_level = detect_header_level(prev_font, base_size, heading_tiers); + let prev_level = effective_heading_level(prev, base_size, heading_tiers, struct_roles); let same_page = prev.page == line.page; let same_level = prev_level == Some(curr_level); let y_gap = prev.y - line.y; // Merge if gap is within ~2x the font size (normal line wrap spacing) let close_enough = y_gap > 0.0 && y_gap < line_font * 2.0; - same_page && same_level && close_enough + // Don't merge if combined text would be too long — real headings are short. + // This prevents merging body-text lines that are mis-tagged as headings. + let prev_words = prev.text().split_whitespace().count(); + let curr_words = line.text().split_whitespace().count(); + let not_too_long = prev_words + curr_words <= 20; + same_page && same_level && close_enough && not_too_long } else { false }; @@ -377,9 +423,13 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec // (a) its individual text matches a candidate, OR // (b) its Y-band's coalesced text matches a band candidate, OR // (c) any sibling in its Y-band was removed (propagation). + // + // The first occurrence (lowest page number) of each repeated header/footer + // is kept so that document titles, column headers, etc. appear once. let mut removal_set: HashSet = HashSet::new(); - // (a) Lines matching individual candidates at edge positions + // Track which page first shows each candidate (to preserve first occurrence) + let mut first_page_individual: HashMap = HashMap::new(); for (idx, line) in lines.iter().enumerate() { if !is_y_at_edge(line.y, line.page, &page_sorted_ys, EDGE_LINE_COUNT) { continue; @@ -387,11 +437,18 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec let text = line.text(); let normalized = normalize_for_comparison(&text); if candidates.contains(&normalized) { - removal_set.insert(idx); + let first = first_page_individual.entry(normalized).or_insert(line.page); + if line.page > *first { + removal_set.insert(idx); + } else if line.page == *first { + // Keep this occurrence (first page) + } } } - // (b) Lines in Y-bands whose coalesced text matches a band candidate + // Track first page for band candidates + let mut first_page_band: HashMap = HashMap::new(); + // First pass: find first page for each band candidate for (&(page, _), indices) in &y_bands { if indices.len() < 2 { continue; @@ -409,8 +466,35 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec .join(" "); let normalized = normalize_for_comparison(&coalesced); if band_candidates.contains(&normalized) { - for &idx in &sorted_indices { - removal_set.insert(idx); + let first = first_page_band.entry(normalized).or_insert(page); + if page < *first { + *first = page; + } + } + } + // Second pass: mark for removal (skip first page) + for (&(page, _), indices) in &y_bands { + if indices.len() < 2 { + continue; + } + let band_y = lines[indices[0]].y; + if !is_y_at_edge(band_y, page, &page_sorted_ys, EDGE_LINE_COUNT) { + continue; + } + let mut sorted_indices = indices.clone(); + sorted_indices.sort(); + let coalesced: String = sorted_indices + .iter() + .map(|&i| lines[i].text()) + .collect::>() + .join(" "); + let normalized = normalize_for_comparison(&coalesced); + if band_candidates.contains(&normalized) { + let first = first_page_band.get(&normalized).copied().unwrap_or(0); + if page > first { + for &idx in &sorted_indices { + removal_set.insert(idx); + } } } } @@ -440,3 +524,163 @@ pub(crate) fn strip_repeated_lines(lines: Vec, page_count: u32) -> Vec .map(|(_, line)| line) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ItemType, TextItem}; + + fn make_item(text: &str, font_size: f32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x: 0.0, + y: 0.0, + width: 100.0, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid, + } + } + + fn make_line(text: &str, font_size: f32, page: u32, y: f32, mcid: Option) -> TextLine { + TextLine { + items: vec![make_item(text, font_size, mcid)], + y, + page, + adaptive_threshold: 0.10, + } + } + + #[test] + fn test_merge_struct_tree_headings() { + // Two consecutive lines tagged as H2 via struct tree, same font size as body + let lines = vec![ + make_line( + "Historical Context for Operations in Snow", + 12.0, + 1, + 700.0, + Some(10), + ), + make_line("Lake District", 12.0, 1, 686.0, Some(11)), + make_line("Body text paragraph.", 12.0, 1, 660.0, Some(12)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H2); + roles.insert(11i64, StructRole::H2); + roles.insert(12i64, StructRole::P); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should merge two H2 lines into one"); + let merged_text = result[0].text(); + assert!( + merged_text.contains("Snow") && merged_text.contains("Lake"), + "merged heading should contain both fragments: {merged_text}" + ); + } + + #[test] + fn test_no_merge_different_struct_levels() { + // Two consecutive lines tagged as different heading levels + let lines = vec![ + make_line("Chapter 1", 12.0, 1, 700.0, Some(10)), + make_line("Introduction", 12.0, 1, 686.0, Some(11)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H1); + roles.insert(11i64, StructRole::H2); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should not merge different heading levels"); + } + + #[test] + fn test_no_merge_heading_with_body() { + // A heading line followed by a body paragraph line + let lines = vec![ + make_line("Introduction", 12.0, 1, 700.0, Some(10)), + make_line("This is body text.", 12.0, 1, 686.0, Some(11)), + ]; + + let mut page_roles = HashMap::new(); + let mut roles = HashMap::new(); + roles.insert(10i64, StructRole::H1); + roles.insert(11i64, StructRole::P); + page_roles.insert(1u32, roles); + + let result = merge_heading_lines(lines, 12.0, &[], Some(&page_roles)); + assert_eq!(result.len(), 2, "should not merge heading with body text"); + } + + #[test] + fn test_merge_font_headings_still_works() { + // Original font-size based merging should still work without struct roles + let lines = vec![ + make_line("A Very Long Heading That", 18.0, 1, 700.0, None), + make_line("Wraps to Next Line", 18.0, 1, 682.0, None), + make_line("Body text.", 12.0, 1, 660.0, None), + ]; + + let heading_tiers = vec![18.0]; + let result = merge_heading_lines(lines, 12.0, &heading_tiers, None); + assert_eq!(result.len(), 2, "should merge font-based heading lines"); + } + + #[test] + fn test_strip_repeated_keeps_first_occurrence() { + // Simulate a repeated page header on 10 pages. + // Each page has a running header at y=750 and many unique body lines. + let mut lines = Vec::new(); + for page in 1..=10u32 { + // Header at top + lines.push(make_line( + "VOICE OF SOUTH MARION May fifteen twenty twenty five", + 10.0, + page, + 750.0, + None, + )); + // Body content — unique text per line per page (no digits to strip) + for j in 0..20u32 { + lines.push(make_line( + &format!( + "parcel r-{:04}-{:03} owner smith address oak street", + page * 100 + j, + page + ), + 10.0, + page, + 600.0 - j as f32 * 15.0, + None, + )); + } + } + + let result = strip_repeated_lines(lines, 10); + + // The header should appear exactly once (page 1) + let header_count = result + .iter() + .filter(|l| l.text().contains("VOICE OF SOUTH MARION")) + .count(); + assert_eq!(header_count, 1, "repeated header should be kept once"); + + // First occurrence should be on page 1 + let first_header = result + .iter() + .find(|l| l.text().contains("VOICE OF SOUTH MARION")) + .unwrap(); + assert_eq!(first_header.page, 1, "first occurrence should be on page 1"); + } +} diff --git a/src/structure_tree.rs b/src/structure_tree.rs new file mode 100644 index 0000000..fa40110 --- /dev/null +++ b/src/structure_tree.rs @@ -0,0 +1,1140 @@ +//! Tagged PDF structure tree parser. +//! +//! Reads the `/StructTreeRoot` from the document catalog and builds an +//! in-memory tree of [`StructElement`] nodes. Each leaf maps back to +//! content-stream marked content via MCID (Marked Content ID), which lets +//! downstream code attach semantic roles (heading, paragraph, table cell, +//! list item, …) to extracted [`TextItem`]s. + +use log::debug; +use lopdf::{Document, Object, ObjectId}; +use std::borrow::Cow; +use std::collections::HashMap; + +// ─── Standard structure types ──────────────────────────────────────── + +/// Standard PDF structure element types (ISO 32000-1, Table 333–340). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum StructRole { + Document, + Part, + Art, + Sect, + Div, + BlockQuote, + Caption, + TOC, + TOCI, + Index, + NonStruct, + Private, + // Heading & paragraph + H, + H1, + H2, + H3, + H4, + H5, + H6, + P, + // List + L, + LI, + Lbl, + LBody, + // Table + Table, + TR, + TH, + TD, + THead, + TBody, + TFoot, + // Inline + Span, + Quote, + Note, + Reference, + BibEntry, + Code, + Link, + Annot, + // Illustration + Figure, + Formula, + Form, + // Ruby / Warichu (CJK) + Ruby, + RB, + RT, + RP, + Warichu, + WT, + WP, + // Fallback + Other(String), +} + +impl StructRole { + fn from_name(name: &str) -> Self { + match name { + "Document" => Self::Document, + "Part" => Self::Part, + "Art" => Self::Art, + "Sect" => Self::Sect, + "Div" => Self::Div, + "BlockQuote" => Self::BlockQuote, + "Caption" => Self::Caption, + "TOC" => Self::TOC, + "TOCI" => Self::TOCI, + "Index" => Self::Index, + "NonStruct" => Self::NonStruct, + "Private" => Self::Private, + "H" => Self::H, + "H1" => Self::H1, + "H2" => Self::H2, + "H3" => Self::H3, + "H4" => Self::H4, + "H5" => Self::H5, + "H6" => Self::H6, + "P" => Self::P, + "L" => Self::L, + "LI" => Self::LI, + "Lbl" => Self::Lbl, + "LBody" => Self::LBody, + "Table" => Self::Table, + "TR" => Self::TR, + "TH" => Self::TH, + "TD" => Self::TD, + "THead" => Self::THead, + "TBody" => Self::TBody, + "TFoot" => Self::TFoot, + "Span" => Self::Span, + "Quote" => Self::Quote, + "Note" => Self::Note, + "Reference" => Self::Reference, + "BibEntry" => Self::BibEntry, + "Code" => Self::Code, + "Link" => Self::Link, + "Annot" => Self::Annot, + "Figure" => Self::Figure, + "Formula" => Self::Formula, + "Form" => Self::Form, + "Ruby" => Self::Ruby, + "RB" => Self::RB, + "RT" => Self::RT, + "RP" => Self::RP, + "Warichu" => Self::Warichu, + "WT" => Self::WT, + "WP" => Self::WP, + other => Self::Other(other.to_string()), + } + } + + /// Resolve a possibly-custom tag name through a role map. + fn from_name_with_role_map(name: &str, role_map: &HashMap) -> Self { + // Follow role map chain (max 8 hops to avoid cycles) + let mut current = name.to_string(); + for _ in 0..8 { + let role = Self::from_name(¤t); + if !matches!(role, Self::Other(_)) { + return role; + } + if let Some(mapped) = role_map.get(current.as_str()) { + current = mapped.clone(); + } else { + return role; + } + } + Self::Other(name.to_string()) + } +} + +// ─── Marked content reference ──────────────────────────────────────── + +/// A leaf reference linking a structure element to content-stream content. +#[derive(Debug, Clone)] +pub struct MarkedContentRef { + /// The Marked Content ID used in the content stream's `BDC`/`BMC`. + pub mcid: i64, + /// Page ObjectId this content belongs to (from `/Pg` key). + pub page_id: Option, +} + +// ─── Structure element ─────────────────────────────────────────────── + +/// A node in the PDF structure tree. +#[derive(Debug, Clone)] +pub struct StructElement { + /// Semantic role (H1, P, Table, TD, …). + pub role: StructRole, + /// Alternative text for figures / illustrations. + pub alt_text: Option, + /// Actual text override (e.g. for ligatures). + pub actual_text: Option, + /// Language override (e.g. "en-US"). + pub lang: Option, + /// Direct marked-content references (leaf content). + pub content_refs: Vec, + /// Child structure elements. + pub children: Vec, +} + +// ─── Structure tree (top level) ────────────────────────────────────── + +/// Parsed PDF structure tree. +/// +/// Built from `/StructTreeRoot` in the document catalog. Use +/// [`StructTree::from_doc`] to parse, then [`StructTree::mcid_to_roles`] +/// to get per-page MCID → role lookup tables. +#[derive(Debug, Clone)] +pub struct StructTree { + /// Root children (the top-level structure elements). + pub children: Vec, +} + +impl StructTree { + /// Attempt to parse the structure tree from a PDF document. + /// + /// Returns `None` if the PDF is not tagged (no `/StructTreeRoot`). + pub fn from_doc(doc: &Document) -> Option { + let catalog = doc.catalog().ok()?; + let struct_root_obj = catalog.get(b"StructTreeRoot").ok()?; + let struct_root = resolve_dict(doc, struct_root_obj)?; + + // Parse role map: custom tag → standard tag + let role_map = parse_role_map(doc, struct_root); + debug!("structure tree: {} role map entries", role_map.len()); + + // Parse child elements from /K + let children = parse_kids(doc, struct_root, &role_map, None, 0); + debug!("structure tree: {} top-level elements", children.len()); + + if children.is_empty() { + return None; + } + + Some(StructTree { children }) + } + + /// Build per-page MCID → StructRole lookup. + /// + /// Returns a map: page_number (1-indexed) → (MCID → StructRole). + /// The `page_ids` map should come from `doc.get_pages()`. + pub fn mcid_to_roles( + &self, + page_ids: &std::collections::BTreeMap, + ) -> HashMap> { + // Invert: ObjectId → page number + let obj_to_page: HashMap = + page_ids.iter().map(|(&num, &id)| (id, num)).collect(); + + let mut result: HashMap> = HashMap::new(); + self.collect_mcid_roles(&self.children, &obj_to_page, &mut result); + result + } + + fn collect_mcid_roles( + &self, + elements: &[StructElement], + obj_to_page: &HashMap, + result: &mut HashMap>, + ) { + for elem in elements { + for mcref in &elem.content_refs { + if let Some(page_id) = mcref.page_id { + if let Some(&page_num) = obj_to_page.get(&page_id) { + result + .entry(page_num) + .or_default() + .insert(mcref.mcid, elem.role.clone()); + } + } + } + self.collect_mcid_roles(&elem.children, obj_to_page, result); + } + } + + /// Count total marked-content references across the tree. + pub fn mcid_count(&self) -> usize { + fn count(elements: &[StructElement]) -> usize { + elements + .iter() + .map(|e| e.content_refs.len() + count(&e.children)) + .sum() + } + count(&self.children) + } + + /// Build a flat list of structure elements with their roles and MCIDs, + /// preserving document order. Useful for structure-aware markdown generation. + pub fn flatten(&self) -> Vec { + let mut out = Vec::new(); + flatten_recursive(&self.children, &mut out, 0); + out + } + + /// Extract table structures from the tagged PDF tree. + /// + /// Walks the tree to find `/Table` elements with `/TR` > `/TD|TH` children, + /// collecting MCIDs at each cell. Returns structured descriptors that can + /// be matched against extracted [`TextItem`]s to build tables without + /// relying on geometry-based detection. + pub fn extract_tables( + &self, + page_ids: &std::collections::BTreeMap, + ) -> Vec { + let obj_to_page: HashMap = + page_ids.iter().map(|(&num, &id)| (id, num)).collect(); + let mut tables = Vec::new(); + collect_tables(&self.children, &obj_to_page, &mut tables); + tables + } +} + +// ─── Tagged table structures ──────────────────────────────────────── + +/// A table cell extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTableCell { + /// Whether this cell is a header cell (`/TH`). + pub is_header: bool, + /// MCIDs with their resolved page numbers. + pub mcids: Vec<(i64, u32)>, +} + +/// A table row extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTableRow { + pub cells: Vec, +} + +/// A complete table extracted from the structure tree. +#[derive(Debug, Clone)] +pub struct StructTable { + pub rows: Vec, +} + +fn collect_tables( + elements: &[StructElement], + obj_to_page: &HashMap, + tables: &mut Vec, +) { + for elem in elements { + if elem.role == StructRole::Table { + let mut rows = Vec::new(); + collect_rows(&elem.children, obj_to_page, &mut rows); + if rows.len() >= 2 && rows.iter().any(|r| !r.cells.is_empty()) { + tables.push(StructTable { rows }); + } + } else { + collect_tables(&elem.children, obj_to_page, tables); + } + } +} + +/// Collect rows from Table children, transparently descending through +/// THead/TBody/TFoot grouping elements. +fn collect_rows( + elements: &[StructElement], + obj_to_page: &HashMap, + rows: &mut Vec, +) { + for elem in elements { + match elem.role { + StructRole::TR => { + let mut cells = Vec::new(); + for child in &elem.children { + if child.role == StructRole::TD || child.role == StructRole::TH { + let is_header = child.role == StructRole::TH; + let mut mcids = Vec::new(); + collect_mcids_recursive(child, obj_to_page, &mut mcids); + cells.push(StructTableCell { is_header, mcids }); + } + } + rows.push(StructTableRow { cells }); + } + StructRole::THead | StructRole::TBody | StructRole::TFoot => { + collect_rows(&elem.children, obj_to_page, rows); + } + _ => {} + } + } +} + +/// Recursively collect all MCIDs from an element and its descendants. +fn collect_mcids_recursive( + elem: &StructElement, + obj_to_page: &HashMap, + mcids: &mut Vec<(i64, u32)>, +) { + for mcref in &elem.content_refs { + if let Some(page_id) = mcref.page_id { + if let Some(&page_num) = obj_to_page.get(&page_id) { + mcids.push((mcref.mcid, page_num)); + } + } + } + for child in &elem.children { + collect_mcids_recursive(child, obj_to_page, mcids); + } +} + +/// A flattened view of a structure element for linear traversal. +#[derive(Debug, Clone)] +pub struct FlatStructElement { + /// Semantic role. + pub role: StructRole, + /// Nesting depth (0 = top-level). + pub depth: usize, + /// Alt text (figures). + pub alt_text: Option, + /// Direct MCIDs with page ObjectIds. + pub content_refs: Vec, + /// Number of child elements (in the original tree). + pub child_count: usize, +} + +fn flatten_recursive(elements: &[StructElement], out: &mut Vec, depth: usize) { + for elem in elements { + out.push(FlatStructElement { + role: elem.role.clone(), + depth, + alt_text: elem.alt_text.clone(), + content_refs: elem.content_refs.clone(), + child_count: elem.children.len(), + }); + flatten_recursive(&elem.children, out, depth + 1); + } +} + +// ─── Parsing helpers ───────────────────────────────────────────────── + +/// Parse the `/RoleMap` dictionary (custom tag → standard tag). +fn parse_role_map(doc: &Document, struct_root: &lopdf::Dictionary) -> HashMap { + let mut map = HashMap::new(); + let Ok(rm_obj) = struct_root.get(b"RoleMap") else { + return map; + }; + let Some(rm_dict) = resolve_dict(doc, rm_obj) else { + return map; + }; + for (key, val) in rm_dict.iter() { + let key_str = String::from_utf8_lossy(key).to_string(); + if let Ok(name) = val.as_name() { + let val_str = String::from_utf8_lossy(name).to_string(); + map.insert(key_str, val_str); + } + } + map +} + +/// Max recursion depth for structure tree parsing (prevents stack overflow on +/// malformed PDFs). +const MAX_DEPTH: usize = 64; + +/// Parse child elements from a `/K` entry. +fn parse_kids( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, +) -> Vec { + if depth >= MAX_DEPTH { + return Vec::new(); + } + + let Ok(k_obj) = dict.get(b"K") else { + return Vec::new(); + }; + + // /Pg on this element (inherited by children) + let page_id = get_page_ref(doc, dict).or(inherited_page); + + match k_obj { + Object::Array(arr) => { + let mut children = Vec::new(); + for item in arr { + let resolved = resolve_obj(doc, item); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + } + children + } + other => { + let resolved = resolve_obj(doc, other); + let mut children = Vec::new(); + parse_kid(doc, resolved, role_map, page_id, depth, &mut children); + children + } + } +} + +/// Parse a single child (either a struct element dict or an MCID integer). +fn parse_kid( + doc: &Document, + obj: &Object, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + match obj { + // Direct MCID integer — create a leaf wrapper + Object::Integer(mcid) => { + // This is a bare MCID at the struct-element level. + // We attach it to the parent element, so we create a wrapper struct element. + // Actually, bare MCIDs inside /K are content refs for the parent, + // not separate child elements. We handle this at the caller level. + // For now, create a minimal Span wrapper. + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id: inherited_page, + }], + children: Vec::new(), + }); + } + Object::Dictionary(d) => { + parse_struct_element_dict(doc, d, role_map, inherited_page, depth, out); + } + Object::Stream(s) => { + // Some PDFs wrap struct elements in streams (rare) + parse_struct_element_dict(doc, &s.dict, role_map, inherited_page, depth, out); + } + _ => {} + } +} + +/// Parse a dictionary that could be either a struct element or a marked-content +/// reference (MCR) dictionary. +fn parse_struct_element_dict( + doc: &Document, + dict: &lopdf::Dictionary, + role_map: &HashMap, + inherited_page: Option, + depth: usize, + out: &mut Vec, +) { + if depth >= MAX_DEPTH { + return; + } + // Check if this is a marked-content reference dict (has /Type /MCR) + if is_mcr_dict(dict) { + if let Ok(Object::Integer(mcid)) = dict.get(b"MCID") { + let page_id = get_page_ref(doc, dict).or(inherited_page); + out.push(StructElement { + role: StructRole::Span, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: *mcid, + page_id, + }], + children: Vec::new(), + }); + } + return; + } + + // Check if this is an object reference dict (has /Type /OBJR) — skip these + if is_objr_dict(dict) { + return; + } + + // It's a struct element — parse its /S (structure type) + let role_name = match dict.get(b"S") { + Ok(s_obj) => { + let resolved = resolve_obj(doc, s_obj); + match resolved.as_name() { + Ok(name) => String::from_utf8_lossy(name).to_string(), + Err(_) => return, + } + } + Err(_) => return, + }; + + let role = StructRole::from_name_with_role_map(&role_name, role_map); + let page_id = get_page_ref(doc, dict).or(inherited_page); + + // Extract optional attributes + let alt_text = get_text_string(dict, b"Alt"); + let actual_text = get_text_string(dict, b"ActualText"); + let lang = get_text_string(dict, b"Lang"); + + // Parse children from /K + let mut content_refs = Vec::new(); + let mut children = Vec::new(); + + if let Ok(k_obj) = dict.get(b"K") { + let k_resolved = resolve_obj(doc, k_obj); + match k_resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Array(arr) => { + for item in arr { + let resolved = resolve_obj(doc, item); + match resolved { + Object::Integer(mcid) => { + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id, + }); + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else if is_objr_dict(d) { + // Skip object references + } else { + parse_struct_element_dict( + doc, + d, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + } + Object::Stream(s) => { + parse_struct_element_dict( + doc, + &s.dict, + role_map, + page_id, + depth + 1, + &mut children, + ); + } + _ => {} + } + } + } + Object::Dictionary(d) => { + if is_mcr_dict(d) { + if let Ok(Object::Integer(mcid)) = d.get(b"MCID") { + let pg = get_page_ref(doc, d).or(page_id); + content_refs.push(MarkedContentRef { + mcid: *mcid, + page_id: pg, + }); + } + } else { + parse_struct_element_dict(doc, d, role_map, page_id, depth + 1, &mut children); + } + } + _ => {} + } + } + + out.push(StructElement { + role, + alt_text, + actual_text, + lang, + content_refs, + children, + }); +} + +/// Check if dict has `/Type /MCR`. +fn is_mcr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"MCR") +} + +/// Check if dict has `/Type /OBJR`. +fn is_objr_dict(dict: &lopdf::Dictionary) -> bool { + dict.get(b"Type") + .ok() + .and_then(|o| o.as_name().ok()) + .is_some_and(|n| n == b"OBJR") +} + +/// Get the `/Pg` page reference from a dictionary. +fn get_page_ref(doc: &Document, dict: &lopdf::Dictionary) -> Option { + let pg = dict.get(b"Pg").ok()?; + match pg { + Object::Reference(id) => Some(*id), + _ => { + let resolved = resolve_obj(doc, pg); + if let Object::Reference(id) = resolved { + Some(*id) + } else { + None + } + } + } +} + +/// Extract a text string from a dictionary key (handles PDF text encoding). +fn get_text_string(dict: &lopdf::Dictionary, key: &[u8]) -> Option { + let obj = dict.get(key).ok()?; + match obj { + Object::String(bytes, _) => Some(crate::text_utils::decode_text_string(bytes)), + _ => None, + } +} + +/// Resolve an Object reference, returning the target object. +fn resolve_obj<'a>(doc: &'a Document, obj: &'a Object) -> &'a Object { + match obj { + Object::Reference(id) => doc.get_object(*id).unwrap_or(obj), + _ => obj, + } +} + +/// Resolve an Object to a dictionary (handling references). +fn resolve_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a lopdf::Dictionary> { + match obj { + Object::Dictionary(d) => Some(d), + Object::Reference(id) => doc.get_dictionary(*id).ok(), + _ => None, + } +} + +// ─── PDF byte pre-processing ──────────────────────────────────────── + +/// Fix malformed structure element `/S` entries in raw PDF bytes. +/// +/// Some PDF generators (notably fpdf2) write bare names like `/S Code` +/// instead of the correct `/S /Code`. lopdf cannot parse dictionaries +/// containing bare tokens, so the entire object is silently dropped. +/// +/// This function scans for the pattern `/S ` inside struct +/// element dictionaries and prepends `/` to make them valid PDF names. +/// Returns `Cow::Borrowed` if no fixes were needed. +pub fn fix_bare_struct_names(buf: &[u8]) -> Cow<'_, [u8]> { + // Quick check: if no StructTreeRoot, nothing to fix + if !contains_bytes(buf, b"/StructTreeRoot") { + return Cow::Borrowed(buf); + } + + // Known struct type names that may appear as bare tokens. + // We only fix names that are valid PDF structure types to avoid + // false positives on arbitrary dictionary values. + const KNOWN_NAMES: &[&[u8]] = &[ + b"Document", + b"Part", + b"Art", + b"Sect", + b"Div", + b"BlockQuote", + b"Caption", + b"TOC", + b"TOCI", + b"Index", + b"NonStruct", + b"Private", + b"H", + b"H1", + b"H2", + b"H3", + b"H4", + b"H5", + b"H6", + b"P", + b"L", + b"LI", + b"Lbl", + b"LBody", + b"Table", + b"TR", + b"TH", + b"TD", + b"THead", + b"TBody", + b"TFoot", + b"Span", + b"Quote", + b"Note", + b"Reference", + b"BibEntry", + b"Code", + b"Link", + b"Annot", + b"Figure", + b"Formula", + b"Form", + b"Ruby", + b"RB", + b"RT", + b"RP", + b"Warichu", + b"WT", + b"WP", + ]; + + let pattern = b"/S "; + let mut result: Option> = None; + let mut pos = 0; + + while pos + pattern.len() < buf.len() { + let Some(idx) = find_bytes(&buf[pos..], pattern).map(|i| i + pos) else { + break; + }; + + let after = idx + pattern.len(); + // Check if the next char is already '/' (correct name) or not + if after < buf.len() && buf[after] == b'/' { + pos = after; + continue; + } + + // Try to match a known bare struct name at this position + let mut matched = false; + for name in KNOWN_NAMES { + let end = after + name.len(); + if end <= buf.len() + && &buf[after..end] == *name + // Must be followed by a delimiter (whitespace, newline, /, >) + && (end >= buf.len() || matches!(buf[end], b'\n' | b'\r' | b' ' | b'/' | b'>')) + { + // Found a bare name — lazily allocate output buffer + let out = result.get_or_insert_with(|| buf[..after].to_vec()); + // Append everything from last position up to the bare name + if out.len() < after { + out.extend_from_slice(&buf[out.len()..after]); + } + out.push(b'/'); + out.extend_from_slice(name); + pos = end; + matched = true; + debug!( + "fix_bare_struct_names: patched /S {} → /S /{}", + String::from_utf8_lossy(name), + String::from_utf8_lossy(name) + ); + break; + } + } + + if !matched { + pos = after; + } + } + + match result { + Some(mut out) => { + // Append remaining bytes + if out.len() < buf.len() { + out.extend_from_slice(&buf[out.len()..]); + } + Cow::Owned(out) + } + None => Cow::Borrowed(buf), + } +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + find_bytes(haystack, needle).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_struct_role_from_name() { + assert_eq!(StructRole::from_name("H1"), StructRole::H1); + assert_eq!(StructRole::from_name("P"), StructRole::P); + assert_eq!(StructRole::from_name("Table"), StructRole::Table); + assert_eq!(StructRole::from_name("TD"), StructRole::TD); + assert_eq!( + StructRole::from_name("CustomTag"), + StructRole::Other("CustomTag".to_string()) + ); + } + + #[test] + fn test_struct_role_with_role_map() { + let mut role_map = HashMap::new(); + role_map.insert("Heading1".to_string(), "H1".to_string()); + role_map.insert("Body".to_string(), "P".to_string()); + // Chain: MyTag → Heading1 → H1 + role_map.insert("MyTag".to_string(), "Heading1".to_string()); + + assert_eq!( + StructRole::from_name_with_role_map("Heading1", &role_map), + StructRole::H1 + ); + assert_eq!( + StructRole::from_name_with_role_map("Body", &role_map), + StructRole::P + ); + assert_eq!( + StructRole::from_name_with_role_map("MyTag", &role_map), + StructRole::H1 + ); + // Standard names bypass the map + assert_eq!( + StructRole::from_name_with_role_map("H2", &role_map), + StructRole::H2 + ); + } + + #[test] + fn test_struct_role_role_map_cycle() { + // A→B→A cycle should not infinite-loop + let mut role_map = HashMap::new(); + role_map.insert("A".to_string(), "B".to_string()); + role_map.insert("B".to_string(), "A".to_string()); + + let role = StructRole::from_name_with_role_map("A", &role_map); + // Should terminate (as Other) rather than loop forever + assert!(matches!(role, StructRole::Other(_))); + } + + #[test] + fn test_flat_struct_element() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + let flat = tree.flatten(); + assert_eq!(flat.len(), 3); + assert_eq!(flat[0].role, StructRole::Document); + assert_eq!(flat[0].depth, 0); + assert_eq!(flat[1].role, StructRole::H1); + assert_eq!(flat[1].depth, 1); + assert_eq!(flat[2].role, StructRole::P); + assert_eq!(flat[2].depth, 1); + } + + #[test] + fn test_mcid_count() { + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![ + MarkedContentRef { + mcid: 0, + page_id: Some((1, 0)), + }, + MarkedContentRef { + mcid: 1, + page_id: Some((1, 0)), + }, + ], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 2, + page_id: Some((1, 0)), + }], + children: Vec::new(), + }, + ], + }], + }; + + assert_eq!(tree.mcid_count(), 3); + } + + #[test] + fn test_mcid_to_roles() { + use std::collections::BTreeMap; + + let page_id: ObjectId = (5, 0); + let mut page_ids = BTreeMap::new(); + page_ids.insert(1u32, page_id); + + let tree = StructTree { + children: vec![StructElement { + role: StructRole::Document, + alt_text: None, + actual_text: None, + lang: None, + content_refs: Vec::new(), + children: vec![ + StructElement { + role: StructRole::H1, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 0, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + StructElement { + role: StructRole::P, + alt_text: None, + actual_text: None, + lang: None, + content_refs: vec![MarkedContentRef { + mcid: 1, + page_id: Some(page_id), + }], + children: Vec::new(), + }, + ], + }], + }; + + let roles = tree.mcid_to_roles(&page_ids); + let page1 = roles.get(&1).unwrap(); + assert_eq!(page1.get(&0), Some(&StructRole::H1)); + assert_eq!(page1.get(&1), Some(&StructRole::P)); + } + + #[test] + fn test_fix_bare_struct_names() { + // Verify the byte-level pre-processor fixes bare names. + // All inputs include /StructTreeRoot to pass the early-return guard. + let input = b"/StructTreeRoot /S Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!( + fixed.windows(b"/S /Code".len()).any(|w| w == b"/S /Code"), + "Should fix bare Code: {:?}", + String::from_utf8_lossy(&fixed) + ); + + // Already correct — should return borrowed + let input = b"/StructTreeRoot /S /Code\n/Type /StructElem"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + + // Multiple bare names + let input = b"/StructTreeRoot /S H1\n/foo\n/S P\n/bar"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S /H1"), "Should fix H1: {s}"); + assert!(s.contains("/S /P"), "Should fix P: {s}"); + + // Unknown name should not be touched + let input = b"/StructTreeRoot /S FooBar\n"; + let fixed = fix_bare_struct_names(input); + let s = String::from_utf8_lossy(&fixed); + assert!(s.contains("/S FooBar"), "Should not fix unknown: {s}"); + + // No StructTreeRoot — skip entirely + let input = b"/S Code\nno struct tree"; + let fixed = fix_bare_struct_names(input); + assert!(matches!(fixed, std::borrow::Cow::Borrowed(_))); + } + + #[test] + fn test_bare_name_struct_types() { + // Some PDF generators (e.g. fpdf2) write /S Code instead of /S /Code. + // lopdf silently drops objects with invalid tokens. Our pre-processor + // fixes these before loading. + let raw = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap(); + let fixed = fix_bare_struct_names(&raw); + let doc = Document::load_mem(fixed.as_ref()).unwrap(); + + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "Should parse bare-name struct tree"); + let tree = tree.unwrap(); + + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + + assert!( + roles.iter().any(|r| matches!(r, StructRole::H1)), + "Should find H1 from bare name: {:?}", + roles + ); + assert!( + roles.iter().any(|r| matches!(r, StructRole::Code)), + "Should find Code from bare name: {:?}", + roles + ); + } + + #[test] + fn test_parse_real_tagged_pdf() { + let doc = Document::load("tests/fixtures/2013-app2.pdf").unwrap(); + let tree = StructTree::from_doc(&doc); + assert!(tree.is_some(), "2013-app2.pdf should have a structure tree"); + let tree = tree.unwrap(); + + // Should have a non-trivial structure + assert!(!tree.children.is_empty()); + assert!( + tree.mcid_count() > 0, + "Should have marked content references" + ); + + // Flatten and verify we get heading/paragraph/table elements + let flat = tree.flatten(); + let roles: Vec<&StructRole> = flat.iter().map(|e| &e.role).collect(); + assert!( + roles.iter().any(|r| matches!(r, StructRole::P)), + "Should contain paragraph elements" + ); + + // Verify mcid_to_roles produces a populated map + let page_ids = doc.get_pages(); + let role_map = tree.mcid_to_roles(&page_ids); + assert!(!role_map.is_empty(), "Should have MCID→role mappings"); + } +} diff --git a/src/tables/detect_heuristic.rs b/src/tables/detect_heuristic.rs index 873caa0..d2579f2 100644 --- a/src/tables/detect_heuristic.rs +++ b/src/tables/detect_heuristic.rs @@ -105,6 +105,7 @@ pub(crate) fn merge_adjacent_items(items: &[TextItem]) -> (Vec, Vec= 9 { let regions = find_table_regions_strict(&body_candidates); + log::debug!("body-font: {} strict regions found", regions.len()); - for (y_min, y_max, x_min, x_max) in regions { + for (y_min, y_max, _x_min, _x_max) in ®ions { + // Use full X range for region items — the strict X bounds from + // qualifying rows can exclude continuation lines in wrapped cells. + // Y bounds from the region are sufficient to scope the table area. let region_items: Vec<(usize, &TextItem)> = body_candidates .iter() - .filter(|(_, item)| { - item.y >= y_min && item.y <= y_max && item.x >= x_min && item.x <= x_max - }) + .filter(|(_, item)| item.y >= *y_min && item.y <= *y_max) .cloned() .collect(); + log::debug!( + " region y={:.0}..{:.0}: {} items of {} candidates", + y_min, + y_max, + region_items.len(), + body_candidates.len() + ); + if region_items.len() < 9 { continue; } @@ -240,6 +266,12 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo .collect(); table.item_indices = original_indices.into_iter().collect(); table.item_indices.sort_unstable(); + log::debug!( + " heuristic table: {}x{}, {} item indices", + table.rows.len(), + table.columns.len(), + table.item_indices.len() + ); } tables @@ -330,24 +362,43 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 } } - if cluster_starts.len() >= 3 { + if cluster_starts.len() >= 2 { qualifying_rows.push((*y, cluster_starts)); } } + log::debug!( + "find_table_regions_strict: {} row groups, {} qualifying (2+ X-clusters)", + row_groups.len(), + qualifying_rows.len() + ); if qualifying_rows.len() < 3 { return vec![]; } - // Step 3: Find contiguous runs of qualifying rows (25pt max Y-gap) + // Step 3: Find contiguous runs of qualifying rows. + // Use adaptive gap: median spacing × 3 (handles wrapped cells where + // qualifying rows are spaced further apart), with a floor of 25pt. qualifying_rows.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + let max_gap = if qualifying_rows.len() >= 3 { + let mut gaps: Vec = qualifying_rows + .windows(2) + .map(|w| (w[1].0 - w[0].0).abs()) + .collect(); + gaps.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_gap = gaps[gaps.len() / 2]; + (median_gap * 3.0).max(25.0) + } else { + 25.0 + }; + let mut candidate_regions: Vec)>> = Vec::new(); let mut current_region: Vec<&(f32, Vec)> = vec![&qualifying_rows[0]]; for row in qualifying_rows.iter().skip(1) { let prev_y = current_region.last().unwrap().0; - if row.0 - prev_y > 25.0 { + if row.0 - prev_y > max_gap { if current_region.len() >= 3 { candidate_regions.push(current_region); } @@ -397,6 +448,11 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 } else { 0.0 }; + log::debug!( + " candidate region: {} rows, avg alignment score={:.2}", + num_rows, + avg_score + ); if avg_score >= 0.5 { let y_min = region_rows.first().unwrap().0; let y_max = region_rows.last().unwrap().0; @@ -422,24 +478,35 @@ fn find_table_regions_strict(items: &[(usize, &TextItem)]) -> Vec<(f32, f32, f32 fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode) -> Option { // Find column boundaries let columns = find_column_boundaries(items, mode); - let min_cols = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; + let min_cols = 2; if columns.len() < min_cols || columns.len() > 25 { + log::debug!( + " detect_table_in_region: rejected {} cols (need {}..25)", + columns.len(), + min_cols + ); return None; } // Find row boundaries let rows = find_row_boundaries(items); - let min_rows = match mode { - TableDetectionMode::SmallFont => 2, - TableDetectionMode::BodyFont => 3, - }; + let min_rows = 2; if rows.len() < min_rows { + log::debug!( + " detect_table_in_region: rejected {} rows (need {}+)", + rows.len(), + min_rows + ); return None; } + log::debug!( + " detect_table_in_region: {} cols, {} rows, {} items", + columns.len(), + rows.len(), + items.len() + ); + // Verify this looks like a table: multiple items should align to columns let col_alignment = check_column_alignment(items, &columns, mode); let min_alignment = match mode { @@ -447,6 +514,13 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode TableDetectionMode::BodyFont => 0.7, }; if col_alignment < min_alignment { + log::debug!( + " detect_table_in_region: rejected alignment {:.2} < {:.2} ({} cols, {} rows)", + col_alignment, + min_alignment, + columns.len(), + rows.len() + ); return None; } @@ -506,9 +580,16 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode cells.push(row_cells); } - // Validation 1: most rows should have content in first column + // Validation 1: some rows should have content in first column. + // Use a lower threshold (25%) for tables with wrapped cells where + // continuation lines leave the first column empty. let rows_with_first_col = cells.iter().filter(|row| !row[0].is_empty()).count(); - if rows_with_first_col < rows.len() / 2 { + if rows_with_first_col < rows.len() / 4 { + log::debug!( + " validation 1 fail: {}/{} rows have first col", + rows_with_first_col, + rows.len() + ); return None; } @@ -522,6 +603,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode TableDetectionMode::BodyFont => (rows.len() / 2).max(1), // 50% }; if rows_with_multi_cols < multi_col_threshold { + log::debug!( + " validation 2 fail: {}/{} rows multi-col (need {})", + rows_with_multi_cols, + rows.len(), + multi_col_threshold + ); return None; } @@ -540,36 +627,43 @@ fn detect_table_in_region(items: &[(usize, &TextItem)], mode: TableDetectionMode .map(|row| row.iter().filter(|c| !c.is_empty()).count()) .sum(); let avg_cells_per_row = total_filled as f32 / rows.len() as f32; - let min_avg_cells = match mode { - TableDetectionMode::SmallFont => 1.5, - TableDetectionMode::BodyFont => 2.5, - }; + let min_avg_cells = 1.5; if avg_cells_per_row < min_avg_cells { + log::debug!( + " validation 4 fail: avg_cells={:.1} < {:.1}", + avg_cells_per_row, + min_avg_cells + ); return None; } // Validation 5: Check for key-value pair layout (NOT a table) if is_key_value_layout(&cells) { + log::debug!(" validation 5 fail: key-value layout"); return None; } // Validation 6: Check column count consistency if !has_consistent_columns(&cells) { + log::debug!(" validation 6 fail: inconsistent columns"); return None; } // Validation 7: Tables should have some numeric/data content if !has_table_like_content(&cells, mode) { + log::debug!(" validation 7 fail: no table-like content"); return None; } // Validation 8: Check for Table of Contents pattern if is_table_of_contents(&cells) { + log::debug!(" validation 8 fail: table of contents"); return None; } // Validation 9: Reject paragraph-like content falsely detected as tables if is_paragraph_content(&cells) { + log::debug!(" validation 9 fail: paragraph content"); return None; } @@ -706,9 +800,10 @@ fn has_table_like_content(cells: &[Vec], mode: TableDetectionMode) -> bo TableDetectionMode::BodyFont => 0.3, }; - // For SmallFont, bypass content check for wide tables (5+ columns may have text headers). - // For BodyFont, always require data-like content to prevent paragraph false positives. - pct_data > min_pct || (mode == TableDetectionMode::SmallFont && num_cols >= 5) + // Bypass content check for wide tables (3+ columns) — text-only tables + // (category lists, program descriptions) are legitimate if they passed + // all structural validations (alignment, consistency, not key-value). + pct_data > min_pct || num_cols >= 3 } /// Check if a cell value looks like table data @@ -794,12 +889,18 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { return false; } + let num_cols = cells[0].len(); let mut dot_cells = 0; let mut page_number_cells = 0; let mut total_cells = 0; + // Track which columns contain dots vs numbers to distinguish + // TOC (dots span middle, page number at end) from data tables + // (dots only in label column, many number columns). + let mut dot_cols = vec![0u32; num_cols]; + let mut numeric_cols = vec![0u32; num_cols]; for row in cells { - for cell in row { + for (ci, cell) in row.iter().enumerate() { let trimmed = cell.trim(); if trimmed.is_empty() { continue; @@ -812,6 +913,9 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3; if is_mostly_dots { dot_cells += 1; + if ci < num_cols { + dot_cols[ci] += 1; + } } // Check for standalone page numbers (1-4 digits, possibly with spaces) @@ -821,6 +925,9 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { && digits_only.chars().all(|c| c.is_ascii_digit()) { page_number_cells += 1; + if ci < num_cols { + numeric_cols[ci] += 1; + } } } } @@ -829,6 +936,17 @@ fn is_table_of_contents(cells: &[Vec]) -> bool { return false; } + // Data tables with dot leaders (e.g. "1973....") have dots concentrated + // in one column (the label column) while many other columns contain numbers. + // True TOCs have dots spanning the middle and one page-number column at the end. + // If dots are confined to ≤1 column AND there are ≥3 columns with numbers, + // this is a data table, not a TOC. + let cols_with_dots = dot_cols.iter().filter(|&&c| c >= 2).count(); + let cols_with_numbers = numeric_cols.iter().filter(|&&c| c >= 2).count(); + if cols_with_dots <= 1 && cols_with_numbers >= 3 { + return false; + } + // If a significant portion of cells are dots or page numbers, it's likely a TOC let dot_ratio = dot_cells as f32 / total_cells as f32; let page_num_ratio = page_number_cells as f32 / total_cells as f32; @@ -1072,3 +1190,196 @@ pub(crate) fn find_first_table_row( (first_table_row, excluded_items) } + +/// Try to recover a label column for numeric-only tables. +/// +/// Financial balance sheets often have text labels (row descriptions) to the +/// left of numeric columns. The label X-positions vary due to indentation, +/// so they don't form a consistent column cluster and are excluded from the +/// initial table detection. This function finds unclaimed items at matching +/// Y-positions to the left of the table and prepends them as column 0. +fn try_add_label_column( + table: &mut Table, + all_candidates: &[(usize, &TextItem)], + claimed_indices: &std::collections::HashSet, + y_min: f32, + y_max: f32, +) { + // Only apply to tables with 2-3 numeric columns and ≥5 rows + if table.columns.len() < 2 || table.columns.len() > 3 || table.rows.len() < 5 { + return; + } + + // Check if the table is predominantly numeric (no text labels in any column) + let numeric_cells = table + .cells + .iter() + .flat_map(|row| row.iter()) + .filter(|cell| { + let text = cell.trim(); + if text.is_empty() { + return false; + } + let data_chars = text + .chars() + .filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c)) + .count(); + let total_chars = text.chars().count(); + total_chars > 0 && data_chars as f32 / total_chars as f32 >= 0.6 + }) + .count(); + let total_non_empty = table + .cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + if total_non_empty == 0 || (numeric_cells as f32 / total_non_empty as f32) < 0.7 { + return; + } + + let table_x_min = table.columns.first().copied().unwrap_or(f32::MAX); + let y_tol = 5.0; + + // For each table row, find unclaimed items to the left at the same Y + let mut label_items_per_row: Vec> = Vec::new(); + let mut found_count = 0; + for &row_y in &table.rows { + let mut row_labels: Vec<(usize, &TextItem)> = all_candidates + .iter() + .filter(|(idx, item)| { + !claimed_indices.contains(idx) + && !table.item_indices.contains(idx) + && (item.y - row_y).abs() < y_tol + && item.x < table_x_min - 10.0 + && item.y >= y_min + && item.y <= y_max + }) + .map(|(idx, item)| (*idx, *item)) + .collect(); + row_labels.sort_by(|a, b| { + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal) + }); + if !row_labels.is_empty() { + found_count += 1; + } + label_items_per_row.push(row_labels); + } + + // Require labels for at least 40% of rows + if found_count < table.rows.len() * 2 / 5 { + return; + } + + debug!( + "recovering label column: {}/{} rows have labels to the left", + found_count, + table.rows.len() + ); + + // Prepend label column + let label_col_x = label_items_per_row + .iter() + .flat_map(|items| items.iter().map(|(_, i)| i.x)) + .fold(f32::INFINITY, f32::min); + + table.columns.insert(0, label_col_x); + for (row_idx, row_labels) in label_items_per_row.iter().enumerate() { + let label_text = row_labels + .iter() + .map(|(_, item)| item.text.as_str()) + .collect::>() + .join(" "); + table.cells[row_idx].insert(0, label_text); + for (idx, _) in row_labels { + table.item_indices.push(*idx); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_table_of_contents_rejects_toc() { + // TOC with separate dot-leader cells and page number cells + let cells = vec![ + vec![ + "Chapter 1".to_string(), + "....................".to_string(), + "1".to_string(), + ], + vec![ + "Chapter 2".to_string(), + "....................".to_string(), + "15".to_string(), + ], + vec![ + "Chapter 3".to_string(), + "....................".to_string(), + "42".to_string(), + ], + vec![ + "Appendix".to_string(), + "....................".to_string(), + "100".to_string(), + ], + ]; + assert!(is_table_of_contents(&cells)); + } + + #[test] + fn is_table_of_contents_allows_data_table_with_dot_leaders() { + // Simulates ERP appendix tables where the first column has year + dots + // (e.g. "1973..........") and other columns have numeric data. + let cells = vec![ + vec![ + "1973..........".to_string(), + "0.80".to_string(), + "1.08".to_string(), + "1.05".to_string(), + "0.02".to_string(), + "-0.28".to_string(), + "-0.33".to_string(), + "5.16".to_string(), + ], + vec![ + "1974..........".to_string(), + "73".to_string(), + "56".to_string(), + "49".to_string(), + "08".to_string(), + "17".to_string(), + "17".to_string(), + "-.28".to_string(), + ], + vec![ + "1975..........".to_string(), + "86".to_string(), + "-.05".to_string(), + "-.14".to_string(), + "09".to_string(), + "91".to_string(), + "85".to_string(), + "1.03".to_string(), + ], + vec![ + "1976..........".to_string(), + "-1.05".to_string(), + "36".to_string(), + "34".to_string(), + "02".to_string(), + "-1.41".to_string(), + "-1.31".to_string(), + "4.01".to_string(), + ], + ]; + assert!( + !is_table_of_contents(&cells), + "data table with dot-leader labels should not be rejected as TOC" + ); + } +} diff --git a/src/tables/detect_lines.rs b/src/tables/detect_lines.rs index e72746d..8dfea17 100644 --- a/src/tables/detect_lines.rs +++ b/src/tables/detect_lines.rs @@ -72,6 +72,13 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 let v_xs: Vec = verticals.iter().map(|(x, _, _)| *x).collect(); let col_edges = snap_edges(&v_xs, 3.0); + log::debug!( + "detect_lines p{}: {} row edges, {} col edges after snap", + page, + row_edges.len(), + col_edges.len() + ); + // Require at least 2 columns (3 col edges) and 2 rows (3 row edges). // A single column of horizontal lines is just separator rules, not a table. if row_edges.len() < 3 || col_edges.len() < 3 { @@ -80,6 +87,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 // Cap grid size: >20 columns is almost certainly a diagram, not a table if col_edges.len() > 21 || row_edges.len() > 80 { + log::debug!( + "detect_lines p{}: rejected — too many edges ({}x{})", + page, + row_edges.len(), + col_edges.len() + ); return Vec::new(); } @@ -101,26 +114,54 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 // dimension in both axes, it's a border frame, not a table. // Standard pages are ~595×842 (A4) or ~612×792 (Letter). if table_width > 500.0 && table_height > 700.0 { + log::debug!( + "detect_lines p{}: rejected — page-spanning frame ({:.0}×{:.0})", + page, + table_width, + table_height + ); return Vec::new(); } - // Validate horizontal lines: at least 3 should span >50% of table width. - // This filters out short segments that accidentally align. + // Validate horizontal lines: at least 3 should span a meaningful width. + // Full-width spanning (>50%) is ideal, but tables with partial horizontal + // rules (column-level separators) are also valid if there are enough. let spanning_h = horizontals .iter() .filter(|(_, x_min, x_max)| (x_max - x_min) > table_width * 0.5) .count(); - if spanning_h < 3 { + let partial_h = horizontals + .iter() + .filter(|(_, x_min, x_max)| (x_max - x_min) > table_width * 0.15) + .count(); + if spanning_h < 3 && partial_h < 6 { + log::debug!( + "detect_lines p{}: rejected — {} spanning + {} partial H lines", + page, + spanning_h, + partial_h + ); return Vec::new(); } - // Validate vertical lines: at least 2 should span >30% of table height. - // Real table columns extend most of the table height. + // Validate vertical lines: at least 2 should span a meaningful height. + // Full spanning (>30%) is ideal, but accept many shorter lines (>10%) + // for tables with partial column separators. let spanning_v = verticals .iter() .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3) .count(); - if spanning_v < 2 { + let partial_v = verticals + .iter() + .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10) + .count(); + if spanning_v < 2 && partial_v < 4 { + log::debug!( + "detect_lines p{}: rejected — {} spanning + {} partial V lines", + page, + spanning_v, + partial_v + ); return Vec::new(); } @@ -248,6 +289,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index a043cde..cfdac1e 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -8,10 +8,11 @@ use crate::types::{PdfRect, TextItem}; use super::Table; -/// Disjoint-set (union-find) for clustering indices. +/// Disjoint-set (union-find) with component sizes for clustering indices. struct UnionFind { parent: Vec, rank: Vec, + size: Vec, } impl UnionFind { @@ -19,6 +20,7 @@ impl UnionFind { Self { parent: (0..n).collect(), rank: vec![0; n], + size: vec![1; n], } } @@ -35,15 +37,24 @@ impl UnionFind { if ra == rb { return; } + let new_size = self.size[ra] + self.size[rb]; if self.rank[ra] < self.rank[rb] { self.parent[ra] = rb; + self.size[rb] = new_size; } else if self.rank[ra] > self.rank[rb] { self.parent[rb] = ra; + self.size[ra] = new_size; } else { self.parent[rb] = ra; + self.size[ra] = new_size; self.rank[ra] += 1; } } + + fn component_size(&mut self, x: usize) -> usize { + let root = self.find(x); + self.size[root] + } } /// Check if two rects overlap after expanding each by `tol` on all sides. @@ -64,8 +75,19 @@ pub(crate) fn rects_overlap(a: &(f32, f32, f32, f32), b: &(f32, f32, f32, f32), !(a_right < b_left || b_right < a_left || a_top < b_bottom || b_top < a_bottom) } +/// Maximum component size for rect clustering. No real table has thousands +/// of cell rects — once a component exceeds this, it is a vector drawing or +/// page-spanning clipping path. We skip overlap checks for rects already in +/// an oversized component, keeping the original O(n²) loop but making it +/// effectively O(n) for pathological pages. +const MAX_CLUSTER_RECTS: usize = 2000; + /// Cluster rects by spatial overlap using union-find. /// Returns groups of rect indices; only groups with ≥ `min_size` rects are returned. +/// +/// Skips overlap checks for rects whose component has already exceeded +/// [`MAX_CLUSTER_RECTS`], so pages with tens of thousands of vector-drawing +/// rects complete in milliseconds instead of minutes. pub(crate) fn cluster_rects( rects: &[(f32, f32, f32, f32)], tolerance: f32, @@ -75,9 +97,20 @@ pub(crate) fn cluster_rects( let mut uf = UnionFind::new(n); for i in 0..n { + // If rect i is already in an oversized component, no point comparing + // it against further rects — the component won't be used for table + // detection anyway. + if uf.component_size(i) >= MAX_CLUSTER_RECTS { + continue; + } for j in (i + 1)..n { if rects_overlap(&rects[i], &rects[j], tolerance) { uf.union(i, j); + // Check if the merged component just exceeded the cap — + // if so, no need to test more pairs for rect i. + if uf.component_size(i) >= MAX_CLUSTER_RECTS { + break; + } } } } @@ -253,26 +286,31 @@ pub fn detect_tables_from_rects( // Only remove when the container is a similarly-sized cell (height // ratio < 4×), NOT when the container is a table-wide background // that dwarfs the sub-rect. - let before = page_rects.len(); - let snapshot = page_rects.clone(); - page_rects.retain(|&(ax, ay, aw, ah)| { - let tol = 2.0; - !snapshot.iter().any(|&(bx, by, bw, bh)| { - // b must strictly contain a (b is larger in area) - bw * bh > aw * ah * 1.2 - && bh < ah * 4.0 // container must be similarly sized, not a table background - && bx <= ax + tol - && (bx + bw) >= (ax + aw) - tol - && by <= ay + tol - && (by + bh) >= (ay + ah) - tol - }) - }); - if page_rects.len() < before { - debug!( - "page {}: removed {} contained sub-rects", - page, - before - page_rects.len(), - ); + // + // Skip this O(n²) dedup when there are too many rects — pages with + // thousands of vector-drawing rects won't benefit from cell dedup. + if page_rects.len() < MAX_CLUSTER_RECTS { + let before = page_rects.len(); + let snapshot = page_rects.clone(); + page_rects.retain(|&(ax, ay, aw, ah)| { + let tol = 2.0; + !snapshot.iter().any(|&(bx, by, bw, bh)| { + // b must strictly contain a (b is larger in area) + bw * bh > aw * ah * 1.2 + && bh < ah * 4.0 // container must be similarly sized, not a table background + && bx <= ax + tol + && (bx + bw) >= (ax + aw) - tol + && by <= ay + tol + && (by + bh) >= (ay + ah) - tol + }) + }); + if page_rects.len() < before { + debug!( + "page {}: removed {} contained sub-rects", + page, + before - page_rects.len(), + ); + } } } @@ -285,12 +323,47 @@ pub fn detect_tables_from_rects( let mut tables = Vec::new(); let mut hint_regions = Vec::new(); + let mut failed_clusters: Vec> = Vec::new(); // Full grid detection requires ≥ 6 rects if page_rects.len() >= 6 { - let clusters = cluster_rects(&page_rects, 3.0, 6); - debug!("page {}: {} clusters with >= 6 rects", page, clusters.len()); + // Identify origin-anchored page-background rects (clipping paths or + // page fills) that would bridge separate table regions if included in + // clustering. Exclude them from adjacency but add them back to each + // cluster they overlap, so grid detection still has their edges. + let is_page_bg = { + let mut heights: Vec = page_rects.iter().map(|&(_, _, _, h)| h).collect(); + heights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let median_height = heights[heights.len() / 2]; + let height_threshold = median_height * 20.0; + let flags: Vec = page_rects + .iter() + .map(|&(x, y, _, h)| x < 5.0 && y < 5.0 && h > height_threshold) + .collect(); + if flags.iter().any(|&b| b) { + debug!( + "page {}: {} origin-anchored page-bg rects excluded from clustering", + page, + flags.iter().filter(|&&b| b).count(), + ); + } + flags + }; + // Build filtered rect list for clustering (excluding page backgrounds) + let non_bg_indices: Vec = + (0..page_rects.len()).filter(|&i| !is_page_bg[i]).collect(); + let non_bg_rects: Vec<(f32, f32, f32, f32)> = + non_bg_indices.iter().map(|&i| page_rects[i]).collect(); + let raw_clusters = cluster_rects(&non_bg_rects, 3.0, 6); + + // Map cluster indices back to page_rects indices + let clusters: Vec> = raw_clusters + .iter() + .map(|cluster| cluster.iter().map(|&i| non_bg_indices[i]).collect()) + .collect(); + + debug!("page {}: {} clusters with >= 6 rects", page, clusters.len()); for cluster_indices in &clusters { let group_rects: Vec<(f32, f32, f32, f32)> = cluster_indices.iter().map(|&i| page_rects[i]).collect(); @@ -307,13 +380,21 @@ pub fn detect_tables_from_rects( left.len(), right.len() ); + let mut split_found = false; for sub in [&left, &right] { if let Some(table) = detect_table_from_rect_group(items, sub, page) { tables.push(table); + split_found = true; } else if let Some(table) = detect_row_stripe_table(items, sub, page) { tables.push(table); + split_found = true; } } + if !split_found { + failed_clusters.push(group_rects); + } + } else { + failed_clusters.push(group_rects); } } @@ -349,6 +430,27 @@ pub fn detect_tables_from_rects( } } + // Cell-rect fallback: when per-cluster attempts all fail, try using + // rect Y-edges for rows + text X-positions for columns on each failed + // cluster. Handles tables with cell-background rects that don't form + // a clean grid (variable column widths, decoration fills). + if tables.is_empty() { + debug!( + "page {}: cell-rect fallback: {} failed clusters", + page, + failed_clusters.len() + ); + for fc_rects in &failed_clusters { + if fc_rects.len() >= 6 { + if let Some(table) = + detect_row_stripe_table_from_cell_rects(items, fc_rects, page) + { + tables.push(table); + } + } + } + } + // Row-stripe fallback: when clustering produces no large clusters // (row stripes don't overlap so each is its own cluster of 1), // try all page rects directly as a row-stripe table. @@ -379,8 +481,11 @@ pub fn detect_tables_from_rects( // from cluster bounding boxes to scope heuristic table detection. // This handles both large decorative-rect clusters (calendars, forms) // and small cell-border clusters on rect-sparse pages. + let mut has_failed_cluster_hints = false; if page_rects.len() >= 6 { let clusters = cluster_rects(&page_rects, 3.0, 6); + + // Generate hints from large clusters (≥30 rects, decorative/calendar style) for cluster_indices in &clusters { let group_rects: Vec<(f32, f32, f32, f32)> = cluster_indices.iter().map(|&i| page_rects[i]).collect(); @@ -415,12 +520,62 @@ pub fn detect_tables_from_rects( }); } } + + // Generate hints from failed clusters (≥6 rects that had valid bounding + // boxes but insufficient grid structure — e.g. outer border or header + // divider with 2x2 edges). These tell us WHERE a table is even though + // the rects don't define column structure. + for fc_rects in &failed_clusters { + if fc_rects.len() < 6 { + continue; + } + let x_left = fc_rects.iter().map(|r| r.0).reduce(f32::min).unwrap(); + let x_right = fc_rects.iter().map(|r| r.0 + r.2).reduce(f32::max).unwrap(); + let y_bottom = fc_rects.iter().map(|r| r.1).reduce(f32::min).unwrap(); + let y_top = fc_rects.iter().map(|r| r.1 + r.3).reduce(f32::max).unwrap(); + let h = y_top - y_bottom; + // Require reasonable height and text items inside the region + let padding = 15.0; + let items_inside = items + .iter() + .filter(|item| { + item.y >= y_bottom - padding + && item.y <= y_top + padding + && item.x >= x_left - padding + && item.x <= x_right + padding + }) + .count(); + let w = x_right - x_left; + // Require reasonable dimensions: height ≥100pt (≈5+ rows), + // height ≤600pt (not full page). + // Width check: ≤500pt normally, but allow wider for large + // clusters (≥30 rects) that are clearly structured. + let max_w = if fc_rects.len() >= 30 { 800.0 } else { 500.0 }; + if (100.0..=600.0).contains(&h) && w <= max_w && items_inside >= 6 { + debug!( + "page {}: failed-cluster hint from {} rects ({} items): x={:.1}..{:.1} y={:.1}..{:.1} ({:.0}×{:.0})", + page, fc_rects.len(), items_inside, x_left, x_right, y_bottom, y_top, + x_right - x_left, h + ); + hint_regions.push(RectHintRegion { + y_top, + y_bottom, + x_left, + x_right, + cluster_rects: fc_rects.clone(), + }); + has_failed_cluster_hints = true; + } + } + // Deduplicate overlapping hints hint_regions = merge_overlapping_hints(hint_regions); // Require multiple hint regions to confirm a multi-zone layout // (calendars, forms). A single hint is likely a decorative cluster // that would interfere with full-page heuristic detection. - if hint_regions.len() < 2 { + // Exception: failed-cluster hints represent real table boundaries + // confirmed by rect presence, so a single one is meaningful. + if hint_regions.len() < 2 && !has_failed_cluster_hints { hint_regions.clear(); } if !hint_regions.is_empty() { @@ -835,16 +990,52 @@ fn try_build_grid( } } - // Reject tables with any completely empty column — indicates a bad grid. - for col in 0..num_cols { + // Trim empty outer columns (rect edges beyond text), reject if any + // interior column is empty — that indicates a bad grid. + let first_non_empty = (0..num_cols).find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let last_non_empty = (0..num_cols).rev().find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let (first_col, last_col) = match (first_non_empty, last_non_empty) { + (Some(f), Some(l)) if l > f => (f, l), + _ => { + debug!(" rejected: no content columns"); + return GridResult::Failed; + } + }; + // Check interior columns + for col in first_col..=last_col { let col_has_content = cells .iter() .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); if !col_has_content { - debug!(" rejected: column {} is completely empty", col); + debug!(" rejected: interior column {} is completely empty", col); return GridResult::Failed; } } + // Trim outer empty columns + let (columns, cells) = if first_col > 0 || last_col < num_cols - 1 { + let trimmed_cols: Vec = columns[first_col..=last_col].to_vec(); + let trimmed_cells: Vec> = cells + .iter() + .map(|row| row[first_col..=last_col].to_vec()) + .collect(); + debug!( + " trimmed {} empty outer columns ({}..={})", + (num_cols - 1 - last_col + first_col), + first_col, + last_col + ); + (trimmed_cols, trimmed_cells) + } else { + (columns, cells) + }; GridResult::Ok(Table { columns, @@ -1180,16 +1371,62 @@ fn detect_row_stripe_table( return None; } - // No empty columns - for col in 0..num_cols { + // Reject if any cell has excessive text — layout background rects (sidebar, + // header, section bands) produce "cells" that contain paragraphs of body text. + // Real alternating-row-stripe data tables have short cell content. + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + // Allow longer cells for multi-column tables (descriptions in one column + // are common). Single-column or 2-column "tables" with giant cells are + // almost always layout backgrounds. + let max_allowed = if num_cols >= 3 { 2000 } else { 500 }; + if max_cell_len > max_allowed { + debug!( + " row-stripe rejected: max cell length {} > {} (layout background)", + max_cell_len, max_allowed + ); + return None; + } + + // Trim empty outer columns, reject if interior columns are empty + let first_col = (0..num_cols).find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let last_col = (0..num_cols).rev().find(|&col| { + cells + .iter() + .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())) + }); + let (first_col, last_col) = match (first_col, last_col) { + (Some(f), Some(l)) if l > f => (f, l), + _ => return None, + }; + for col in first_col..=last_col { let col_has_content = cells .iter() .any(|row| row.get(col).is_some_and(|c| !c.trim().is_empty())); if !col_has_content { - debug!(" row-stripe rejected: column {} is empty", col); + debug!(" row-stripe rejected: interior column {} is empty", col); return None; } } + let (col_edges, cells) = if first_col > 0 || last_col < num_cols - 1 { + let new_edges: Vec = col_edges[first_col..=last_col + 1].to_vec(); + let new_cells: Vec> = cells + .iter() + .map(|row| row[first_col..=last_col].to_vec()) + .collect(); + (new_edges, new_cells) + } else { + (col_edges, cells) + }; + let num_cols = col_edges.len() - 1; let column_centers: Vec = (0..num_cols) .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) @@ -1213,6 +1450,255 @@ fn detect_row_stripe_table( }) } +/// Detect a table from cell-background rects that failed grid detection. +/// +/// Uses rect Y-edges for row boundaries and text X-position clustering for +/// columns. Handles tables with cell backgrounds that don't form a clean +/// X-edge grid (variable column widths, decorative fills). +fn detect_row_stripe_table_from_cell_rects( + items: &[TextItem], + group_rects: &[(f32, f32, f32, f32)], + page: u32, +) -> Option
{ + if group_rects.len() < 6 { + return None; + } + + // Extract Y-edges from rects + let mut y_edges: Vec = Vec::new(); + for &(_, y, _, h) in group_rects { + y_edges.push(y); + y_edges.push(y + h); + } + let y_edges = snap_edges(&y_edges, 6.0); + + // If rect Y-edges are insufficient for row structure, use the rect + // bounding box to scope items and derive rows from text Y-positions. + let row_edges = if y_edges.len() >= 4 { + let mut edges = y_edges; + edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + edges + } else { + // Fall back: gather items in the rect region and cluster by Y + let y_min = y_edges.first().copied().unwrap_or(0.0); + let y_max = y_edges.last().copied().unwrap_or(0.0); + let x_min = group_rects + .iter() + .map(|r| r.0) + .reduce(f32::min) + .unwrap_or(0.0); + let x_max = group_rects + .iter() + .map(|r| r.0 + r.2) + .reduce(f32::max) + .unwrap_or(0.0); + let region_items: Vec<&TextItem> = items + .iter() + .filter(|i| { + i.page == page + && i.y >= y_min - 5.0 + && i.y <= y_max + 5.0 + && i.x >= x_min - 5.0 + && i.x <= x_max + 5.0 + }) + .collect(); + if region_items.len() < 4 { + return None; + } + // Cluster Y positions using median font height as threshold + let median_h = { + let mut hs: Vec = region_items.iter().map(|i| i.height).collect(); + hs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + hs[hs.len() / 2] + }; + let mut ys: Vec = region_items.iter().map(|i| i.y).collect(); + ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + let mut edges = Vec::new(); + let threshold = median_h * 0.8; + let mut cluster_start = ys[0]; + let mut cluster_sum = ys[0]; + let mut cluster_count = 1.0f32; + for &y in &ys[1..] { + if (cluster_sum / cluster_count - y).abs() > threshold { + let center = cluster_sum / cluster_count; + edges.push(center + median_h * 0.5); + edges.push(center - median_h * 0.5); + cluster_start = y; + cluster_sum = y; + cluster_count = 1.0; + } else { + cluster_sum += y; + cluster_count += 1.0; + } + } + let center = cluster_sum / cluster_count; + edges.push(center + median_h * 0.5); + edges.push(center - median_h * 0.5); + let _ = cluster_start; // suppress unused warning + edges = snap_edges(&edges, 3.0); + edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + if edges.len() < 4 { + return None; + } + edges + }; + + // Compute bounding box from non-full-page rects + let median_h = { + let mut heights: Vec = group_rects.iter().map(|&(_, _, _, h)| h).collect(); + heights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + heights[heights.len() / 2] + }; + let content_rects: Vec<_> = group_rects + .iter() + .filter(|&&(_, _, _, h)| h < median_h * 10.0) + .collect(); + if content_rects.is_empty() { + return None; + } + + let x_left = content_rects + .iter() + .map(|&&(x, _, _, _)| x) + .reduce(f32::min)?; + let x_right = content_rects + .iter() + .map(|&&(x, _, w, _)| x + w) + .reduce(f32::max)?; + let y_top = row_edges[0]; + let y_bottom = *row_edges.last()?; + + // Gather items within the rect region + let page_items: Vec<(usize, &TextItem)> = items + .iter() + .enumerate() + .filter(|(_, item)| { + item.page == page + && item.y >= y_bottom - 2.0 + && item.y <= y_top + 2.0 + && item.x >= x_left - 5.0 + && item.x + item.width <= x_right + 5.0 + }) + .collect(); + + if page_items.is_empty() { + return None; + } + + // Derive columns from text X-position clustering + let columns = cluster_x_positions(&page_items, 15.0); + if columns.len() < 2 { + return None; + } + + // Build column edges + let mut col_edges: Vec = Vec::with_capacity(columns.len() + 1); + let min_x = page_items.iter().map(|(_, i)| i.x).reduce(f32::min)?; + col_edges.push(min_x - 5.0); + for pair in columns.windows(2) { + col_edges.push((pair[0] + pair[1]) / 2.0); + } + let max_x_right = page_items + .iter() + .map(|(_, i)| i.x + i.width) + .reduce(f32::max)?; + col_edges.push(max_x_right + 5.0); + + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + debug!( + " cell-rect table: {}x{} from {} rects, {} items", + num_rows, + num_cols, + group_rects.len(), + page_items.len() + ); + + let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + + if item_indices.is_empty() { + return None; + } + + // Validate: >=2 non-empty rows, >=25% density + let non_empty_rows = cells + .iter() + .filter(|row| row.iter().any(|c| !c.trim().is_empty())) + .count(); + if non_empty_rows < 2 { + debug!( + " cell-rect rejected: only {} non-empty rows", + non_empty_rows + ); + return None; + } + + let total_cells = (num_cols * num_rows) as f32; + let non_empty_cells = cells + .iter() + .flat_map(|row| row.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let density = if total_cells > 0.0 { + non_empty_cells as f32 / total_cells + } else { + 0.0 + }; + if density < 0.25 { + debug!( + " cell-rect rejected: density {:.0}% < 25%", + density * 100.0 + ); + return None; + } + + // Reject tables with paragraph-length cells (layout backgrounds, not tables) + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + if max_cell_len > 500 { + debug!( + " cell-rect rejected: max cell length {} > 500", + max_cell_len + ); + return None; + } + + // Reject wildly disproportionate grids (e.g. 68x6 from decorative rects) + if num_rows > 20 && num_cols < 4 { + debug!( + " cell-rect rejected: disproportionate grid {}x{}", + num_rows, num_cols + ); + return None; + } + + let column_centers: Vec = (0..num_cols) + .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) + .collect(); + let row_centers: Vec = (0..num_rows) + .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) + .collect(); + + debug!( + " cell-rect table accepted: {}x{}, {:.0}% density", + num_rows, + num_cols, + non_empty_cells as f32 / total_cells * 100.0 + ); + + Some(Table { + columns: column_centers, + rows: row_centers, + cells, + item_indices, + }) +} + /// Detect a table by merging all cluster rects into one group. /// /// This handles clip-path PDFs where each column's cell rects form a separate @@ -1348,6 +1834,22 @@ fn detect_merged_cluster_table( return None; } + // Reject if any cell has excessive text — layout background rects produce + // "cells" containing paragraphs, not short data-table values. + let max_cell_len = cells + .iter() + .flat_map(|row| row.iter()) + .map(|c| c.len()) + .max() + .unwrap_or(0); + if max_cell_len > 500 { + debug!( + " merged-cluster rejected: max cell length {} > 500 (layout background)", + max_cell_len + ); + return None; + } + // No empty columns for col in 0..num_cols { let col_has_content = cells @@ -1450,6 +1952,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -1755,6 +2258,28 @@ mod tests { assert!(!is_row_stripe_pattern(&rects)); } + #[test] + fn test_row_stripe_rejects_layout_background_long_cells() { + // Simulate a newsletter page with wide background rects (sidebar, header, body) + // that look like row stripes but contain paragraphs of body text. + let rects = vec![ + (10.0, 700.0, 550.0, 50.0), // header band + (10.0, 640.0, 550.0, 50.0), // nav band + (10.0, 200.0, 550.0, 430.0), // body background + ]; + let items = vec![ + make_item("General News", 20.0, 650.0, 10.0), + make_item("People News", 20.0, 710.0, 10.0), + // Simulate a long body text (>500 chars) in the main content area + make_item(&"A".repeat(600), 200.0, 650.0, 10.0), + ]; + let result = detect_row_stripe_table(&items, &rects, 1); + assert!( + result.is_none(), + "layout background rects should not be detected as a table" + ); + } + // --- propagate_merged_cells --- #[test] @@ -2293,4 +2818,209 @@ mod tests { assert!((merged[0].x_left - 20.0).abs() < 0.01); assert!((merged[0].x_right - 340.0).abs() < 0.01); } + + #[test] + fn failed_cluster_generates_hint_with_items() { + // A cluster of rects forming an outer border (2 x-edges after snapping) + // that fails grid detection should produce a hint when items are inside. + // Use overlapping rects with the same left/right edges but varied heights + // so row-stripe detection also fails. + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 100.0, 400.0, 200.0), // outer border + (52.0, 102.0, 396.0, 196.0), // inner border (within snap tolerance) + (51.0, 101.0, 398.0, 198.0), // another border variant + (50.0, 100.0, 400.0, 10.0), // top divider (thin) + (50.0, 290.0, 400.0, 10.0), // bottom divider (thin) + (50.0, 195.0, 400.0, 10.0), // middle divider + ]; + // Create text items inside the bounding box (≥6 items) + let mut items: Vec = Vec::new(); + for row in 0..4 { + for col in 0..3 { + items.push(TextItem { + text: format!("cell{}_{}", row, col), + x: 60.0 + col as f32 * 120.0, + y: 120.0 + row as f32 * 40.0, + width: 50.0, + height: 10.0, + font: String::new(), + font_size: 10.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid: None, + }); + } + } + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&items, &rects, 1); + // Grid detection should fail (2 x-edges after snapping: ~50 and ~450) + // If detection fails, we should get a failed-cluster hint + if tables.is_empty() { + assert_eq!(hints.len(), 1, "failed cluster should produce one hint"); + assert!(!hints[0].cluster_rects.is_empty()); + } + // If tables were detected, that's also acceptable + } + + #[test] + fn failed_cluster_no_hint_without_items() { + // Rects with no text items inside → no failed-cluster hint generated. + // Use >6 rects to avoid the rect-sparse path (4-6 rects). + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 100.0, 400.0, 200.0), + (52.0, 102.0, 396.0, 196.0), + (51.0, 101.0, 398.0, 198.0), + (50.0, 100.0, 400.0, 10.0), + (50.0, 290.0, 400.0, 10.0), + (50.0, 195.0, 400.0, 10.0), + (50.0, 150.0, 400.0, 10.0), + (50.0, 250.0, 400.0, 10.0), + ]; + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&[], &rects, 1); + // No items → no table, no hint (items_inside check fails) + if tables.is_empty() { + assert!(hints.is_empty(), "no items inside → no hint"); + } + } + + #[test] + fn failed_cluster_no_hint_narrow_height() { + // Cluster with only 20pt height (header band) should not produce hint + // even with items inside (height < 100pt threshold) + let page_rects: Vec<(f32, f32, f32, f32)> = vec![ + (50.0, 650.0, 50.0, 20.0), + (100.0, 650.0, 50.0, 20.0), + (150.0, 650.0, 50.0, 20.0), + (200.0, 650.0, 50.0, 20.0), + (250.0, 650.0, 50.0, 20.0), + (300.0, 650.0, 50.0, 20.0), + (350.0, 650.0, 50.0, 20.0), + (400.0, 650.0, 50.0, 20.0), + ]; + let mut items: Vec = Vec::new(); + for col in 0..8 { + items.push(TextItem { + text: format!("hdr{}", col), + x: 55.0 + col as f32 * 50.0, + y: 655.0, + width: 40.0, + height: 10.0, + font: String::new(), + font_size: 10.0, + page: 1, + is_bold: false, + is_italic: false, + item_type: crate::types::ItemType::Text, + mcid: None, + }); + } + let rects: Vec = page_rects + .iter() + .map(|&(x, y, w, h)| crate::types::PdfRect { + x, + y, + width: w, + height: h, + page: 1, + }) + .collect(); + let (tables, hints) = detect_tables_from_rects(&items, &rects, 1); + assert!(tables.is_empty()); + assert!( + hints.is_empty(), + "narrow header band (20pt) should not produce hint" + ); + } + + // --- page-bg clustering exclusion --- + + #[test] + fn page_bg_rects_do_not_bridge_separate_clusters() { + // Simulate page 27 scenario: two groups of row stripes at different Y + // ranges, connected by full-page background rects at (0,0). + // Without exclusion, all rects cluster into one group. + // With exclusion, two separate clusters form. + let mut rects = Vec::new(); + let page = 1; + + // Group 1: 7 row stripes at Y=444..537 (Reference Group table) + for i in 0..7 { + let y = 444.0 + i as f32 * 15.5; + rects.push(PdfRect { + x: 44.0, + y, + width: 505.0, + height: 15.5, + page, + }); + } + + // Group 2: 4 row stripes at Y=176..238 (smaller table) + for i in 0..4 { + let y = 176.0 + i as f32 * 15.5; + rects.push(PdfRect { + x: 44.0, + y, + width: 505.0, + height: 15.5, + page, + }); + } + + // 3 full-page background rects at origin + for _ in 0..3 { + rects.push(PdfRect { + x: 0.0, + y: 0.0, + width: 594.0, + height: 774.0, + page, + }); + } + + // Items in group 1 region for row-stripe detection + let mut items = Vec::new(); + for i in 0..7 { + let y = 449.0 + i as f32 * 15.5; + items.push(make_item("Company Name", 50.0, y, 9.0)); + items.push(make_item("P", 320.0, y, 9.0)); + items.push(make_item("P", 450.0, y, 9.0)); + } + + let (tables, _hints) = detect_tables_from_rects(&items, &rects, page); + // Should detect the group 1 table (7 row stripes) without being + // confused by group 2 stripes bridged via page-bg rects. + assert!( + !tables.is_empty(), + "should detect table from row stripes when page-bg rects are excluded from clustering" + ); + // The table should have rows from group 1 only, not spanning to group 2 + let table = &tables[0]; + assert!( + table.rows.len() <= 8, + "table should have at most ~7 rows from group 1, got {}", + table.rows.len() + ); + } } diff --git a/src/tables/detect_struct.rs b/src/tables/detect_struct.rs new file mode 100644 index 0000000..182c543 --- /dev/null +++ b/src/tables/detect_struct.rs @@ -0,0 +1,377 @@ +//! Structure-tree-based table detection. +//! +//! When a PDF has a well-formed structure tree with `/Table` > `/TR` > `/TD|TH` +//! elements linked to MCIDs, this module builds `Table` structs directly from +//! the semantic hierarchy — no geometry heuristics needed. + +use std::collections::HashMap; + +use log::debug; + +use crate::structure_tree::StructTable; +use crate::types::TextItem; + +use super::Table; + +/// Build tables from structure-tree table descriptors by matching MCIDs to TextItems. +/// +/// Returns tables for the given page. Tables where fewer than 50% of cells +/// resolve to TextItems are rejected (stale or broken structure tree). +pub fn detect_tables_from_struct_tree( + items: &[TextItem], + struct_tables: &[StructTable], + page: u32, +) -> Vec
{ + if struct_tables.is_empty() { + return Vec::new(); + } + + // Build MCID → item indices for this page + let mut mcid_to_items: HashMap> = HashMap::new(); + for (idx, item) in items.iter().enumerate() { + if item.page == page { + if let Some(mcid) = item.mcid { + mcid_to_items.entry(mcid).or_default().push(idx); + } + } + } + + let mut tables = Vec::new(); + + for st in struct_tables { + // Filter rows to this page + let page_rows: Vec<_> = st + .rows + .iter() + .filter(|row| { + row.cells + .iter() + .any(|cell| cell.mcids.iter().any(|&(_, p)| p == page)) + }) + .collect(); + + debug!( + "page {}: struct table has {} rows on this page (from {} total)", + page, + page_rows.len(), + st.rows.len() + ); + + if page_rows.len() < 2 { + continue; + } + + // Determine column count from max cells per row + let num_cols = page_rows.iter().map(|r| r.cells.len()).max().unwrap_or(0); + if num_cols < 2 { + continue; + } + + // Build cell text and collect item indices + let mut cells: Vec> = Vec::new(); + let mut all_item_indices: Vec = Vec::new(); + let mut total_cells = 0u32; + let mut matched_cells = 0u32; + + for row in &page_rows { + let mut row_cells = Vec::with_capacity(num_cols); + for (col_idx, cell) in row.cells.iter().enumerate() { + if col_idx >= num_cols { + break; + } + total_cells += 1; + + // Collect all items for this cell's MCIDs + let mut cell_items: Vec<(usize, &TextItem)> = Vec::new(); + for &(mcid, p) in &cell.mcids { + if p == page { + if let Some(indices) = mcid_to_items.get(&mcid) { + for &idx in indices { + cell_items.push((idx, &items[idx])); + } + } + } + } + + if !cell_items.is_empty() { + matched_cells += 1; + } + + // Sort by Y (descending = top-to-bottom) then X + cell_items.sort_by(|a, b| { + b.1.y + .partial_cmp(&a.1.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + + let text: String = cell_items + .iter() + .map(|(_, item)| item.text.as_str()) + .collect::>() + .join(" "); + + for (idx, _) in &cell_items { + all_item_indices.push(*idx); + } + + row_cells.push(text); + } + + // Pad to num_cols + while row_cells.len() < num_cols { + row_cells.push(String::new()); + } + cells.push(row_cells); + } + + // Reject if too few cells matched (stale structure tree) + let coverage = if total_cells > 0 { + matched_cells as f32 / total_cells as f32 + } else { + 0.0 + }; + debug!( + "page {}: struct table {}x{}, {}/{} cells matched ({:.0}%)", + page, + page_rows.len(), + num_cols, + matched_cells, + total_cells, + coverage * 100.0 + ); + if total_cells == 0 || coverage < 0.3 { + continue; + } + + // Derive row/column positions from item geometry + let mut row_positions: Vec = Vec::new(); + for row in &page_rows { + let y = row + .cells + .iter() + .flat_map(|c| c.mcids.iter()) + .filter(|(_, p)| *p == page) + .filter_map(|(mcid, _)| mcid_to_items.get(mcid)) + .flatten() + .map(|&idx| items[idx].y) + .reduce(f32::max) + .unwrap_or(0.0); + row_positions.push(y); + } + + // Column positions: use X positions of first non-empty cell in each column + let mut col_positions: Vec = vec![0.0; num_cols]; + for (col, col_pos) in col_positions.iter_mut().enumerate() { + for row in &page_rows { + if col < row.cells.len() { + if let Some(x) = row.cells[col] + .mcids + .iter() + .filter(|(_, p)| *p == page) + .filter_map(|(mcid, _)| mcid_to_items.get(mcid)) + .flatten() + .map(|&idx| items[idx].x) + .reduce(f32::min) + { + *col_pos = x; + break; + } + } + } + } + + all_item_indices.sort_unstable(); + all_item_indices.dedup(); + + tables.push(Table { + columns: col_positions, + rows: row_positions, + cells, + item_indices: all_item_indices, + }); + } + + tables +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::structure_tree::{StructTableCell, StructTableRow}; + use crate::types::ItemType; + + fn make_item(text: &str, x: f32, y: f32, page: u32, mcid: Option) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * 5.0, + height: 10.0, + font: "Test".to_string(), + font_size: 10.0, + page, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid, + } + } + + #[test] + fn basic_struct_table() { + let items = vec![ + make_item("Name", 50.0, 700.0, 1, Some(10)), + make_item("Age", 200.0, 700.0, 1, Some(11)), + make_item("Alice", 50.0, 680.0, 1, Some(20)), + make_item("30", 200.0, 680.0, 1, Some(21)), + make_item("Bob", 50.0, 660.0, 1, Some(30)), + make_item("25", 200.0, 660.0, 1, Some(31)), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: true, + mcids: vec![(10, 1)], + }, + StructTableCell { + is_header: true, + mcids: vec![(11, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(30, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(31, 1)], + }, + ], + }, + ], + }]; + + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert_eq!(tables.len(), 1); + let table = &tables[0]; + assert_eq!(table.cells.len(), 3); + assert_eq!(table.cells[0], vec!["Name", "Age"]); + assert_eq!(table.cells[1], vec!["Alice", "30"]); + assert_eq!(table.cells[2], vec!["Bob", "25"]); + assert_eq!(table.item_indices.len(), 6); + } + + #[test] + fn rejects_low_mcid_coverage() { + // Items have no MCIDs matching the struct table + let items = vec![ + make_item("Orphan", 50.0, 700.0, 1, Some(999)), + make_item("Text", 200.0, 700.0, 1, None), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(10, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(11, 1)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 1)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 1)], + }, + ], + }, + ], + }]; + + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert!( + tables.is_empty(), + "should reject table with no MCID matches" + ); + } + + #[test] + fn filters_by_page() { + let items = vec![ + make_item("A", 50.0, 700.0, 2, Some(10)), + make_item("B", 200.0, 700.0, 2, Some(11)), + make_item("C", 50.0, 680.0, 2, Some(20)), + make_item("D", 200.0, 680.0, 2, Some(21)), + ]; + + let struct_tables = vec![StructTable { + rows: vec![ + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(10, 2)], + }, + StructTableCell { + is_header: false, + mcids: vec![(11, 2)], + }, + ], + }, + StructTableRow { + cells: vec![ + StructTableCell { + is_header: false, + mcids: vec![(20, 2)], + }, + StructTableCell { + is_header: false, + mcids: vec![(21, 2)], + }, + ], + }, + ], + }]; + + // Page 1 should find nothing + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 1); + assert!(tables.is_empty()); + + // Page 2 should find the table + let tables = detect_tables_from_struct_tree(&items, &struct_tables, 2); + assert_eq!(tables.len(), 1); + } +} diff --git a/src/tables/financial.rs b/src/tables/financial.rs index 046ba17..9c628a2 100644 --- a/src/tables/financial.rs +++ b/src/tables/financial.rs @@ -109,6 +109,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> is_bold: item.is_bold, is_italic: item.is_italic, item_type: item.item_type.clone(), + mcid: item.mcid, }); } Some(sub_items) diff --git a/src/tables/format.rs b/src/tables/format.rs index 8b69169..14e5aa3 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -17,33 +17,21 @@ pub fn table_to_markdown(table: &Table) -> String { let num_cols = cleaned_cells[0].len(); let mut output = String::new(); - // Calculate column widths for alignment (capped to avoid massive whitespace padding) - const MAX_COL_WIDTH: usize = 40; - let col_widths: Vec = (0..num_cols) - .map(|col| { - cleaned_cells - .iter() - .map(|row| row.get(col).map(|c| c.len()).unwrap_or(0)) - .max() - .unwrap_or(3) - .clamp(3, MAX_COL_WIDTH) - }) - .collect(); - - // Output each row + // Compact format: no padding, minimal separators. Optimized for token + // efficiency — AI agents are the primary consumer, not human eyes. for (row_idx, row) in cleaned_cells.iter().enumerate() { output.push('|'); - for (col_idx, cell) in row.iter().enumerate() { - let width = col_widths[col_idx]; - output.push_str(&format!(" {:width$} |", cell, width = width)); + for cell in row.iter() { + output.push_str(cell); + output.push('|'); } output.push('\n'); // Add separator after header row if row_idx == 0 { output.push('|'); - for width in &col_widths { - output.push_str(&format!(" {} |", "-".repeat(*width))); + for _ in 0..num_cols { + output.push_str("---|"); } output.push('\n'); } @@ -119,11 +107,43 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { let looks_like_data_row = non_first_cells.len() >= 2 && avg_cell_len <= 10.0 && numeric_cells > non_first_cells.len() / 2; - let is_continuation = first_cell.is_empty() + // Classic continuation: first cell empty, content in other cells + let is_classic_continuation = first_cell.is_empty() && !non_first_cells.is_empty() && !is_short_subheader && !looks_like_data_row - && cleaned.len() > 1; // Don't merge into the first row (header) + && cleaned.len() > 1; + + // Wrapped-cell continuation: row has fewer filled cells than the header + // row, suggesting it's overflow text from the previous row's cells. + // Only trigger when the previous row has significantly more filled cells. + let num_cols = row.len(); + let filled_cells = row.iter().filter(|c| !c.trim().is_empty()).count(); + let prev_filled = cleaned + .last() + .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) + .unwrap_or(0); + let header_filled = cleaned + .first() + .map(|r| r.iter().filter(|c| !c.trim().is_empty()).count()) + .unwrap_or(num_cols); + // Merge when the row has significantly fewer filled cells than header. + // For wide tables (5+ cols), require ≤50% of header cells. + // For narrow tables (2-4 cols), require fewer than header cells. + // This prevents merging normal data rows in wide tables (6_KE_Chart) + // while allowing continuation merging in narrow tables (178). + let max_filled_for_merge = if header_filled >= 5 { + header_filled / 2 + } else { + header_filled.saturating_sub(1) + }; + let is_wrapped_continuation = cleaned.len() > 1 + && filled_cells <= max_filled_for_merge + && prev_filled > filled_cells + && !looks_like_data_row + && !is_short_subheader; + + let is_continuation = is_classic_continuation || is_wrapped_continuation; if is_continuation { // Merge with previous row @@ -340,10 +360,10 @@ mod tests { item_indices: vec![], }; let md = table_to_markdown(&table); - assert!(md.contains("| Name")); - assert!(md.contains("| ---")); - assert!(md.contains("| Alice")); - assert!(md.contains("| Bob")); + assert!(md.contains("|Name|")); + assert!(md.contains("|---|")); + assert!(md.contains("|Alice|")); + assert!(md.contains("|Bob|")); } #[test] @@ -355,8 +375,8 @@ mod tests { item_indices: vec![], }; let md = table_to_markdown(&table); - assert!(md.contains("| Only")); - assert!(md.contains("| ---")); + assert!(md.contains("|Only|")); + assert!(md.contains("|---|")); } #[test] diff --git a/src/tables/grid.rs b/src/tables/grid.rs index ceb9e69..950dde4 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -66,13 +66,19 @@ pub(crate) fn find_column_boundaries( let threshold = (consec_gaps[best_split] + consec_gaps[(best_split + 1).min(consec_gaps.len() - 1)]) / 2.0; - // Only override for genuinely dense tables: many items packed into - // a wide range (e.g. 1200+ items for a 24-column train schedule). - // Smaller item counts (< 500) with a bimodal gap pattern are usually - // normal tables where center-based clustering works correctly. + // Override for tables with a clear bimodal gap pattern: + // - Dense tables (500+ items, e.g. 24-column train schedule): use + // edge-based clustering with the detected threshold. + // - Smaller tables with a strong bimodal signal (jump > 10pt): + // lower the threshold but keep center-based clustering to avoid + // over-splitting. if threshold < 15.0 && best_jump > 2.0 && x_positions.len() > 500 { cluster_threshold = threshold.clamp(8.0, 25.0); use_edge_clustering = true; + } else if best_jump > 10.0 && threshold < cluster_threshold { + // Strong bimodal signal even with fewer items — the gap between + // within-column jitter and between-column spacing is unambiguous. + cluster_threshold = threshold.max(8.0); } } @@ -116,6 +122,13 @@ pub(crate) fn find_column_boundaries( }) .collect(); + log::debug!( + " find_column_boundaries: {} columns before filter, threshold={:.1}, {} items", + columns.len(), + cluster_threshold, + items.len() + ); + // Anti-paragraph safeguard for BodyFont mode: // Paragraphs concentrate items at the left margin; tables distribute evenly. // Reject if any single column has >60% of all items. @@ -381,6 +394,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -727,6 +741,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); @@ -762,6 +777,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, page: 1, }, )); diff --git a/src/tables/mod.rs b/src/tables/mod.rs index a85418c..ece5e81 100644 --- a/src/tables/mod.rs +++ b/src/tables/mod.rs @@ -5,6 +5,7 @@ mod detect_heuristic; mod detect_lines; mod detect_rects; +mod detect_struct; mod financial; mod format; mod grid; @@ -13,6 +14,7 @@ pub use detect_heuristic::detect_tables; pub use detect_lines::detect_tables_from_lines; pub(crate) use detect_rects::cluster_rects; pub use detect_rects::{detect_tables_from_rects, RectHintRegion}; +pub use detect_struct::detect_tables_from_struct_tree; pub use format::table_to_markdown; use crate::types::TextItem; @@ -231,6 +233,7 @@ fn split_merged_numbers(item: &TextItem, col_boundaries: &[f32]) -> Vec Vec Option
{ + use crate::extractor::{ + detect_columns, group_into_lines_with_thresholds, is_newspaper_layout, ColumnRegion, + }; + use std::collections::HashMap; + + let mut columns = detect_columns(items, page, false); + if columns.len() < 4 { + return None; + } + + // Refine columns: look for header-like rows where multiple items share + // the same Y and are evenly spaced. If a wide column contains two header + // items, split it at the gap between them. + let page_items: Vec<&TextItem> = items.iter().filter(|i| i.page == page).collect(); + let y_tol = 3.0; + + // Find the top-most row with items in multiple columns (likely the header) + let mut ys: Vec = page_items.iter().map(|i| i.y).collect(); + ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + ys.dedup_by(|a, b| (*a - *b).abs() < y_tol); + + for &header_y in ys.iter().take(5) { + let row_items: Vec<&&TextItem> = page_items + .iter() + .filter(|i| (i.y - header_y).abs() < y_tol) + .collect(); + if row_items.len() < columns.len() { + continue; + } + // Check if any column contains 2+ items at this Y — needs splitting + let mut new_columns = Vec::new(); + let mut did_split = false; + for col in &columns { + let col_items: Vec<&&&TextItem> = row_items + .iter() + .filter(|i| i.x >= col.x_min && i.x < col.x_max) + .collect(); + if col_items.len() >= 2 { + // Sort by X and find the split point + let mut sorted: Vec = col_items.iter().map(|i| i.x).collect(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + // Split at the midpoint between the two items + let split_x = (sorted[0] + + col_items.iter().find(|i| i.x == sorted[0]).unwrap().width + + sorted[1]) + / 2.0; + new_columns.push(ColumnRegion { + x_min: col.x_min, + x_max: split_x, + }); + new_columns.push(ColumnRegion { + x_min: split_x, + x_max: col.x_max, + }); + did_split = true; + } else { + new_columns.push(col.clone()); + } + } + if did_split { + log::debug!( + "column refinement: {} -> {} columns from header row at y={:.1}", + columns.len(), + new_columns.len(), + header_y + ); + columns = new_columns; + break; + } + } + + // Group items into per-column lines to check newspaper vs tabular + let mut col_buckets: Vec> = vec![Vec::new(); columns.len()]; + let mut spanning_items: Vec = Vec::new(); + for item in items { + if item.page != page { + continue; + } + // Check if item spans multiple columns + let item_left = item.x; + let item_right = item.x + item.width; + let mut spans = 0; + for col in &columns { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > 0.0 { + spans += 1; + } + } + if spans > 1 { + spanning_items.push(item.clone()); + continue; + } + // Assign to best-overlap column + let mut best_col = 0; + let mut best_overlap = f32::NEG_INFINITY; + for (ci, col) in columns.iter().enumerate() { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > best_overlap { + best_overlap = overlap; + best_col = ci; + } + } + col_buckets[best_col].push(item.clone()); + } + + let thresholds = HashMap::new(); + let per_column_lines: Vec> = col_buckets + .iter() + .map(|bucket| { + group_into_lines_with_thresholds( + bucket.clone(), + &thresholds, + &std::collections::HashSet::new(), + ) + }) + .collect(); + + // Must be tabular (not newspaper) layout + if is_newspaper_layout(&per_column_lines, &columns) { + return None; + } + + // Collect all unique Y positions across all columns (row boundaries) + let y_tol = 5.0; + let mut row_ys: Vec = Vec::new(); + for col_lines in &per_column_lines { + for line in col_lines { + let y = line.y; + if !row_ys.iter().any(|&ry| (ry - y).abs() < y_tol) { + row_ys.push(y); + } + } + } + row_ys.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + if row_ys.len() < 3 || row_ys.len() > 40 { + return None; + } + + // Build cell grid + let col_xs: Vec = columns.iter().map(|c| c.x_min).collect(); + let mut cells: Vec> = vec![vec![String::new(); columns.len()]; row_ys.len()]; + let mut item_indices: Vec = Vec::new(); + + for (item_idx, item) in items.iter().enumerate() { + if item.page != page { + continue; + } + // Find column + let item_left = item.x; + let item_right = item.x + item.width; + let mut best_col = None; + let mut best_overlap = 0.0f32; + let mut span_count = 0; + for (ci, col) in columns.iter().enumerate() { + let overlap = (item_right.min(col.x_max) - item_left.max(col.x_min)).max(0.0); + if overlap > 0.0 { + span_count += 1; + } + if overlap > best_overlap { + best_overlap = overlap; + best_col = Some(ci); + } + } + if span_count > 1 || best_col.is_none() { + continue; // spanning item, skip + } + let col = best_col.unwrap(); + + // Find row + let row = row_ys.iter().position(|&ry| (ry - item.y).abs() < y_tol); + if let Some(row) = row { + if !cells[row][col].is_empty() { + cells[row][col].push(' '); + } + cells[row][col].push_str(&item.text); + item_indices.push(item_idx); + } + } + + // Validate: need reasonable fill rate + let total_cells = row_ys.len() * columns.len(); + let filled_cells = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| !c.trim().is_empty()) + .count(); + let fill_rate = filled_cells as f32 / total_cells as f32; + + if fill_rate < 0.15 { + return None; + } + + // Need at least 40% of rows to have content in 2+ columns + let multi_col_rows = cells + .iter() + .filter(|row| row.iter().filter(|c| !c.trim().is_empty()).count() >= 2) + .count(); + // Need majority (>50%) of rows with content in 2+ columns + if multi_col_rows * 2 < row_ys.len() { + return None; + } + + // Reject prose-like content: if cells are too long on average, this is + // a multi-column text layout, not a data table. Real table cells are + // typically short (≤ 40 chars). Prose paragraphs are much longer. + let cell_lengths: Vec = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| !c.trim().is_empty()) + .map(|c| c.trim().len()) + .collect(); + if !cell_lengths.is_empty() { + let avg_cell_len = cell_lengths.iter().sum::() as f32 / cell_lengths.len() as f32; + if avg_cell_len > 40.0 { + return None; + } + // Reject if any significant number of cells are long prose (> 80 chars) + let long_cells = cell_lengths.iter().filter(|&&len| len > 80).count(); + if long_cells as f32 / cell_lengths.len() as f32 > 0.10 { + return None; + } + } + + // Reject when cells look like prose sentences: if too many cells contain + // sentence-ending punctuation (.!?:) it's prose text, not table data. + let prose_cells = cells + .iter() + .flat_map(|r| r.iter()) + .filter(|c| { + let t = c.trim(); + t.len() > 20 + && (t.ends_with('.') || t.ends_with('!') || t.ends_with('?') || t.ends_with(':')) + }) + .count(); + if filled_cells > 0 && prose_cells as f32 / filled_cells as f32 > 0.15 { + return None; + } + + // Reject when most content is in one column (newspaper-like asymmetry). + // Count items per column; if any column has >60% of items, it's likely + // a body text column with side annotations, not a data table. + let mut items_per_col: Vec = vec![0; columns.len()]; + for row in &cells { + for (ci, cell) in row.iter().enumerate() { + if !cell.trim().is_empty() { + items_per_col[ci] += 1; + } + } + } + let max_col_items = *items_per_col.iter().max().unwrap_or(&0); + if filled_cells > 0 && max_col_items as f32 / filled_cells as f32 > 0.60 { + return None; + } + + log::debug!( + "column-based table: {} cols x {} rows, fill={:.0}%, multi_col_rows={}", + columns.len(), + row_ys.len(), + fill_rate * 100.0, + multi_col_rows + ); + + Some(Table { + columns: col_xs, + rows: row_ys, + cells, + item_indices, + }) +} + /// A detected table. #[derive(Debug, Clone)] pub struct Table { @@ -296,6 +580,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -312,6 +597,7 @@ mod tests { is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -359,9 +645,9 @@ mod tests { }; let md = table_to_markdown(&table); - assert!(md.contains("| Header 1")); - assert!(md.contains("| ---")); - assert!(md.contains("| Cell 1")); + assert!(md.contains("|Header 1|")); + assert!(md.contains("|---|")); + assert!(md.contains("|Cell 1|")); } #[test] @@ -721,12 +1007,12 @@ mod tests { let md = table_to_markdown(&table); // JAN and FEB should be on their own rows, not merged into adjacent rows assert!( - md.contains("| JAN"), + md.contains("|JAN|"), "JAN should be on its own row, got:\n{}", md ); assert!( - md.contains("| FEB"), + md.contains("|FEB|"), "FEB should be on its own row, got:\n{}", md ); diff --git a/src/text_utils.rs b/src/text_utils.rs index a6929bb..702c4bd 100644 --- a/src/text_utils.rs +++ b/src/text_utils.rs @@ -5,6 +5,7 @@ //! and markdown pipelines. use crate::types::TextItem; +use unicode_normalization::UnicodeNormalization; /// Check if a character is CJK (Chinese, Japanese, Korean). /// CJK languages don't use spaces between words, so word-boundary @@ -40,6 +41,12 @@ pub(crate) fn is_rtl_char(c: char) -> bool { ) } +fn is_arabic_presentation_form(c: char) -> bool { + // U+FEFF is BOM/ZWNJ, not an Arabic presentation form despite falling + // in the Presentation Forms-B codepoint range. + matches!(c, '\u{FB50}'..='\u{FDFF}' | '\u{FE70}'..='\u{FEFE}') +} + pub(crate) fn is_rtl_text(texts: I) -> bool where I: Iterator, @@ -105,6 +112,9 @@ pub fn is_italic_font(font_name: &str) -> bool { /// Expand Unicode ligature characters to their component characters. /// This makes extracted text more searchable and semantically correct. +/// Also applies NFKC normalization (converts Arabic presentation forms to base +/// characters, decomposes Latin ligatures, etc.) and reverses visual-order +/// Arabic text back to logical order when presentation forms are detected. pub(crate) fn expand_ligatures(text: &str) -> String { // Strip null bytes and other control characters (except newline/tab) let text = if text @@ -118,9 +128,26 @@ pub(crate) fn expand_ligatures(text: &str) -> String { text.to_string() }; + // Detect Arabic presentation forms before normalization — their presence + // signals visual-order storage that needs reversal after NFKC. + let had_presentation_forms = text.chars().any(is_arabic_presentation_form); + + // Apply NFKC normalization only when Arabic presentation forms are present. + // This converts forms (U+FB50-FDFF, U+FE70-FEFF) back to base Arabic + // (U+0600-06FF). We avoid broad NFKC on all non-ASCII text because it + // would convert NBSP (U+00A0) to regular space, breaking downstream logic. + // Latin ligatures are already handled by the explicit match arms below. + let text = if had_presentation_forms { + text.nfkc().collect::() + } else { + text + }; + let mut result = String::with_capacity(text.len()); for ch in text.chars() { match ch { + // Keep explicit ligature expansion as fallback for fonts that bypass + // NFKC (e.g. custom ToUnicode mappings to PUA codepoints) '\u{FB00}' => result.push_str("ff"), '\u{FB01}' => result.push_str("fi"), '\u{FB02}' => result.push_str("fl"), @@ -141,9 +168,74 @@ pub(crate) fn expand_ligatures(text: &str) -> String { _ => result.push(ch), } } + + // If the original text had Arabic presentation forms, the characters are in + // visual (LTR screen) order. After NFKC normalization, reverse to restore + // logical reading order. + if had_presentation_forms { + result = reverse_visual_arabic(&result); + } + result } +/// Reverse visual-order Arabic text to logical order. +/// +/// Pure RTL text (no ASCII alphanumerics) gets a simple character reversal. +/// Mixed content (embedded numbers or Latin words) splits into LTR and non-LTR +/// runs: run order is reversed, and only non-LTR runs are reversed internally. +fn reverse_visual_arabic(text: &str) -> String { + // Check if there are any LTR runs (ASCII letters or digits) + let has_ltr = text.chars().any(|c| c.is_ascii_alphanumeric()); + + if !has_ltr { + // Pure RTL: simple reversal + return text.chars().rev().collect(); + } + + // Mixed content: split into runs of LTR (ASCII alphanumeric + adjacent + // punctuation like '.', ',', '/', '-') vs non-LTR (Arabic + spaces + other). + let chars: Vec = text.chars().collect(); + let mut runs: Vec<(bool, String)> = Vec::new(); // (is_ltr, content) + + let mut i = 0; + while i < chars.len() { + let is_ltr = chars[i].is_ascii_alphanumeric() + || (chars[i].is_ascii_punctuation() && is_adjacent_to_ascii_alnum(&chars, i)); + + let mut run = String::new(); + while i < chars.len() { + let c = chars[i]; + let c_is_ltr = c.is_ascii_alphanumeric() + || (c.is_ascii_punctuation() && is_adjacent_to_ascii_alnum(&chars, i)); + if c_is_ltr != is_ltr { + break; + } + run.push(c); + i += 1; + } + runs.push((is_ltr, run)); + } + + // Reverse run order and reverse non-LTR runs internally + runs.reverse(); + let mut result = String::with_capacity(text.len()); + for (is_ltr, content) in &runs { + if *is_ltr { + result.push_str(content); + } else { + result.extend(content.chars().rev()); + } + } + result +} + +/// Check if the character at `idx` is adjacent to an ASCII alphanumeric character. +fn is_adjacent_to_ascii_alnum(chars: &[char], idx: usize) -> bool { + (idx > 0 && chars[idx - 1].is_ascii_alphanumeric()) + || (idx + 1 < chars.len() && chars[idx + 1].is_ascii_alphanumeric()) +} + /// Decode a PDF text string (ActualText, etc.) that may be UTF-16BE (BOM \xFE\xFF) /// or PDFDocEncoding (Latin-1 superset). pub(crate) fn decode_text_string(bytes: &[u8]) -> String { @@ -186,12 +278,271 @@ pub(crate) fn is_cid_font(font: &str) -> bool { font.starts_with("C2_") || font.starts_with("C0_") } +/// Detect and fix Canva-style letter-spacing within text items. +/// +/// Canva-generated PDFs render text character-by-character with CSS-style +/// letter-spacing. The TJ handler inserts spaces between each character, +/// producing items like `"a r i b"` instead of `"arib"`. This function +/// detects such items by checking if the text follows a strict pattern of +/// alternating single characters and spaces, then removes the spurious spaces. +/// +/// Only activates when ≥50% of items on the page are letter-spaced, to avoid +/// false positives on normal PDFs with short items like `"a b"`. +/// +/// Returns the adaptive join threshold for this page: DEFAULT (0.10) for normal +/// pages, or a higher Otsu-derived threshold for Canva-style pages. +pub(crate) fn fix_letterspaced_items(items: &mut [TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + + if items.is_empty() { + return DEFAULT; + } + + // Check if the item text matches "x y z" pattern (single chars separated by spaces) + fn is_letterspaced(text: &str) -> bool { + let trimmed = text.trim(); + let chars: Vec = trimmed.chars().collect(); + // Need at least 3 chars: "a b" = ['a', ' ', 'b'] + if chars.len() < 3 { + return false; + } + // Pattern: non-space, space, non-space, space, ... + chars + .iter() + .enumerate() + .all(|(i, &c)| if i % 2 == 0 { c != ' ' } else { c == ' ' }) + } + + // Count how many items are letter-spaced vs total non-trivial items + let mut letterspaced_count = 0u32; + let mut total_text_items = 0u32; + for item in items.iter() { + let trimmed = item.text.trim(); + if trimmed.is_empty() || trimmed.len() < 3 { + continue; + } + total_text_items += 1; + if is_letterspaced(&item.text) { + letterspaced_count += 1; + } + } + + // Only fix if ≥50% of substantial items are letter-spaced + if total_text_items < 4 || letterspaced_count * 2 < total_text_items { + // Second detection path: per-character rendering without embedded spaces. + // Canva sometimes emits each character as a separate TextItem (no "a b c" + // pattern within items). Detect by checking if >50% of items are single chars. + let single_char_count = items + .iter() + .filter(|i| i.text.trim().chars().count() == 1) + .count(); + if items.len() >= 10 && single_char_count * 2 >= items.len() { + let threshold = compute_canva_join_threshold(items); + if threshold > 0.40 { + return threshold; + } + } + return DEFAULT; + } + // Compute threshold BEFORE removing spaces. Since we've confirmed this + // is a Canva-style page (≥50% letterspaced), use the ungated variant + // that includes all pairs — the char-count guard in the normal function + // would filter out long letterspaced items like "i s s i o n" (11 chars). + let threshold = compute_canva_join_threshold(items); + + // Remove spaces from letter-spaced items + for item in items.iter_mut() { + if is_letterspaced(&item.text) { + let fixed: String = item.text.chars().filter(|&c| c != ' ').collect(); + item.text = fixed; + } + } + + threshold +} + +/// Compute join threshold for a confirmed Canva-style page. +/// +/// Uses `median × 1.55` on the gap/font_size ratio distribution. The page-level +/// threshold is used for multi-char item pairs; single-char pairs use +/// character-width–based joining in `should_join_items` instead. +fn compute_canva_join_threshold(items: &[TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + const MIN_SAMPLES: usize = 8; + + let ratios = collect_gap_ratios(items); + if ratios.len() < MIN_SAMPLES { + return DEFAULT; + } + + let mut sorted: Vec = ratios; + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + if sorted[sorted.len() - 1] < 0.40 || sorted[0] < 0.40 { + return DEFAULT; + } + + let median = sorted[sorted.len() / 2]; + (median * 1.55).clamp(0.50, 2.0) +} + +/// Collect positive gap/font_size ratios from adjacent item pairs, +/// filtering out CJK, zero-width, and out-of-range values. +fn collect_gap_ratios(items: &[TextItem]) -> Vec { + let mut ratios: Vec = Vec::new(); + for pair in items.windows(2) { + let prev = &pair[0]; + let curr = &pair[1]; + + let prev_c = prev.text.trim().chars().last(); + let curr_c = curr.text.trim().chars().next(); + if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) { + continue; + } + + if prev.width <= 0.0 || prev.font_size <= 0.0 { + continue; + } + + let gap = if prev.x <= curr.x { + curr.x - (prev.x + prev.width) + } else { + prev.x - (curr.x + curr.width) + }; + + let ratio = gap / prev.font_size; + + if (0.0..=3.0).contains(&ratio) { + ratios.push(ratio); + } + } + ratios +} + +/// Compute an adaptive join threshold for text items on a line. +/// +/// Uses Otsu's method on the gap/font_size ratio distribution to find the +/// natural split between intra-word and inter-word gaps. With per-pair +/// char-count guard (both items ≥ 5 chars → skip). Used only in tests; +/// production code uses `compute_canva_join_threshold` via `fix_letterspaced_items`. +#[cfg(test)] +fn compute_single_char_join_threshold(items: &[TextItem]) -> f32 { + const DEFAULT: f32 = 0.10; + const MIN_SAMPLES: usize = 8; + + // Collect gap/font_size ratios for adjacent pairs involving at least one + // short fragment (< 5 chars). This detects per-character rendering + // (Canva-style) without being fooled by uniform word-level spacing. + let mut ratios: Vec = Vec::new(); + for pair in items.windows(2) { + let prev = &pair[0]; + let curr = &pair[1]; + + let prev_chars = prev.text.trim().chars().count(); + let curr_chars = curr.text.trim().chars().count(); + + // Require at least one item to be a short fragment. + // Pairs of long words (both ≥ 5 chars) indicate normal text. + if prev_chars >= 5 && curr_chars >= 5 { + continue; + } + + // Skip CJK pairs + let prev_c = prev.text.trim().chars().last(); + let curr_c = curr.text.trim().chars().next(); + if prev_c.is_some_and(is_cjk_char) || curr_c.is_some_and(is_cjk_char) { + continue; + } + + if prev.width <= 0.0 || prev.font_size <= 0.0 { + continue; + } + + let gap = if prev.x <= curr.x { + curr.x - (prev.x + prev.width) + } else { + prev.x - (curr.x + curr.width) + }; + + let ratio = gap / prev.font_size; + + // Skip negative gaps and huge gaps (> 3× font_size) + if !(0.0..=3.0).contains(&ratio) { + continue; + } + + ratios.push(ratio); + } + + if ratios.len() < MIN_SAMPLES { + return DEFAULT; + } + + ratios.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + // If all gaps are tight (max < 0.40), use default — normal PDF + let max_ratio = ratios[ratios.len() - 1]; + if max_ratio < 0.40 { + return DEFAULT; + } + + // If the minimum gap is below 0.40, there's a mix of tight and wide gaps, + // meaning this isn't a uniform letter-spacing PDF — use default. + // Canva-style letter-spacing has min gaps ≈ 0.5× font_size; normal + // justified text gaps are ≈ 0.15–0.30× font_size. + if ratios[0] < 0.40 { + return DEFAULT; + } + + // All gaps are wide (≥0.25× font_size) — Canva-style letter-spacing. + // Use Otsu to find the split between intra-word and inter-word gaps. + let n = ratios.len() as f32; + let total_sum: f32 = ratios.iter().sum(); + + let mut best_threshold = DEFAULT; + let mut best_variance = f32::NEG_INFINITY; + + let mut w0: f32 = 0.0; + let mut sum0: f32 = 0.0; + + for i in 0..ratios.len() - 1 { + w0 += 1.0; + sum0 += ratios[i]; + + let w1 = n - w0; + if w1 == 0.0 { + break; + } + + let mean0 = sum0 / w0; + let mean1 = (total_sum - sum0) / w1; + let variance = w0 * w1 * (mean0 - mean1).powi(2); + + // Only consider thresholds at value boundaries (skip duplicates) + if i + 1 < ratios.len() && (ratios[i + 1] - ratios[i]).abs() < 1e-6 { + continue; + } + + if variance > best_variance { + best_variance = variance; + // Place threshold midway between the two classes + best_threshold = (ratios[i] + ratios[i + 1]) / 2.0; + } + } + + best_threshold.clamp(0.05, 2.0) +} + /// Determine if two adjacent text items should be joined without a space /// based on their physical positions on the page and character case. /// Uses a hybrid approach: position-based with case-aware thresholds. /// CID fonts emit one word per text operator with gaps ≈ 0 between words. /// Non-CID (Type1/TrueType) fonts emit phrases or fragments. -pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> bool { +pub(crate) fn should_join_items( + prev_item: &TextItem, + curr_item: &TextItem, + single_char_threshold: f32, +) -> bool { // If either text explicitly has leading/trailing spaces, respect them if prev_item.text.ends_with(' ') || curr_item.text.starts_with(' ') { return false; @@ -228,8 +579,10 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b }; let font_size = prev_item.font_size; - // Never join across column-scale gaps - if gap > font_size * 3.0 { + // Never join across column-scale gaps or large overlaps. + // Large negative gaps arise when Tc/Tw inflate item widths past + // where adjacent items actually start. + if gap > font_size * 3.0 || gap < -font_size { return false; } @@ -261,18 +614,44 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b // are positioned close together are almost always a single number. // e.g., "34,20" + "8" → "34,208", "+13." + "0" + "%" → "+13.0%" // Use a generous threshold since word spaces in numbers are rare. + // The lower bound (-font_size) rejects large overlaps caused by + // Tc/Tw–inflated item widths that make adjacent items appear to + // occupy the same space. if let (Some(p), Some(c)) = (prev_last, curr_first) { let prev_is_numeric = p.is_ascii_digit() || p == ',' || p == '.'; let curr_is_numeric = c.is_ascii_digit() || c == '%' || c == '.'; if prev_is_numeric && curr_is_numeric { - return gap < font_size * 0.3; + return gap > -font_size && gap < font_size * 0.3; } // Sign characters (+/-) followed by digits if (p == '+' || p == '-') && c.is_ascii_digit() { - return gap < font_size * 0.3; + return gap > -font_size && gap < font_size * 0.3; } } + // When the adaptive threshold indicates Canva-style letter-spacing + // (all gaps wide), use character-width–based joining. + // + // Canva renders text character-by-character with CSS-style letter-spacing. + // For single-char prev items, gap/char_width gives a clean separation + // (~0.9–1.05 for letter gaps, ~1.5+ for word gaps). + // For multi-char prev, avg_char_width normalizes for character mix. + // Multi→multi pairs use the page-level threshold (gap/font_size). + if single_char_threshold > 0.20 { + if prev_chars == 1 { + // Single-char prev: its rendered width is an accurate reference + return gap < prev_item.width * 1.25; + } + if curr_chars == 1 { + // Multi→single: avg char width of prev normalises for + // wide/narrow character mix (e.g. "ilw" includes i,l,w) + let avg_char_width = prev_item.width / prev_chars as f32; + return gap < avg_char_width * 1.25; + } + // Both multi-char: use page-level threshold + return gap < font_size * single_char_threshold; + } + // Single-character fragment joined to a multi-character item: use a // moderately generous threshold to rejoin split words like "b" + "illion" // or "C" + "ultural". Gap near 0 = same word; gap ~0.2+ = different words. @@ -293,7 +672,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b return gap < font_size * 0.25; } } - return gap < font_size * 0.10; + return gap < font_size * single_char_threshold; } // With accurate widths, a gap < 15% of font size means glyphs are @@ -381,6 +760,7 @@ pub(crate) fn should_join_items(prev_item: &TextItem, curr_item: &TextItem) -> b #[cfg(test)] mod tests { use super::*; + use crate::types::ItemType; #[test] fn strip_soft_hyphen() { @@ -425,4 +805,356 @@ mod tests { // NBSP (U+00A0) should NOT be normalized assert_eq!(expand_ligatures("a\u{00A0}b"), "a\u{00A0}b"); } + + #[test] + fn nfkc_arabic_presentation_forms() { + // Arabic Presentation Form-B: FEE1 = MEEM medial, FEF3 = YEH initial + // NFKC maps these to base Arabic + reversal restores logical order + let input = "\u{FEE1}\u{FEF3}"; // visual order: medial meem, initial yeh + let result = expand_ligatures(input); + // After NFKC: base Arabic chars; after reversal: logical order + assert!( + !result.chars().any(is_arabic_presentation_form), + "presentation forms should be normalized: {result:?}" + ); + assert!( + result.chars().any(|c| matches!(c, '\u{0600}'..='\u{06FF}')), + "should contain base Arabic characters: {result:?}" + ); + } + + #[test] + fn no_reversal_for_base_arabic() { + // Base Arabic already in logical order — no presentation forms means no reversal + let input = "\u{0645}\u{0631}\u{062D}\u{0628}\u{0627}"; // مرحبا + let result = expand_ligatures(input); + assert_eq!(result, input, "base Arabic should pass through unchanged"); + } + + #[test] + fn latin_text_unaffected() { + assert_eq!(expand_ligatures("Hello World"), "Hello World"); + } + + #[test] + fn reverse_visual_arabic_pure_rtl() { + // Pure RTL: simple reversal + let input = "\u{0628}\u{0627}"; // ba (visual order) + let result = reverse_visual_arabic(input); + assert_eq!(result, "\u{0627}\u{0628}"); // ab (logical order) + } + + #[test] + fn reverse_visual_arabic_with_ltr_run() { + // Mixed: Arabic + embedded number "123" + Arabic + // Visual order: أ 123 ب → runs: [أ], [123], [ب] + // Reversed runs: [ب], [123], [أ] + // Non-LTR reversed internally: ب, 123, أ + let input = "\u{0623}123\u{0628}"; + let result = reverse_visual_arabic(input); + assert_eq!(result, "\u{0628}123\u{0623}"); + } + + #[test] + fn arabic_presentation_form_detection() { + // Presentation Forms-A range + assert!(is_arabic_presentation_form('\u{FB50}')); + assert!(is_arabic_presentation_form('\u{FDFF}')); + // Presentation Forms-B range (excludes U+FEFF which is BOM) + assert!(is_arabic_presentation_form('\u{FE70}')); + assert!(is_arabic_presentation_form('\u{FEFE}')); + assert!(!is_arabic_presentation_form('\u{FEFF}')); + // Base Arabic — NOT presentation form + assert!(!is_arabic_presentation_form('\u{0645}')); + // Latin + assert!(!is_arabic_presentation_form('A')); + } + + /// Helper to create a single-char TextItem at a given x position with width. + fn make_char_item(ch: char, x: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: ch.to_string(), + x, + y: 100.0, + width, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn otsu_threshold_sec_style_tight_gaps() { + // SEC-style: intra-word gaps ≈ 0, word gap ≈ 0.15× font_size + // All gaps tight → should return default 0.10 + let fs = 12.0; + let char_w = fs * 0.5; + let mut items = Vec::new(); + // 15 chars with gap ≈ 0 (intra-word) + for i in 0..15 { + let x = 100.0 + i as f32 * (char_w + fs * 0.01); + items.push(make_char_item('a', x, char_w, fs)); + } + // Word gap + let word_x = items.last().unwrap().x + char_w + fs * 0.15; + items.push(make_char_item('b', word_x, char_w, fs)); + // 5 more tight chars + for i in 1..5 { + let x = word_x + i as f32 * (char_w + fs * 0.01); + items.push(make_char_item('c', x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + // Max gap is 0.15, but most are 0.01 → max < 0.20 → default + assert!( + (threshold - 0.10).abs() < 0.01, + "SEC-style should return default ~0.10, got {threshold}" + ); + } + + #[test] + fn otsu_threshold_canva_style_wide_gaps() { + // Canva-style: intra-word gaps ≈ 0.6× font_size, word gaps ≈ 1.2× font_size + let fs = 12.0; + let char_w = fs * 0.5; + let intra_gap = fs * 0.6; + let word_gap = fs * 1.2; + let mut items = Vec::new(); + + // Word 1: 8 chars with intra-word spacing + for i in 0..8 { + let x = 100.0 + i as f32 * (char_w + intra_gap); + items.push(make_char_item('K', x, char_w, fs)); + } + // Word gap + let word_x = items.last().unwrap().x + char_w + word_gap; + items.push(make_char_item('T', word_x, char_w, fs)); + // Word 2: 7 more chars + for i in 1..7 { + let x = word_x + i as f32 * (char_w + intra_gap); + items.push(make_char_item('o', x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + // Should find threshold between 0.6 and 1.2 → roughly 0.9 + assert!( + threshold > 0.5 && threshold < 1.1, + "Canva-style should find threshold ~0.9, got {threshold}" + ); + } + + #[test] + fn otsu_threshold_few_samples_returns_default() { + // < 8 single-char pairs → default + let fs = 12.0; + let char_w = fs * 0.5; + let items: Vec = (0..5) + .map(|i| make_char_item('x', 100.0 + i as f32 * (char_w + 1.0), char_w, fs)) + .collect(); + + let threshold = compute_single_char_join_threshold(&items); + assert!( + (threshold - 0.10).abs() < 0.01, + "few samples should return default 0.10, got {threshold}" + ); + } + + #[test] + fn fix_letterspaced_items_returns_adaptive_threshold() { + // Simulate Canva page with many letter-spaced items and word gaps. + // Needs ≥8 inter-item gaps for the threshold to be computed. + let fs = 12.0; + let char_w = fs * 0.5; + let letter_gap = fs * 0.6; // 0.6× font_size between items + let word_gap = fs * 1.2; // 1.2× font_size between words + + let words: Vec<&str> = vec![ + "H e l l o", + "W o r l d", + "F o o", + "B a r", + "B a z", + "Q u x", + "T e s t", + "D a t a", + "M o r e", + "T e x t", + ]; + + let mut items = Vec::new(); + let mut x = 100.0; + for (wi, word) in words.iter().enumerate() { + let char_count = word.chars().filter(|c| !c.is_whitespace()).count(); + let w = char_count as f32 * char_w + (char_count - 1) as f32 * letter_gap; + items.push(TextItem { + text: word.to_string(), + x, + y: 100.0, + width: w, + height: fs, + font: "TestFont".to_string(), + font_size: fs, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + }); + // Alternate between letter-gap and word-gap to create bimodal distribution + x += w + if wi % 3 == 2 { word_gap } else { letter_gap }; + } + + let threshold = fix_letterspaced_items(&mut items); + + // Threshold should be above default (Canva-style detected) + assert!( + threshold > 0.50, + "Canva page should get threshold > 0.50, got {threshold}" + ); + + // Spaces should be removed from letter-spaced items + assert_eq!(items[0].text, "Hello"); + assert_eq!(items[1].text, "World"); + assert_eq!(items[2].text, "Foo"); + assert_eq!(items[9].text, "Text"); + } + + #[test] + fn canva_style_items_join_correctly() { + // Simulate Canva PDF: "Hello" with 0.6× font_size letter-spacing + let fs = 12.0; + let char_w = fs * 0.5; + let intra_gap = fs * 0.6; + let word_gap = fs * 1.2; + + let mut items = Vec::new(); + let chars = ['H', 'e', 'l', 'l', 'o']; + for (i, &ch) in chars.iter().enumerate() { + let x = 100.0 + i as f32 * (char_w + intra_gap); + items.push(make_char_item(ch, x, char_w, fs)); + } + // Space then "W" + let w_x = items.last().unwrap().x + char_w + word_gap; + items.push(make_char_item('W', w_x, char_w, fs)); + let chars2 = ['o', 'r', 'l', 'd']; + for (i, &ch) in chars2.iter().enumerate() { + let x = w_x + (i + 1) as f32 * (char_w + intra_gap); + items.push(make_char_item(ch, x, char_w, fs)); + } + + let threshold = compute_single_char_join_threshold(&items); + + // Intra-word pairs should join + assert!( + should_join_items(&items[0], &items[1], threshold), + "H+e should join with threshold {threshold}" + ); + assert!( + should_join_items(&items[3], &items[4], threshold), + "l+o should join with threshold {threshold}" + ); + // Word boundary should NOT join + assert!( + !should_join_items(&items[4], &items[5], threshold), + "o+W (word boundary) should NOT join with threshold {threshold}" + ); + } + + /// Helper to create a multi-char TextItem at a given position. + fn make_text_item(text: &str, x: f32, width: f32, font_size: f32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y: 100.0, + width, + height: font_size, + font: "TestFont".to_string(), + font_size, + page: 1, + is_bold: false, + is_italic: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn canva_width_based_single_char_prev_join() { + // Canva-style: single-char prev uses gap/prev.width < 1.25 + let fs = 12.0; + let threshold = 0.90; // Canva page threshold + + // "K" (w=7.9) → "a" (gap=8.12): letter gap, ratio=1.028 → JOIN + let k = make_text_item("K", 100.0, 7.9, fs); + let a = make_text_item("a", 115.9, 6.0, fs); + assert!( + should_join_items(&k, &a, threshold), + "K→a: gap/width={:.3}, should join", + (a.x - (k.x + k.width)) / k.width + ); + + // "f" (w=4.0) → "K" (gap=10.47): word boundary, ratio=2.618 → SPLIT + let f = make_text_item("f", 193.0, 4.0, fs); + let k2 = make_text_item("K", 207.47, 7.9, fs); + assert!( + !should_join_items(&f, &k2, threshold), + "f→K: gap/width={:.3}, should split", + (k2.x - (f.x + f.width)) / f.width + ); + } + + #[test] + fn canva_width_based_multi_to_single_join() { + // Multi→single: uses avg_char_width of prev + let fs = 12.0; + let threshold = 0.90; + + // "ilw" (w=23.6, 3 chars) → "a" (gap=9.42): intra-word, avg=7.87, ratio=1.197 → JOIN + let ilw = make_text_item("ilw", 320.0, 23.6, fs); + let a = make_text_item("a", 353.0, 6.0, fs); + assert!( + should_join_items(&ilw, &a, threshold), + "ilw→a: avg_ratio={:.3}, should join (intra-word 'railway')", + (a.x - (ilw.x + ilw.width)) / (ilw.width / 3.0) + ); + + // "rich" (w=34.8, 4 chars) → "m" (gap=14.01): word boundary, avg=8.7, ratio=1.610 → SPLIT + let rich = make_text_item("rich", 229.0, 34.8, fs); + let m = make_text_item("m", 277.8, 10.7, fs); + assert!( + !should_join_items(&rich, &m, threshold), + "rich→m: avg_ratio={:.3}, should split (word boundary)", + (m.x - (rich.x + rich.width)) / (rich.width / 4.0) + ); + } + + #[test] + fn canva_width_based_multi_to_multi_page_threshold() { + // Multi→multi: uses page-level threshold (gap/font_size < threshold) + let fs = 12.0; + let threshold = 0.90; + + // "rib" (w=25.0) → "ib" (gap=7.01): intra-word, r=0.584 → JOIN + let rib = make_text_item("rib", 236.0, 25.0, fs); + let ib = make_text_item("ib", 268.0, 14.0, fs); + assert!( + should_join_items(&rib, &ib, threshold), + "rib→ib: ratio={:.3}, should join (intra-word)", + (ib.x - (rib.x + rib.width)) / fs + ); + + // "ized" (w=35.9) → "fo" (gap=13.92): word boundary, r=1.160 → SPLIT + let ized = make_text_item("ized", 142.0, 35.9, fs); + let fo = make_text_item("fo", 191.8, 13.8, fs); + assert!( + !should_join_items(&ized, &fo, threshold), + "ized→fo: ratio={:.3}, should split (word boundary)", + (fo.x - (ized.x + ized.width)) / fs + ); + } } diff --git a/src/tounicode.rs b/src/tounicode.rs index eeb3e10..6e13f4a 100644 --- a/src/tounicode.rs +++ b/src/tounicode.rs @@ -18,6 +18,9 @@ pub struct ToUnicodeCMap { pub ranges: Vec<(u16, u16, u32)>, /// Byte width of source codes (1 or 2), determined from codespace and CMap entries pub code_byte_length: u8, + /// When true, unmapped CIDs are interpreted as Unicode codepoints directly. + /// Used as a last resort for Identity-H fonts without ToUnicode/cmap/glyph names. + pub cid_passthrough: bool, } pub(crate) fn build_cmap_entry_from_stream( @@ -467,10 +470,24 @@ impl ToUnicodeCMap { match self.lookup(cid) { Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s), _ => { - // Do NOT blindly interpret CIDs as Unicode codepoints. - // CIDs are font-internal indices, not Unicode values. - // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. - unmapped_count += 1; + if self.cid_passthrough { + // Last-resort: treat CID as Unicode codepoint. + // Valid for Identity-H fonts where the PDF generator + // used Unicode values as CIDs but stripped the cmap. + if let Some(ch) = char::from_u32(cid as u32) { + if !ch.is_control() || ch == '\t' || ch == '\n' { + result.push(ch); + } else { + unmapped_count += 1; + } + } else { + unmapped_count += 1; + } + } else { + // CIDs are font-internal indices, not Unicode values. + // Unmapped 2-byte CIDs are skipped to avoid CJK garbage. + unmapped_count += 1; + } } } } @@ -815,6 +832,28 @@ fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option { } } } + // Fallback: Windows Unicode BMP (3,1) — maps Unicode codepoints to GIDs. + // For single-byte fonts, try each byte value as a Unicode codepoint. + // Common in OCR-generated PDFs where byte values correspond to Unicode + // codepoints but the declared encoding (WinAnsiEncoding) is wrong. + if !used_encoding_cmap { + for subtable in cmap_table.subtables { + if subtable.platform_id == ttf_parser::PlatformId::Windows + && subtable.encoding_id == 1 + { + for code in 0x20..=0xFF_u32 { + if let Some(gid) = subtable.glyph_index(code) { + if let Some(&ch) = gid_to_unicode.get(&gid.0) { + let ch = strip_pua_char(ch); + cmap.char_map.entry(code as u16).or_insert(ch.to_string()); + } + } + } + used_encoding_cmap = true; + break; + } + } + } } if !used_encoding_cmap { @@ -1606,6 +1645,68 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap base } +/// Check if a CIDFont's /W (widths) array contains CID values that look like +/// Unicode codepoints rather than low-value GIDs. +/// +/// Returns true if the median CID is >= 0x41 (letter 'A'), indicating +/// the PDF generator likely used Unicode codepoints as CIDs. +fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool { + let w_arr = match cid_font_dict.get(b"W").ok() { + Some(Object::Array(arr)) => arr, + _ => return false, + }; + + // The /W array format: [cid [w1 w2 ...]] or [cid_start cid_end w] + // We extract all CID values (the first element of each group). + let mut cids: Vec = Vec::new(); + let mut i = 0; + while i < w_arr.len() { + if let Ok(cid) = w_arr[i].as_i64() { + cids.push(cid as u16); + // Skip the width data + if i + 1 < w_arr.len() { + match &w_arr[i + 1] { + Object::Array(widths) => { + // [cid [w1 w2 ...]] — CIDs are cid, cid+1, ..., cid+len-1 + for j in 1..widths.len() { + cids.push((cid as u16).wrapping_add(j as u16)); + } + i += 2; + } + _ => { + // [cid_start cid_end w] — range of CIDs + if i + 2 < w_arr.len() { + if let Ok(cid_end) = w_arr[i + 1].as_i64() { + for c in (cid as u16)..=(cid_end as u16) { + cids.push(c); + } + } + i += 3; + } else { + i += 1; + } + } + } + } else { + i += 1; + } + } else { + i += 1; + } + } + + if cids.is_empty() { + return false; + } + + cids.sort_unstable(); + let median = cids[cids.len() / 2]; + // Unicode text CIDs are typically >= 0x20 (space) with letters at 0x41+. + // GID-based subsets typically start at low values (0-based). + // Use median >= 0x41 as a heuristic for Unicode CIDs. + median >= 0x41 +} + /// Build a ToUnicodeCMap from predefined CID→Unicode mapping based on CIDSystemInfo. /// /// Supports Adobe-Korea1 (Korean) character collection. Can be extended for @@ -1712,7 +1813,7 @@ impl FontCMaps { }; let data = match stream.decompressed_content() { Ok(d) => d, - Err(_) => continue, + Err(_) => stream.content.clone(), }; if let Some(cmap) = ToUnicodeCMap::parse(&data) { debug!( @@ -1852,23 +1953,25 @@ impl FontCMaps { // Try parsing embedded TrueType/OpenType cmap if let Some(ff_ref) = font_file_ref { if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) { - if let Ok(data) = stream.decompressed_content() { - if let Some(cmap) = build_cmap_from_truetype(&data) { - debug!( - "TrueType CMap obj={:<6} (embedded font) char_map={}", - lookup_key, - cmap.char_map.len() - ); - by_obj_num.insert( - lookup_key, - CMapEntry { - primary: cmap, - remapped: None, - fallback: None, - }, - ); - resolved = true; - } + let data = match stream.decompressed_content() { + Ok(d) => d, + Err(_) => stream.content.clone(), + }; + if let Some(cmap) = build_cmap_from_truetype(&data) { + debug!( + "TrueType CMap obj={:<6} (embedded font) char_map={}", + lookup_key, + cmap.char_map.len() + ); + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + resolved = true; } } } @@ -1889,6 +1992,38 @@ impl FontCMaps { fallback: None, }, ); + resolved = true; + } + } + + // Last resort: CID-as-Unicode passthrough. + // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H encoding where + // CID values ARE Unicode codepoints, but strip the cmap table and omit + // ToUnicode. We detect this by checking the /W (widths) array: if CID values + // fall in typical Unicode letter/digit ranges (0x41+), CIDs are likely Unicode. + // If CIDs are low values (< 0x41), they're GIDs in a subset font. + if !resolved { + if cid_values_look_like_unicode(cid_font_dict) { + debug!( + "Identity-H font obj={}: W array CIDs look like Unicode — using passthrough", + lookup_key + ); + let mut cmap = ToUnicodeCMap::new(); + cmap.code_byte_length = 2; + cmap.cid_passthrough = true; + by_obj_num.insert( + lookup_key, + CMapEntry { + primary: cmap, + remapped: None, + fallback: None, + }, + ); + } else { + debug!( + "Identity-H font obj={}: no decoding possible (stripped cmap, GID-based CIDs)", + lookup_key + ); } } } diff --git a/src/types.rs b/src/types.rs index 7798f7b..78fe2b0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,7 +8,8 @@ use std::collections::HashMap; use crate::text_utils::should_join_items; -/// Result tuple returned by page-level text extraction: text items, rectangles, and line segments. +/// Result tuple returned by page-level text extraction: text items, rectangles, line segments, +/// and whether fonts with unresolvable gid-encoded glyphs were encountered. pub(crate) type PageExtraction = (Vec, Vec, Vec); // ── Font types (crate-internal) ────────────────────────────────────── @@ -117,6 +118,9 @@ pub struct TextItem { pub is_italic: bool, /// Type of item (text, image, link) pub item_type: ItemType, + /// Marked Content ID from the content stream's BDC/BMC operator. + /// Used to link this item to the PDF structure tree for tagged PDFs. + pub mcid: Option, } /// A line of text (grouped text items) @@ -125,6 +129,10 @@ pub struct TextLine { pub items: Vec, pub y: f32, pub page: u32, + /// Adaptive join threshold from page-level letter-spacing detection. + /// Default 0.10 for normal PDFs; higher for Canva-style PDFs. + #[doc(hidden)] + pub adaptive_threshold: f32, } impl TextLine { @@ -138,6 +146,8 @@ impl TextLine { return self.text_plain(); } + let single_char_threshold = self.adaptive_threshold; + let mut result = String::new(); let mut current_bold = false; let mut current_italic = false; @@ -156,7 +166,7 @@ impl TextLine { false } else { let prev_item = &self.items[i - 1]; - self.needs_space_between(prev_item, item, &result) + self.needs_space_between(prev_item, item, &result, single_char_threshold) }; // Preserve leading whitespace from the item text. @@ -211,6 +221,8 @@ impl TextLine { /// Get plain text without formatting fn text_plain(&self) -> String { + let single_char_threshold = self.adaptive_threshold; + let mut result = String::new(); for (i, item) in self.items.iter().enumerate() { let text = item.text.as_str(); @@ -218,7 +230,7 @@ impl TextLine { result.push_str(text); } else { let prev_item = &self.items[i - 1]; - if self.needs_space_between(prev_item, item, &result) { + if self.needs_space_between(prev_item, item, &result, single_char_threshold) { result.push(' '); } result.push_str(text); @@ -228,7 +240,13 @@ impl TextLine { } /// Determine if a space is needed between two items - fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool { + fn needs_space_between( + &self, + prev_item: &TextItem, + item: &TextItem, + result: &str, + single_char_threshold: f32, + ) -> bool { let text = item.text.as_str(); // Don't add space before/after hyphens for hyphenated words @@ -245,7 +263,7 @@ impl TextLine { let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0; // Use position-based spacing detection - let should_join = should_join_items(prev_item, item); + let should_join = should_join_items(prev_item, item, single_char_threshold); // Check if space already exists let prev_ends_with_space = result.ends_with(' '); diff --git a/tests/fixtures/bare_name_struct.pdf b/tests/fixtures/bare_name_struct.pdf new file mode 100644 index 0000000..bb7af19 Binary files /dev/null and b/tests/fixtures/bare_name_struct.pdf differ diff --git a/tests/fixtures/firecrawl_docs_tagged.pdf b/tests/fixtures/firecrawl_docs_tagged.pdf new file mode 100644 index 0000000..e4b3a78 Binary files /dev/null and b/tests/fixtures/firecrawl_docs_tagged.pdf differ diff --git a/tests/fixtures/shinagawa_identity_h.pdf b/tests/fixtures/shinagawa_identity_h.pdf new file mode 100644 index 0000000..b6639f0 Binary files /dev/null and b/tests/fixtures/shinagawa_identity_h.pdf differ diff --git a/tests/fixtures/tnagriculture_06_12.pdf b/tests/fixtures/tnagriculture_06_12.pdf new file mode 100644 index 0000000..51bf3f2 Binary files /dev/null and b/tests/fixtures/tnagriculture_06_12.pdf differ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2864a2d..467e42c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -4,8 +4,8 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy}; use pdf_inspector::extractor::group_into_lines; use pdf_inspector::types::TextLine; use pdf_inspector::{ - detect_pdf_type, extract_text, extract_text_with_positions, to_markdown, MarkdownOptions, - PdfError, PdfType, TextItem, + detect_pdf_type, extract_text, extract_text_with_positions, process_pdf_with_options, + to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem, }; // Helper to create test TextItems @@ -23,6 +23,7 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text is_bold: false, is_italic: false, item_type: ItemType::Text, + mcid: None, } } @@ -47,6 +48,7 @@ fn make_text_item_with_font( is_bold: is_bold_font(font), is_italic: is_italic_font(font), item_type: ItemType::Text, + mcid: None, } } @@ -138,6 +140,7 @@ fn test_text_line_text_method() { items, y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), "Hello World"); } @@ -149,6 +152,7 @@ fn test_text_line_single_item() { items, y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), "Single"); } @@ -159,6 +163,7 @@ fn test_text_line_empty() { items: vec![], y: 700.0, page: 1, + adaptive_threshold: 0.10, }; assert_eq!(line.text(), ""); } @@ -473,11 +478,13 @@ fn test_markdown_from_lines_basic() { items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)], y: 700.0, page: 1, + adaptive_threshold: 0.10, }, TextLine { items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)], y: 680.0, page: 1, + adaptive_threshold: 0.10, }, ]; let md = to_markdown_from_lines(lines, MarkdownOptions::default()); @@ -994,3 +1001,109 @@ startxref ); } } + +#[test] +fn test_firecrawl_tagged_pdf_struct_tree() { + use lopdf::Document; + use pdf_inspector::structure_tree::{StructRole, StructTree}; + + let doc = Document::load("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let tree = StructTree::from_doc(&doc).expect("Should have a structure tree"); + + // Verify structure tree contains expected roles + let page_ids = doc.get_pages(); + let roles = tree.mcid_to_roles(&page_ids); + assert!(!roles.is_empty(), "Should have MCID roles across pages"); + + let flat = tree.flatten(); + let has_code = flat.iter().any(|e| matches!(e.role, StructRole::Code)); + let has_h1 = flat.iter().any(|e| matches!(e.role, StructRole::H1)); + let has_li = flat.iter().any(|e| matches!(e.role, StructRole::LI)); + let has_caption = flat.iter().any(|e| matches!(e.role, StructRole::Caption)); + assert!(has_code, "Should have Code elements"); + assert!(has_h1, "Should have H1 elements"); + assert!(has_li, "Should have LI elements"); + assert!(has_caption, "Should have Caption elements"); + + // Full conversion: code fences should be generated from Code struct elements + let buf = std::fs::read("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap(); + let result = pdf_inspector::process_pdf_mem(&buf).unwrap(); + let md = result.markdown.unwrap(); + let fence_count = md.matches("```").count(); + assert!( + fence_count > 0, + "Should produce code fences from tagged Code elements" + ); + // Fences come in open/close pairs + assert_eq!(fence_count % 2, 0, "Code fences should be balanced"); +} + +#[test] +fn test_identity_h_no_tounicode_suppresses_garbage() { + // shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no + // ToUnicode CMap. The raw CID values look like random Latin characters. + // We should suppress the garbage and flag the page for OCR. + let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap(); + let result = pdf_inspector::process_pdf_mem(&buf).unwrap(); + + // Page 1 should be flagged for OCR + assert!( + result.pages_needing_ocr.contains(&1), + "Page with Identity-H font without ToUnicode should be flagged for OCR" + ); + + // Markdown should be empty (garbage suppressed) + let md = result.markdown.unwrap_or_default(); + assert!( + md.trim().is_empty(), + "Garbage CID text should be suppressed, got {} chars: {:?}", + md.len(), + &md[..md.len().min(100)] + ); +} + +#[test] +fn test_rotated_table_layout_correction() { + // tnagriculture_06_12.pdf has landscape content in a portrait page via + // a 90° CCW text matrix [0, b, -b, 0, tx, ty]. Without rotation + // correction, the table is read sideways (jumbled numbers). + let result = + process_pdf_with_options("tests/fixtures/tnagriculture_06_12.pdf", PdfOptions::new()) + .unwrap(); + let md = result.markdown.unwrap_or_default(); + + // Title should appear near the top + assert!( + md.contains("DISTRICT WISE PRODUCTION OF SPICES AND CONDIMENTS"), + "Should extract the table title" + ); + + // District names should be readable (not jumbled with numbers) + assert!( + md.contains("Ariyalur"), + "Should extract district name Ariyalur" + ); + assert!( + md.contains("Coimbatore"), + "Should extract district name Coimbatore" + ); + + // Spice column headers should appear + assert!( + md.contains("CARDAMOM"), + "Should extract spice header CARDAMOM" + ); + assert!( + md.contains("RED CHILLIES"), + "Should extract spice header RED CHILLIES" + ); + + // Table should be formatted as markdown table (has pipe delimiters) + let has_table_row = md + .lines() + .any(|l: &str| l.contains('|') && l.contains("Ariyalur")); + assert!( + has_table_row, + "District data should be in a markdown table row" + ); +} diff --git a/tests/snapshots/2013-app2.md b/tests/snapshots/2013-app2.md index e595beb..dfaf416 100644 --- a/tests/snapshots/2013-app2.md +++ b/tests/snapshots/2013-app2.md @@ -1,298 +1,299 @@ -| | Date | Procurement Title | PE | Bidder | Amount | -| --- | ---- | ---------------------------------------- | --------- | ----------------------------------- | ---------------------------------------- | -| | JAN | | | | | -| 1 | 8/1 | Procurement of Criticals Spare Parts for Engine Maintenance on Mahe | PUC | Wartsila Eastern Africa Ltd | Euro97,922.30 | -| 2 | 8/1 | Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin | PUC | Wartsila Eastern Africa Ltd | Euro270,982.90 | -| 3 | 15/1 | Manufacturing and Deliveries of 900 Students Desks | MOE | | SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00 | -| 4 | 15/1 | Renovation Works at Mont Fleuri Secondary School | MOE | Sai-Fu Enterprise | SR2,815,771.00 | -| 5 | 15/1 | La Gogue to Mont Simpson Raw Water Transfer | PUC | Vijay Construction | SR7,816,058.00 | -| 6 | 15/1 | Storm Water Channel Project at Au Cap-Additional Works | DOE | United Concrete Products (Sey)Ltd | SR184,300.00 | -| 7 | 22/1 | Procurement of Security at ex-Maritime Training Centre | SFA | Elite Surveillance Security Agency | SR30,000.00 | -| 8 | 22/1 | Installation of Sewerage Treatment Plant at Anse Gaulette | MLUH | Green Island Construction Compnay | SR4,524,884.85 | -| 9 | 29/1 | Procurement of Cylinder Liner | PUC | Wartsila Eastern Africa Ltd | Euro28,186.75 | -| 10 | 29/1 | Procurement of Service Pack for Coupling | PUC | Wartsila Eastern Africa Ltd | Euro11,151.00 | -| 11 | 29/1 | Construction of Drainage for Roads A & B Eve Island Praslin | MLUH | Ascent Projects Sey | SR2,931,174.00 | -| 12 | 29/1 | Procurement of Electric Cables for Perseverance Infrastructure-Variations | PUC | Indian Ocean Export Company Pty Ltd | USD768.12 | -| | FEB | | | | | -| 13 | 5/2 | Procurement of DI Pipes and Fittings for Le Rocher Refurbishment | PUC | Legend General Supply | USD43,807.50 | -| 14 | 5/2 | Procurement of Bearings for ABB Turbo Charger | PUC | ABB France | EURO 28,966.15 | -| 15 | 5/2 | Procurement of Alfa Laval Separator Spares | PUC | ALFA LAVAL (Pty) Ltd | Euro 41,191,30 | -| 16 | 5/2 | constraction of 6*2 bedroom Houses- Mont Buxton | MLUH | O-NIVO Construction | SR3,688,844.00 | -| 17 | 5/2 | Procurement of Critical Spares frr Major Overhaul-Set A41 PUC | PUC | Wartsila Eastern Africa Ltd | EURO 104,880.10 | -| 18 | 5/2 | Procurement of Vehcile x1 | MOE | Abhaye Valabhji Pty Ltd | SR 650,000.00 | -| 19 | 5/2 | Procurement of Vehicle x1 | SBFA | Abhaye Valabhji Pty Ltd | SR 585,000.00 | -| 20 | 12/2 | Procurement of Turbo charger Rotor for Engine on Praslin | PUC | Marine Power International FZC | Euro 835,669.00 | -| 21 | 12/2 | Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin | PUC | Overseae Tractor S.A | USD 20,546.12 | -| 22 | 12/2 | Procurement of Air Cooler Cartridge for Wartsila Engine | PUC | Marine Power International FZC | Euro 36,508.93 | -| 23 | 12/2 | Procurement of Spares for Genset 8P on Praslin | PUC | Wartsila Eastern Africa Ltd | Euro 288,202.50 | -| 24 | 19/2 | Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka | SLTA | Esparon's Enterprise | SR 1,105,069.00 | -| 25 | 19/2 | Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41 | PUC | Marine Power International | Euro 144,726.26 | -| 26 | 19/2 | Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31 | PUC | Marine Power International FZC | Euro 143,746.75 | -| 27 | 19/2 | Procurement of Critical Spares for Wartsila Engine-Engine Set B11 | PUC | Marine Power International FZC | Euro 44,089.42 | -| 28 | 26/2 | Procurement for Vehcile x 2 | Judiciary | PMC Auto | SR1,349,551.00 | -| 29 | 26/2 | Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares) | PUC | Wartsila Eastern Africa Ltd | Euro 202,898.40 | -| 30 | 26/2 | Procurement of Safety Spare Patrs for 8MW Engines - (Turbo) | PUC | ABB France | Euro 132,833.33 | -| | MAR | | | | | -| 31 | 5/3 | Procurement of Spare Parts for the Asphalt Plant- Petite Paris | SLTA | Astec Factory (USA) | USD 101,337.81 | -| 32 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines | PUC | Marine Power International FZC | Euro 44,050.00 | -| 33 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines B11 | PUC | Marine Power International FZC | Euro 98,743.00 | -| 34 | 5/3 | Procurement of Non-Critical Spare Parts for Wartsila Engines 8B | PUC | Marine Power International FZC | Euro 103,215.00 | -| 35 | 5/3 | Procurement of Critical Spare Parts for Wartsila Engines 8B | PUC | Wartsila Global Logistic | Euro 97,979.10 | -| 36 | 5/3 | Procurement Spare Parts for the 8MW Engines | PUC | Wartsila Global Logistic | Euro 41,444.00 | -| 37 | 5/3 | Procurement of Voltage Districbution Boxes and Fuses-Variations | PUC | Indian Ocean Export Company Pty Ltd | GBP1,300.00 | -| 38 | 5/3 | 33Kv South Mahe Project-Variations | PUC | United Concrete Products (Sey)Ltd | SR459,684.00 | -| 39 | 5/3 | construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations | SFRS | Furui Construction Pty Ltd | SR184,836.03 | -| 40 | 5/3 | Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services | MLHU | GIBB ( Mauritius) | USD138,050.00 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 1 Roy & Sons Import | SR 396,000.00 | -| | | | | | 1 | -| 41 | 12/3 | Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools | MOE | Lot 3 Roy & Sons Import | SR650,000.00 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 4 HIS & PJ Enterprise | SR 1,753,614.70 | -| 41 | 12/3 | Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools | MOE | Lot 5 Roy & Sons Import | SR 1,080.000.00 | -| 42 | 19/3 | Procurement of Medium Voltage Cables | PUC | Nexans France Lens | Euro 52,354.45 | -| 43 | 19/3 | Procurement of Brass/Bronze Fitting | PUC | Jainsons Industries | Euro 97,718.48 | -| 44 | 19/3 | Procurement of Training Vessel for Maritime Training Centre | SFA | Neil Marine (Sri Lanka) | USD 284,521.15 | -| 44 | 19/3 | Procurement of Training Vessel for Maritime Training Centre-Equipment | SFA | Neil Marine (Sri Lanka) | USD 97,861.50 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 1 | MOE | Print V Care | SR 349,450.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 2 | MOE | Print V Care | SR 392,500.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 3 | MOE | JD's Stationery & Educational Centre | SR 19,150.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 5 | MOE | Vitoria Computer Services Pty LTd | SR 94,500.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 6 | MOE | Roy & Sons Imports | SR 114,750.00 | -| 45 | 19/3 | Procurement of Stationery Items-Lot 7 | MOE | Trade Supplies Pty Ltd | SR 35,224.00 | -| 46 | 26/3 | Procurement of Microsoft Windows License | STB | Victoria Computer Service | SR 977,150.40 | -| 47 | 26/3 | | Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations PUC | Ras Tek Pvt Ltd | Euro 58,000.00 | -| | APR | | | | | -| 48 | 2/4 | Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin | PUC | Vijay Construction | SR 623,025.00 | -| 49 | 2/4 | Supply of Fish to the Prison Department | | Prison Service Mr. Danny Loizeau | SR 27.50 per KG | -| 50 | 2/4 | Supply of Polo T- Shirt and Jeans -Lot 1 | PUC | Magilyn Ltee | USD 28,885.00 | -| 50 | 2/4 | Supply of Polo T- Shirt and Jeans -Lot 2 | PUC | Magilyn Ltee | USD 35,910.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps for Rocher Caiman Pump Stations | PUC | Bluezone Mauritius | Euro 30,761.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3 | PUC | Bluezone Mauritius | Euro 24,254.00 | -| 51 | 2/4 | Procurement of Grundfos Pumps Sewerage Pump Station | PUC | Bluezone Mauritius | Euro 65,203.00 | -| 52 | 2/4 | Procurement of Virtual Studio Equipment | SBC | New Tek Europe | Euro 27,240.00 | -| 53 | 2/4 | Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works | PUC | Tracetebel Engineering | Euro 244,810.00 | -| 54 | 2/4 | Security in MOH's Institution-Lot 1 | MOH | Alliance Security | SR 58,968.00 | -| 55 | 2/4 | Procurement of Bearings for TurboCharger Wartsila Engines | PUC | ABB France | Euro 57,191.50 | -| 56 | 2/4 | Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine | PUC | Marine Power International | Euro 33,769.00 | -| 57 | 2/4 | Procurement of Lighting Equipments | PUC | Thorn Europhane | Euro 38,982.30 | -| 58 | 9/4 | Construction of Fooothpath at Olivier Maradan Street | SLTA | Benoiton Construction | SR 1,548.520.00 | -| 59 | 9/4 | Procurement of Utrasonic Cleaning Machine | PUC | IOP Marine (Denmark) | Euro 31,740.00 | -| 60 | 9/4 | Implementation of Nwe Financial System | SCAA | Blanche Birger | Euro 65,779.50 | -| 61 | 9/4 | Construction of Fuel station and Admin Block | SPTC | Onivo Construction | SR2,196.748.00 | -| 62 | 9/4 | Procurement of Plastic Chairs | MOE | J Galt International | ZAR 498,560.00 | -| 63 | 9/4 | Security Service in All Education Institution -Lot 2 | MOE | Xtreme Security Service | SR 28,000.00 | -| 64 | 16/4 | Procurement of Critical Spare Parts for Wartsila Engines B41 | PUC | Wartsila Global Logistic Service | Euro 37,460.92 | -| 65 | 16/4 | Security Service STC Premises-Supermarket | STC | Isles Security Agency Ltd | SR 908,107.20 | -| 65 | 16/4 | Security Service STC Premises-Meat & Veg | STC | Allaince Security | SR 456,183.36 | -| 65 | 16/4 | Security Service STC Premises-Warehouse, Duty free, BDR Complex | STC | Isles Security Agency Ltd | SR 490,752.00 | -| 65 | 16/4 | Security Service STC Premises-Head Office | STC | Isles Security Agency Ltd | SR 503,712.00 | -| 65 | 16/4 | Security Service STC Premises-Praslin ( Amitie Store /Duty-Free | STC | Alliance Security | SR 327,598.56 | -| 66 | 16/4 | Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract | MLUH | Allied Builders | SR 1,732,989.80 | -| 67 | 16/4 | Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials | MLUH | Ascent Projects (Sey) Pty Ltd | SR 3,888,632.42 | -| 68 | 23/4 | Procurement of Vehicle | SFA | Exel Motors | SR 519,869.79 | -| 69 | 23/4 | Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson | SCAA | Smiths Detection | Euro 368,000.00 | -| | | | | | 2 | -| 71 | 23/4 | Procurement of Ultracsonic Cleaning Machine | PUC | IOP Marine (Denmark) | Euro 1,250.00 | -| 72 | 30/4 | Procurement of Vehicle | PA | Abhaye Valabhji Pty Ltd | SR575,000.00 | -| 73 | 30/4 | Procurement of ID Cards | DICT | Blanche Birger Bureautique | Euro 43,500.00 | -| 74 | 30/4 | Procurement of Electrical Meters | PUC | ISKRAEMECO | Euro 55,733.60 | -| 75 | 30/4 | Procurement of Grunfos Pump for Le Rocher Pump Station | PUC | Bluezone Mauritius Ltd | Euro 37,910.60 | -| 76 | 30/4 | Procurement of A3 and A4 Photocopr Paper | MOE | Islad Motors Co Ltd | SR552,000.00 | -| | MAY | | | | | -| 77 | 7/5 | Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power | PUC | Technoglass | US$ 156, 920.00 | -| 77 | 7/5 | Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite | PUC | Technoglass | US$ 83,980.00 | -| 78 | 7/5 | Procurement of Connecting Rod (Variations) | PUC | Wartsila Eastern Africa Ltd | Euro 1,200.00 | -| 79 | 7/5 | Project Management Consultancy-Extension of contract | SIBA | Philippe Adrienne Consultancy Engineers | SR 225.000.00 | -| 80 | 7/5 | Procurement of Liquid Chlorine | PUC | Al Afaq LLC | USD 77,088.00 | -| 81 | 21/5 | Procurement of Agriculture Inputs | SAA | Rodley Mathieu | SR1,241,225.00 | -| 82 | 21/5 | Procurement of Metelogical Equipment | MEE | Vaisala | Euro 33,815.00 | -| 83 | 21/5 | Procurement of Turbocharger for Rotor Shaft for Wartsila Engine | PUC | Marine Power International | Euro 39,945.00 | -| 84 | 21/5 | Provision of Security Srevices for MOH's Institutions | MOH | Alliance Security | SR 19,656.00 | -| 85 | 21/5 | Procurement of Low Voltage ABC Cables | PUC | Nextans | Euro 50,118.69 | -| 86 | 28/5 | Procurement of HDPE pipes-Pipe for water applications | PUC | STR Marketing- | USD 68,030.02 | -| 86 | 28/5 | Procurement of HDPE pipes-Pipe for sewerage applications | PUC | STR Marketing | USD 12,509.55 | -| 87 | 28/5 | Procurement of Incenerator for Baie Ste Anne Praslin Hospital | MOH | Incinco Limited | GBP 101,992.00 | -| 88 | 28/5 | Geotechnical Survey on Ile Soleil | | 2020 Development Ltd Geoconsul Ltee | SR 741,520.00 | -| 89 | 28/5 | Extra Works at Palais De Justice | | The Judiciary Quingjian Group Co | SR 1,614,226.00 | -| 90 | 28/5 | Fire Fighting and rescue training Course | SFRS | Emergency Training Solution Pty Ltd | ZAR 824,453.80 | -| 91 | 28/5 | Procurement of Charger Air Cooler for Wartsila Engine at Power Station C | PUC | Marine Power International FZC | Euro 38,217.00 | -| | JUN | | | | | -| 92 | 4/6 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brits Pale | ZAR 438,081.83 | -| 93 | 4/6 | | Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC | Wartsila Global Service | Euro 74,768.40 | -| 94 | 4/6 | Procurement of Cylinder Liner and Pistons | PUC | Wartsila Eastern Africa | Euro 70,082.73 | -| 95 | 4/6 | Procurement of Digital Microwave Equipment | SBC | MOCHINO | Euro 125,845.00 | -| 96 | 4/6 | Procurement of Vehicle 1 | LWMA | Abhaye Valabhji Pty Ltd-Jeep | SR 475,000.00 | -| 96 | 4/6 | Procurement of Vehicle 1 | LWMA | EHW Seychelles Ltd- Car | SR 267,850.00 | -| 97 | 4/6 | Procurement of Forged filter Ball-Valves | PUC | Jainsons Malleables | USD 11,000.00 | -| 98 | 11/6 | Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka | SLTA | Bazil Construction | Sr 1,081,530.00 | -| 99 | 11/6 | Procurement of Seychelles Paswsports | DIA | Groupe Impimerie Nationale | Euro 102,400.00 | -| 100 | 19/6 | Anse Boileau Footpath and Drainage Construction Phase II | SLTA | TCH Building Contractor | SR 960,107.50 | -| 101 | 21/6 | Procurement of Vehicle x 1 | PSD | Abhaye Valabhji-Bus x1 | SR 475,000.00 | -| 101 | 21/6 | Procurement of Vehicle x 1 | PSD | PMC Auto- Car x1 | SR 267,850.00 | -| 102 | 21/6 | Renovation Works on Block A- Beau Vollon Secondary School | MOE | Sai-Fu Enterprise Company Ltd | SR 2,253,300.00 | -| 103 | 21/6 | Procurement of 35 VMS Terminal Accessories | SFA | Communication Specialist Ltd | Euro 63,770.00 | -| 104 | 21/6 | Renewal of SFA's Themis FMC Services | SFA | CLS- France | Euro 36,000.00 | -| 105 | 21/6 | Procurement of Electrical Spares for Wartsila Engine | PUC | Wartsila Eastern Africa | Euro 1,757.00 | -| 106 | 21/6 | Procurement of Class D water Meters | PUC | Elster Metering Limited (Pty) Ltd | USD 109,7051.00 | -| 107 | 21/6 | Procurement of Sludge Incinerator | PUC | Atlas Incinerator | € 94,470.00 | -| 108 | 21/6 | Procurement of Technical Services for Crankshaft Grinding | PUC | Goltens | USD 76,075.00 | -| 109 | 21/6 | Procurement of WAS Pumps | PUC | Netzsch Southern Africa Pty Ltd | Euro 56,657.20 | -| | | | | | 3 | -| 111 | 21/6 | Procurement of Fitting and Pipes ( Stock Replenishment) | PUC | STR Marketing Ltee | USD 60,688.90 | -| 112 | 21/6 | Refurbishment of Pharmaceutical Production Unit-Contract Extension | MOH | Mahe Design | SR808,375.00 | -| 113 | 21/6 | Technical Service for Repair on Generator-1B | PUC | Goltens | USD 75,600.00 | -| 114 | 21/6 | Procurement of Gate Valves | PUC | AVK Valves Southern africa (Pty) Ltd | ZAR 461,608.44 | -| 115 | 25/6 | Procurement of Hot-Dipped Galvanised Materials | PUC | HDSA Shipping (Pty) Ltd | Zar 435,614.00 | -| 116 | 25/6 | Procurement of Critical Spares for Wartsila Engine B51- Lot 1 | PUC | Wartsila Global Services | Euro 92,472.70 | -| 117 | 25/6 | Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin | PUC | RUYSCH | Euro 83,262.64 | -| 118 | 25/6 | | Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013 PUC | Tornado Group | USD 266,820.82 | -| 119 | 25/6 | Procurement of X-ray Screening Machine for VVIP Lounge | SCAA | Smiths Detection | Euro 101,800.00 | -| | JUL | | | | | -| 120 | 2/7 | Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road) | SLTA | Esparon's Enterprise | SR 183,410.00 | -| 121 | 2/7 | Bridge Renovation at Cascade | SLTA | Benioton Construction | SR 1,361,100.00 | -| 122 | 2/7 | Procurement of Bitumen | SLTA | Termcotank S.A | USD 519,418.20 | -| 123 | 2/7 | Procurement of Vehicle x 2 | STB | PMC Auto Pty Ltd | SR 960,126.00 | -| 124 | 2/7 | | Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning MFA | John Nevill | SR 180,000.00 | -| 125 | 9/7 | Procurement of Stationery Items Lot 4 | MOE | JD's Stationey EDU Centre | SR 629,950.00 | -| 126 | 9/7 | Procurement of Atomic Spectrometer | SBS | SMM Instrument (Pty) Ltd | USD 181,147.00 | -| 127 | 9/7 | Operation and Maintenance of Containerised Desalination Units by Tornado-2012 | PUC | Tornado Group | USD 45,174.62 | -| 128 | 9/7 | Procurement of Sensors and Transmitters for Wartsila Engines | PUC | Wartsila Global Logistic Services | Euro 50,360.00 | -| 129 | 9/7 | Procurement of Bulk Water Meters Strainers | PUC | Elster Metering Limited (Pty) Ltd | ZAR 931,966.00 | -| 130 | 9/7 | Procurement of Gudgeon Pins for engine 8P | PUC | Wartsila Global Logistic Services | Euro 23,460.00 | -| 131 | 9/7 | Procurement of Piston for Wartsila Engine A21 | PUC | Wartsila Global Logistic Services | Euro 219,622.00 | -| 132 | 9/7 | Procurement of Piston for Wartsila Engine B11 | PUC | Marine Power International FZC | Euro 243,799.98 | -| 133 | 9/7 | Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance | MLUH | Sai-Fu Enterprise Company Ltd | SR 17,376,392.65 | -| 134 | 9/7 | La Louise Non- Performance Pipieline Replacement | PUC | Ascent Projects Sey Pty Ltd | SR 1,289,375.00 | -| 135 | 9/7 | Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital | MOH | Des Iles Environment Solutions | SR 1,462,875.00 | -| 136 | 9/7 | Consultancy Services for Quality management System (QMS) | DE | Mr. John Horack | USD 22,325.00 | -| 137 | 9/7 | Construction of New Road at Cascade Primary School | MLUH | Esparon's Enterprise | SR 2,443,814.00 | -| 138 | 9/7 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brits Pale Pty Ltd | ZAR 518,694.02 | -| 139 | 9/7 | Procurement of Non-Critical Spares- Specialized Tools | PUC | Marine Power International FZC | Euro 10,866.55 | -| 140 | 9/7 | Upgarding of Roche Caiman Road and Roundabout | SLTA | Bazil Construction | SR 1,592,002.00 | -| 141 | 9/7 | Procurement of Crankshaft for Engine 5B at New Port Station | PUC | A&D Sales | GBP 65,375.00 | -| 142 | 16/7 | Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital | MOH | Des Iles Environment Solutions | SR1,462,875.00 | -| 143 | 16/7 | Consultancy Service for Quality Management System (QMS) | DE | Mr. John Horack | CND$22,325.00 | -| 144 | 16/7 | Construction of New Road at Cascade Primary School | SLTA | Esparon's Enterprise | SR 2,443,814.00 | -| 145 | 16/7 | Procurement of Tanalisth Treated Wooden Poles | PUC | Brit Pale Pty Ltd | ZAR518,694.02 | -| 146 | 16/7 | Procurement od Non- critcal spares-specialized tools | PUC | Marine International FZC | Euro10,866.55 | -| 147 | 16/7 | Upgrading of Roche Caiman Road and roundabout | SLTA | Bazil Construction | SR1,592,002.00 | -| 148 | 16/7 | Procurement of Crankshaft for Engine 5B at New Port Station | PUC | A&D Sales | GBP 65,375.00 | -| 149 | 23/7 | Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area. | MLUH | LC International | SR2,814,108.00 | -| 150 | 23/7 | Procurement of CT Scan Tube | MOH | Ireland Blyth Ltd from Mauritius | Euro 122,000.00 | -| 151 | 23/7 | Procurement of Ultraviolet disinfection System for Sewerage Treatment | PUC | Orica Wtercare | SR851,104.72 | -| 152 | 23/7 | | Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps PUC | Hidrostal Sewage Sa Pty Ltd | Euro97,498.32 | -| 153 | 23/7 | (B)Procurement of Grundfos Pumps for water pumping | PUC | Bluezone Mauritius | Euro 13,410.00 | -| 154 | 23/7 | Procurement of spare for Mirrlees Radiator | PUC | Covard Heat Transfer Ltd | GBP32,042.43 | -| | | | | | 4 | -| 156 | 23/7 | General renovation works to Block B at Belonie Secondary School | MOE | Belvedere Builders | SR869,505.75 | -| 157 | 30/7 | Procurement of Engine Block and Crankshaft for Engine A11 | PUC | Ras Tek Pvt Ltd | Euro798,650.00 | -| 158 | 30/7 | procurement of Wartsila Engine spares | PUC | Wartsila Eastern Africa ltd | Euro158,424.00 | -| 159 | 30/7 | Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) | SLTA | G&S Enterpise | SR1,113,010.00 | -| 160 | 30/7 | Procurement of transfer pump control panel | PUC | CA Engineering Consultancy Pte Ltd | SGD14,600.00 | -| 161 | 30/7 | Consultancy service for North to South Victoria Bye- Pass road and utilities organisation | MLUH | Sonnel Seychelles LTD | SR1,332,000.00 | -| 162 | 30/7 | Procurement of the supply of sodium cardonate | PUC | HPL Chemical LTD | USD42,600.00 | -| | AUG | | | | | -| 163 | 6/8 | Procurement of Vehichels X 4 | MOH | Kim-Koom & Co Pty Ltd | SR1,100,000.00 | -| 164 | 6/8 | | Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe WMF | Mr. Donal Ernesta | | -| 164 | 6/8 | | Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe WMF | Mr. Kali Deenudayali | | -| 165 | 13/8 | Procurement of exercise books | MOE | JD's Stationey EDU Centre | SR,1,700,000.00 | -| 166 | 13/8 | Procurement of High Pressure pump spares -BZM00003022 | PUC | Bluezone Mauritius Ltd | Euro27,712.73 | -| 166 | 13/8 | Procurement of High Pressure pump spares -BZM00003023 | PUC | Bluezone Mauritius Ltd | Euro12,639.55 | -| 167 | 13/8 | Procurement of CR64 pump spares | PUC | Bluezone Mauritius Ltd | Euro31,822.00 | -| 168 | 13/8 | Construction of access road at Ex-Deltel- Anse Royale | MLUH | Benoiton Construction Pty Ltd | SR4,585,441.12 | -| 169 | 13/8 | Renovation work to one classroom block at Pionte Larue Secondary School | MOE | Belverdere Builders | SR1,114,575.00 | -| 170 | 13/8 | Completion of Amitie Housing Project 12 x 3 Bedrooms | MLUH | Allied Builders Sey Ltd | 6,006,410.37 | -| 171 | 13/8 | Copolia Road widening-Phase 2 | SLTA | Belverdere Builders | SR1,037,235.00 | -| 172 | 20/8 | Procurement of ABB Turbocharger Cartridge | PUC | ABB France | Euro106,266.66 | -| 173 | 20/8 | Procurement of services to carry out the full refit and overhaul of tug Alouette | SPA | SECREN (Madagascar) | Euro189,618.42 | -| 174 | 20/8 | Awarding of cranshatf and Block replacement solution for A11 engine | PUC | Wartsila | Euro800,000.00 | -| 175 | 29/8 | Servicing of geartrain for Wartsila Engine 18V32LN | PUC | Wartsila Eastern Africa | Euro 21,141.90 | -| 176 | 29/8 | Spare parts for Wartsila Engine 18V32LN | PUC | Wartsila Eastern Africa | Euro38,440.00 | -| 177 | 29/8 | Procurement for sience equipment and chemical for 2013 | MOE | Findel Education | GBP37,831.26 | -| 178 | 29/8 | Installation of fencing at Mont Fleuri Secodary School | MOE | Donald Builbing & Contractor Pty Ltd | SR1,761,034.00 | -| 179 | 29/8 | | Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA | Furui Construction Pty Ltd | SR277,521.45 | -| 180 | 29/8 | Propsed road widening and drainage improvement at Quincy Vilage | SLTA | TCH Contractor | SR1,015,806.24 | -| 181 | 3/9 | Tender for procurement of windows licenses | STB | Victoria Computer Service | SR788,808.00 | -| 182 | 3/9 | Procurement of Bitumen in drums for the asphalt production Praslin | SLTA | Benzene International Pte Ltd | Euro87,220.00 | -| 183 | 3/9 | Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S | PUC | Wartsila Eastern Africa | Euro9,861.00 | -| 184 | 3/9 | Procurement of gear train parts for Stork Wartsila Engine SW280 | PUC | Wartsila Eastern Africa | Euro7,931.00 | -| | SEP | | | | | -| 185 | 10/9 | Cleaning and maintenance of wetlands and rivers on Praslin | ED | "W" Cleaning Service | SR769,590.00 | -| 186 | 10/9 | Re-construction and partition of Independence House | MLUH | Green Island Construction Co Pty | SR868,022.94 | -| 187 | 10/9 | Procurement of critical spare fro major overhaul | PUC | Wartsila Eastern Afirca | Euro33,623.00 | -| 188 | 10/9 | Procurement of control panel for six pumps station | PUC | CA Engineering Consultancy Pte Ltd | SGD37,400.00 | -| 189 | 17/9 | Security service for Wellness Centre | MOH | Alliance Security | SR78,624.00 | -| 190 | 17/9 | Procurement of reinforce plastic (FRP) handrails and grating | PUC | Webforge Group | SR839,795.02 | -| 191 | 17/9 | Procurement of helital fitting | PUC | Cu A1 Engineering (Pty) Ltd | ZAR343,000.00 | -| 192 | 17/9 | Procurement of Flygt pumps | PUC | Aqualia DPI LTD | Euro56,080.00 | -| 193 | 17/9 | Procurement of black-up and emergency pumps | PUC | M.A.H.Y Khoory & Co | UAE186,650.00 | -| 194 | 17/9 | Procurement of additional requirement of polo t-shirts and jeans | PUC | Magilyn Ltee | USD42,127.50 | -| 195 | 17/9 | Procurement of Pressure filters | PUC | Barr +Wray | GBP379,921.00 | -| 196 | 17/9 | Procurement of grunfos pump for water pumping | PUC | Blue Zone Mauritius | Euro108,574.00 | -| | | | | | 5 | -| 198 | 24/9 | Procurement of Technical Services for Crankshaft Grinding on Engine 6B | PUC | Golten Co Ltd | USD92,654.00 | -| 199 | 24/9 | Procurement of technical service for repair of generator 1B- varations | PUC | Golten Co Ltd | USD84,337.00 | -| 200 | 24/9 | Procurement of spare parts fro Wartsila Engine A21 | PUC | Wartsila Global Logistic | Euro235,579.30 | -| 201 | 24/9 | Procurement of vehicle x 2 | SLTA | Abhaye Valabhji Pty Ltd | SR1000.000.00 | -| | OCT | | | | | -| 202 | 1/10 | Proposed new traffic lane to 5th June Avenue | SLTA | Divy Constrution | SR2,864,589.00 | -| 203 | 1/10 | | Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA | G & S Enterprise | SR200,448.00 | -| 204 | 1/10 | Proposed Reconstrcution of Burnt House-Au Cap | MLUH | Furui Construction | SR946,130.00 | -| 205 | 1/10 | Variation on the project associated with the procurement of seven 100m3/day containerised plant | PUC | Tornado Group (UAE) | USD172,500.00 | -| 206 | 1/10 | Works on the breaker system at Bel Omber desalination plant | PUC | United Concrete Products (Sey)Ltd | SR1,998,993.11 | -| 207 | 1/10 | Procurement of Viking Johnson fittings | PUC | Viking Johnson (UK) | GBP92,286.50 | -| 208 | 1/10 | Procurement of HDPE Pipes and Fittings | PUC | STR Marketing Ltee | USD196,672.57 | -| 209 | 1/10 | Procurement of Piston and Gudgeon Pins | PUC | Marine Power International FZC | Euro277,098.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 1 | PUC | Ms. Suzanne Edmond | SR500.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 2 | PUC | Ms. Suzanne Edmond | SR200.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 3 | PUC | Ms. Suzanne Edmond | SR475.00 | -| 210 | 8/10 | Procurement of Services for Sewing of Uniform for Office Staff-Lot 4 | PUC | Sey Sytle | SR450.00 | -| 211 | 8/10 | Procurement of vehicle x 2 | FIU | PMC Auto Pty Ltd | SR526,864.00 | -| 212 | 15/10 | Renovation Works to Le Chantier Mall-Variation Works | SSF | Allied Builders Sey Ltd | SR1,673,752.51 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 1 | MSACDS | Eagle Watch Security Services | SR96,600.00 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 2 | MSACDS | Alliance Security Services | SR103,012.00 | -| 213 | 15/10 | Security Services for District's Administration Offices and Community Centres-Lot 4 | MSACDS | Security Protection Services | SR71,000.00 | -| 214 | 15/10 | Anse Boileau Footpath and Drainage Construction Phase 2 - Variations | SLTA | TCH Contractor | SR326,780.00 | -| 215 | 15/10 | Construction of Fooothpath at Olivier Maradan Street-Variations | | Benoiton Construction Pty Ltd | SR368,642.50 | -| 216 | 15/10 | Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014 | PUC | Tornado Group (UAE) | USD1,214,400.00 | -| 217 | 15/10 | Procurement of Non Critical Spare Parts for Wartsila Engine 5B | | MAN Diesel & Turbo (UK) | GDP88,360.88 | -| 218 | 15/10 | Procurement of Meter Boxes | PUC | CAHORS (France) | Euro35,221.20 | -| 219 | 15/10 | Construction of Services Road to Commercial Zone West of Inter Island Quay | MLUH | Allied Builders Sey Ltd | SR6,234,343.36 | -| 220 | 15/10 | Proposed New traffic lane from Roche Caiman to Eden Island | SLTA | Franky's Constrcution | SR3,017,155.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | Seytex | SR695,810.58 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | VCS Pty Ltd | SR36,430.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | KIS Distribution Company | SR10,500.00 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | MNM General Supply | SR16,216.20 | -| 221 | 22/10 | Procurement of Textbooks for Primary and Secondary Schools | MOE | Roy & Sons Import | SR107,625.00 | -| 222 | 22/10 | Road Widening and Drainage Improvement at Quincy Village-Variations | SLTA | TCH Building Contractor | SR328,213.75 | -| 223 | 22/10 | Procurement of Grunfos Pumps for Mare Aux Cochons | PUC | Bluezone Mauritius | Euro44,914.00 | -| 224 | 22/10 | Procurement of Vehicle x 3 | MLUH | Sun Motors | SR2,310,000.00 | -| 225 | 22/10 | Procurement of Agriculture Inputs | SAA | Launch Export (SA) | ZAR720,100.00 | -| 226 | 22/10 | Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09 | | Allied Builders Sey Ltd | SR14,964,125.91 | -| 227 | 22/10 | Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08 | | Allied Builders Sey Ltd | SR35,927,978.92 | -| 228 | 29/10 | Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri | MLUH | Allied Builders Sey Ltd | SR10,372,824.02 | -| 229 | 29/10 | Construction of One Block of Condominium Flats at Pointe Larue | MLUH | Franky's Constrcution | SR18,063,876.00 | -| 230 | 29/10 | Proposed Construction of Micro-Enterprise Building at Providence | MLUH | O Nivo Construction | SR28,462,412.00 | -| 231 | 29/10 | Renovation of English River Secondary Schools | MLUH | Divy Constrution | SR2,133,560.00 | -| 232 | 29/10 | Renovation and Partitioning of Independence House-Variation | MLUH | Green Island Construction Co Pty | SR1,847,477.43 | -| | | | | | 6 | +|Date|Procurement Title|PE|Bidder|Amount|| +|---|---|---|---|---|---| +|JAN|||||| +|1|8/1|Procurement of Criticals Spare Parts for Engine Maintenance on Mahe|PUC|Wartsila Eastern Africa Ltd|Euro97,922.30| +|2|8/1|Procurement of Criticals Spare Parts for Caterpillar Engine on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro270,982.90| +|3|15/1|Manufacturing and Deliveries of 900 Students Desks|MOE|SPR Richard, Building & Furniture Contractor Pty LtdSR1,080,000.00|| +|4|15/1|Renovation Works at Mont Fleuri Secondary School|MOE|Sai-Fu Enterprise|SR2,815,771.00| +|5|15/1|La Gogue to Mont Simpson Raw Water Transfer|PUC|Vijay Construction|SR7,816,058.00| +|6|15/1|Storm Water Channel Project at Au Cap-Additional Works|DOE|United Concrete Products (Sey)Ltd|SR184,300.00| +|7|22/1|Procurement of Security at ex-Maritime Training Centre|SFA|Elite Surveillance Security Agency|SR30,000.00| +|8|22/1|Installation of Sewerage Treatment Plant at Anse Gaulette|MLUH|Green Island Construction Compnay|SR4,524,884.85| +|9|29/1|Procurement of Cylinder Liner|PUC|Wartsila Eastern Africa Ltd|Euro28,186.75| +|10|29/1|Procurement of Service Pack for Coupling|PUC|Wartsila Eastern Africa Ltd|Euro11,151.00| +|11|29/1|Construction of Drainage for Roads A & B Eve Island Praslin|MLUH|Ascent Projects Sey|SR2,931,174.00| +|12 FEB|29/1|Procurement of Electric Cables for Perseverance Infrastructure-Variations|PUC|Indian Ocean Export Company Pty Ltd|USD768.12| +|13|5/2|Procurement of DI Pipes and Fittings for Le Rocher Refurbishment|PUC|Legend General Supply|USD43,807.50| +|14|5/2|Procurement of Bearings for ABB Turbo Charger|PUC|ABB France|EURO 28,966.15| +|15|5/2|Procurement of Alfa Laval Separator Spares|PUC|ALFA LAVAL (Pty) Ltd|Euro 41,191,30| +|16|5/2|constraction of 6*2 bedroom Houses- Mont Buxton|MLUH|O-NIVO Construction|SR3,688,844.00| +|17|5/2|Procurement of Critical Spares frr Major Overhaul-Set A41 PUC|PUC|Wartsila Eastern Africa Ltd|EURO 104,880.10| +|18|5/2|Procurement of Vehcile x1|MOE|Abhaye Valabhji Pty Ltd|SR 650,000.00| +|19|5/2|Procurement of Vehicle x1|SBFA|Abhaye Valabhji Pty Ltd|SR 585,000.00| +|20|12/2|Procurement of Turbo charger Rotor for Engine on Praslin|PUC|Marine Power International FZC|Euro 835,669.00| +|21|12/2|Procurement of Critical Spares for Genset M4 Major Overhaul on Praslin|PUC|Overseae Tractor S.A|USD 20,546.12| +|22|12/2|Procurement of Air Cooler Cartridge for Wartsila Engine|PUC|Marine Power International FZC|Euro 36,508.93| +|23|12/2|Procurement of Spares for Genset 8P on Praslin|PUC|Wartsila Eastern Africa Ltd|Euro 288,202.50| +|24|19/2|Construction of Stone Masonry Retaining Wall at Jean Larue Road, Takamaka|SLTA|Esparon's Enterprise|SR 1,105,069.00| +|25|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A41|PUC|Marine Power International|Euro 144,726.26| +|26|19/2|Procurement of Non Critical Spares for Wartsila Engine at Power Station C- Engine Set A31|PUC|Marine Power International FZC|Euro 143,746.75| +|27|19/2|Procurement of Critical Spares for Wartsila Engine-Engine Set B11|PUC|Marine Power International FZC|Euro 44,089.42| +|28|26/2|Procurement for Vehcile x 2|Judiciary|PMC Auto|SR1,349,551.00| +|29|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (safety Spares)|PUC|Wartsila Eastern Africa Ltd|Euro 202,898.40| +|30 MAR|26/2|Procurement of Safety Spare Patrs for 8MW Engines - (Turbo)|PUC|ABB France|Euro 132,833.33| +|31|5/3|Procurement of Spare Parts for the Asphalt Plant- Petite Paris|SLTA|Astec Factory (USA)|USD 101,337.81| +|32|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines|PUC|Marine Power International FZC|Euro 44,050.00| +|33|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines B11|PUC|Marine Power International FZC|Euro 98,743.00| +|34|5/3|Procurement of Non-Critical Spare Parts for Wartsila Engines 8B|PUC|Marine Power International FZC|Euro 103,215.00| +|35|5/3|Procurement of Critical Spare Parts for Wartsila Engines 8B|PUC|Wartsila Global Logistic|Euro 97,979.10| +|36|5/3|Procurement Spare Parts for the 8MW Engines|PUC|Wartsila Global Logistic|Euro 41,444.00| +|37|5/3|Procurement of Voltage Districbution Boxes and Fuses-Variations|PUC|Indian Ocean Export Company Pty Ltd|GBP1,300.00| +|38|5/3|33Kv South Mahe Project-Variations|PUC|United Concrete Products (Sey)Ltd|SR459,684.00| +|39|5/3|construction of New Fire Station Fuel Store Bin Site and Boundary Wall- la Digue-Variations|SFRS|Furui Construction Pty Ltd|SR184,836.03| +|40|5/3|Provision of Utilities and Infrastructture on Ile Preseverance- Extention of Consulancy Services|MLHU|GIBB ( Mauritius)|USD138,050.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 1 Roy & Sons Import|SR 396,000.00| -| 298 | 17/12 | Renovation of Glacis Health Centre-Variations | MOH | F & P Construction | SR400,000.00 | -| --- | ----- | ---------------------------------------- | ---- | ---------------------------------------- | -------------- | -| 299 | 17/12 | Supply of New Equipment for Kitchen | | Prison Service K. K. Chua | SR943,000.00 | -| 300 | 23/12 | Procurement of Vehicle x 1 | MLUH | Sun Motors | SR770,000.00 | -| 300 | 23/12 | Procurement of Vehicle x 1 | MLUH | Abhaye Valabhji | SR435,000.00 | -| 301 | 23/12 | Procurement of Spares for LT Water Circulating Pump | PUC | Wartsila Global Logistcs Services | Euro 19,389.00 | -| 302 | 23/12 | Procurement of Filter Cartridge-Desalination Plants | PUC | Trans Crescent Technical Equipment Company | Euro22,030.00 | -| 303 | 23/12 | Procurement of LED Streetlights | SLTA | Lighting Orient (China) | USD104,282.00 | -| 304 | 23/12 | Proposed New Road at Cascade Primary School-Variations | SLTA | Esparon's Enterprise | SR861,400.00 | -| 305 | 23/12 | General renovation at School Section at MOE Headquarters-Variations | MOE | Prime Builders | SR2,761,935.75 | -| 306 | 23/12 | Remedial Works at Mont Fleuri Primary School and Creche-Variations | MOE | Sai-Fu Enterprise Company Ltd | SR1,006,778.50 | -| 307 | 23/12 | General Renovation of Toilet at La Digue School | MOE | Furui Construction | SR338,785.40 | -| 308 | 26/12 | Procurement of 2500 Ream of A4 Paper | MOE | Island Motors | SR1,200,000.00 | -| 309 | 26/12 | Procurement of Additional of 400 Desktop Computers | MOE | Orion Computers | SR3,100,000.00 | -| 310 | 26/12 | Procuremet of Vehcile x 5 | MOE | PMC Auto Pty Ltd | SR1,374,897.00 | -| 311 | 26/12 | Procurement of Canon Ink / Riso Meter | MOE | Paradise Computer Services | SR473,000.00 | -| 312 | 26/12 | Supply of Metal Fencing | MOE | BBT (UK) | GBP105,278.00 | -| 313 | 26/12 | Procurement of Mobile Dental Clinic / Surgeries x 2 | MOH | Quayle Dental (UK) | GBP233,842.00 | -| | | | | | 9 | +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 2 Roy & Sons Import|SR 359,200.00| +|---|---|---|---|---|---| +|41|12/3|Procurement of Uniform Materials fro Primary, Secondary and Post Secondary Schools|MOE|Lot 3 Roy & Sons Import|SR650,000.00| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 4 HIS & PJ Enterprise|SR 1,753,614.70| +|41|12/3|Procurement of Uniform Materials for Primary, Secondary and Post Secondary Schools|MOE|Lot 5 Roy & Sons Import|SR 1,080.000.00| +|42|19/3|Procurement of Medium Voltage Cables|PUC|Nexans France Lens|Euro 52,354.45| +|43|19/3|Procurement of Brass/Bronze Fitting|PUC|Jainsons Industries|Euro 97,718.48| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre|SFA|Neil Marine (Sri Lanka)|USD 284,521.15| +|44|19/3|Procurement of Training Vessel for Maritime Training Centre-Equipment|SFA|Neil Marine (Sri Lanka)|USD 97,861.50| +|45|19/3|Procurement of Stationery Items-Lot 1|MOE|Print V Care|SR 349,450.00| +|45|19/3|Procurement of Stationery Items-Lot 2|MOE|Print V Care|SR 392,500.00| +|45|19/3|Procurement of Stationery Items-Lot 3|MOE|JD's Stationery & Educational Centre|SR 19,150.00| +|45|19/3|Procurement of Stationery Items-Lot 5|MOE|Vitoria Computer Services Pty LTd|SR 94,500.00| +|45|19/3|Procurement of Stationery Items-Lot 6|MOE|Roy & Sons Imports|SR 114,750.00| +|45|19/3|Procurement of Stationery Items-Lot 7|MOE|Trade Supplies Pty Ltd|SR 35,224.00| +|46|26/3|Procurement of Microsoft Windows License|STB|Victoria Computer Service|SR 977,150.40| +|47 APR|26/3|Procurement of Technical Services for Maintenance of Cenerators at Roche Caiman, New Port & praslin Power Stations|PUC|Ras Tek Pvt Ltd|Euro 58,000.00| +|48|2/4|Site Preparation for 2000m GRP Tank at Fond B'Offay Praslin|PUC|Vijay Construction|SR 623,025.00| +|49|2/4|Supply of Fish to the Prison Department|Prison Service|Mr. Danny Loizeau|SR 27.50 per KG| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 1|PUC|Magilyn Ltee|USD 28,885.00| +|50|2/4|Supply of Polo T- Shirt and Jeans -Lot 2|PUC|Magilyn Ltee|USD 35,910.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations|PUC|Bluezone Mauritius|Euro 30,761.00| +|51|2/4|Procurement of Grundfos Pumps for Rocher Caiman Pump Stations No 3|PUC|Bluezone Mauritius|Euro 24,254.00| +|51|2/4|Procurement of Grundfos Pumps Sewerage Pump Station|PUC|Bluezone Mauritius|Euro 65,203.00| +|52|2/4|Procurement of Virtual Studio Equipment|SBC|New Tek Europe|Euro 27,240.00| +|53|2/4|Consultancy Service for Inspection of Rochon Dam & Desisn of Remedial Works|PUC|Tracetebel Engineering|Euro 244,810.00| +|54|2/4|Security in MOH's Institution-Lot 1|MOH|Alliance Security|SR 58,968.00| +|55|2/4|Procurement of Bearings for TurboCharger Wartsila Engines|PUC|ABB France|Euro 57,191.50| +|56|2/4|Procurement for Turbocharger Rotor Refurbishment- Wartsila Engine|PUC|Marine Power International|Euro 33,769.00| +|57|2/4|Procurement of Lighting Equipments|PUC|Thorn Europhane|Euro 38,982.30| +|58|9/4|Construction of Fooothpath at Olivier Maradan Street|SLTA|Benoiton Construction|SR 1,548.520.00| +|59|9/4|Procurement of Utrasonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 31,740.00| +|60|9/4|Implementation of Nwe Financial System|SCAA|Blanche Birger|Euro 65,779.50| +|61|9/4|Construction of Fuel station and Admin Block|SPTC|Onivo Construction|SR2,196.748.00| +|62|9/4|Procurement of Plastic Chairs|MOE|J Galt International|ZAR 498,560.00| +|63|9/4|Security Service in All Education Institution -Lot 2|MOE|Xtreme Security Service|SR 28,000.00| +|64|16/4|Procurement of Critical Spare Parts for Wartsila Engines B41|PUC|Wartsila Global Logistic Service|Euro 37,460.92| +|65|16/4|Security Service STC Premises-Supermarket|STC|Isles Security Agency Ltd|SR 908,107.20| +|65|16/4|Security Service STC Premises-Meat & Veg|STC|Allaince Security|SR 456,183.36| +|65|16/4|Security Service STC Premises-Warehouse, Duty free, BDR Complex|STC|Isles Security Agency Ltd|SR 490,752.00| +|65|16/4|Security Service STC Premises-Head Office|STC|Isles Security Agency Ltd|SR 503,712.00| +|65|16/4|Security Service STC Premises-Praslin ( Amitie Store /Duty-Free|STC|Alliance Security|SR 327,598.56| +|66|16/4|Concrete Works for Roads A & B Eve Island, Baie Ste Anne Praslin- Extension of Contract|MLUH|Allied Builders|SR 1,732,989.80| +|67|16/4|Supply of Electrical Cable, Transformewr Equipment & Service Connection Materials|MLUH|Ascent Projects (Sey) Pty Ltd|SR 3,888,632.42| +|68|23/4|Procurement of Vehicle|SFA|Exel Motors|SR 519,869.79| +|69|23/4|Procurement of X-Ray Bulk Cargo Screening Machine for Import Cargo Warehouse Extenson|SCAA|Smiths Detection|Euro 368,000.00| + +|70|23/4|Procurement of ADOBE CSE Editing Software for Audio and Video|SBC|Nuclei|Euro 91,187.70| +|---|---|---|---|---|---| +|71|23/4|Procurement of Ultracsonic Cleaning Machine|PUC|IOP Marine (Denmark)|Euro 1,250.00| +|72|30/4|Procurement of Vehicle|PA|Abhaye Valabhji Pty Ltd|SR575,000.00| +|73|30/4|Procurement of ID Cards|DICT|Blanche Birger Bureautique|Euro 43,500.00| +|74|30/4|Procurement of Electrical Meters|PUC|ISKRAEMECO|Euro 55,733.60| +|75|30/4|Procurement of Grunfos Pump for Le Rocher Pump Station|PUC|Bluezone Mauritius Ltd|Euro 37,910.60| +|76 MAY|30/4|Procurement of A3 and A4 Photocopr Paper|MOE|Islad Motors Co Ltd|SR552,000.00| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment -Calcium Hypochlorite Power|PUC|Technoglass|US$ 156, 920.00| +|77|7/5|Supply of Chemicals for use in Drinking Water Treatment-Calcium Hypochlorite|PUC|Technoglass|US$ 83,980.00| +|78|7/5|Procurement of Connecting Rod (Variations)|PUC|Wartsila Eastern Africa Ltd|Euro 1,200.00| +|79|7/5|Project Management Consultancy-Extension of contract|SIBA|Philippe Adrienne Consultancy Engineers|SR 225.000.00| +|80|7/5|Procurement of Liquid Chlorine|PUC|Al Afaq LLC|USD 77,088.00| +|81|21/5|Procurement of Agriculture Inputs|SAA|Rodley Mathieu|SR1,241,225.00| +|82|21/5|Procurement of Metelogical Equipment|MEE|Vaisala|Euro 33,815.00| +|83|21/5|Procurement of Turbocharger for Rotor Shaft for Wartsila Engine|PUC|Marine Power International|Euro 39,945.00| +|84|21/5|Provision of Security Srevices for MOH's Institutions|MOH|Alliance Security|SR 19,656.00| +|85|21/5|Procurement of Low Voltage ABC Cables|PUC|Nextans|Euro 50,118.69| +|86|28/5|Procurement of HDPE pipes-Pipe for water applications|PUC|STR Marketing-|USD 68,030.02| +|86|28/5|Procurement of HDPE pipes-Pipe for sewerage applications|PUC|STR Marketing|USD 12,509.55| +|87|28/5|Procurement of Incenerator for Baie Ste Anne Praslin Hospital|MOH|Incinco Limited|GBP 101,992.00| +|88|28/5|Geotechnical Survey on Ile Soleil|2020 Development Ltd|Geoconsul Ltee|SR 741,520.00| +|89|28/5|Extra Works at Palais De Justice|The Judiciary|Quingjian Group Co|SR 1,614,226.00| +|90|28/5|Fire Fighting and rescue training Course|SFRS|Emergency Training Solution Pty Ltd|ZAR 824,453.80| +|91 JUN|28/5|Procurement of Charger Air Cooler for Wartsila Engine at Power Station C|PUC|Marine Power International FZC|Euro 38,217.00| +|92|4/6|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale|ZAR 438,081.83| +|93|4/6|Procurement of Spare Parts for Maintenance on Generator at Baie Ste Anne Anne Praslin Power StationPUC||Wartsila Global Service|Euro 74,768.40| +|94|4/6|Procurement of Cylinder Liner and Pistons|PUC|Wartsila Eastern Africa|Euro 70,082.73| +|95|4/6|Procurement of Digital Microwave Equipment|SBC|MOCHINO|Euro 125,845.00| +|96|4/6|Procurement of Vehicle 1|LWMA|Abhaye Valabhji Pty Ltd-Jeep|SR 475,000.00| +|96|4/6|Procurement of Vehicle 1|LWMA|EHW Seychelles Ltd- Car|SR 267,850.00| +|97|4/6|Procurement of Forged filter Ball-Valves|PUC|Jainsons Malleables|USD 11,000.00| +|98|11/6|Completion of Stone Masonry Retaining Wall at Jean Larue Road Takamaka|SLTA|Bazil Construction|Sr 1,081,530.00| +|99|11/6|Procurement of Seychelles Paswsports|DIA|Groupe Impimerie Nationale|Euro 102,400.00| +|100|19/6|Anse Boileau Footpath and Drainage Construction Phase II|SLTA|TCH Building Contractor|SR 960,107.50| +|101|21/6|Procurement of Vehicle x 1|PSD|Abhaye Valabhji-Bus x1|SR 475,000.00| +|101|21/6|Procurement of Vehicle x 1|PSD|PMC Auto- Car x1|SR 267,850.00| +|102|21/6|Renovation Works on Block A- Beau Vollon Secondary School|MOE|Sai-Fu Enterprise Company Ltd|SR 2,253,300.00| +|103|21/6|Procurement of 35 VMS Terminal Accessories|SFA|Communication Specialist Ltd|Euro 63,770.00| +|104|21/6|Renewal of SFA's Themis FMC Services|SFA|CLS- France|Euro 36,000.00| +|105|21/6|Procurement of Electrical Spares for Wartsila Engine|PUC|Wartsila Eastern Africa|Euro 1,757.00| +|106|21/6|Procurement of Class D water Meters|PUC|Elster Metering Limited (Pty) Ltd|USD 109,7051.00| +|107|21/6|Procurement of Sludge Incinerator|PUC|Atlas Incinerator|€ 94,470.00| +|108|21/6|Procurement of Technical Services for Crankshaft Grinding|PUC|Goltens|USD 76,075.00| +|109|21/6|Procurement of WAS Pumps|PUC|Netzsch Southern Africa Pty Ltd|Euro 56,657.20| + +|110|21/6|Refurbishment of Turbo Charger for Rotor Shaft|PUC|Marine Power International FZC|Euro 57,326.00| +|---|---|---|---|---|---| +|111|21/6|Procurement of Fitting and Pipes ( Stock Replenishment)|PUC|STR Marketing Ltee|USD 60,688.90| +|112|21/6|Refurbishment of Pharmaceutical Production Unit-Contract Extension|MOH|Mahe Design|SR808,375.00| +|113|21/6|Technical Service for Repair on Generator-1B|PUC|Goltens|USD 75,600.00| +|114|21/6|Procurement of Gate Valves|PUC|AVK Valves Southern africa (Pty) Ltd|ZAR 461,608.44| +|115|25/6|Procurement of Hot-Dipped Galvanised Materials|PUC|HDSA Shipping (Pty) Ltd|Zar 435,614.00| +|116|25/6|Procurement of Critical Spares for Wartsila Engine B51- Lot 1|PUC|Wartsila Global Services|Euro 92,472.70| +|117|25/6|Procurement of Pistons for Replacement on Wartsila Engined- 8p on Praslin|PUC|RUYSCH|Euro 83,262.64| +|118|25/6|Procurement of services Operation and Maintenance of Containerised Desalination Units on Mahe -2013|PUC|Tornado Group|USD 266,820.82| +|119 JUL|25/6|Procurement of X-ray Screening Machine for VVIP Lounge|SCAA|Smiths Detection|Euro 101,800.00| +|120|2/7|Construction of Motorable Road at Anse Aux Pins- Capucin (Nourrice Road)|SLTA|Esparon's Enterprise|SR 183,410.00| +|121|2/7|Bridge Renovation at Cascade|SLTA|Benioton Construction|SR 1,361,100.00| +|122|2/7|Procurement of Bitumen|SLTA|Termcotank S.A|USD 519,418.20| +|123|2/7|Procurement of Vehicle x 2|STB|PMC Auto Pty Ltd|SR 960,126.00| +|124|2/7|Consutancy Services for Technical Assistance for Elaboration of Theme on Natioal and International Positioning|MFA|John Nevill|SR 180,000.00| +|125|9/7|Procurement of Stationery Items Lot 4|MOE|JD's Stationey EDU Centre|SR 629,950.00| +|126|9/7|Procurement of Atomic Spectrometer|SBS|SMM Instrument (Pty) Ltd|USD 181,147.00| +|127|9/7|Operation and Maintenance of Containerised Desalination Units by Tornado-2012|PUC|Tornado Group|USD 45,174.62| +|128|9/7|Procurement of Sensors and Transmitters for Wartsila Engines|PUC|Wartsila Global Logistic Services|Euro 50,360.00| +|129|9/7|Procurement of Bulk Water Meters Strainers|PUC|Elster Metering Limited (Pty) Ltd|ZAR 931,966.00| +|130|9/7|Procurement of Gudgeon Pins for engine 8P|PUC|Wartsila Global Logistic Services|Euro 23,460.00| +|131|9/7|Procurement of Piston for Wartsila Engine A21|PUC|Wartsila Global Logistic Services|Euro 219,622.00| +|132|9/7|Procurement of Piston for Wartsila Engine B11|PUC|Marine Power International FZC|Euro 243,799.98| +|133|9/7|Constrcution of 6 Blocks of 6 Units of Flats- Ilse Preseverance|MLUH|Sai-Fu Enterprise Company Ltd|SR 17,376,392.65| +|134|9/7|La Louise Non- Performance Pipieline Replacement|PUC|Ascent Projects Sey Pty Ltd|SR 1,289,375.00| +|135|9/7|Refurbishment Sewage Treatment Plant Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR 1,462,875.00| +|136|9/7|Consultancy Services for Quality management System (QMS)|DE|Mr. John Horack|USD 22,325.00| +|137|9/7|Construction of New Road at Cascade Primary School|MLUH|Esparon's Enterprise|SR 2,443,814.00| +|138|9/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brits Pale Pty Ltd|ZAR 518,694.02| +|139|9/7|Procurement of Non-Critical Spares- Specialized Tools|PUC|Marine Power International FZC|Euro 10,866.55| +|140|9/7|Upgarding of Roche Caiman Road and Roundabout|SLTA|Bazil Construction|SR 1,592,002.00| +|141|9/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|142|16/7|Refurnishment sewage treatment plant, Baie Ste Anne Praslin Hospital|MOH|Des Iles Environment Solutions|SR1,462,875.00| +|143|16/7|Consultancy Service for Quality Management System (QMS)|DE|Mr. John Horack|CND$22,325.00| +|144|16/7|Construction of New Road at Cascade Primary School|SLTA|Esparon's Enterprise|SR 2,443,814.00| +|145|16/7|Procurement of Tanalisth Treated Wooden Poles|PUC|Brit Pale Pty Ltd|ZAR518,694.02| +|146|16/7|Procurement od Non- critcal spares-specialized tools|PUC|Marine International FZC|Euro10,866.55| +|147|16/7|Upgrading of Roche Caiman Road and roundabout|SLTA|Bazil Construction|SR1,592,002.00| +|148|16/7|Procurement of Crankshaft for Engine 5B at New Port Station|PUC|A&D Sales|GBP 65,375.00| +|149|23/7|Consultancy services for infrastructure PIE (Z18,Z6,Z20, link Z20-pie star area.|MLUH|LC International|SR2,814,108.00| +|150|23/7|Procurement of CT Scan Tube|MOH|Ireland Blyth Ltd from Mauritius|Euro 122,000.00| +|151|23/7|Procurement of Ultraviolet disinfection System for Sewerage Treatment|PUC|Orica Wtercare|SR851,104.72| +|152|23/7|Procurement of pumps, Electrical panels and spares for water pumping stations and spares for sewage pumps (a) Procurement of spares for Hidrostal sewega pumps|PUC|Hidrostal Sewage Sa Pty Ltd|Euro97,498.32| +|153|23/7|(B)Procurement of Grundfos Pumps for water pumping|PUC|Bluezone Mauritius|Euro 13,410.00| +|154|23/7|Procurement of spare for Mirrlees Radiator|PUC|Covard Heat Transfer Ltd|GBP32,042.43| + +|155|23/7|Renovation work at the schools section MOE headquater|MOE|Prime Builders|SR1,768,620.00| +|---|---|---|---|---|---| +|156|23/7|General renovation works to Block B at Belonie Secondary School|MOE|Belvedere Builders|SR869,505.75| +|157|30/7|Procurement of Engine Block and Crankshaft for Engine A11|PUC|Ras Tek Pvt Ltd|Euro798,650.00| +|158|30/7|procurement of Wartsila Engine spares|PUC|Wartsila Eastern Africa ltd|Euro158,424.00| +|159|30/7|Proposed walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg)|SLTA|G&S Enterpise|SR1,113,010.00| +|160|30/7|Procurement of transfer pump control panel|PUC|CA Engineering Consultancy Pte Ltd|SGD14,600.00| +|161|30/7|Consultancy service for North to South Victoria Bye- Pass road and utilities organisation|MLUH|Sonnel Seychelles LTD|SR1,332,000.00| +|162 AUG|30/7|Procurement of the supply of sodium cardonate|PUC|HPL Chemical LTD|USD42,600.00| +|163|6/8|Procurement of Vehichels X 4|MOH|Kim-Koom & Co Pty Ltd|SR1,100,000.00| +|164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the North Mahe|WMF|Mr. Donal Ernesta|| +|164|6/8|Tender for the collection of redeem center for the collection of pet plastic produts and empty aluminum beverage cans for the Central Mahe|WMF|Mr. Kali Deenudayali|| +|165|13/8|Procurement of exercise books|MOE|JD's Stationey EDU Centre|SR,1,700,000.00| +|166|13/8|Procurement of High Pressure pump spares -BZM00003022|PUC|Bluezone Mauritius Ltd|Euro27,712.73| +|166|13/8|Procurement of High Pressure pump spares -BZM00003023|PUC|Bluezone Mauritius Ltd|Euro12,639.55| +|167|13/8|Procurement of CR64 pump spares|PUC|Bluezone Mauritius Ltd|Euro31,822.00| +|168|13/8|Construction of access road at Ex-Deltel- Anse Royale|MLUH|Benoiton Construction Pty Ltd|SR4,585,441.12| +|169|13/8|Renovation work to one classroom block at Pionte Larue Secondary School|MOE|Belverdere Builders|SR1,114,575.00| +|170|13/8|Completion of Amitie Housing Project 12 x 3 Bedrooms|MLUH|Allied Builders Sey Ltd|6,006,410.37| +|171|13/8|Copolia Road widening-Phase 2|SLTA|Belverdere Builders|SR1,037,235.00| +|172|20/8|Procurement of ABB Turbocharger Cartridge|PUC|ABB France|Euro106,266.66| +|173|20/8|Procurement of services to carry out the full refit and overhaul of tug Alouette|SPA|SECREN (Madagascar)|Euro189,618.42| +|174|20/8|Awarding of cranshatf and Block replacement solution for A11 engine|PUC|Wartsila|Euro800,000.00| +|175|29/8|Servicing of geartrain for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro 21,141.90| +|176|29/8|Spare parts for Wartsila Engine 18V32LN|PUC|Wartsila Eastern Africa|Euro38,440.00| +|177|29/8|Procurement for sience equipment and chemical for 2013|MOE|Findel Education|GBP37,831.26| +|178|29/8|Installation of fencing at Mont Fleuri Secodary School|MOE|Donald Builbing & Contractor Pty Ltd|SR1,761,034.00| +|179|29/8|Construction ot New Fire Station, fuel store, bin site and boundary wall at Lapasse La Digue-Variations SFRSA||Furui Construction Pty Ltd|SR277,521.45| +|180|29/8|Propsed road widening and drainage improvement at Quincy Vilage|SLTA|TCH Contractor|SR1,015,806.24| +|181|3/9|Tender for procurement of windows licenses|STB|Victoria Computer Service|SR788,808.00| +|182|3/9|Procurement of Bitumen in drums for the asphalt production Praslin|SLTA|Benzene International Pte Ltd|Euro87,220.00| +|183|3/9|Procurement of spre parts for Sulzer Engine 8ZAL40 and 8ZAL40S|PUC|Wartsila Eastern Africa|Euro9,861.00| +|184 SEP|3/9|Procurement of gear train parts for Stork Wartsila Engine SW280|PUC|Wartsila Eastern Africa|Euro7,931.00| +|185|10/9|Cleaning and maintenance of wetlands and rivers on Praslin|ED|"W" Cleaning Service|SR769,590.00| +|186|10/9|Re-construction and partition of Independence House|MLUH|Green Island Construction Co Pty|SR868,022.94| +|187|10/9|Procurement of critical spare fro major overhaul|PUC|Wartsila Eastern Afirca|Euro33,623.00| +|188|10/9|Procurement of control panel for six pumps station|PUC|CA Engineering Consultancy Pte Ltd|SGD37,400.00| +|189|17/9|Security service for Wellness Centre|MOH|Alliance Security|SR78,624.00| +|190|17/9|Procurement of reinforce plastic (FRP) handrails and grating|PUC|Webforge Group|SR839,795.02| +|191|17/9|Procurement of helital fitting|PUC|Cu A1 Engineering (Pty) Ltd|ZAR343,000.00| +|192|17/9|Procurement of Flygt pumps|PUC|Aqualia DPI LTD|Euro56,080.00| +|193|17/9|Procurement of black-up and emergency pumps|PUC|M.A.H.Y Khoory & Co|UAE186,650.00| +|194|17/9|Procurement of additional requirement of polo t-shirts and jeans|PUC|Magilyn Ltee|USD42,127.50| +|195|17/9|Procurement of Pressure filters|PUC|Barr +Wray|GBP379,921.00| +|196|17/9|Procurement of grunfos pump for water pumping|PUC|Blue Zone Mauritius|Euro108,574.00| + +|197|17/9|Supply of pipes and fittings for Network diversion in Victoria|PUC|Ascent Projects (Sey) Pty Ltd|USD181,661.00| +|---|---|---|---|---|---| +|198|24/9|Procurement of Technical Services for Crankshaft Grinding on Engine 6B|PUC|Golten Co Ltd|USD92,654.00| +|199|24/9|Procurement of technical service for repair of generator 1B- varations|PUC|Golten Co Ltd|USD84,337.00| +|200|24/9|Procurement of spare parts fro Wartsila Engine A21|PUC|Wartsila Global Logistic|Euro235,579.30| +|201|24/9|Procurement of vehicle x 2|SLTA|Abhaye Valabhji Pty Ltd|SR1000.000.00| +||OCT||||| +|202|1/10|Proposed new traffic lane to 5th June Avenue|SLTA|Divy Constrution|SR2,864,589.00| +|203|1/10||Proposed Walkway, Drain, rock armoring , road and Bridge widening at Anse Talbot( Ex-Golden Egg) - Variations SLTA|G & S Enterprise|SR200,448.00| +|204|1/10|Proposed Reconstrcution of Burnt House-Au Cap|MLUH|Furui Construction|SR946,130.00| +|205|1/10|Variation on the project associated with the procurement of seven 100m3/day containerised plant|PUC|Tornado Group (UAE)|USD172,500.00| +|206|1/10|Works on the breaker system at Bel Omber desalination plant|PUC|United Concrete Products (Sey)Ltd|SR1,998,993.11| +|207|1/10|Procurement of Viking Johnson fittings|PUC|Viking Johnson (UK)|GBP92,286.50| +|208|1/10|Procurement of HDPE Pipes and Fittings|PUC|STR Marketing Ltee|USD196,672.57| +|209|1/10|Procurement of Piston and Gudgeon Pins|PUC|Marine Power International FZC|Euro277,098.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 1|PUC|Ms. Suzanne Edmond|SR500.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 2|PUC|Ms. Suzanne Edmond|SR200.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 3|PUC|Ms. Suzanne Edmond|SR475.00| +|210|8/10|Procurement of Services for Sewing of Uniform for Office Staff-Lot 4|PUC|Sey Sytle|SR450.00| +|211|8/10|Procurement of vehicle x 2|FIU|PMC Auto Pty Ltd|SR526,864.00| +|212|15/10|Renovation Works to Le Chantier Mall-Variation Works|SSF|Allied Builders Sey Ltd|SR1,673,752.51| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 1|MSACDS|Eagle Watch Security Services|SR96,600.00| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 2|MSACDS|Alliance Security Services|SR103,012.00| +|213|15/10|Security Services for District's Administration Offices and Community Centres-Lot 4|MSACDS|Security Protection Services|SR71,000.00| +|214|15/10|Anse Boileau Footpath and Drainage Construction Phase 2 - Variations|SLTA|TCH Contractor|SR326,780.00| +|215|15/10|Construction of Fooothpath at Olivier Maradan Street-Variations||Benoiton Construction Pty Ltd|SR368,642.50| +|216|15/10|Operation and Maintenance of Containerised Desalination Units on Mahe 2013-2014|PUC|Tornado Group (UAE)|USD1,214,400.00| +|217|15/10|Procurement of Non Critical Spare Parts for Wartsila Engine 5B||MAN Diesel & Turbo (UK)|GDP88,360.88| +|218|15/10|Procurement of Meter Boxes|PUC|CAHORS (France)|Euro35,221.20| +|219|15/10|Construction of Services Road to Commercial Zone West of Inter Island Quay|MLUH|Allied Builders Sey Ltd|SR6,234,343.36| +|220|15/10|Proposed New traffic lane from Roche Caiman to Eden Island|SLTA|Franky's Constrcution|SR3,017,155.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Seytex|SR695,810.58| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|VCS Pty Ltd|SR36,430.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|KIS Distribution Company|SR10,500.00| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|MNM General Supply|SR16,216.20| +|221|22/10|Procurement of Textbooks for Primary and Secondary Schools|MOE|Roy & Sons Import|SR107,625.00| +|222|22/10|Road Widening and Drainage Improvement at Quincy Village-Variations|SLTA|TCH Building Contractor|SR328,213.75| +|223|22/10|Procurement of Grunfos Pumps for Mare Aux Cochons|PUC|Bluezone Mauritius|Euro44,914.00| +|224|22/10|Procurement of Vehicle x 3|MLUH|Sun Motors|SR2,310,000.00| +|225|22/10|Procurement of Agriculture Inputs|SAA|Launch Export (SA)|ZAR720,100.00| +|226|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 09||Allied Builders Sey Ltd|SR14,964,125.91| +|227|22/10|Provision of Utilities and Infrastructture on Ile Preseverance Island-Civil 08||Allied Builders Sey Ltd|SR35,927,978.92| +|228|29/10|Proposed Completion of Corgate Estate Re-Development-Phase A (Zone A2) - Mont Fleuri|MLUH|Allied Builders Sey Ltd|SR10,372,824.02| +|229|29/10|Construction of One Block of Condominium Flats at Pointe Larue|MLUH|Franky's Constrcution|SR18,063,876.00| +|230|29/10|Proposed Construction of Micro-Enterprise Building at Providence|MLUH|O Nivo Construction|SR28,462,412.00| +|231|29/10|Renovation of English River Secondary Schools|MLUH|Divy Constrution|SR2,133,560.00| +|232|29/10|Renovation and Partitioning of Independence House-Variation|MLUH|Green Island Construction Co Pty|SR1,847,477.43| +||||||6| + +|298|17/12|Renovation of Glacis Health Centre-Variations|MOH|F & P Construction|SR400,000.00| +|---|---|---|---|---|---| +|299|17/12|Supply of New Equipment for Kitchen|Prison Service|K. K. Chua|SR943,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Sun Motors|SR770,000.00| +|300|23/12|Procurement of Vehicle x 1|MLUH|Abhaye Valabhji|SR435,000.00| +|301|23/12|Procurement of Spares for LT Water Circulating Pump|PUC|Wartsila Global Logistcs Services|Euro 19,389.00| +|302|23/12|Procurement of Filter Cartridge-Desalination Plants|PUC|Trans Crescent Technical Equipment Company|Euro22,030.00| +|303|23/12|Procurement of LED Streetlights|SLTA|Lighting Orient (China)|USD104,282.00| +|304|23/12|Proposed New Road at Cascade Primary School-Variations|SLTA|Esparon's Enterprise|SR861,400.00| +|305|23/12|General renovation at School Section at MOE Headquarters-Variations|MOE|Prime Builders|SR2,761,935.75| +|306|23/12|Remedial Works at Mont Fleuri Primary School and Creche-Variations|MOE|Sai-Fu Enterprise Company Ltd|SR1,006,778.50| +|307|23/12|General Renovation of Toilet at La Digue School|MOE|Furui Construction|SR338,785.40| +|308|26/12|Procurement of 2500 Ream of A4 Paper|MOE|Island Motors|SR1,200,000.00| +|309|26/12|Procurement of Additional of 400 Desktop Computers|MOE|Orion Computers|SR3,100,000.00| +|310|26/12|Procuremet of Vehcile x 5|MOE|PMC Auto Pty Ltd|SR1,374,897.00| +|311|26/12|Procurement of Canon Ink / Riso Meter|MOE|Paradise Computer Services|SR473,000.00| +|312|26/12|Supply of Metal Fencing|MOE|BBT (UK)|GBP105,278.00| +|313|26/12|Procurement of Mobile Dental Clinic / Surgeries x 2|MOH|Quayle Dental (UK)|GBP233,842.00| diff --git a/tests/snapshots/nexo-price-en.md b/tests/snapshots/nexo-price-en.md index 2279856..5bc4ae2 100644 --- a/tests/snapshots/nexo-price-en.md +++ b/tests/snapshots/nexo-price-en.md @@ -1,10 +1,9 @@ 본 가격표는 국내 거주 중인 외국인을 위한 한국어 가격표의 비공식 번역본입니다. ※ The post-tax benefit sales price is provided for your reference only, reflecting the current tax benefits and eco-friendly vehicle individual consumption tax reductions. 본 가격표와 한국어 가격표의 내용이 상이한 경우 한국어 가격표의 내용이 우선하므로, 반드시 한국어 가격표의 내용을 확인하십시오. The final sales price may vary depending on the addition of optional items and whether the eco-friendly vehicle criteria are met, so please be sure to check the quotation. This price list is an unofficial translation of the Korean price list for the convenience of foreign residents in South Korea. ※ Please check the Korean price list for information on colors, details, and fuel consumption for each model. If the price list differs from the Korean price list, please check the contents of the Korean price list first. ※ All optional item prices are listed based on pre-tax reduction amounts. The actual sales price, which reflects the total individual consumption tax reduction including optional items, may differ depending on applicable tax benefits. ※ The items (specifications, colors, etc.) and prices listed in this pricing table are subject to change without prior notice depending on the holding of new car launch events, improvements made in automobile performance, introduction of related laws and regulations, and changes in company circumstances. The all-new NEXO Release Date: June 10, 2025 / (Unit: KRW) -| Classification Exclusive Exclusive | Selling price before tax benefit Supply value(surtax) 80,509,000 73,190,000(7,319,000) | Selling price after tax benefit 76,435,000 | Standard equipment • Powertrain/Performance: Fuel cell system(150kW drive motor, lithium-ion battery, and reducer), Regenerative braking system, Column-Type Shift By Wire(vibration warning), Drive mode select • Safety: 9 airbag system(1st-row advanced/center side airbags, 1st/2nd-row side airbags, and rollover-resistant curtain airbags), Multi-Collision Brake System, Active hood system(for pedestrian protection), Safety unlock function, Artificial engine sound(for pedestrian protection), Child seat fastening system (2 in 2nd-row), Fire extinguisher for vehicles, Pedal Misapplication Safety Assist • Smart Safety Technology: Forward Collision-avoidance Assist(vehicles/ pedestrians/two-wheeled vehicles/junction turning/front oncoming), Smart Cruise Control with Stop & Go, Lane Keeping Assist, Lane Following Assist 2, Blind-spot Collision Warning(driving), Blind-spot Collision-avoidance Assist(forward exit), Rear Cross-traffic Collision-avoidance Assist, Safety Exit Assist, Driver Attention Warning, High Beam Assist, Advanced Rear Occupant Alert, Intelligent Speed Limit Assist, Hands-On Detection, Highway Driving Assist, Navigation-based Smart Cruise Control(safety speed zone/curve control), Vibration warning steering wheel • Exterior: Full LED headlamps(projection type), LED turn signal lamps (front and rear), LED Daytime Running Lights, LED positioning lights, LED rear combination lamps, LED third brake lights, 18-inch alloy wheels & tires, Solar glass(windshield), Double-glazed soundproof glass(windshield, and 1st/ 2nd-row doors), Outside mirror(heating, power-folding, power adjustment, | Options (before tax benefit) ▶ Hi-pass(e hi-pass) [200,000] | -| ---------------------------------- | ---------------------------------------- | ---------------------------------------- | ---------------------------------------- | ---------------------------------------- | -| | with 3.5% individual consumption tax applied 79,287,000 | with 3.5% individual consumption tax applied 76,435,000 | and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather- | | -| Special | 83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000 | 79,275,000 with 3.5% individual consumption tax applied 79,275,000 | upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's | ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN | -| Prestige | 87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000 | 83,445,000 with 3.5% individual consumption tax applied 83,445,000 | seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(fron t and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design | sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires | +|Classification Exclusive Exclusive|Selling price before tax benefit Supply value(surtax) 80,509,000 73,190,000(7,319,000)|Selling price after tax benefit 76,435,000|Standard equipment • Powertrain/Performance: Fuel cell system(150kW drive motor, lithium-ion battery, and reducer), Regenerative braking system, Column-Type Shift By Wire(vibration warning), Drive mode select • Safety: 9 airbag system(1st-row advanced/center side airbags, 1st/2nd-row side airbags, and rollover-resistant curtain airbags), Multi-Collision Brake System, Active hood system(for pedestrian protection), Safety unlock function, Artificial engine sound(for pedestrian protection), Child seat fastening system (2 in 2nd-row), Fire extinguisher for vehicles, Pedal Misapplication Safety Assist • Smart Safety Technology: Forward Collision-avoidance Assist(vehicles/ pedestrians/two-wheeled vehicles/junction turning/front oncoming), Smart Cruise Control with Stop & Go, Lane Keeping Assist, Lane Following Assist 2, Blind-spot Collision Warning(driving), Blind-spot Collision-avoidance Assist(forward exit), Rear Cross-traffic Collision-avoidance Assist, Safety Exit Assist, Driver Attention Warning, High Beam Assist, Advanced Rear Occupant Alert, Intelligent Speed Limit Assist, Hands-On Detection, Highway Driving Assist, Navigation-based Smart Cruise Control(safety speed zone/curve control), Vibration warning steering wheel • Exterior: Full LED headlamps(projection type), LED turn signal lamps (front and rear), LED Daytime Running Lights, LED positioning lights, LED rear combination lamps, LED third brake lights, 18-inch alloy wheels & tires, Solar glass(windshield), Double-glazed soundproof glass(windshield, and 1st/ 2nd-row doors), Outside mirror(heating, power-folding, power adjustment,|Options (before tax benefit) ▶ Hi-pass(e hi-pass) [200,000]| +|---|---|---|---|---| +|Special|with 3.5% individual consumption tax applied 79,287,000 83,500,000 75,909,091(7,590,909) with 3.5% individual consumption tax applied 82,232,000|with 3.5% individual consumption tax applied 76,435,000 79,275,000 with 3.5% individual consumption tax applied 79,275,000|and LED turn signal lamps), Auto flush door handles, Black door garnish • Interior: Panoramic curved display, 12.3-inch color LCD cluster, Leather- upholstered steering wheel(with heating, two-tone color, and Interactive Pixel Lights), LED interior lamp (map lamp, personal lamp, sun visor lamp, and luggage lamp), Metallic door scuff plate • Seat: Synthetic leather seats, 1st-row manual seats, Heated 1st-row seats, 2nd-row 60/40-split folding seats(reclining) • Convenience: Proximity key with push-button start, Smart key remote start, Electronic Parking Brake(with automatic vehicle hold), Paddle shift(regenerative control), Dual-zone full automatic air conditioning(with high-performance antibacterial combination filter, auto defog system, fine dust sensor, air cleaning mode, and after-blow function), 2nd-row seat air vent, Auto light control system, USB Type-C Ports(1×27W switchable charging/data port in 1st-row, and 2×100W charging ports in both 1st and 2nd-row), ECM room mirror(frameless), Rain sensor, Power windows with pinch protection(1st/2nd-row), Power outlet (1 in 1st-row), Parking Distance Warning-Forward/Reverse, Rear View Monitor, Wireless phone charger(single), Walk-away lock, Route planner, Hyundai AI Assistant • Infotainment: 12.3-inch navigation(Bluelink, phone projection, Bluetooth hands-free, and In-car Payment), Audio system(6 speakers), Over-The-Air navigation updates ▶ Standard equipment of Exclusive plus • Smart Safety Technology: Forward Collision-avoidance Assist(intersection crossing/changing lanes in oncoming traffic/approaching from either side/ evasive steering assist), Highway Driving Assist 2, Navigation-based Smart Cruise Control(access road) • Exterior: Roof rack • Interior: Metallic pedal, Driving mode-dependent ambient mood lighting(crash pad, 1st/2nd-row door trim) • Seat: Synthetic leather seats(patch applied), Power-adjustable driver's|▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [950,000] Parking Assist ▶ [1,150,000] Audio by BANG & OLUFSEN| +|Prestige|87,893,000 79,902,727(7,990,273) with 3.5% individual consumption tax applied 86,559,000|83,445,000 with 3.5% individual consumption tax applied 83,445,000|seat(8-way, lumbar support, and Integrated Memory System(driver's seat and outside mirror connected)), Power-adjustable front passenger’s seat(8-way), Ventilated 1st-row seats, Heated 2nd-row seats • Convenience: Hi-pass(e hi-pass), In-car fingerprint authentication system(personalization, startup, payment, and etc.), Smart power tailgate ▶ Standard equipment of Exclusive Special plus • Smart Safety Technology: Remote Smart Parking Assist 2, Parking Collison- avoidance Assist(front/side/rear) • Exterior: Intelligent Front-Lighting System(IFS), Dynamic welcome/escort lighting(1 type), Sequential turn signals(front and rear), Ambient lighting auto flush door handles, Two-tone door garnish, Glossy black rear diffuser • Interior: Recycled PET suede interior materials(headlining/sunvisor), Fabric upholstered crash pad • Seat: BIO-processed natural leather seats(metal patch applied, embossed design punching), Passenger's seat walk-in device, 1st-row relaxation comfort seats(leg rest included), Ventilated 2nd-row seats • Convenience: Parking Distance Warning-Side, Head-Up Display, Digital key 2, Wireless phone charger(dual), Surround View Monitor, Blind-spot View Monitor, LED reverse light guide • Infotainment: Audio by BANG & OLUFSEN sound system(14 speakers, including external amp), Active road noise control, Active Sound Design|sound system ▶ [250,000] 19-inch alloy wheels & tires ▶ [600,000] Built-in Cam 2 Plus, Augmented reality navigation ▶ [850,000] Indoor/outdoor V2L ▶ [900,000] Vision roof ▶ [1,380,000] Digital side mirror ▶ [750,000] Camera package ▶ [250,000] 19-inch alloy wheels & tires| -**Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR +**Classification Details** **Indoor/outdoor V2L** Indoor V2L, Outdoor V2L(connectorless type) **Parking Assist** Surround View Monitor, Blind-spot View Monitor, Parking Distance Warning-Side, Parking Collison-avoidance Assist-Rear **Audio by BANG & OLUFSEN** Audio by BANG & OLUFSEN sound system(14 speakers, including external amp.), Active road noise control, Active Sound Design **sound system** **Camera package** Digital center mirror(with camera sensor cleaning system), Driver monitoring system THE ALL-NEW NEXO /// ECO-FRIENDLY CAR diff --git a/tests/snapshots/p1244-1996.md b/tests/snapshots/p1244-1996.md index a80af44..1b64aed 100644 --- a/tests/snapshots/p1244-1996.md +++ b/tests/snapshots/p1244-1996.md @@ -8,7 +8,7 @@ Department of the Treasury **Internal Revenue Service** # and Report to Employer -**This publication contains:** **Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer +**This publication contains:** **Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer For the period @@ -20,7 +20,7 @@ Name and address of employee **Publication 1244 (Rev. 7-96)** Cat. No. 44472W -**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A , Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.— If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. +**Instructions** You must keep sufficient proof to show the amount of your tip income for the year. A daily record of your tip income is considered sufficient proof. Keep a daily record for each workday showing the amount of cash and credit card tips received directly from customers or other employees. Also keep a record of the amount of tips, if any, you paid to other employees through tip sharing, tip pooling or other arrangements, and the names of employees to whom you paid tips. Show the date that each entry is made. This date should be on or near the date you received the tip income. You may use Form 4070A, Employee’s Daily Record of Tips, or any other daily record to record your tips. **Reporting Tips to Your Employer.—If you** receive tips that total $20 or more for any month while working for one employer, you must report the tips to your employer. Tips include cash left by customers, tips customers add to credit card charges, and tips you receive from other employees. You must report your tips for any one month by the 10th day of the next month. If the 10th day falls on a Saturday, Sunday, or legal holiday, you may give the report to your employer on the next business day that is not a Saturday, Sunday, or legal holiday. You must report tips that total $20 or more every month regardless of your total wages and tips for the year. You may use Form 4070, Employee’s Report of Tips to Employer, to report your tips to your employer. See the instructions on the back of Form 4070. You must include all tips, including tips not reported to your employer, as wages on your income tax return. You may use the last page of this publication to total your tips for the year. Your employer must withhold income, social security, and Medicare (or railroad retirement) taxes on tips you report. Your employer usually deducts the withholding due on tips from your regular wages. *(continued on inside of back cover)* @@ -33,6 +33,11 @@ Date Date **a. Tips received** **b. Credit card tips c. Tips paid out to d. Names of employees to whom you** tips of directly from customers received other employees paid tips rec’d. entry and other employees 1 2 3 4 5 **Subtotals** **For Paperwork Reduction Act Notice, see Instructions on the back of Form 4070. Page 1** +Date Date **a. Tips received** + +**b. Credit card tips c. Tips paid out to d. Names of employees to whom you** +tips of directly from customers received other employees paid tips rec’d. entry and other employees + 7 8 9 10 11 12 13 14 15 **Subtotals** **Page 2** @@ -43,9 +48,9 @@ tips of directly from customers received other employees paid tips rec’d. entr 27 28 29 30 31 **Subtotals** **from pages** **1, 2, and 3** **Totals** -**1.** Report total cash tips (col. a) on Form 4070, line 1. -**2.** Report total credit card tips (col. b) on Form 4070, line 2. -**3.** Report total tips paid out (col. c) on Form 4070, line 3. **Page 4** +**1.** Report total cash tips (col. a) on Form 4070, line 1. +**2.** Report total credit card tips (col. b) on Form 4070, line 2. +**3.** Report total tips paid out (col. c) on Form 4070, line 3. **Page 4** Form Employee’s Report (Rev. July 1996) @@ -59,15 +64,15 @@ Employer’s name and address (include establishment name, if different) **1** C **3** Tips paid out -Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3 ) from, 19, to, 19 Signature Date +Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3) from, 19, to, 19 Signature Date -**Paperwork Reduction Act Notice.— We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law —each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these +**Paperwork Reduction Act Notice.—We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; Learning **about the law—each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these -forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676). +forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676). **Instructions (continued)** -**Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law. +**Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law. **Instructions (continued)** diff --git a/tests/snapshots/real-estate-pricing.md b/tests/snapshots/real-estate-pricing.md index 75f2112..b3962da 100644 --- a/tests/snapshots/real-estate-pricing.md +++ b/tests/snapshots/real-estate-pricing.md @@ -28,9 +28,9 @@ R E V I E W 8 5 -800 -| 1982 | 1986 | 1990 | 1998 | 2006 | -| ---- | --------- | ---------- | ---------- | ------ | -| | Apartment | Industrial | Office-CBD | Retail | +|1982|1986|1990|1998|2006| +|---|---|---|---|---| +||Apartment|Industrial|Office-CBD|Retail| 1982 1986 1990 1994 1998 2002 2006 @@ -38,11 +38,11 @@ R E V I E W 8 5 **Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office** -| | Multifamily | Industrial | CBD Office | -| ---------- | ----------- | ---------- | ---------- | -| Industrial | 0.937 | | | -| CBDOffice | 0.924 | | | -| Retail | 0.922 | 0.969 | 0.964 | +||Multifamily|Industrial|CBD Office| +|---|---|---|---| +|Industrial|0.937||| +|CBDOffice|0.924||| +|Retail|0.922|0.969|0.964| more about investing in tax losses than burst, cap rates spreads steadily com- real estate cash streams. When tax laws pressed, recently falling to approximately dramatically changed in 1986, cap rate zero. And if NOI cap rate spreads are spreads rose, though they generally roughly zero, cash flow cap rate spreads remained negative due to the availability (after reserves for tenant improvements, of excess leverage through 1990 and pro-leasing commissions, and capital expendi- jections of strong cash flow growth, in tures) are well below zero. spite of weak fundamentals. This compression of cap rates and cap Throughout the first two-thirds of the rate spreads over the past five years has 1990s, spreads substantially widened as generated enormous wealth for real estate capital abandoned real estate. Spreads fur-owners. In fact, the combination of cheap ther widened in the latter part of the debt and cap rate compression covered a 1990s, as investors scorned cash flow dur-multitude of property underwriting ing the tech bubble and treasury rates errors made during the past five years, as drifted downward. As the tech bubble neither cap rate compression nor narrow- diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index ca2e20c..8161541 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -1,235 +1,196 @@ -(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows: +(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows: §1.1563-3T Rules for determining stock ownership (temporary). -(a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a) -through (d)(2)(iii). (iv) Statement. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include-- +(a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a) +through (d)(2)(iii). (iv) Statement. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include-- (A) A description of each of the controlled groups in which the corporation -could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and +could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and (B) The following representation: [INSERT NAME AND EMPLOYER IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER OF THE [INSERT DESIGNATION OF GROUP]. -(v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of +(v) Election-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in -termination of membership in the controlled group in which such corporation has been included. +|termination of membership in the controlled group in which such corporation has been included. (B) Election not filed.|In the event no election is filed in accordance with the| +|---|---| +|provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service|| +|will determine the group in which such corporation is to be included. Such|| +|determination will be binding for all subsequent years unless the corporation files a|| +|valid election with respect to any such subsequent year or until a change in the|| +|stock ownership of the corporation results in termination of membership in the|| +|controlled group in which such corporation has been included.|| +|(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3).|| +|(e) Effective date-- (1) Applicability date.|This section applies to any original| -(B) Election not filed. In the event no election is filed in accordance with the -provisions of paragraph (d)(2)(iv) of this section, then the Internal Revenue Service will determine the group in which such corporation is to be included. Such determination will be binding for all subsequent years unless the corporation files a valid election with respect to any such subsequent year or until a change in the stock ownership of the corporation results in termination of membership in the controlled group in which such corporation has been included. - -(d)(3) [Reserved]. For further guidance, see §1.1563-3(d)(3). -(e) Effective date-- (1) Applicability date. This section applies to any original Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30, 2006. -(2) Expiration date. The applicability of this section will expire on May 26, -2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: §1.6012-2 Corporations required to make returns of income. +(2) Expiration date. The applicability of this section will expire on May 26, +2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: §1.6012-2 Corporations required to make returns of income. * * * * * (c) [Reserved]. For further guidance, see §1.6012-2T(c). * * * * * -(k) [Reserved]. For further guidance, see §1.6012-2T(k)(1). +(k) [Reserved]. For further guidance, see §1.6012-2T(k)(1). -Par. 52. Section 1.6012-2T is added to read as follows: §1.6012-2T Corporations required to make returns of income (temporary). +Par. 52. Section 1.6012-2T is added to read as follows: §1.6012-2T Corporations required to make returns of income (temporary). -(a) through (b) [Reserved]. For further guidance, see §1.6012-2(a) through +(a) through (b) [Reserved]. For further guidance, see §1.6012-2(a) through (b). -(c) Insurance companies-- (1) Domestic life insurance companies-- (i) In -general. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return-- +(c) Insurance companies-- (1) Domestic life insurance companies-- (i) In +general. A life insurance company subject to tax under section 801 shall make a return on Form 1120L. Except as provided in paragraph (c)(4) of this section, such company shall file with its return-- (A) A copy of its annual statement which shows the reserves used by the company in computing the taxable income reported on its return; and (B) A copy of Schedule A (real estate) and of Schedule D (bonds and stocks), -or any successor thereto, of such annual statement. (ii) Mutual savings banks. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L. +or any successor thereto, of such annual statement. (ii) Mutual savings banks. Mutual savings banks conducting life insurance business and meeting the requirements of section 594 are subject to partial tax computed on Form 1120 and partial tax computed on Form 1120L. The Form 1120L is attached as a schedule to Form 1120, together with the annual statement and schedules required to be filed with Form 1120L. -(2) Domestic nonlife insurance companies. Every domestic insurance -company other than a life insurance company shall make a return on Form 1120PC. This includes organizations described in section 501(m)(1) that provide commercial- type insurance and organizations described in section 833. Except as provided in paragraph (c)(4) of this section, such company shall file with its return a copy of its +(2) Domestic nonlife insurance companies. Every domestic insurance +company other than a life insurance company shall make a return on Form 1120PC. This includes organizations described in section 501(m)(1) that provide commercial- type insurance and organizations described in section 833. Except as provided in paragraph (c)(4) of this section, such company shall file with its return a copy of its annual statement (or a pro forma annual statement), including the underwriting and investment exhibit for the year covered by such return. -(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and -(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. -(4) Exception for insurance companies filing their Federal income tax returns -electronically. If an insurance company described in paragraph (c)(1), (c)(2), or +||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and| +|---|---| +||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of| -(c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). -(5) Definition. For purposes of this section, the term annual statement means -the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of +Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. -Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. - -(d) through (j) [Reserved]. For further guidance, see §1.6012-2(d) through (j). +(d) through (j) [Reserved]. For further guidance, see §1.6012-2(d) through (j). (k) Effective date-- (1) Applicability date. This section applies to any original Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30, 2006. -(2) Expiration date. The applicability of this section will expire on May 26, -2009. Par. 53. For each entry in the “Location” column of the following table, remove the language in the “Remove” column and add the language in the “Add” column in its place: Location Remove Add The last sentence of the The following rules shall The rules described in introductory text to be applicable in paragraph (a) of §1.302- §1.302-4 determining whether the 4T and in paragraphs (b) -specific requirements of through (g) of this section section 302(c)(2) are apply in determining met: whether the specific requirements of section 302(c)(2) are met. §1.338(h)(10)-1(f) §1.331-1(d), and §1.332-§1.331-1T(d) and §1.332- 6 6T +(2) Expiration date. The applicability of this section will expire on May 26, +2009. -| The last sentence of | paragraph (a)(2)(ii) of this | paragraph (a) of §1.382- | -| ----------------------- | ---------------------------- | ------------------------- | -| §1.382-2T(h)(4)(vi)(B) | section | 11T | -| The first sentence of | §1.382-2T(a)(2)(ii) | §1.382-11T(a) | +|||Par. 53. For each entry in the “Location” column of the following table,| +|---|---|---| +||remove the language in the “Remove” column and add the language in the “Add”|| +|column in its place:||| +|Location|Remove|Add| +|The last sentence of the|The following rules shall|The rules described in| +|introductory text to|be applicable in|paragraph (a) of §1.302-| +|§1.302-4|determining whether the specific requirements of section 302(c)(2) are met:|4T and in paragraphs (b) through (g) of this section apply in determining whether the specific| -§1.382-6(b)(2)(i) The second sentence of paragraph (c) of this paragraphs (c)(1), (c)(3), §1.382-8(a) section (c)(4) and (c)(5) of this section and paragraph +requirements of section 302(c)(2) are met. + +|§1.338(h)(10)-1(f)|§1.331-1(d), and §1.332-|§1.331-1T(d) and §1.332-| +|---|---|---| +||6|6T| +|The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| +|§1.382-2T(h)(4)(vi)(B)|section|11T| +|The first sentence of §1.382-6(b)(2)(i)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The second sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|§1.382-8(a)|section|(c)(4) and (c)(5) of this| + +section and paragraph (c)(2) of §1.382-8T -The third sentence of §1.382-8(a) +|The third sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|---|---|---| +|§1.382-8(a)|section|(c)(4) and (c)(5) of this section and paragraph (c)(2) of §1.382-8T| +|§1.382-8(c)(3)|paragraph (c)(2) of this section|paragraph (c)(2) of §1.382-8T| +|The first sentence of|paragraphs (c)(1), (2),|paragraphs (c)(1) and| +|§1.382-8(c)(4)|and (3) of this section|(c)(3) of this section and paragraph (c)(2) of §1.382-8T| +|§1.382-8(c)(5)|this paragraph (c)|paragraphs (c)(1), (c)(3),| -§1.382-8(c)(3) The first sentence of §1.382-8(c)(4) - -§1.382-8(c)(5) - -The fifth sentence of §1.382-8(f) - -§1.382-8(g), Example - -(1)(b)(2) The second sentence of §1.382-8(g), Example -(1)(c) -paragraph (c) of this section - -paragraph (c)(2) of this section paragraphs (c)(1), (2), and (3) of this section - -this paragraph (c) - -paragraphs (c)(1), (c)(3), - -(c)(4) and (c)(5) of this section and paragraph -(c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T paragraphs (c)(1) and -(c)(3) of this section and paragraph (c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), -(c)(4), and (c)(5) of this section, and paragraph -(c)(2) of §1.382-8T paragraphs (c)(1), (c)(3), (c)(4), and (c)(5) of this section, and paragraph (c)(2) of §1.382-8T -| §1.382-8(g), Example | paragraph (c)(2) of this | paragraph (c)(2) of | -| --------------------- | ------------------------ | ------------------- | -| The first sentence of | paragraph (c)(2) of this | paragraph (c)(2) of | -| §1.382-8(g), Example | section | §1.382-8T | +|The fifth sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|---|---|---| +|§1.382-8(f)|section|(c)(4), and (c)(5) of this section, and paragraph (c)(2) of §1.382-8T| +|§1.382-8(g), Example|paragraph (c) of this|paragraphs (c)(1), (c)(3), section, and paragraph (c)(2) of §1.382-8T| +|The second sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| +|§1.382-8(g), Example|section|(c)(4), and (c)(5) of this| + +(1)(b)(2) section (c)(4), and (c)(5) of this +(1)(c) section, and paragraph + +(c)(2) of §1.382-8T + +|§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| +|---|---|---| +|The first sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| +|§1.382-8(g), Example|section|§1.382-8T| +|§1.382-8(g), Example|paragraph (c)(2) of this|paragraph (c)(2) of| +|§1.382-8(g), Example|paragraphs (c)(1) and (2)|paragraph (c)(1) of this| (2)(c) section §1.382-8T - (2)(e) - -| §1.382-8(g), Example | paragraph (c)(2) of this | paragraph (c)(2) of | -| --------------------- | ------------------------- | ------------------------ | -| §1.382-8(g), Example | paragraphs (c)(1) and (2) | paragraph (c)(1) of this | - (3)(b) section §1.382-8T (3)(c)(1)(B) of this section section and paragraph -The second sentence of §1.382-8(g), Example +(c)(2) of §1.382-8T -(4)(c) The second sentence of §1.382-8(g), Example +|The second sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| +|---|---|---| +|§1.382-8(g), Example|section|§1.382-8T| +|The second sentence of|paragraph (c)(2) of this|paragraph (c)(2) of| +|§1.382-8(g), Example|section|§1.382-8T| +|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| +|§1.1502-32(b)(4)(v)(A)|this section|§1.1502-32T| +|The first sentence of|paragraph (b)(4)(iv) of|paragraph (b)(4)(iv) of| +|§1.1502-32(b)(4)(v)(B)|this section|§1.1502-32T| + +(4)(c) (5)(c) -paragraph (c) of this section -paragraph (c) of this section +|§1.1502-35(c)(4)(ii)(B)|§1.1502-76(b)(2)(ii)(D)|§1.1502-76T(b)(2)(ii)(D)| +|---|---|---| +|§1.1502-76(b)(2)(ii)(A)(2)|paragraph (b)(2)(ii)(D) of this section|paragraph (b)(2)(ii)(D) of §1.1502-76T| +|§1.1502-92(e)(1)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The first sentence of §1.1502-92(e)(2)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The first sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The second sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| +|The last sentence of|paragraph (f) of this|paragraph (f) of §1.1502-| +|§1.1502-95(b)(3)|section|95T| +|The last sentence of|subdivision (ii) of this|paragraph (c)(2)(i) of| +|§1.1563-1(c)(2)(iv), Example (1)|subparagraph|§1.1563-1T| +|The last sentence of|the district director with|the Internal Revenue| +|§1.1563-1(c)(2)(iv), Example (1)|audit jurisdiction of N’s return|Service| +|The third sentence of|subdivision (iii) of this|paragraph (c)(2)(ii) of| +|§1.1563-1(c)(2)(iv), Example (2)|subparagraph|§1.1563-1T| +|The third sentence of|the district director with|the Internal Revenue| +|§1.1563-1(c)(2)(iv), Example (2)|audit jurisdiction of the return of the corporation whose taxable year ends on the earliest date|Service| +|The last sentence of|district director|Internal Revenue Service| -paragraph (c) of this section +§1.1563-1(c)(2)(iv), Example (2) -paragraph (c)(2) of this section paragraph (c)(2) of this section +|The second sentence of|subdivisions (ii), (iii), and|paragraphs (d)(2)(ii) and| +|---|---|---| +|§1.1563-3(d)(2)(i)|(iv) of this subparagraph|(iii) of this section, and paragraph (d)(2)(iv) of §1.1563-3T| +|The first sentence of|§1.332-6(b), 1.368-3(a),|§1.332-6T(a), §1.368-| +|§1.6043-2(a)|or 1.1081-11|3T(a), or §1.1081-11T| +|The first sentence of §301.6011-5T(a) (twice)|§1.6012-2|paragraphs (a), (b) and (d) through (j) of §1.6012- 2, and paragraph (c) of §1.6012-2T| -(c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T paragraph (c)(2) of §1.382-8T +|||PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK|| +|---|---|---|---| +||REDUCTION ACT Authority: 26 U.S.C. 7805. 1. The following entries to the table are removed: §602.101 OMB Control numbers.|Par. 54. The authority citation for part 602 continues to read as follows: Par. 55. In §602.101, paragraph (b) is amended to read as follows:|| +|* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| +|* * * * *|1.332-6………………………………………………………………….|1.382-11……………………………………………………………….. 1545-2019 1.351-3…………………………………………………………………. 1545-2019 1.355-5…………………………………………………………………. 1545-2019 1.368-3…………………………………………………………………. 1545-2019 1.1081-11………………………………………………………………. 1545-2019|1545-2019| +|* * * * *|§602.101 OMB Control numbers.|______________________________________________________________ 2. The following entries are added in numerical order to the table:|| +|* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| +|* * * * *|1.302-2T………………………………………………………………… 1545 1.302-4T………………………………………………………………… 1545||-2019 -2019| -| The first sentence of | paragraph (b)(4)(iv) of | paragraph (b)(4)(iv) of | -| ----------------------- | ----------------------- | ----------------------- | -| §1.1502-32(b)(4)(v)(A) | this section | §1.1502-32T | -| The first sentence of | paragraph (b)(4)(iv) of | paragraph (b)(4)(iv) of | -| §1.1502-32(b)(4)(v)(B) | this section | §1.1502-32T | +|1.331-1T………………………………………………………………… 1545|-2019| +|---|---| +||1.332-6T………………………………………………………………... 1545-2019 1.338-10T………………………………………………………………. 1545-2019| +|1.351-3T………………………………………………………………… 1545|-2019| +|1.355-5T………………………………………………………………… 1545|-2019| +|1.368-3T………………………………………………………………… 1545|-2019 1.381(b)-1T…………………………………………………………….. 1545-2019| +|1.382-8T………………………………………………………………… 1545|-2019 1.382-11T………………………………………………………………. 1545-2019 1.1081-11T……………………………………………………………… 1545-2019 1.1221-2T……………………………………………………………….. 1545-2019| +|1.1502-13T………………………………………………………………|1545-2019 1.1502-31T……………………………………………………………… 1545-2019 1.1502-32T……………………………………………………………… 1545-2019 1.1502-33T……………………………………………………………… 1545-2019 1.1502-35T……………………………………………………………… 1545-2019 1.1502-76T……………………………………………………………… 1545-2019| +|1.1502-95T……………………………………………………………… 1545|-2019 1.1563-1T……………………………………………………………….. 1545-2019| -§1.1502-35(c)(4)(ii)(B) §1.1502-76(b)(2)(ii)(D) §1.1502-76(b)(2)(ii)(A)(2) paragraph (b)(2)(ii)(D) of this section §1.1502-92(e)(1) §1.382-2T(a)(2)(ii) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-92(e)(2) The first sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) The second sentence of §1.382-2T(a)(2)(ii) §1.1502-94(d) - -| The last sentence of | paragraph (f) of this | paragraph (f) of §1.1502- | -| --------------------- | ------------------------ | -------------------------- | -| §1.1502-95(b)(3) | section | 95T | -| The last sentence of | subdivision (ii) of this | paragraph (c)(2)(i) of | -| §1.1563-1(c)(2)(iv), | subparagraph | §1.1563-1T | - -Example (1) The last sentence of §1.1563-1(c)(2)(iv), Example (1) The third sentence of §1.1563-1(c)(2)(iv), Example (2) The third sentence of §1.1563-1(c)(2)(iv), Example (2) - -The last sentence of §1.1563-1(c)(2)(iv), Example (2) The second sentence of §1.1563-3(d)(2)(i) - -the district director with audit jurisdiction of N’s return subdivision (iii) of this subparagraph the district director with audit jurisdiction of the return of the corporation whose taxable year ends on the earliest date district director - -subdivisions (ii), (iii), and (iv) of this subparagraph - -§1.1502-76T(b)(2)(ii)(D) paragraph (b)(2)(ii)(D) of §1.1502-76T §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) §1.382-11T(a) - -| The first sentence of | §1.332-6(b), 1.368-3(a), | §1.332-6T(a), §1.368- | -| --------------------- | ------------------------- | ----------------------- | -| §1.6043-2(a) | or 1.1081-11 | 3T(a), or §1.1081-11T | -| The first sentence of | §1.6012-2 | paragraphs (a), (b) and | - -§301.6011-5T(a) (twice) - -the Internal Revenue Service paragraph (c)(2)(ii) of §1.1563-1T the Internal Revenue Service - -Internal Revenue Service - -paragraphs (d)(2)(ii) and (iii) of this section, and paragraph (d)(2)(iv) of §1.1563-3T - -(d) through (j) of §1.6012- 2, and paragraph (c) of §1.6012-2T - -PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK REDUCTION ACT Par. 54. The authority citation for part 602 continues to read as follows: Authority: 26 U.S.C. 7805. Par. 55. In §602.101, paragraph (b) is amended to read as follows: - -1. The following entries to the table are removed: -§602.101 OMB Control numbers. - -* * * * * -(b) * * * -CFR part or section where Current OMB identified or described control No. - -* * * * * -1.332-6…………………………………………………………………. 1545-2019 -1.382-11……………………………………………………………….. 1545-2019 -1.351-3…………………………………………………………………. 1545-2019 -1.355-5…………………………………………………………………. 1545-2019 -1.368-3…………………………………………………………………. 1545-2019 -1.1081-11………………………………………………………………. 1545-2019 -* * * * * **______________________________________________________________** -2. The following entries are added in numerical order to the table: -§602.101 OMB Control numbers. - -* * * * * -(b) * * * -CFR part or section where Current OMB identified or described control No. - -* * * * * -1.302-2T………………………………………………………………… 1545-2019 -1.302-4T………………………………………………………………… 1545-2019 - -1.331-1T………………………………………………………………… 1545-2019 -1.332-6T………………………………………………………………... 1545-2019 -1.338-10T………………………………………………………………. 1545-2019 -1.351-3T………………………………………………………………… 1545-2019 -1.355-5T………………………………………………………………… 1545-2019 -1.368-3T………………………………………………………………… 1545-2019 -1.381(b)-1T…………………………………………………………….. 1545-2019 -1.382-8T………………………………………………………………… 1545-2019 -1.382-11T………………………………………………………………. 1545-2019 -1.1081-11T……………………………………………………………… 1545-2019 -1.1221-2T……………………………………………………………….. 1545-2019 -1.1502-13T……………………………………………………………… 1545-2019 -1.1502-31T……………………………………………………………… 1545-2019 -1.1502-32T……………………………………………………………… 1545-2019 -1.1502-33T……………………………………………………………… 1545-2019 -1.1502-35T……………………………………………………………… 1545-2019 -1.1502-76T……………………………………………………………… 1545-2019 -1.1502-95T……………………………………………………………… 1545-2019 -1.1563-1T……………………………………………………………….. 1545-2019 - -1.1563-3T……………………………………………………………….. 1545-2019 -1.6012-2T……………………………………………………………….. 1545-2019 +1.1563-3T……………………………………………………………….. 1545-2019 +1.6012-2T……………………………………………………………….. 1545-2019 * * * * * Mark E. Matthews Deputy Commissioner for Services and Enforcement. -Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy). +Approved: May 19, 2006 Eric Solomon Acting Deputy Assistant Secretary of the Treasury (Tax Policy). diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 1c3ba51..25ea607 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -1,6 +1,6 @@ **Technical Information** -## l T-12 SI +## l T-12 SI DuPont Fluorochemicals @@ -16,10 +16,10 @@ DuPont Fluorochemicals **®** **Thermodynamic Properties of Freon 12 Refrigerant** **(R-12)** **SI Units** -Tables of the thermodynamic **Units** properties of R-12 have been developed and are presented here. P = Pressure in kPa. Absolute This information is based on values calculated using the NIST REFPROP T = Temperature in Celcius Database (McLinden, M.O., Klein, +Tables of the thermodynamic **Units** properties of R-12 have been developed and are presented here. P = Pressure in kPa. Absolute This information is based on values calculated using the NIST REFPROP T = Temperature in Celcius Database (McLinden, M.O., Klein, -S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume -A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter +S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume +A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter H = Enthalpy (kJ/kg) S = Entropy (kJ/kg.K) @@ -46,65 +46,63 @@ l **Freon** **®** **12 Saturation Properties-Temperature Table** -| Temp | Pressure | Volume | | Density | | | Enthalpy | | Entropy | | Temp | -| ---- | -------- | ----------- | ------------- | ---------------- | -------- | -------- | ---------------- | -------- | ------------------ | -------- | ---- | -| °C | [kPa] | [m Liquid v | /kg] Vapour v | [kg/m3] Liquid d | Vapour d | Liquid H | [kJ/kg] Latent H | Vapour H | [kJ/K-kg] Liquid S | Vapour S | °C | -| -100 | 1.2 | 0.0006 | 10.0000 | 1679.0 | 0.100 | 113.3 | 192.8 | 306.1 | 0.6077 | 1.7210 | -100 | -| -99 | 1.3 | 0.0006 | 9.1670 | 1677.0 | 0.109 | 114.1 | 192.4 | 306.5 | 0.6124 | 1.7170 | -99 | -| -98 | 1.4 | 0.0006 | 8.4100 | 1674.0 | 0.119 | 115.0 | 192.0 | 307.0 | 0.6171 | 1.7130 | -98 | -| -97 | 1.6 | 0.0006 | 7.7250 | 1671.0 | 0.129 | 115.8 | 191.6 | 307.4 | 0.6218 | 1.7100 | -97 | -| -96 | 1.7 | 0.0006 | 7.1040 | 1669.0 | 0.141 | 116.6 | 191.3 | 307.9 | 0.6264 | 1.7060 | -96 | -| -95 | 1.9 | 0.0006 | 6.5400 | 1666.0 | 0.153 | 117.4 | 190.9 | 308.3 | 0.6310 | 1.7030 | -95 | -| -94 | 2.0 | 0.0006 | 6.0270 | 1663.0 | 0.166 | 118.2 | 190.6 | 308.8 | 0.6356 | 1.6990 | -94 | -| -93 | 2.2 | 0.0006 | 5.5610 | 1661.0 | 0.180 | 119.1 | 190.1 | 309.2 | 0.6402 | 1.6960 | -93 | -| -92 | 2.4 | 0.0006 | 5.1360 | 1658.0 | 0.195 | 119.9 | 189.8 | 309.7 | 0.6448 | 1.6920 | -92 | -| -91 | 2.6 | 0.0006 | 4.7480 | 1655.0 | 0.211 | 120.7 | 189.4 | 310.1 | 0.6493 | 1.6890 | -91 | -| -90 | 2.9 | 0.0006 | 4.3950 | 1653.0 | 0.228 | 121.5 | 189.1 | 310.6 | 0.6538 | 1.6860 | -90 | -| -89 | 3.1 | 0.0006 | 4.0720 | 1650.0 | 0.246 | 122.4 | 188.6 | 311.0 | 0.6583 | 1.6830 | -89 | -| -88 | 3.4 | 0.0006 | 3.7760 | 1648.0 | 0.265 | 123.2 | 188.3 | 311.5 | 0.6628 | 1.6800 | -88 | -| -87 | 3.6 | 0.0006 | 3.5050 | 1645.0 | 0.285 | 124.0 | 188.0 | 312.0 | 0.6672 | 1.6770 | -87 | -| -86 | 3.9 | 0.0006 | 3.2570 | 1642.0 | 0.307 | 124.8 | 187.6 | 312.4 | 0.6716 | 1.6740 | -86 | -| -85 | 4.3 | 0.0006 | 3.0290 | 1640.0 | 0.330 | 125.7 | 187.2 | 312.9 | 0.6761 | 1.6710 | -85 | -| -84 | 4.6 | 0.0006 | 2.8190 | 1637.0 | 0.355 | 126.5 | 186.8 | 313.3 | 0.6804 | 1.6680 | -84 | -| -83 | 5.0 | 0.0006 | 2.6270 | 1634.0 | 0.381 | 127.3 | 186.5 | 313.8 | 0.6848 | 1.6660 | -83 | -| -82 | 5.3 | 0.0006 | 2.4490 | 1632.0 | 0.408 | 128.1 | 186.2 | 314.3 | 0.6892 | 1.6630 | -82 | -| -81 | 5.8 | 0.0006 | 2.2860 | 1629.0 | 0.437 | 129.0 | 185.7 | 314.7 | 0.6935 | 1.6600 | -81 | -| -80 | 6.2 | 0.0006 | 2.1360 | 1626.0 | 0.468 | 129.8 | 185.4 | 315.2 | 0.6978 | 1.6580 | -80 | -| -79 | 6.7 | 0.0006 | 1.9970 | 1624.0 | 0.501 | 130.6 | 185.1 | 315.7 | 0.7021 | 1.6550 | -79 | -| -78 | 7.1 | 0.0006 | 1.8680 | 1621.0 | 0.535 | 131.5 | 184.6 | 316.1 | 0.7064 | 1.6530 | -78 | -| -77 | 7.7 | 0.0006 | 1.7490 | 1618.0 | 0.572 | 132.3 | 184.3 | 316.6 | 0.7106 | 1.6500 | -77 | -| -76 | 8.2 | 0.0006 | 1.6390 | 1616.0 | 0.610 | 133.1 | 184.0 | 317.1 | 0.7149 | 1.6480 | -76 | -| -75 | 8.8 | 0.0006 | 1.5380 | 1613.0 | 0.650 | 134.0 | 183.5 | 317.5 | 0.7191 | 1.6450 | -75 | -| -74 | 9.4 | 0.0006 | 1.4430 | 1610.0 | 0.693 | 134.8 | 183.2 | 318.0 | 0.7233 | 1.6430 | -74 | -| -73 | 10.1 | 0.0006 | 1.3560 | 1608.0 | 0.738 | 135.6 | 182.9 | 318.5 | 0.7275 | 1.6410 | -73 | -| -72 | 10.8 | 0.0006 | 1.2740 | 1605.0 | 0.785 | 136.5 | 182.4 | 318.9 | 0.7317 | 1.6390 | -72 | -| -71 | 11.5 | 0.0006 | 1.1990 | 1602.0 | 0.834 | 137.3 | 182.1 | 319.4 | 0.7358 | 1.6370 | -71 | -| -70 | 12.3 | 0.0006 | 1.1290 | 1600.0 | 0.886 | 138.2 | 181.7 | 319.9 | 0.7400 | 1.6340 | -70 | -| -69 | 13.1 | 0.0006 | 1.0630 | 1597.0 | 0.941 | 139.0 | 181.3 | 320.3 | 0.7441 | 1.6320 | -69 | -| -68 | 14.0 | 0.0006 | 1.0020 | 1594.0 | 0.998 | 139.8 | 181.0 | 320.8 | 0.7482 | 1.6300 | -68 | -| -67 | 14.9 | 0.0006 | 0.9455 | 1591.0 | 1.058 | 140.7 | 180.6 | 321.3 | 0.7523 | 1.6280 | -67 | -| -66 | 15.8 | 0.0006 | 0.8925 | 1589.0 | 1.120 | 141.5 | 180.3 | 321.8 | 0.7564 | 1.6260 | -66 | -| -65 | 16.8 | 0.0006 | 0.8430 | 1586.0 | 1.186 | 142.4 | 179.8 | 322.2 | 0.7604 | 1.6250 | -65 | -| -64 | 17.9 | 0.0006 | 0.7968 | 1583.0 | 1.255 | 143.2 | 179.5 | 322.7 | 0.7645 | 1.6230 | -64 | -| -63 | 19.0 | 0.0006 | 0.7536 | 1581.0 | 1.327 | 144.1 | 179.1 | 323.2 | 0.7685 | 1.6210 | -63 | -| -62 | 20.1 | 0.0006 | 0.7132 | 1578.0 | 1.402 | 144.9 | 178.8 | 323.7 | 0.7726 | 1.6190 | -62 | -| -61 | 21.3 | 0.0006 | 0.6754 | 1575.0 | 1.481 | 145.8 | 178.3 | 324.1 | 0.7766 | 1.6170 | -61 | -| -60 | 22.6 | 0.0006 | 0.6399 | 1572.0 | 1.563 | 146.6 | 178.0 | 324.6 | 0.7806 | 1.6160 | -60 | -| -59 | 24.0 | 0.0006 | 0.6067 | 1570.0 | 1.648 | 147.5 | 177.6 | 325.1 | 0.7845 | 1.6140 | -59 | -| -58 | 25.4 | 0.0006 | 0.5755 | 1567.0 | 1.738 | 148.3 | 177.3 | 325.6 | 0.7885 | 1.6120 | -58 | -| -57 | 26.8 | 0.0006 | 0.5463 | 1564.0 | 1.831 | 149.2 | 176.8 | 326.0 | 0.7924 | 1.6110 | -57 | -| -56 | 28.4 | 0.0006 | 0.5188 | 1561.0 | 1.928 | 150.0 | 176.5 | 326.5 | 0.7964 | 1.6090 | -56 | -| -55 | 30.0 | 0.0006 | 0.4930 | 1559.0 | 2.029 | 150.9 | 176.1 | 327.0 | 0.8003 | 1.6080 | -55 | -| -54 | 31.6 | 0.0006 | 0.4687 | 1556.0 | 2.134 | 151.7 | 175.8 | 327.5 | 0.8042 | 1.6060 | -54 | -| -53 | 33.4 | 0.0006 | 0.4458 | 1553.0 | 2.243 | 152.6 | 175.4 | 328.0 | 0.8081 | 1.6050 | -53 | -| -52 | 35.2 | 0.0007 | 0.4243 | 1550.0 | 2.357 | 153.5 | 174.9 | 328.4 | 0.8120 | 1.6030 | -52 | -| -51 | 37.1 | 0.0007 | 0.4040 | 1548.0 | 2.475 | 154.3 | 174.6 | 328.9 | 0.8159 | 1.6020 | -51 | -| -50 | 39.1 | 0.0007 | 0.3849 | 1545.0 | 2.598 | 155.2 | 174.2 | 329.4 | 0.8197 | 1.6000 | -50 | -| -49 | 41.2 | 0.0007 | 0.3669 | 1542.0 | 2.725 | 156.0 | 173.9 | 329.9 | 0.8236 | 1.5990 | -49 | -| -48 | 43.4 | 0.0007 | 0.3499 | 1539.0 | 2.858 | 156.9 | 173.4 | 330.3 | 0.8274 | 1.5980 | -48 | -| -47 | 45.6 | 0.0007 | 0.3339 | 1536.0 | 2.995 | 157.8 | 173.0 | 330.8 | 0.8313 | 1.5960 | -47 | +|Temp|Pressure||Volume|||Density||Enthalpy|||Entropy|Temp| +|---|---|---|---|---|---|---|---|---|---|---|---|---| +|°C|[kPa]|[m3 Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m3] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C| -**3** - -**f g f g f** fg **g f g** +|-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| +|---|---|---|---|---|---|---|---|---|---|---|---| +|-99|1.3|0.0006|9.1670|1677.0|0.109|114.1|192.4|306.5|0.6124|1.7170|-99| +|-98|1.4|0.0006|8.4100|1674.0|0.119|115.0|192.0|307.0|0.6171|1.7130|-98| +|-97|1.6|0.0006|7.7250|1671.0|0.129|115.8|191.6|307.4|0.6218|1.7100|-97| +|-96|1.7|0.0006|7.1040|1669.0|0.141|116.6|191.3|307.9|0.6264|1.7060|-96| +|-95|1.9|0.0006|6.5400|1666.0|0.153|117.4|190.9|308.3|0.6310|1.7030|-95| +|-94|2.0|0.0006|6.0270|1663.0|0.166|118.2|190.6|308.8|0.6356|1.6990|-94| +|-93|2.2|0.0006|5.5610|1661.0|0.180|119.1|190.1|309.2|0.6402|1.6960|-93| +|-92|2.4|0.0006|5.1360|1658.0|0.195|119.9|189.8|309.7|0.6448|1.6920|-92| +|-91|2.6|0.0006|4.7480|1655.0|0.211|120.7|189.4|310.1|0.6493|1.6890|-91| +|-90|2.9|0.0006|4.3950|1653.0|0.228|121.5|189.1|310.6|0.6538|1.6860|-90| +|-89|3.1|0.0006|4.0720|1650.0|0.246|122.4|188.6|311.0|0.6583|1.6830|-89| +|-88|3.4|0.0006|3.7760|1648.0|0.265|123.2|188.3|311.5|0.6628|1.6800|-88| +|-87|3.6|0.0006|3.5050|1645.0|0.285|124.0|188.0|312.0|0.6672|1.6770|-87| +|-86|3.9|0.0006|3.2570|1642.0|0.307|124.8|187.6|312.4|0.6716|1.6740|-86| +|-85|4.3|0.0006|3.0290|1640.0|0.330|125.7|187.2|312.9|0.6761|1.6710|-85| +|-84|4.6|0.0006|2.8190|1637.0|0.355|126.5|186.8|313.3|0.6804|1.6680|-84| +|-83|5.0|0.0006|2.6270|1634.0|0.381|127.3|186.5|313.8|0.6848|1.6660|-83| +|-82|5.3|0.0006|2.4490|1632.0|0.408|128.1|186.2|314.3|0.6892|1.6630|-82| +|-81|5.8|0.0006|2.2860|1629.0|0.437|129.0|185.7|314.7|0.6935|1.6600|-81| +|-80|6.2|0.0006|2.1360|1626.0|0.468|129.8|185.4|315.2|0.6978|1.6580|-80| +|-79|6.7|0.0006|1.9970|1624.0|0.501|130.6|185.1|315.7|0.7021|1.6550|-79| +|-78|7.1|0.0006|1.8680|1621.0|0.535|131.5|184.6|316.1|0.7064|1.6530|-78| +|-77|7.7|0.0006|1.7490|1618.0|0.572|132.3|184.3|316.6|0.7106|1.6500|-77| +|-76|8.2|0.0006|1.6390|1616.0|0.610|133.1|184.0|317.1|0.7149|1.6480|-76| +|-75|8.8|0.0006|1.5380|1613.0|0.650|134.0|183.5|317.5|0.7191|1.6450|-75| +|-74|9.4|0.0006|1.4430|1610.0|0.693|134.8|183.2|318.0|0.7233|1.6430|-74| +|-73|10.1|0.0006|1.3560|1608.0|0.738|135.6|182.9|318.5|0.7275|1.6410|-73| +|-72|10.8|0.0006|1.2740|1605.0|0.785|136.5|182.4|318.9|0.7317|1.6390|-72| +|-71|11.5|0.0006|1.1990|1602.0|0.834|137.3|182.1|319.4|0.7358|1.6370|-71| +|-70|12.3|0.0006|1.1290|1600.0|0.886|138.2|181.7|319.9|0.7400|1.6340|-70| +|-69|13.1|0.0006|1.0630|1597.0|0.941|139.0|181.3|320.3|0.7441|1.6320|-69| +|-68|14.0|0.0006|1.0020|1594.0|0.998|139.8|181.0|320.8|0.7482|1.6300|-68| +|-67|14.9|0.0006|0.9455|1591.0|1.058|140.7|180.6|321.3|0.7523|1.6280|-67| +|-66|15.8|0.0006|0.8925|1589.0|1.120|141.5|180.3|321.8|0.7564|1.6260|-66| +|-65|16.8|0.0006|0.8430|1586.0|1.186|142.4|179.8|322.2|0.7604|1.6250|-65| +|-64|17.9|0.0006|0.7968|1583.0|1.255|143.2|179.5|322.7|0.7645|1.6230|-64| +|-63|19.0|0.0006|0.7536|1581.0|1.327|144.1|179.1|323.2|0.7685|1.6210|-63| +|-62|20.1|0.0006|0.7132|1578.0|1.402|144.9|178.8|323.7|0.7726|1.6190|-62| +|-61|21.3|0.0006|0.6754|1575.0|1.481|145.8|178.3|324.1|0.7766|1.6170|-61| +|-60|22.6|0.0006|0.6399|1572.0|1.563|146.6|178.0|324.6|0.7806|1.6160|-60| +|-59|24.0|0.0006|0.6067|1570.0|1.648|147.5|177.6|325.1|0.7845|1.6140|-59| +|-58|25.4|0.0006|0.5755|1567.0|1.738|148.3|177.3|325.6|0.7885|1.6120|-58| +|-57|26.8|0.0006|0.5463|1564.0|1.831|149.2|176.8|326.0|0.7924|1.6110|-57| +|-56|28.4|0.0006|0.5188|1561.0|1.928|150.0|176.5|326.5|0.7964|1.6090|-56| +|-55|30.0|0.0006|0.4930|1559.0|2.029|150.9|176.1|327.0|0.8003|1.6080|-55| +|-54|31.6|0.0006|0.4687|1556.0|2.134|151.7|175.8|327.5|0.8042|1.6060|-54| +|-53|33.4|0.0006|0.4458|1553.0|2.243|152.6|175.4|328.0|0.8081|1.6050|-53| +|-52|35.2|0.0007|0.4243|1550.0|2.357|153.5|174.9|328.4|0.8120|1.6030|-52| +|-51|37.1|0.0007|0.4040|1548.0|2.475|154.3|174.6|328.9|0.8159|1.6020|-51| +|-50|39.1|0.0007|0.3849|1545.0|2.598|155.2|174.2|329.4|0.8197|1.6000|-50| +|-49|41.2|0.0007|0.3669|1542.0|2.725|156.0|173.9|329.9|0.8236|1.5990|-49| +|-48|43.4|0.0007|0.3499|1539.0|2.858|156.9|173.4|330.3|0.8274|1.5980|-48| +|-47|45.6|0.0007|0.3339|1536.0|2.995|157.8|173.0|330.8|0.8313|1.5960|-47|