Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a5ad20072 | ||
|
|
35cf88eae5 | ||
|
|
44b58a1f6b | ||
|
|
c409e3b8ca | ||
|
|
704ca588f5 | ||
|
|
6f4a523a06 | ||
|
|
2f23f07f6e | ||
|
|
0a9c120a6b | ||
|
|
7c8b09be67 | ||
|
|
35445c3208 |
@@ -2,14 +2,41 @@ name: Publish npm package
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
branches: [main]
|
||||
paths: ['napi/package.json']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(node -p "require('./napi/package.json').version")
|
||||
OLD_VERSION=$(git show HEAD~1:napi/package.json | node -p "JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')).version")
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
if [ "$NEW_VERSION" != "$OLD_VERSION" ]; then
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
build:
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true'
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
@@ -68,7 +95,7 @@ jobs:
|
||||
|
||||
publish:
|
||||
name: Publish to npm
|
||||
needs: build
|
||||
needs: [check-version, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -55,12 +55,12 @@ print(result.markdown) # Markdown string or None
|
||||
### Node.js
|
||||
|
||||
```bash
|
||||
npm install @firecrawl/pdf-inspector-js
|
||||
npm install firecrawl-pdf-inspector
|
||||
```
|
||||
|
||||
```javascript
|
||||
import { readFileSync } from 'fs';
|
||||
import { processPdf, classifyPdf } from '@firecrawl/pdf-inspector-js';
|
||||
import { processPdf, classifyPdf } from 'firecrawl-pdf-inspector';
|
||||
|
||||
const result = processPdf(readFileSync('document.pdf'));
|
||||
console.log(result.pdfType); // "TextBased", "Scanned", "ImageBased", "Mixed"
|
||||
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { createRequire } from "module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { version } = require("../package.json");
|
||||
|
||||
const HELP = `pdf-inspector v${version} — Fast PDF text extraction to Markdown
|
||||
|
||||
Usage:
|
||||
pdf-inspector <file> Extract markdown (default)
|
||||
pdf-inspector detect <file> Classify PDF type
|
||||
|
||||
Options:
|
||||
--json Output as JSON
|
||||
--pages <pages> Comma-separated page numbers (e.g. 1,3,5)
|
||||
-o, --output <file> Write output to file instead of stdout
|
||||
-h, --help Show this help
|
||||
-v, --version Show version
|
||||
|
||||
Examples:
|
||||
pdf-inspector document.pdf
|
||||
pdf-inspector document.pdf --json
|
||||
pdf-inspector document.pdf --pages 1,2,3
|
||||
pdf-inspector detect document.pdf --json
|
||||
cat document.pdf | pdf-inspector -`;
|
||||
|
||||
function die(msg) {
|
||||
process.stderr.write(`error: ${msg}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { json: false, pages: null, output: null, file: null, command: "extract" };
|
||||
let i = 0;
|
||||
|
||||
// Check for subcommand
|
||||
if (argv[0] === "detect") {
|
||||
opts.command = "detect";
|
||||
i = 1;
|
||||
}
|
||||
|
||||
while (i < argv.length) {
|
||||
const arg = argv[i];
|
||||
if (arg === "-h" || arg === "--help") {
|
||||
process.stdout.write(HELP + "\n");
|
||||
process.exit(0);
|
||||
} else if (arg === "-v" || arg === "--version") {
|
||||
process.stdout.write(`${version}\n`);
|
||||
process.exit(0);
|
||||
} else if (arg === "--json") {
|
||||
opts.json = true;
|
||||
} else if (arg === "--pages") {
|
||||
i++;
|
||||
if (!argv[i]) die("--pages requires a value (e.g. 1,3,5)");
|
||||
opts.pages = argv[i].split(",").map((p) => {
|
||||
const n = parseInt(p.trim(), 10);
|
||||
if (Number.isNaN(n) || n < 1) die(`invalid page number: ${p}`);
|
||||
return n;
|
||||
});
|
||||
} else if (arg === "-o" || arg === "--output") {
|
||||
i++;
|
||||
if (!argv[i]) die("-o requires a filename");
|
||||
opts.output = argv[i];
|
||||
} else if (arg === "-" || !arg.startsWith("-")) {
|
||||
if (opts.file) die(`unexpected argument: ${arg}`);
|
||||
opts.file = arg;
|
||||
} else {
|
||||
die(`unknown option: ${arg}`);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
|
||||
return opts;
|
||||
}
|
||||
|
||||
function readInput(file) {
|
||||
if (file === "-") {
|
||||
return readFileSync(0); // stdin fd
|
||||
}
|
||||
try {
|
||||
return readFileSync(file);
|
||||
} catch (err) {
|
||||
if (err.code === "ENOENT") die(`file not found: ${file}`);
|
||||
die(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
function output(text, outputPath) {
|
||||
if (outputPath) {
|
||||
writeFileSync(outputPath, text);
|
||||
} else {
|
||||
process.stdout.write(text);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- main ----
|
||||
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (!opts.file) {
|
||||
// Check if stdin is piped
|
||||
if (process.stdin.isTTY !== false) {
|
||||
process.stderr.write(HELP + "\n");
|
||||
process.exit(1);
|
||||
}
|
||||
opts.file = "-";
|
||||
}
|
||||
|
||||
const { processPdf, classifyPdf } = await import("../index.js");
|
||||
const buffer = readInput(opts.file);
|
||||
|
||||
if (opts.command === "detect") {
|
||||
const result = classifyPdf(buffer);
|
||||
if (opts.json) {
|
||||
output(JSON.stringify(result, null, 2) + "\n", opts.output);
|
||||
} else {
|
||||
const ocr = result.pagesNeedingOcr.length > 0
|
||||
? `, ${result.pagesNeedingOcr.length} pages need OCR`
|
||||
: "";
|
||||
output(`${result.pdfType} (${result.pageCount} pages, confidence: ${result.confidence.toFixed(2)}${ocr})\n`, opts.output);
|
||||
}
|
||||
} else {
|
||||
const result = processPdf(buffer, opts.pages ?? undefined);
|
||||
if (opts.json) {
|
||||
output(JSON.stringify(result, null, 2) + "\n", opts.output);
|
||||
} else {
|
||||
output((result.markdown ?? "") + "\n", opts.output);
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -1,9 +1,12 @@
|
||||
{
|
||||
"name": "firecrawl-pdf-inspector",
|
||||
"version": "0.7.1",
|
||||
"version": "1.0.0",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
"bin": {
|
||||
"pdf-inspector": "bin/pdf-inspector.mjs"
|
||||
},
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"pdf",
|
||||
@@ -20,6 +23,7 @@
|
||||
"index.js",
|
||||
"index.d.ts",
|
||||
"*.node",
|
||||
"bin/",
|
||||
"README.md"
|
||||
],
|
||||
"repository": {
|
||||
|
||||
+1954
-79
File diff suppressed because it is too large
Load Diff
@@ -216,6 +216,7 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut marked_content_stack: Vec<MarkedContentEntry> = Vec::new();
|
||||
let mut suppress_glyph_extraction = false;
|
||||
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
|
||||
let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
|
||||
/// Get the innermost MCID from the marked content stack.
|
||||
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
|
||||
stack.iter().rev().find_map(|e| e.mcid)
|
||||
@@ -349,8 +350,15 @@ pub(crate) fn extract_page_text_items(
|
||||
)
|
||||
})
|
||||
});
|
||||
// ActualText: suppress glyph extraction, just advance text matrix
|
||||
// ActualText: suppress glyph extraction, just advance text matrix.
|
||||
// Capture the FIRST glyph's text matrix as the rendering position
|
||||
// for the ActualText item. Td ops between BDC and the first Tj
|
||||
// may have moved the position to the correct line — the BDC-entry
|
||||
// position (actual_text_start_tm) can be on the previous line.
|
||||
if suppress_glyph_extraction {
|
||||
if actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
}
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
text_matrix[5] += w_ts * text_matrix[1];
|
||||
@@ -425,6 +433,10 @@ pub(crate) fn extract_page_text_items(
|
||||
let font_info = font_widths.get(¤t_font);
|
||||
let is_invisible = (text_rendering_mode == 3 && !include_invisible)
|
||||
|| suppress_glyph_extraction;
|
||||
// Capture first-glyph position for ActualText
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
}
|
||||
|
||||
// Compute space threshold based on font metrics when available
|
||||
let space_threshold = if let Some(font_info) = font_info {
|
||||
@@ -700,6 +712,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if actual_text.is_some() {
|
||||
suppress_glyph_extraction = true;
|
||||
actual_text_start_tm = Some(text_matrix);
|
||||
actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
|
||||
}
|
||||
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
|
||||
}
|
||||
@@ -707,8 +720,13 @@ pub(crate) fn extract_page_text_items(
|
||||
// End Marked Content — emit ActualText item with correct width
|
||||
if let Some(entry) = marked_content_stack.pop() {
|
||||
if let Some(at) = entry.actual_text {
|
||||
// Compute width from text matrix advancement during BDC..EMC
|
||||
if let Some(start_tm) = actual_text_start_tm.take() {
|
||||
// Use the first-glyph position (if available) instead of the
|
||||
// BDC-entry position. Td operators between BDC and the first
|
||||
// Tj may have moved the text position to the correct line —
|
||||
// the BDC-entry position can be on the previous line.
|
||||
let glyph_tm = actual_text_glyph_tm.take();
|
||||
let entry_tm = actual_text_start_tm.take();
|
||||
if let Some(start_tm) = glyph_tm.or(entry_tm) {
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
// Rust 1.95 introduced collapsible_match for `if` inside match arms.
|
||||
// The content-stream parsers use this pattern extensively (match on operator
|
||||
// name, then check `in_text_block && !op.operands.is_empty()`). Collapsing
|
||||
// these into match guards would hurt readability. Allow crate-wide.
|
||||
#![allow(clippy::collapsible_match)]
|
||||
|
||||
//! Smart PDF detection and text extraction using lopdf
|
||||
//!
|
||||
//! # Quick start
|
||||
|
||||
Reference in New Issue
Block a user