Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
3081f94e72 |
@@ -1,87 +0,0 @@
|
||||
name: Publish Rust crate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['Cargo.toml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("Cargo.toml").read_text())["package"]["version"])')
|
||||
OLD_VERSION=$(git show HEAD~1:Cargo.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["package"]["version"])')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/crate-version.json --write-out "%{http_code}" \
|
||||
-H "User-Agent: firecrawl/pdf-inspector publish workflow (https://github.com/firecrawl/pdf-inspector)" \
|
||||
"https://crates.io/api/v1/crates/pdf-inspector/$NEW_VERSION")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/crate-version.json
|
||||
echo "Unexpected crates.io response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
publish:
|
||||
name: Publish to crates.io
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
environment: crates-io
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Verify package
|
||||
run: cargo publish --dry-run
|
||||
|
||||
- name: Authenticate with crates.io
|
||||
id: auth
|
||||
uses: rust-lang/crates-io-auth-action@v1
|
||||
|
||||
- name: Publish crate
|
||||
run: cargo publish
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
|
||||
+1
-1
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2.0"
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# pdf-inspector
|
||||
|
||||
[](https://crates.io/crates/pdf-inspector)
|
||||
[](https://www.npmjs.com/package/@firecrawl/pdf-inspector)
|
||||
|
||||
Fast Rust library for PDF classification and text extraction. Detects whether a PDF is text-based or scanned, extracts text with position awareness, and converts to clean Markdown — all without OCR. Includes bindings for [Python](docs/python.md) and [Node.js](napi/README.md).
|
||||
|
||||
Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in under 200ms, skipping expensive OCR services for the ~54% of PDFs that don't need them.
|
||||
@@ -74,17 +71,9 @@ console.log(result.markdown); // Markdown string or null
|
||||
|
||||
### Rust
|
||||
|
||||
Install from [crates.io](https://crates.io/crates/pdf-inspector):
|
||||
|
||||
```bash
|
||||
cargo add pdf-inspector
|
||||
```
|
||||
|
||||
Or add it manually:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pdf-inspector = "0.1"
|
||||
pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
|
||||
```
|
||||
|
||||
```rust
|
||||
@@ -102,34 +91,29 @@ if let Some(markdown) = &result.markdown {
|
||||
### CLI
|
||||
|
||||
```bash
|
||||
# Install the CLI tools
|
||||
cargo install pdf-inspector
|
||||
|
||||
# Convert PDF to Markdown
|
||||
pdf2md document.pdf
|
||||
cargo run --bin pdf2md -- document.pdf
|
||||
|
||||
# JSON output (for piping)
|
||||
pdf2md document.pdf --json
|
||||
cargo run --bin pdf2md -- document.pdf --json
|
||||
|
||||
# Raw markdown only (no headers)
|
||||
pdf2md document.pdf --raw
|
||||
cargo run --bin pdf2md -- document.pdf --raw
|
||||
|
||||
# Insert page break markers (<!-- Page N -->)
|
||||
pdf2md document.pdf --pages
|
||||
cargo run --bin pdf2md -- document.pdf --pages
|
||||
|
||||
# Process only specific pages
|
||||
pdf2md document.pdf --select-pages 1,3,5-10
|
||||
cargo run --bin pdf2md -- document.pdf --select-pages 1,3,5-10
|
||||
|
||||
# Detection only (no extraction)
|
||||
detect-pdf document.pdf
|
||||
detect-pdf document.pdf --json
|
||||
cargo run --bin detect-pdf -- document.pdf
|
||||
cargo run --bin detect-pdf -- document.pdf --json
|
||||
|
||||
# Detection + layout analysis (tables, columns)
|
||||
detect-pdf document.pdf --analyze --json
|
||||
cargo run --bin detect-pdf -- document.pdf --analyze --json
|
||||
```
|
||||
|
||||
From a source checkout, use `cargo run --bin pdf2md -- document.pdf` or `cargo run --bin detect-pdf -- document.pdf` instead.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
# Publishing
|
||||
|
||||
The Rust crate is published to [crates.io](https://crates.io/crates/pdf-inspector) with trusted publishing from GitHub Actions. The first release was published manually; future releases publish from `.github/workflows/publish-crate.yml` when a `Cargo.toml` version change lands on `main`.
|
||||
|
||||
## crates.io Trusted Publisher
|
||||
|
||||
Configure the trusted publisher for the `pdf-inspector` crate with:
|
||||
|
||||
- Repository: `firecrawl/pdf-inspector`
|
||||
- Workflow: `publish-crate.yml`
|
||||
- Environment: `crates-io`
|
||||
|
||||
The workflow uses `rust-lang/crates-io-auth-action@v1` to exchange GitHub's OIDC token for a short-lived crates.io token, then passes it to `cargo publish`.
|
||||
|
||||
## Release Steps
|
||||
|
||||
1. Update `version` in `Cargo.toml`.
|
||||
2. Merge the version bump to `main`.
|
||||
3. The publish workflow compares the new `Cargo.toml` version with `HEAD~1`, runs `cargo publish --dry-run`, then publishes if that version is not already on crates.io.
|
||||
|
||||
If `Cargo.toml` changes without a package version bump, the workflow exits without publishing.
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.9.5",
|
||||
"version": "1.9.3",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+21
-7
@@ -858,13 +858,6 @@ pub(crate) fn extract_text_from_operand(
|
||||
// unmapped. Don't fall through to text-interpretation fallbacks
|
||||
// (Latin-1, UTF-16, etc.) which would misinterpret CID bytes as
|
||||
// character codes (e.g. CID 0x01A9 → Latin-1 "©").
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
|
||||
// Try our custom encoding map from Differences arrays.
|
||||
// The Differences array overrides specific codes in a base encoding (typically
|
||||
@@ -973,6 +966,27 @@ pub(crate) fn extract_text_from_operand(
|
||||
return Some(symbol_text);
|
||||
}
|
||||
|
||||
// Latin-1 fallback. Safe ONLY for fonts that use single-byte
|
||||
// encodings — for these, an unmapped byte is a valid character
|
||||
// code in Latin-1/WinAnsi space. CID fonts (Type0 / Identity-H)
|
||||
// emit multi-byte CIDs that aren't characters; per-byte Latin-1
|
||||
// produces mojibake (e.g. 2-byte CID 0xCDD9 → "ÍÙ" for the
|
||||
// production scrape_id 019de78c-... samples).
|
||||
//
|
||||
// For a CID font (has_cmap is set OR a /ToUnicode reference
|
||||
// exists) with any non-ASCII bytes, emit a single U+FFFD per
|
||||
// CID instead. This both replaces the mojibake with a proper
|
||||
// "decode failed" marker AND keeps `detect_encoding_issues`
|
||||
// tripping so the page is flagged for OCR — the existing
|
||||
// garbage-detection path that the high-Latin-1 mojibake used
|
||||
// to satisfy by accident.
|
||||
if is_type0_cid_font && bytes.iter().any(|&b| b > 0x7F) {
|
||||
// 2-byte CIDs (Identity-H) are by far the common case; for
|
||||
// an odd byte count we still emit at least one marker so
|
||||
// detection downstream fires.
|
||||
let cid_count = (bytes.len() / 2).max(1);
|
||||
return Some("\u{FFFD}".repeat(cid_count));
|
||||
}
|
||||
// Pure ASCII bytes round-trip safely (Latin-1 == ASCII for
|
||||
// 0x00..=0x7F), and non-CID (Type1 / TrueType / Type3) fonts
|
||||
// use single-byte encodings where Latin-1 fallback is the
|
||||
|
||||
+2
-232
@@ -149,79 +149,6 @@ fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32)
|
||||
set
|
||||
}
|
||||
|
||||
/// Pre-scan body-size all-bold runs that are too long to be headings.
|
||||
///
|
||||
/// Some academic PDFs use an all-bold abstract/summary paragraph immediately
|
||||
/// after the author block. A line-local bold heading heuristic sees each
|
||||
/// wrapped visual line as "standalone" once the first line is misclassified,
|
||||
/// producing a stack of `##` headings. Multi-line body-size bold runs with a
|
||||
/// paragraph-sized word count should stay paragraph text.
|
||||
fn find_wrapped_bold_paragraph_lines(
|
||||
lines: &[TextLine],
|
||||
base_size: f32,
|
||||
para_threshold: f32,
|
||||
) -> HashSet<usize> {
|
||||
let mut set = HashSet::new();
|
||||
let mut i = 0usize;
|
||||
|
||||
while i < lines.len() {
|
||||
if !is_body_size_all_bold_line(&lines[i], base_size) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let start = i;
|
||||
let mut end = i;
|
||||
let mut word_count = lines[i].text().split_whitespace().count();
|
||||
|
||||
while end + 1 < lines.len()
|
||||
&& is_body_size_all_bold_line(&lines[end + 1], base_size)
|
||||
&& is_wrapped_same_style_line(&lines[end], &lines[end + 1], para_threshold)
|
||||
{
|
||||
end += 1;
|
||||
word_count += lines[end].text().split_whitespace().count();
|
||||
}
|
||||
|
||||
let line_count = end - start + 1;
|
||||
if line_count >= 3 && word_count > 20 {
|
||||
for idx in start..=end {
|
||||
set.insert(idx);
|
||||
}
|
||||
}
|
||||
|
||||
i = end + 1;
|
||||
}
|
||||
|
||||
set
|
||||
}
|
||||
|
||||
fn is_body_size_all_bold_line(line: &TextLine, base_size: f32) -> bool {
|
||||
let Some(first) = line.items.first() else {
|
||||
return false;
|
||||
};
|
||||
first.font_size >= base_size * 0.95
|
||||
&& first.font_size < base_size * 1.2
|
||||
&& line
|
||||
.items
|
||||
.iter()
|
||||
.all(|item| item.is_bold && (item.font_size - first.font_size).abs() < 0.5)
|
||||
}
|
||||
|
||||
fn is_wrapped_same_style_line(prev: &TextLine, next: &TextLine, para_threshold: f32) -> bool {
|
||||
if prev.page != next.page {
|
||||
return false;
|
||||
}
|
||||
|
||||
let y_gap = prev.y - next.y;
|
||||
if !(y_gap > 0.0 && y_gap <= para_threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let prev_x = prev.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
let next_x = next.items.first().map(|item| item.x).unwrap_or(0.0);
|
||||
(prev_x - next_x).abs() <= 40.0
|
||||
}
|
||||
|
||||
/// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
|
||||
///
|
||||
/// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
|
||||
@@ -470,8 +397,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
// between paragraphs at body font size. Inspired by opendataloader's
|
||||
// lookahead in HeadingProcessor (prevNode/nextNode context).
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
// Detect struct heading levels that are overused (body text mistagged as headings)
|
||||
let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
|
||||
@@ -485,7 +410,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut in_code_block = false;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
|
||||
let mut inserted_images: HashSet<(u32, usize)> = HashSet::new();
|
||||
|
||||
@@ -551,7 +475,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
current_page = line.page;
|
||||
prev_y = f32::MAX;
|
||||
prev_x = 0.0;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -566,7 +489,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(table_md);
|
||||
@@ -584,7 +506,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push('\n');
|
||||
output.push_str(image_md);
|
||||
@@ -606,18 +527,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
&& y_gap.abs() <= para_threshold
|
||||
&& (prev_x - line_x).abs() > 50.0
|
||||
&& prev_y < f32::MAX;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_band_switch || is_bold_to_regular_break) && in_paragraph {
|
||||
if (is_para_break || is_band_switch) && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -660,7 +572,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -714,9 +625,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -748,7 +656,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -771,7 +678,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("- {}", trimmed));
|
||||
output.push('\n');
|
||||
@@ -785,7 +691,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -832,7 +737,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(&format!("> {}\n", trimmed));
|
||||
continue;
|
||||
@@ -843,7 +747,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
if !in_code_block {
|
||||
output.push_str("```\n");
|
||||
@@ -864,11 +767,6 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -938,8 +836,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let para_threshold = compute_paragraph_threshold(&lines, base_size);
|
||||
|
||||
let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
|
||||
let wrapped_bold_paragraph_lines =
|
||||
find_wrapped_bold_paragraph_lines(&lines, base_size, para_threshold);
|
||||
|
||||
let mut output = String::new();
|
||||
let mut current_page = 0u32;
|
||||
@@ -948,7 +844,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
let mut in_paragraph = false;
|
||||
let mut last_list_x: Option<f32> = None;
|
||||
let mut prev_had_dot_leaders = false;
|
||||
let mut paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
for (line_idx, line) in lines.iter().enumerate() {
|
||||
// Page break
|
||||
@@ -965,7 +860,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
in_list = false;
|
||||
last_list_x = None;
|
||||
prev_had_dot_leaders = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
|
||||
if options.include_page_numbers {
|
||||
output.push_str(&format!("<!-- Page {} -->\n\n", current_page));
|
||||
@@ -976,18 +870,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
// (newspaper columns emitted sequentially on the same page).
|
||||
let y_gap = prev_y - line.y;
|
||||
let is_para_break = y_gap.abs() > para_threshold;
|
||||
let line_all_bold = !line.items.is_empty() && line.items.iter().all(|item| item.is_bold);
|
||||
let line_in_wrapped_bold_run = wrapped_bold_paragraph_lines.contains(&line_idx);
|
||||
let is_bold_to_regular_break = in_paragraph
|
||||
&& paragraph_in_wrapped_bold_run
|
||||
&& !line_in_wrapped_bold_run
|
||||
&& !line_all_bold
|
||||
&& y_gap > base_size * 1.2
|
||||
&& y_gap <= para_threshold;
|
||||
if (is_para_break || is_bold_to_regular_break) && in_paragraph {
|
||||
if is_para_break && in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Don't immediately end list on paragraph break
|
||||
// Let the continuation check below decide if we're still in a list
|
||||
@@ -1011,7 +896,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
output.push_str("\n\n");
|
||||
@@ -1034,9 +918,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if !(1..=15).contains(&word_count) {
|
||||
return None;
|
||||
}
|
||||
if wrapped_bold_paragraph_lines.contains(&line_idx) {
|
||||
return None;
|
||||
}
|
||||
let rarity = font_size_rarity(line_font_size, &font_stats);
|
||||
let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
|
||||
let standalone = !in_paragraph;
|
||||
@@ -1054,7 +935,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let prefix = "#".repeat(header_level);
|
||||
// Use plain text for headers to avoid redundant formatting
|
||||
@@ -1069,7 +949,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
let formatted = format_list_item(trimmed);
|
||||
output.push_str(&formatted);
|
||||
@@ -1114,7 +993,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
if in_paragraph {
|
||||
output.push_str("\n\n");
|
||||
in_paragraph = false;
|
||||
paragraph_in_wrapped_bold_run = false;
|
||||
}
|
||||
// Use plain text for code blocks
|
||||
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
|
||||
@@ -1132,11 +1010,6 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
||||
}
|
||||
}
|
||||
output.push_str(trimmed);
|
||||
paragraph_in_wrapped_bold_run = if in_paragraph {
|
||||
paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run
|
||||
} else {
|
||||
line_in_wrapped_bold_run
|
||||
};
|
||||
in_paragraph = true;
|
||||
prev_had_dot_leaders = cur_dot_leaders;
|
||||
}
|
||||
@@ -1483,109 +1356,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wrapped_bold_abstract_is_not_split_into_headings() {
|
||||
// Regression for arXiv 1107.1353: the opening abstract paragraph is
|
||||
// entirely bold at body size. The first wrapped lines used to become
|
||||
// separate H2 headings, and the following body paragraph was joined to
|
||||
// the bold abstract because the paragraph gap is modest.
|
||||
let make = |text: &str, y: f32, font_size: f32, bold: bool| {
|
||||
let mut item = make_item(text, 1, None);
|
||||
item.y = y;
|
||||
item.font_size = font_size;
|
||||
item.height = font_size;
|
||||
item.is_bold = bold;
|
||||
item
|
||||
};
|
||||
|
||||
let lines = vec![
|
||||
make_line(vec![make(
|
||||
"Quantum Nature of Light Measured With a Single Detector",
|
||||
747.7,
|
||||
25.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Gesine A. Steudle1*, Stefan Schietinger1, David Höckel1",
|
||||
651.1,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Zwiller2, and Oliver Benson1",
|
||||
638.5,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"The introduction of light quanta by Einstein in 1905 triggered strong efforts to",
|
||||
607.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"demonstrate the quantum properties of light directly, without involving matter",
|
||||
594.8,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"quantization. It however took more than seven decades for the quantum granularity",
|
||||
582.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"of light to be observed in the fluorescence of single atoms. Single atoms emit",
|
||||
569.5,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"photons one at a time, this is typically demonstrated with a Hanbury-Brown-Twiss",
|
||||
556.9,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"Our work significantly simplifies a widely used photon-correlation technique.",
|
||||
544.2,
|
||||
11.0,
|
||||
true,
|
||||
)]),
|
||||
make_line(vec![make(
|
||||
"A photon is a single excitation of a mode of the electromagnetic field.",
|
||||
528.7,
|
||||
11.0,
|
||||
false,
|
||||
)]),
|
||||
];
|
||||
|
||||
let md = to_markdown_from_lines_with_tables_and_images(
|
||||
lines,
|
||||
MarkdownOptions::default(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
&std::collections::HashSet::new(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(
|
||||
md.contains("# Quantum Nature of Light Measured With a Single Detector"),
|
||||
"title should remain a heading: {md}"
|
||||
);
|
||||
assert!(
|
||||
!md.contains("## The introduction")
|
||||
&& !md.contains("## demonstrate")
|
||||
&& !md.contains("## quantization"),
|
||||
"bold abstract lines should not become headings: {md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains("technique.**\n\nA photon is a single excitation"),
|
||||
"body paragraph should be separated from bold abstract: {md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_role_code_multiline_accumulation() {
|
||||
let mut line1 = make_item("fn main() {", 1, Some(0));
|
||||
|
||||
+46
-564
@@ -566,7 +566,7 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if page_items.len() < 2 {
|
||||
if page_items.len() < 4 {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -575,12 +575,15 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
.max(1.0);
|
||||
let y_tol = (median_font_size * 0.75).clamp(4.0, 9.0);
|
||||
let rows = group_key_value_visual_rows(page_items, y_tol);
|
||||
if rows.is_empty() || rows.len() > 80 {
|
||||
if rows.len() < 2 || rows.len() > 80 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let split_x = infer_key_value_split_x(&rows, median_font_size)?;
|
||||
let mut kv_rows: Vec<KeyValueRow> = Vec::new();
|
||||
let mut paired_rows = 0usize;
|
||||
let mut section_rows = 0usize;
|
||||
let mut left_label_like = 0usize;
|
||||
let mut left_starts = Vec::new();
|
||||
let mut right_starts = Vec::new();
|
||||
|
||||
@@ -606,12 +609,18 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
item_indices.dedup();
|
||||
|
||||
if !left.is_empty() && !right.is_empty() {
|
||||
paired_rows += 1;
|
||||
if looks_like_key_value_label(&left) {
|
||||
left_label_like += 1;
|
||||
}
|
||||
if let Some(x) = left_items.first().map(|ri| ri.item.x) {
|
||||
left_starts.push(x);
|
||||
}
|
||||
if let Some(x) = right_items.first().map(|ri| ri.item.x) {
|
||||
right_starts.push(x);
|
||||
}
|
||||
} else if !left.is_empty() {
|
||||
section_rows += 1;
|
||||
}
|
||||
|
||||
kv_rows.push(KeyValueRow {
|
||||
@@ -622,75 +631,11 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
});
|
||||
}
|
||||
|
||||
if kv_rows.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let raw_left_only_rows = kv_rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && row.right.is_empty())
|
||||
.count();
|
||||
let raw_right_only_rows = kv_rows
|
||||
.iter()
|
||||
.filter(|row| row.left.is_empty() && !row.right.is_empty())
|
||||
.count();
|
||||
let edgar_tag_rows = key_value_rows_look_like_edgar_tags(&kv_rows);
|
||||
if edgar_tag_rows {
|
||||
kv_rows.retain(|row| !row.right.is_empty() || !is_edgar_table_boundary_cell(&row.left));
|
||||
}
|
||||
let header_inferred = !edgar_tag_rows && key_value_first_pair_is_header(&kv_rows);
|
||||
kv_rows = normalize_key_value_rows(kv_rows, header_inferred);
|
||||
|
||||
let paired_rows = kv_rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
.count();
|
||||
let section_rows = kv_rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && row.right.is_empty())
|
||||
.count();
|
||||
let dangling_right_rows = kv_rows
|
||||
.iter()
|
||||
.filter(|row| row.left.is_empty() && !row.right.is_empty())
|
||||
.count();
|
||||
let left_label_like = kv_rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
.filter(|row| looks_like_key_value_label(&row.left))
|
||||
.count();
|
||||
if paired_rows < 1 {
|
||||
return None;
|
||||
}
|
||||
if dangling_right_rows > 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let left_x = median_f32(left_starts).unwrap_or_else(|| {
|
||||
rows.iter()
|
||||
.flat_map(|row| row.items.iter().map(|ri| ri.item.x))
|
||||
.fold(f32::INFINITY, f32::min)
|
||||
});
|
||||
let right_x = median_f32(right_starts).unwrap_or(split_x);
|
||||
if !left_x.is_finite() || !right_x.is_finite() || right_x - left_x < 40.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let single_pair_allowed = key_value_single_pair_allowed(
|
||||
KeyValueSinglePairStats {
|
||||
paired_rows,
|
||||
section_rows,
|
||||
raw_left_only_rows,
|
||||
raw_right_only_rows,
|
||||
},
|
||||
&kv_rows,
|
||||
header_inferred,
|
||||
left_x,
|
||||
right_x,
|
||||
);
|
||||
if (kv_rows.len() < 2 || paired_rows < 2) && !single_pair_allowed {
|
||||
if kv_rows.len() < 2 || paired_rows < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let header_inferred = key_value_first_pair_is_header(&kv_rows);
|
||||
let data_pairs = if header_inferred {
|
||||
paired_rows.saturating_sub(1)
|
||||
} else {
|
||||
@@ -700,7 +645,7 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
return None;
|
||||
}
|
||||
|
||||
if section_rows > paired_rows * 2 + 2 && !single_pair_allowed {
|
||||
if section_rows > paired_rows * 2 + 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -714,25 +659,28 @@ pub(crate) fn try_build_key_value_table_from_rows(items: &[TextItem], page: u32)
|
||||
} else {
|
||||
left_label_like
|
||||
};
|
||||
if !header_inferred
|
||||
&& !edgar_tag_rows
|
||||
&& label_rows_for_score >= 2
|
||||
&& label_like_for_score * 2 < label_rows_for_score
|
||||
{
|
||||
if label_rows_for_score >= 2 && label_like_for_score * 2 < label_rows_for_score {
|
||||
return None;
|
||||
}
|
||||
|
||||
let left_x = median_f32(left_starts).unwrap_or_else(|| {
|
||||
rows.iter()
|
||||
.flat_map(|row| row.items.iter().map(|ri| ri.item.x))
|
||||
.fold(f32::INFINITY, f32::min)
|
||||
});
|
||||
let right_x = median_f32(right_starts).unwrap_or(split_x);
|
||||
if !left_x.is_finite() || !right_x.is_finite() || right_x - left_x < 40.0 {
|
||||
return None;
|
||||
}
|
||||
let right_cluster_count = significant_side_x_clusters(&rows, split_x, false);
|
||||
let marker_rows = marker_matrix_value_rows(&kv_rows);
|
||||
if !single_pair_allowed
|
||||
&& !edgar_tag_rows
|
||||
&& ((right_cluster_count >= 5 && paired_rows >= 3)
|
||||
|| (right_cluster_count >= 3 && marker_rows >= 3 && marker_rows * 2 >= paired_rows))
|
||||
if (right_cluster_count >= 5 && paired_rows >= 3)
|
||||
|| (right_cluster_count >= 3 && marker_rows >= 3 && marker_rows * 2 >= paired_rows)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
if !edgar_tag_rows && key_value_rows_look_like_prose(&kv_rows, header_inferred) {
|
||||
if key_value_rows_look_like_prose(&kv_rows, header_inferred) {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -817,82 +765,6 @@ struct KeyValueRow {
|
||||
item_indices: Vec<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct KeyValueSinglePairStats {
|
||||
paired_rows: usize,
|
||||
section_rows: usize,
|
||||
raw_left_only_rows: usize,
|
||||
raw_right_only_rows: usize,
|
||||
}
|
||||
|
||||
fn normalize_key_value_rows(rows: Vec<KeyValueRow>, header_inferred: bool) -> Vec<KeyValueRow> {
|
||||
let mut normalized: Vec<KeyValueRow> = Vec::with_capacity(rows.len());
|
||||
|
||||
for row in rows {
|
||||
if row.left.is_empty() && row.right.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if row.left.is_empty() && !row.right.is_empty() {
|
||||
if let Some(last) = normalized.last_mut() {
|
||||
if !last.right.is_empty() {
|
||||
append_key_value_text(&mut last.right, &row.right);
|
||||
last.item_indices.extend(row.item_indices);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
normalized.push(row);
|
||||
continue;
|
||||
}
|
||||
|
||||
if !row.left.is_empty() && row.right.is_empty() {
|
||||
let normalized_len = normalized.len();
|
||||
if let Some(last) = normalized.last_mut() {
|
||||
let last_is_header = header_inferred && normalized_len == 1;
|
||||
if !last_is_header
|
||||
&& !last.left.is_empty()
|
||||
&& !last.right.is_empty()
|
||||
&& key_value_left_continuation_allowed(&last.left, &row.left)
|
||||
{
|
||||
append_key_value_text(&mut last.left, &row.left);
|
||||
last.item_indices.extend(row.item_indices);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
normalized.push(row);
|
||||
}
|
||||
|
||||
normalized
|
||||
}
|
||||
|
||||
fn append_key_value_text(target: &mut String, addition: &str) {
|
||||
let addition = addition.trim();
|
||||
if addition.is_empty() {
|
||||
return;
|
||||
}
|
||||
if !target.trim().is_empty() {
|
||||
target.push(' ');
|
||||
}
|
||||
target.push_str(addition);
|
||||
}
|
||||
|
||||
fn key_value_left_continuation_allowed(previous_left: &str, continuation: &str) -> bool {
|
||||
let trimmed = continuation.trim();
|
||||
if trimmed.is_empty() || looks_like_key_value_section_label(trimmed) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let previous = previous_left.trim_end();
|
||||
let continuation_chars = trimmed.chars().count();
|
||||
let continuation_words = word_count_simple(trimmed);
|
||||
previous.ends_with(['-', '/', ',', ';', ':'])
|
||||
|| first_alpha_is_lowercase(trimmed)
|
||||
|| continuation_chars > 28
|
||||
|| continuation_words > 4
|
||||
}
|
||||
|
||||
fn group_key_value_visual_rows(mut items: Vec<RowItem>, y_tol: f32) -> Vec<VisualRow> {
|
||||
items.sort_by(|a, b| {
|
||||
b.item
|
||||
@@ -956,13 +828,6 @@ fn infer_key_value_split_x(rows: &[VisualRow], median_font_size: f32) -> Option<
|
||||
}
|
||||
|
||||
if splits.len() < 2 {
|
||||
let paired_visual_rows = rows.iter().filter(|row| row.items.len() >= 2).count();
|
||||
if splits.len() == 1
|
||||
&& paired_visual_rows == 1
|
||||
&& (rows.len() == 1 || rows.iter().all(|row| row.items.len() <= 2))
|
||||
{
|
||||
return splits.into_iter().next();
|
||||
}
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -1037,169 +902,16 @@ fn looks_like_key_value_label(cell: &str) -> bool {
|
||||
trimmed.chars().any(|c| c.is_alphabetic())
|
||||
}
|
||||
|
||||
fn key_value_rows_look_like_edgar_tags(rows: &[KeyValueRow]) -> bool {
|
||||
let paired_rows = rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
.count();
|
||||
if paired_rows < 2 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let tag_pairs = rows
|
||||
.iter()
|
||||
.filter(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
.filter(|row| is_edgar_tag_cell(&row.left))
|
||||
.count();
|
||||
let first_marker = rows.first().is_some_and(|row| {
|
||||
row.left.eq_ignore_ascii_case("<S>") && row.right.eq_ignore_ascii_case("<C>")
|
||||
});
|
||||
|
||||
tag_pairs >= 3 || (first_marker && tag_pairs >= 2)
|
||||
}
|
||||
|
||||
fn is_edgar_tag_cell(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
let Some(inner) = trimmed.strip_prefix('<').and_then(|s| s.strip_suffix('>')) else {
|
||||
return false;
|
||||
};
|
||||
!inner.is_empty()
|
||||
&& inner.len() <= 48
|
||||
&& inner
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || matches!(ch, '-' | '_'))
|
||||
}
|
||||
|
||||
fn is_edgar_table_boundary_cell(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
trimmed.eq_ignore_ascii_case("<TABLE>") || trimmed.eq_ignore_ascii_case("</TABLE>")
|
||||
}
|
||||
|
||||
fn key_value_single_pair_allowed(
|
||||
stats: KeyValueSinglePairStats,
|
||||
rows: &[KeyValueRow],
|
||||
header_inferred: bool,
|
||||
left_x: f32,
|
||||
right_x: f32,
|
||||
) -> bool {
|
||||
if header_inferred || stats.paired_rows != 1 || stats.section_rows != 0 || rows.len() != 1 {
|
||||
return false;
|
||||
}
|
||||
if right_x - left_x < 60.0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(row) = rows
|
||||
.iter()
|
||||
.find(|row| !row.left.is_empty() && !row.right.is_empty())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let left_chars = row.left.chars().count();
|
||||
let right_chars = row.right.chars().count();
|
||||
if !(2..=120).contains(&left_chars) || right_chars == 0 {
|
||||
return false;
|
||||
}
|
||||
if key_value_cell_looks_like_sentence(&row.left) {
|
||||
return false;
|
||||
}
|
||||
if stats.raw_left_only_rows == 0
|
||||
&& stats.raw_right_only_rows >= 2
|
||||
&& left_chars <= 70
|
||||
&& right_chars <= 1_500
|
||||
&& looks_like_key_value_label(&row.left)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if right_chars > 80 {
|
||||
return false;
|
||||
}
|
||||
if key_value_cell_looks_like_sentence(&row.right) && !compact_key_value_scalar(&row.right) {
|
||||
return false;
|
||||
}
|
||||
|
||||
(looks_like_key_value_label(&row.left) || left_chars <= 90)
|
||||
&& compact_key_value_scalar(&row.right)
|
||||
}
|
||||
|
||||
fn compact_key_value_scalar(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
let chars = trimmed.chars().count();
|
||||
let words = word_count_simple(trimmed);
|
||||
if trimmed.is_empty() || chars > 60 || words > 6 || trimmed.ends_with(['.', '!', '?']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
trimmed.chars().any(|ch| ch.is_ascii_digit())
|
||||
|| matches!(
|
||||
lower.as_str(),
|
||||
"yes" | "no" | "true" | "false" | "none" | "n/a" | "na"
|
||||
)
|
||||
|| words <= 4
|
||||
}
|
||||
|
||||
fn looks_like_key_value_section_label(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
let chars = trimmed.chars().count();
|
||||
let words = word_count_simple(trimmed);
|
||||
if !(1..=5).contains(&words) || !(2..=48).contains(&chars) {
|
||||
return false;
|
||||
}
|
||||
if trimmed.ends_with(['.', ',', ';', ':']) || first_alpha_is_lowercase(trimmed) {
|
||||
return false;
|
||||
}
|
||||
if trimmed
|
||||
.chars()
|
||||
.any(|ch| matches!(ch, '.' | ',' | ';' | '(' | ')' | '[' | ']'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
trimmed.chars().any(|ch| ch.is_alphabetic())
|
||||
}
|
||||
|
||||
fn first_alpha_is_lowercase(cell: &str) -> bool {
|
||||
cell.chars()
|
||||
.find(|ch| ch.is_alphabetic())
|
||||
.is_some_and(|ch| ch.is_lowercase())
|
||||
}
|
||||
|
||||
fn key_value_cell_looks_like_sentence(cell: &str) -> bool {
|
||||
let trimmed = cell.trim();
|
||||
let chars = trimmed.chars().count();
|
||||
chars > 90
|
||||
|| word_count_simple(trimmed) > 12
|
||||
|| (chars > 42 && trimmed.ends_with(['.', '!', '?']))
|
||||
}
|
||||
|
||||
fn key_value_rows_look_like_prose(rows: &[KeyValueRow], header_inferred: bool) -> bool {
|
||||
let mut left_cells = 0usize;
|
||||
let mut left_prose_cells = 0usize;
|
||||
let mut left_label_like = 0usize;
|
||||
let mut total_left_chars = 0usize;
|
||||
let mut long_sentence_cells = 0usize;
|
||||
let mut total_cells = 0usize;
|
||||
let mut total_chars = 0usize;
|
||||
let mut paired_rows = 0usize;
|
||||
let mut paired_sentence_rows = 0usize;
|
||||
let mut solo_prose_rows = 0usize;
|
||||
|
||||
for row in rows.iter().skip(usize::from(header_inferred)) {
|
||||
if !row.left.is_empty() && !row.right.is_empty() {
|
||||
paired_rows += 1;
|
||||
let left = row.left.trim();
|
||||
let right = row.right.trim();
|
||||
let left_prose = key_value_cell_looks_like_sentence(left);
|
||||
let right_prose = key_value_cell_looks_like_sentence(right);
|
||||
left_cells += 1;
|
||||
total_left_chars += left.chars().count();
|
||||
if looks_like_key_value_label(left) {
|
||||
left_label_like += 1;
|
||||
}
|
||||
if left_prose {
|
||||
left_prose_cells += 1;
|
||||
}
|
||||
if left_prose && right_prose {
|
||||
paired_sentence_rows += 1;
|
||||
}
|
||||
} else {
|
||||
let solo = if row.left.is_empty() {
|
||||
row.right.trim()
|
||||
@@ -1213,23 +925,30 @@ fn key_value_rows_look_like_prose(rows: &[KeyValueRow], header_inferred: bool) -
|
||||
solo_prose_rows += 1;
|
||||
}
|
||||
}
|
||||
for cell in [&row.left, &row.right] {
|
||||
let trimmed = cell.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
total_cells += 1;
|
||||
total_chars += trimmed.chars().count();
|
||||
if trimmed.chars().count() > 100
|
||||
|| (trimmed.chars().count() > 55 && trimmed.ends_with(['.', '!', '?']))
|
||||
{
|
||||
long_sentence_cells += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if paired_rows < 1 || left_cells == 0 {
|
||||
if paired_rows < 1 || total_cells == 0 {
|
||||
return true;
|
||||
}
|
||||
if solo_prose_rows >= 3 {
|
||||
return true;
|
||||
}
|
||||
if paired_rows >= 2 && paired_sentence_rows * 2 >= paired_rows {
|
||||
return true;
|
||||
}
|
||||
if !header_inferred && left_prose_cells * 2 >= left_cells {
|
||||
return true;
|
||||
}
|
||||
|
||||
let avg_left_chars = total_left_chars as f32 / left_cells as f32;
|
||||
!header_inferred && avg_left_chars > 70.0 && left_label_like * 2 < left_cells
|
||||
let avg_chars = total_chars as f32 / total_cells as f32;
|
||||
avg_chars > 75.0 || long_sentence_cells * 2 >= total_cells
|
||||
}
|
||||
|
||||
fn marker_matrix_value_rows(rows: &[KeyValueRow]) -> usize {
|
||||
@@ -1635,243 +1354,6 @@ mod tests {
|
||||
assert!(md.contains("|Engine Code|1ZR-FAE|"), "{md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_merges_wrapped_value_continuations() {
|
||||
let items = vec![
|
||||
make_char("Storage", 80.0, 700.0, 9.0, 42.0),
|
||||
make_char(
|
||||
"Store under normal conditions in dry rooms.",
|
||||
250.0,
|
||||
700.0,
|
||||
9.0,
|
||||
210.0,
|
||||
),
|
||||
make_char(
|
||||
"Protect from heat and humidity in the original packaging material.",
|
||||
250.0,
|
||||
686.0,
|
||||
9.0,
|
||||
315.0,
|
||||
),
|
||||
make_char("Shelf Life", 80.0, 668.0, 9.0, 48.0),
|
||||
make_char(
|
||||
"To obtain best performance use within 24 months.",
|
||||
250.0,
|
||||
668.0,
|
||||
9.0,
|
||||
255.0,
|
||||
),
|
||||
make_char("Technical Information", 80.0, 650.0, 9.0, 104.0),
|
||||
make_char(
|
||||
"The product is designed for repeated industrial use and long service life.",
|
||||
250.0,
|
||||
650.0,
|
||||
9.0,
|
||||
340.0,
|
||||
),
|
||||
make_char(
|
||||
"Additional details are provided for compatibility and installation planning.",
|
||||
250.0,
|
||||
636.0,
|
||||
9.0,
|
||||
350.0,
|
||||
),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.contains("|Field|Value|"), "{md}");
|
||||
assert!(
|
||||
md.contains(
|
||||
"|Storage|Store under normal conditions in dry rooms. Protect from heat and humidity in the original packaging material.|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains(
|
||||
"|Technical Information|The product is designed for repeated industrial use and long service life. Additional details are provided for compatibility and installation planning.|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_merges_wrapped_left_labels() {
|
||||
let items = vec![
|
||||
make_char("Title/Description", 76.0, 700.0, 9.0, 86.0),
|
||||
make_char("Instances", 350.0, 700.0, 9.0, 48.0),
|
||||
make_char("RE: Homes Gerald Ford lived in.", 76.0, 682.0, 9.0, 150.0),
|
||||
make_char("Box 7", 350.0, 682.0, 9.0, 28.0),
|
||||
make_char(
|
||||
"Grand Rapids Remembers Gerald R. Ford issue. Grand",
|
||||
76.0,
|
||||
664.0,
|
||||
9.0,
|
||||
245.0,
|
||||
),
|
||||
make_char("Box 7", 350.0, 664.0, 9.0, 28.0),
|
||||
make_char(
|
||||
"Rapids Magazine, September 1987, p. 65.",
|
||||
76.0,
|
||||
650.0,
|
||||
9.0,
|
||||
196.0,
|
||||
),
|
||||
make_char(
|
||||
"A Workhorse not a show horse: Gerald Ford remembered as humble.",
|
||||
76.0,
|
||||
632.0,
|
||||
9.0,
|
||||
290.0,
|
||||
),
|
||||
make_char("Box 7", 350.0, 632.0, 9.0, 28.0),
|
||||
make_char(
|
||||
"not flashy during his public life.",
|
||||
76.0,
|
||||
618.0,
|
||||
9.0,
|
||||
150.0,
|
||||
),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.starts_with("|Title/Description|Instances|"), "{md}");
|
||||
assert!(
|
||||
md.contains(
|
||||
"|Grand Rapids Remembers Gerald R. Ford issue. Grand Rapids Magazine, September 1987, p. 65.|Box 7|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
assert!(
|
||||
md.contains(
|
||||
"|A Workhorse not a show horse: Gerald Ford remembered as humble. not flashy during his public life.|Box 7|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_allows_tiny_two_cell_region() {
|
||||
let items = vec![
|
||||
make_char(
|
||||
"3M E-A-R Classic Small Earplug Uncorded",
|
||||
80.0,
|
||||
700.0,
|
||||
9.0,
|
||||
210.0,
|
||||
),
|
||||
make_char("02/05/24", 360.0, 700.0, 9.0, 42.0),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.contains("|Field|Value|"), "{md}");
|
||||
assert!(
|
||||
md.contains("|3M E-A-R Classic Small Earplug Uncorded|02/05/24|"),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_allows_single_wrapped_value_region() {
|
||||
let items = vec![
|
||||
make_char("Intrinsic Safety", 42.0, 174.0, 9.0, 60.0),
|
||||
make_char(
|
||||
"The powered air purifying respirator has been tested and classified",
|
||||
311.0,
|
||||
174.0,
|
||||
9.0,
|
||||
260.0,
|
||||
),
|
||||
make_char(
|
||||
"for intrinsic safety in hazardous locations by Underwriters Laboratory",
|
||||
311.0,
|
||||
160.0,
|
||||
9.0,
|
||||
270.0,
|
||||
),
|
||||
make_char(
|
||||
"for the following classes, divisions, groups, and temperature ratings.",
|
||||
311.0,
|
||||
146.0,
|
||||
9.0,
|
||||
275.0,
|
||||
),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.contains("|Field|Value|"), "{md}");
|
||||
assert!(
|
||||
md.contains(
|
||||
"|Intrinsic Safety|The powered air purifying respirator has been tested and classified for intrinsic safety in hazardous locations by Underwriters Laboratory for the following classes, divisions, groups, and temperature ratings.|"
|
||||
),
|
||||
"{md}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_rejects_leading_value_only_prose() {
|
||||
let items = vec![
|
||||
make_char(
|
||||
"3rd Party Authorization documenting the reason for the hardship.",
|
||||
260.0,
|
||||
714.0,
|
||||
9.0,
|
||||
310.0,
|
||||
),
|
||||
make_char("Borrower", 80.0, 696.0, 9.0, 44.0),
|
||||
make_char(
|
||||
"Homeowner has adequate income to support modified payments.",
|
||||
260.0,
|
||||
696.0,
|
||||
9.0,
|
||||
300.0,
|
||||
),
|
||||
make_char("Servicer", 80.0, 678.0, 9.0, 42.0),
|
||||
make_char(
|
||||
"Collects documentation and reviews hardship status.",
|
||||
260.0,
|
||||
678.0,
|
||||
9.0,
|
||||
260.0,
|
||||
),
|
||||
];
|
||||
|
||||
assert!(try_build_key_value_table_from_rows(&items, 1).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_recovers_edgar_tag_value_rows() {
|
||||
let items = vec![
|
||||
make_char("<S>", 70.0, 700.0, 9.0, 18.0),
|
||||
make_char("<C>", 240.0, 700.0, 9.0, 18.0),
|
||||
make_char("<PERIOD-TYPE>", 70.0, 684.0, 9.0, 78.0),
|
||||
make_char("3-MOS", 240.0, 684.0, 9.0, 30.0),
|
||||
make_char("<FISCAL-YEAR-END>", 70.0, 668.0, 9.0, 104.0),
|
||||
make_char("DEC-31-2000", 240.0, 668.0, 9.0, 66.0),
|
||||
make_char("<PERIOD-END>", 70.0, 652.0, 9.0, 76.0),
|
||||
make_char("MAR-31-2000", 240.0, 652.0, 9.0, 66.0),
|
||||
make_char("<CASH>", 70.0, 636.0, 9.0, 38.0),
|
||||
make_char("214", 240.0, 636.0, 9.0, 18.0),
|
||||
make_char("</TABLE>", 70.0, 620.0, 9.0, 46.0),
|
||||
];
|
||||
|
||||
let table = try_build_key_value_table_from_rows(&items, 1).unwrap();
|
||||
let md = table_to_markdown(&table);
|
||||
|
||||
assert!(md.starts_with("|Field|Value|"), "{md}");
|
||||
assert!(md.contains("|<S>|<C>|"), "{md}");
|
||||
assert!(md.contains("|<FISCAL-YEAR-END>|DEC-31-2000|"), "{md}");
|
||||
assert!(md.contains("|<CASH>|214|"), "{md}");
|
||||
assert!(!md.contains("</TABLE>"), "{md}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_key_value_builder_rejects_split_prose() {
|
||||
let items = vec![
|
||||
|
||||
Reference in New Issue
Block a user