Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael MartellandClaude Fable 5
|
59b3fcd9e6 |
@@ -1,166 +0,0 @@
|
||||
name: Publish Python package
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['pyproject.toml']
|
||||
# Manual fallback: re-publish the current version without a version bump
|
||||
# (e.g. first run after PyPI trusted publishing is configured).
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check-version:
|
||||
name: Check version change
|
||||
# Guard manual dispatches too: PyPI trusted publishing matches
|
||||
# repo+workflow+environment but NOT branch, so without this a
|
||||
# workflow_dispatch from any branch could publish unmerged code.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
published: ${{ steps.check.outputs.published }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Check if version changed
|
||||
id: check
|
||||
run: |
|
||||
NEW_VERSION=$(python3 -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')
|
||||
echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
# Manual dispatch always rebuilds and publishes. Combined with
|
||||
# skip-existing on the publish step, this repairs partial releases
|
||||
# (PyPI's version endpoint returns 200 even when only some of the
|
||||
# expected wheels were uploaded).
|
||||
echo "manual dispatch: publishing v$NEW_VERSION (skip-existing handles uploaded files)"
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# .get(): the parent commit may predate the static version field
|
||||
# (pyproject.toml used dynamic = ["version"]) — treat that as a change
|
||||
# so the very first merge of this workflow publishes.
|
||||
OLD_VERSION=$(git show HEAD~1:pyproject.toml | python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"].get("version", ""))')
|
||||
echo "old=$OLD_VERSION new=$NEW_VERSION"
|
||||
|
||||
if [ "$NEW_VERSION" = "$OLD_VERSION" ]; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
HTTP_STATUS=$(curl --silent --show-error --output /tmp/pypi-version.json --write-out "%{http_code}" \
|
||||
"https://pypi.org/pypi/pdf-inspector/$NEW_VERSION/json")
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
echo "published=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pdf-inspector v$NEW_VERSION is already published to PyPI"
|
||||
;;
|
||||
404)
|
||||
echo "published=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
cat /tmp/pypi-version.json
|
||||
echo "Unexpected PyPI response: $HTTP_STATUS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
build:
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
name: Build ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- os: ubuntu-latest
|
||||
target: aarch64-unknown-linux-gnu
|
||||
# macos-13 was retired by GitHub; macos-15-intel is the remaining
|
||||
# Intel runner label (available through 2027).
|
||||
- os: macos-15-intel
|
||||
target: x86_64-apple-darwin
|
||||
- os: macos-14
|
||||
target: aarch64-apple-darwin
|
||||
- os: windows-latest
|
||||
target: x86_64-pc-windows-msvc
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Build wheel
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
args: --release --out dist
|
||||
manylinux: auto
|
||||
|
||||
- name: Upload wheel
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: wheels-${{ matrix.target }}
|
||||
path: dist/*.whl
|
||||
if-no-files-found: error
|
||||
|
||||
sdist:
|
||||
needs: check-version
|
||||
if: needs.check-version.outputs.changed == 'true' && needs.check-version.outputs.published == 'false'
|
||||
name: Build sdist
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build sdist
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
command: sdist
|
||||
args: --out dist
|
||||
|
||||
- name: Upload sdist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sdist
|
||||
path: dist/*.tar.gz
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
name: Publish to PyPI
|
||||
needs: [check-version, build, sdist]
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: List artifacts
|
||||
run: ls -la dist/
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: dist
|
||||
# Tolerate already-uploaded files so a manual re-run can complete
|
||||
# a release that previously failed partway through.
|
||||
skip-existing: true
|
||||
+1
-1
@@ -14,7 +14,7 @@ crate-type = ["lib", "cdylib"]
|
||||
|
||||
[dependencies]
|
||||
# Python bindings
|
||||
pyo3 = { version = "0.25", features = ["extension-module", "abi3-py38"], optional = true }
|
||||
pyo3 = { version = "0.25", features = ["extension-module"], optional = true }
|
||||
|
||||
# PDF parsing
|
||||
lopdf = { version = "0.41.0", features = ["rayon"] }
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@firecrawl/pdf-inspector",
|
||||
"version": "1.10.4",
|
||||
"version": "1.10.1",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
+3
-3
@@ -4,9 +4,9 @@ build-backend = "maturin"
|
||||
|
||||
[project]
|
||||
name = "pdf-inspector"
|
||||
# Bump this to publish to PyPI — CI publishes automatically when the version
|
||||
# changes on main (same flow as napi/package.json for npm).
|
||||
version = "0.2.1"
|
||||
# Version is sourced from Cargo.toml [package] version by maturin so the Python
|
||||
# artifact always tracks the crate release instead of drifting on its own.
|
||||
dynamic = ["version"]
|
||||
description = "Fast PDF inspection, classification, and text extraction with smart scanned vs text-based detection"
|
||||
license = { text = "MIT" }
|
||||
requires-python = ">=3.8"
|
||||
|
||||
+2
-43
@@ -11,37 +11,6 @@ use std::process;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Escape a string for embedding in a JSON string value.
|
||||
fn format_detector_ocr_reasons(reasons: &std::collections::BTreeMap<u32, Vec<String>>) -> String {
|
||||
reasons
|
||||
.iter()
|
||||
.map(|(page, page_reasons)| {
|
||||
let reasons_json = page_reasons
|
||||
.iter()
|
||||
.map(|reason| format!(r#""{}""#, json_escape(reason)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(r#"{{"page":{},"reasons":[{}]}}"#, page, reasons_json)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> String {
|
||||
reasons
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let reasons_json = entry
|
||||
.reasons
|
||||
.iter()
|
||||
.map(|reason| format!(r#""{}""#, json_escape(reason)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(r#"{{"page":{},"reasons":[{}]}}"#, entry.page, reasons_json)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
fn json_escape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len() + 16);
|
||||
for ch in s.chars() {
|
||||
@@ -148,13 +117,11 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
.iter()
|
||||
.map(|p| p.to_string())
|
||||
.collect();
|
||||
let ocr_reasons = format_ocr_reasons_by_page(&result.ocr_reasons_by_page);
|
||||
println!(
|
||||
r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#,
|
||||
r#"{{"pdf_type":"{}","page_count":{},"pages_needing_ocr":[{}],"is_complex":{},"pages_with_tables":[{}],"pages_with_columns":[{}],"detection_time_ms":{}}}"#,
|
||||
pdf_type_str(&result.pdf_type),
|
||||
result.page_count,
|
||||
ocr_pages.join(","),
|
||||
ocr_reasons,
|
||||
result.layout.is_complex,
|
||||
table_pages.join(","),
|
||||
col_pages.join(","),
|
||||
@@ -177,9 +144,6 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
println!("Page count: {}", result.page_count);
|
||||
if !result.pages_needing_ocr.is_empty() {
|
||||
println!("Pages needing OCR: {:?}", result.pages_needing_ocr);
|
||||
for entry in &result.ocr_reasons_by_page {
|
||||
println!(" page {}: {}", entry.page, entry.reasons.join(", "));
|
||||
}
|
||||
}
|
||||
println!();
|
||||
if result.layout.is_complex {
|
||||
@@ -220,9 +184,8 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
.iter()
|
||||
.map(|p| p.to_string())
|
||||
.collect();
|
||||
let ocr_reasons = format_detector_ocr_reasons(&result.ocr_reasons_by_page);
|
||||
println!(
|
||||
r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"ocr_reasons_by_page":[{}],"detection_time_ms":{}}}"#,
|
||||
r#"{{"pdf_type":"{}","page_count":{},"pages_sampled":{},"pages_with_text":{},"confidence":{:.2},"title":{},"ocr_recommended":{},"pages_needing_ocr":[{}],"detection_time_ms":{}}}"#,
|
||||
pdf_type_str(&result.pdf_type),
|
||||
result.page_count,
|
||||
result.pages_sampled,
|
||||
@@ -235,7 +198,6 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
.unwrap_or_else(|| "null".to_string()),
|
||||
result.ocr_recommended,
|
||||
ocr_pages.join(","),
|
||||
ocr_reasons,
|
||||
elapsed.as_millis()
|
||||
);
|
||||
} else {
|
||||
@@ -270,9 +232,6 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
|
||||
result.pages_needing_ocr, result.page_count
|
||||
);
|
||||
}
|
||||
for (page, reasons) in &result.ocr_reasons_by_page {
|
||||
println!(" page {}: {}", page, reasons.join(", "));
|
||||
}
|
||||
}
|
||||
if let Some(title) = &result.title {
|
||||
println!("Title: {}", title);
|
||||
|
||||
+2
-111
@@ -60,10 +60,6 @@ pub struct PdfTypeResult {
|
||||
/// 1-indexed page numbers that need OCR (image-only or insufficient text).
|
||||
/// Empty for TextBased. All pages for Scanned/ImageBased. Specific pages for Mixed.
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Per-page explanation for `pages_needing_ocr`: 1-indexed page → reason
|
||||
/// codes (`scanned`, `no_text`, `vector_text`, `suspected_garbled_text`).
|
||||
/// Only contains pages that need OCR.
|
||||
pub ocr_reasons_by_page: std::collections::BTreeMap<u32, Vec<String>>,
|
||||
}
|
||||
|
||||
/// Configuration for PDF type detection
|
||||
@@ -386,12 +382,7 @@ pub(crate) fn detect_from_document(
|
||||
let analysis = if let Some(cached) = analysis_cache.get(&page_num) {
|
||||
cached.clone()
|
||||
} else if let Some(&page_id) = pages.get(&page_num) {
|
||||
// Cache the fresh analysis so the reason-classification pass
|
||||
// below sees the real signals (vector_text, etc.) instead of
|
||||
// defaulting to "scanned".
|
||||
let a = analyze_page_content(doc, page_id);
|
||||
analysis_cache.insert(page_num, a.clone());
|
||||
a
|
||||
analyze_page_content(doc, page_id)
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
@@ -438,9 +429,6 @@ pub(crate) fn detect_from_document(
|
||||
let analysis = analyze_page_content(doc, page_id);
|
||||
if analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts {
|
||||
pages_needing_ocr.push(page_num);
|
||||
// Cache so the reason pass reports suspected_garbled_text
|
||||
// rather than defaulting to "scanned".
|
||||
analysis_cache.insert(page_num, analysis);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -448,19 +436,6 @@ pub(crate) fn detect_from_document(
|
||||
pages_needing_ocr.sort();
|
||||
pages_needing_ocr.dedup();
|
||||
|
||||
// Explain each OCR-flagged page. Pages we analyzed get a signal-derived
|
||||
// reason; pages flagged only by whole-document classification (unsampled
|
||||
// pages of a Scanned/ImageBased doc) default to `scanned`.
|
||||
let mut ocr_reasons_by_page: std::collections::BTreeMap<u32, Vec<String>> =
|
||||
std::collections::BTreeMap::new();
|
||||
for &page_num in &pages_needing_ocr {
|
||||
let reasons = match analysis_cache.get(&page_num) {
|
||||
Some(analysis) => page_ocr_reasons(analysis),
|
||||
None => vec![crate::OCR_REASON_SCANNED],
|
||||
};
|
||||
ocr_reasons_by_page.insert(page_num, reasons.into_iter().map(String::from).collect());
|
||||
}
|
||||
|
||||
// Try to get title from metadata
|
||||
let title = get_document_title(doc);
|
||||
|
||||
@@ -473,7 +448,6 @@ pub(crate) fn detect_from_document(
|
||||
title,
|
||||
ocr_recommended,
|
||||
pages_needing_ocr,
|
||||
ocr_reasons_by_page,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -513,7 +487,7 @@ fn distribute_pages(n: u32, total: u32) -> Vec<u32> {
|
||||
}
|
||||
|
||||
/// Page content analysis result
|
||||
#[derive(Clone, Default)]
|
||||
#[derive(Clone)]
|
||||
struct PageAnalysis {
|
||||
text_operator_count: u32,
|
||||
has_images: bool,
|
||||
@@ -549,31 +523,6 @@ struct PageAnalysis {
|
||||
has_decodable_text_fonts: bool,
|
||||
}
|
||||
|
||||
/// Explain *why* a page needs OCR, from its content analysis. Priority:
|
||||
/// undecodable fonts (`suspected_garbled_text`) and vector-outlined text
|
||||
/// (`vector_text`) come first because they persist even when a text layer is
|
||||
/// present; otherwise a page with no extractable text is `scanned` when an
|
||||
/// image backs it or `no_text` when nothing does.
|
||||
fn page_ocr_reasons(a: &PageAnalysis) -> Vec<&'static str> {
|
||||
let mut reasons = Vec::new();
|
||||
if a.has_identity_h_no_tounicode || a.has_only_type3_fonts {
|
||||
reasons.push(crate::OCR_REASON_SUSPECTED_GARBLED_TEXT);
|
||||
}
|
||||
if a.has_vector_text {
|
||||
reasons.push(crate::OCR_REASON_VECTOR_TEXT);
|
||||
}
|
||||
if reasons.is_empty() {
|
||||
let has_extractable_text = a.text_operator_count > 0 && a.unique_text_chars > 0;
|
||||
if !has_extractable_text && !a.has_images && !a.has_template_image {
|
||||
reasons.push(crate::OCR_REASON_NO_TEXT);
|
||||
} else {
|
||||
// Image-backed with no usable text, or too little text to trust.
|
||||
reasons.push(crate::OCR_REASON_SCANNED);
|
||||
}
|
||||
}
|
||||
reasons
|
||||
}
|
||||
|
||||
/// Extracted font information from a Resource dictionary entry.
|
||||
/// Stores the properties needed for decodability/identity-h checks
|
||||
/// without holding a reference to the document.
|
||||
@@ -1860,64 +1809,6 @@ fn get_document_title(doc: &Document) -> Option<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn page_ocr_reasons_classify() {
|
||||
// Scanned: no text, full-page image.
|
||||
let scanned = PageAnalysis {
|
||||
has_template_image: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(page_ocr_reasons(&scanned), vec![crate::OCR_REASON_SCANNED]);
|
||||
|
||||
// Image-only page (no template flag, but has an image).
|
||||
let image_only = PageAnalysis {
|
||||
has_images: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
page_ocr_reasons(&image_only),
|
||||
vec![crate::OCR_REASON_SCANNED]
|
||||
);
|
||||
|
||||
// No text, no image → no_text.
|
||||
let blank = PageAnalysis::default();
|
||||
assert_eq!(page_ocr_reasons(&blank), vec![crate::OCR_REASON_NO_TEXT]);
|
||||
|
||||
// Vector-outlined text.
|
||||
let vector = PageAnalysis {
|
||||
has_vector_text: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
page_ocr_reasons(&vector),
|
||||
vec![crate::OCR_REASON_VECTOR_TEXT]
|
||||
);
|
||||
|
||||
// Undecodable fonts → garbled, and it wins over the fall-through.
|
||||
let garbled = PageAnalysis {
|
||||
has_identity_h_no_tounicode: true,
|
||||
has_images: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
page_ocr_reasons(&garbled),
|
||||
vec![crate::OCR_REASON_SUSPECTED_GARBLED_TEXT]
|
||||
);
|
||||
|
||||
// A page with real extractable text and an image is not flagged here
|
||||
// as scanned/no_text (only reached for pages already needing OCR).
|
||||
let text_with_image = PageAnalysis {
|
||||
text_operator_count: 40,
|
||||
unique_text_chars: 120,
|
||||
has_images: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
page_ocr_reasons(&text_with_image),
|
||||
vec![crate::OCR_REASON_SCANNED]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_content_operators() {
|
||||
let mut uchars = HashSet::new();
|
||||
|
||||
+7
-45
@@ -252,47 +252,17 @@ fn suppress_table_underlines(
|
||||
}
|
||||
|
||||
let mut table_item_indices: HashSet<usize> = HashSet::new();
|
||||
// A detected "table" that swallows nearly every text item on the page
|
||||
// is a detection artifact (prose pages with boxed callouts or stacked
|
||||
// underline rules read as one giant grid), not a real table — letting
|
||||
// it through here erased every legitimate underline on the page
|
||||
// (text_dense__underline: rect detection claimed 52/52 items). Real
|
||||
// ruled tables share the page with headings, captions, and body text.
|
||||
let plausible = |table: &crate::tables::Table| {
|
||||
// Content sanity gate: prose pages with boxed callouts and stacked
|
||||
// underline rules can detect as a structurally rich "table" that
|
||||
// swallows every item on the page (text_dense__underline: a 4x8
|
||||
// grid claiming 52/52 items, one "cell" holding 806 chars of body
|
||||
// text) — suppressing there erased every legitimate underline on
|
||||
// the page. Real data-table cells are short values; a cell with
|
||||
// hundreds of characters means the grid captured flowing prose.
|
||||
let lens: Vec<usize> = table
|
||||
.cells
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|cell| !cell.trim().is_empty())
|
||||
.map(|cell| cell.chars().count())
|
||||
.collect();
|
||||
if lens.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let long = lens.iter().filter(|&&n| n > 100).count();
|
||||
(long as f32) < (lens.len() as f32) * 0.3
|
||||
};
|
||||
|
||||
if !rects.is_empty() {
|
||||
let (rect_tables, _) = crate::tables::detect_tables_from_rects(items, rects, page);
|
||||
for table in rect_tables.iter().filter(|t| plausible(t)) {
|
||||
table_item_indices.extend(table.item_indices.iter().copied());
|
||||
for table in rect_tables {
|
||||
table_item_indices.extend(table.item_indices);
|
||||
}
|
||||
}
|
||||
|
||||
if !lines.is_empty() {
|
||||
for table in crate::tables::detect_tables_from_lines(items, lines, page)
|
||||
.iter()
|
||||
.filter(|t| plausible(t))
|
||||
{
|
||||
table_item_indices.extend(table.item_indices.iter().copied());
|
||||
for table in crate::tables::detect_tables_from_lines(items, lines, page) {
|
||||
table_item_indices.extend(table.item_indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -903,17 +873,9 @@ pub(crate) fn merge_subscript_items(items: Vec<TextItem>) -> Vec<TextItem> {
|
||||
.chars()
|
||||
.last()
|
||||
.is_some_and(|c| c.is_alphabetic());
|
||||
// Strikeout boundaries block the merge (a struck word
|
||||
// must not extend its strike over a live footnote digit,
|
||||
// and a struck digit must not lose its own mark). An
|
||||
// underlined parent with an unmarked digit DOES merge:
|
||||
// the drawn rule easily misses the tiny digit's overlap
|
||||
// window, and refusing costs the whole subscript token
|
||||
// ("b"+"2" staying split). Visually the rule spans both.
|
||||
let marks_ok = parent.is_strikeout == item.is_strikeout
|
||||
&& (parent.is_underline == item.is_underline
|
||||
|| (parent.is_underline && !item.is_underline));
|
||||
if parent.font_size >= sub_threshold && ends_with_letter && marks_ok {
|
||||
let same_marks = parent.is_underline == item.is_underline
|
||||
&& parent.is_strikeout == item.is_strikeout;
|
||||
if parent.font_size >= sub_threshold && ends_with_letter && same_marks {
|
||||
let parent_right = parent.x + parent.width;
|
||||
let gap = item.x - parent_right;
|
||||
// Subscripts must be tightly adjacent (within ~1pt)
|
||||
|
||||
+7
-288
@@ -115,13 +115,7 @@ fn rules_from_graphics(rects: &[PdfRect], lines: &[UnderlineLine], page: u32) ->
|
||||
rules
|
||||
}
|
||||
|
||||
fn discard_repeated_ruling_rules(
|
||||
rules: Vec<Rule>,
|
||||
items: &[TextItem],
|
||||
rects: &[PdfRect],
|
||||
lines: &[UnderlineLine],
|
||||
page: u32,
|
||||
) -> Vec<Rule> {
|
||||
fn discard_repeated_ruling_rules(rules: Vec<Rule>) -> Vec<Rule> {
|
||||
if rules.len() < MIN_REPEATED_RULE_LEVELS {
|
||||
return rules;
|
||||
}
|
||||
@@ -129,142 +123,12 @@ fn discard_repeated_ruling_rules(
|
||||
rules
|
||||
.iter()
|
||||
.filter(|rule| {
|
||||
// A rule snugly owned by one text line is an underline even when
|
||||
// span-similar rules repeat down the page — documents that
|
||||
// underline many full-width lines (dense CJK business docs) look
|
||||
// exactly like table rulings to the repetition check, which used
|
||||
// to discard every one of them. Table rulings fail snugness:
|
||||
// row separators extend past their cells' text (or have no text
|
||||
// on the baseline above), and multi-column matches are still
|
||||
// culled by the tabular filter afterwards.
|
||||
// Same-row segmented rules (column-header separators) are
|
||||
// always rulings — each segment snugly owns its column label,
|
||||
// so snugness must not override that check.
|
||||
!is_segmented_row_ruling_rule(rule, &rules)
|
||||
&& ((has_snug_text_owner(rule, items)
|
||||
&& !has_flanking_verticals(rule, rects, lines, page))
|
||||
|| !is_repeated_ruling_rule(rule, &rules))
|
||||
!is_repeated_ruling_rule(rule, &rules) && !is_segmented_row_ruling_rule(rule, &rules)
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// True when a single text item both matches the rule vertically (baseline
|
||||
/// window) and horizontally contains it: the rule may not extend past the
|
||||
/// item's span by more than ~0.75em on either side. Underlines are drawn to
|
||||
/// the width of the text they decorate; table/form rulings span cells or
|
||||
/// full table width and overshoot any single item.
|
||||
/// A rule flanked by vertical strokes at its ends is a table/box border
|
||||
/// row edge, not an underline — underlined text lines have no vertical
|
||||
/// rules rising from their ends. Checked against raw stroked lines: a
|
||||
/// near-vertical segment whose x sits at either end of the rule and whose
|
||||
/// y-range covers the rule's row.
|
||||
fn has_flanking_verticals(
|
||||
rule: &Rule,
|
||||
rects: &[PdfRect],
|
||||
lines: &[UnderlineLine],
|
||||
page: u32,
|
||||
) -> bool {
|
||||
// A drawn rect that CONTAINS the rule vetoes rescue only with GRID
|
||||
// EVIDENCE: another drawn rect abutting it vertically (cell rows tile).
|
||||
// Height alone can't separate a table cell from a decorative callout
|
||||
// panel — genuine underlines live inside isolated filled panels, and
|
||||
// multiline table cells can be arbitrarily tall.
|
||||
let norm = |r: &PdfRect| {
|
||||
let (x_lo, x_hi) = if r.width >= 0.0 {
|
||||
(r.x, r.x + r.width)
|
||||
} else {
|
||||
(r.x + r.width, r.x)
|
||||
};
|
||||
let (y_lo, y_hi) = if r.height >= 0.0 {
|
||||
(r.y, r.y + r.height)
|
||||
} else {
|
||||
(r.y + r.height, r.y)
|
||||
};
|
||||
(x_lo, x_hi, y_lo, y_hi)
|
||||
};
|
||||
let page_rects: Vec<(f32, f32, f32, f32)> = rects
|
||||
.iter()
|
||||
.filter(|r| r.page == page && r.height.abs() > 6.0)
|
||||
.map(norm)
|
||||
.collect();
|
||||
let rect_flank = page_rects.iter().any(|&(x_lo, x_hi, y_lo, y_hi)| {
|
||||
let contains = x_lo <= rule.x1 + 2.0
|
||||
&& x_hi >= rule.x2 - 2.0
|
||||
&& y_lo <= rule.y + 2.0
|
||||
&& y_hi >= rule.y - 2.0;
|
||||
if !contains {
|
||||
return false;
|
||||
}
|
||||
// Grid evidence: a vertically abutting neighbor box with x-overlap.
|
||||
page_rects.iter().any(|&(nx_lo, nx_hi, ny_lo, ny_hi)| {
|
||||
let x_overlap = nx_hi.min(x_hi) - nx_lo.max(x_lo);
|
||||
if x_overlap <= 10.0 {
|
||||
return false;
|
||||
}
|
||||
(ny_lo - y_hi).abs() <= 3.0 || (y_lo - ny_hi).abs() <= 3.0
|
||||
})
|
||||
});
|
||||
if rect_flank {
|
||||
return true;
|
||||
}
|
||||
lines.iter().any(|l| {
|
||||
if l.page != page || (l.x1 - l.x2).abs() > 2.0 {
|
||||
return false;
|
||||
}
|
||||
let x = (l.x1 + l.x2) / 2.0;
|
||||
let near_end = (x - rule.x1).abs() <= 6.0 || (x - rule.x2).abs() <= 6.0;
|
||||
if !near_end {
|
||||
return false;
|
||||
}
|
||||
let (y_lo, y_hi) = if l.y1 <= l.y2 {
|
||||
(l.y1, l.y2)
|
||||
} else {
|
||||
(l.y2, l.y1)
|
||||
};
|
||||
y_lo <= rule.y + 2.0 && y_hi >= rule.y - 2.0
|
||||
})
|
||||
}
|
||||
|
||||
fn has_snug_text_owner(rule: &Rule, items: &[TextItem]) -> bool {
|
||||
// Underlines are drawn to the width of the text they decorate, but the
|
||||
// text may be split into several runs (CJK lines mix scripts and font
|
||||
// switches) — so ownership is judged against the UNION of the runs on
|
||||
// the rule's baseline row. Table/form rulings overshoot their row's
|
||||
// text (row separators span cell padding and empty columns), so they
|
||||
// fail either containment or coverage.
|
||||
let matched: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|item| is_underline_candidate(item) && rule_matches_item(rule, item))
|
||||
.collect();
|
||||
if matched.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let x1 = matched.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
|
||||
let x2 = matched
|
||||
.iter()
|
||||
.map(|i| i.x + i.width)
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
let max_fs = matched.iter().map(|i| i.font_size).fold(0.0, f32::max);
|
||||
let pad = (max_fs * 0.75).max(4.0);
|
||||
if rule.x1 < x1 - pad || rule.x2 > x2 + pad {
|
||||
return false;
|
||||
}
|
||||
let covered: f32 = matched.iter().map(|i| i.width).sum();
|
||||
if covered < rule.width() * 0.6 {
|
||||
return false;
|
||||
}
|
||||
// A table row also unions to the rule's span — but its cells sit apart.
|
||||
// An underlined text line is contiguous runs with word-sized gaps; any
|
||||
// column-sized hole between matched runs means this is a row ruling.
|
||||
let mut sorted = matched;
|
||||
sorted.sort_by(|a, b| a.x.total_cmp(&b.x));
|
||||
sorted.windows(2).all(|pair| {
|
||||
let gap = pair[1].x - (pair[0].x + pair[0].width);
|
||||
gap <= (max_fs * 2.0).max(12.0)
|
||||
})
|
||||
}
|
||||
|
||||
fn is_repeated_ruling_rule(rule: &Rule, rules: &[Rule]) -> bool {
|
||||
let mut y_levels: Vec<f32> = rules
|
||||
.iter()
|
||||
@@ -351,11 +215,9 @@ fn is_underline_candidate(item: &TextItem) -> bool {
|
||||
|
||||
fn rule_matches_item(rule: &Rule, item: &TextItem) -> bool {
|
||||
// Vertical window: underlines sit at or slightly below the baseline.
|
||||
// Latin fonts draw them at roughly 5-15% of the em below; CJK layouts
|
||||
// put them under the full em box, measured up to ~0.67em below the
|
||||
// baseline (text_dense__underline). Allow 0.72em (min 3pt) below and
|
||||
// 1pt above for rounding.
|
||||
let below = (item.font_size * 0.72).max(3.0);
|
||||
// Fonts draw them at roughly 5-15% of the em below; allow up to 35%
|
||||
// (min 3pt) below and 1pt above for rounding.
|
||||
let below = (item.font_size * 0.35).max(3.0);
|
||||
let y_min = item.y - below;
|
||||
let y_max = item.y + 1.0;
|
||||
if rule.y < y_min || rule.y > y_max {
|
||||
@@ -398,48 +260,12 @@ pub(crate) fn mark_underlined_items(
|
||||
lines: &[UnderlineLine],
|
||||
page: u32,
|
||||
) {
|
||||
let rules = discard_repeated_ruling_rules(
|
||||
rules_from_graphics(rects, lines, page),
|
||||
items,
|
||||
rects,
|
||||
lines,
|
||||
page,
|
||||
);
|
||||
let rules = discard_repeated_ruling_rules(rules_from_graphics(rects, lines, page));
|
||||
if rules.is_empty() {
|
||||
return;
|
||||
}
|
||||
let tabular_rules = tabular_row_separator_rule_indices(&rules, items);
|
||||
|
||||
// Math fraction bars are short horizontal lines with the numerator just
|
||||
// above AND the denominator just below — underline geometry from above,
|
||||
// but no underline has text hanging directly beneath it at fraction
|
||||
// distance. Only narrow rules qualify: real underlines under short
|
||||
// labels have their next text line a full line-pitch away.
|
||||
let fraction_rules: HashSet<usize> = rules
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, rule)| {
|
||||
rule.width() <= 60.0
|
||||
&& items.iter().any(|item| {
|
||||
if !is_underline_candidate(item) {
|
||||
return false;
|
||||
}
|
||||
// A denominator HUGS the bar (fraction typesetting
|
||||
// leaves ~0.1-0.2em) and is bar-sized. Both bounds
|
||||
// matter: a short last-line of a paragraph at normal
|
||||
// leading sits further below, and a full next text
|
||||
// line is far wider than the rule.
|
||||
let dy = rule.y - (item.y + item.height);
|
||||
let overlap = rule.x2.min(item.x + item.width) - rule.x1.max(item.x);
|
||||
dy > 0.0
|
||||
&& dy <= item.font_size * 0.3
|
||||
&& overlap > rule.width() * 0.5
|
||||
&& item.width <= rule.width() * 1.5
|
||||
})
|
||||
})
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
|
||||
for item in items.iter_mut() {
|
||||
if !is_underline_candidate(item) {
|
||||
continue;
|
||||
@@ -449,10 +275,7 @@ pub(crate) fn mark_underlined_items(
|
||||
if tabular_rules.contains(&rule_idx) {
|
||||
continue;
|
||||
}
|
||||
// The fraction guard only gates UNDERLINE marking — a rule that
|
||||
// reads as a fraction bar from below can still legitimately
|
||||
// strike through a line above it.
|
||||
if !fraction_rules.contains(&rule_idx) && rule_matches_item(rule, item) {
|
||||
if rule_matches_item(rule, item) {
|
||||
item.is_underline = true;
|
||||
}
|
||||
if rule_strikes_item(rule, item) {
|
||||
@@ -500,16 +323,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn cell_rect(x: f32, y: f32, width: f32, height: f32) -> PdfRect {
|
||||
PdfRect {
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
page: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn thin_rect(x: f32, y: f32, width: f32) -> PdfRect {
|
||||
PdfRect {
|
||||
x,
|
||||
@@ -730,100 +543,6 @@ mod tests {
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_snug_underlines_survive_ruling_filter() {
|
||||
// Dense docs underline many full-width lines: span-similar rules at
|
||||
// 3+ y-levels used to be discarded wholesale as table rulings.
|
||||
// Each rule here snugly matches one text line, so all must mark.
|
||||
let mut items = vec![
|
||||
item("first underlined line of text", 50.0, 700.0, 300.0, 11.0),
|
||||
item("second underlined line here", 50.0, 650.0, 300.0, 11.0),
|
||||
item("third underlined line as well", 50.0, 600.0, 300.0, 11.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(50.0, 350.0, 697.0),
|
||||
hline(50.0, 350.0, 647.0),
|
||||
hline(50.0, 350.0, 597.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snug_rescue_spans_split_runs_on_one_line() {
|
||||
// A single underlined line is often split into several runs (script
|
||||
// or font switches). The union of touching runs owns the rule.
|
||||
let mut items = vec![
|
||||
item("run one", 50.0, 700.0, 100.0, 11.0),
|
||||
item("run two", 150.5, 700.0, 100.0, 11.0),
|
||||
item("run three", 251.0, 700.0, 99.0, 11.0),
|
||||
item("other a", 50.0, 650.0, 300.0, 11.0),
|
||||
item("other b", 50.0, 600.0, 300.0, 11.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(50.0, 350.0, 697.0),
|
||||
hline(50.0, 350.0, 647.0),
|
||||
hline(50.0, 350.0, 597.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items[0].is_underline && items[1].is_underline && items[2].is_underline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snug_rescue_denied_for_row_with_cell_gaps() {
|
||||
// A full-width rule whose baseline row is several items separated by
|
||||
// column-sized gaps is a table row separator, not an underline —
|
||||
// even when span-similar rules repeat down the page.
|
||||
let mut items = vec![
|
||||
item("cell a", 50.0, 700.0, 60.0, 11.0),
|
||||
item("cell b", 190.0, 700.0, 60.0, 11.0),
|
||||
item("cell c", 330.0, 700.0, 70.0, 11.0),
|
||||
item("cell d", 50.0, 650.0, 60.0, 11.0),
|
||||
item("cell e", 190.0, 650.0, 60.0, 11.0),
|
||||
item("cell f", 330.0, 650.0, 70.0, 11.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(50.0, 400.0, 697.0),
|
||||
hline(50.0, 400.0, 647.0),
|
||||
hline(50.0, 400.0, 597.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &[], &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snug_rescue_denied_inside_cell_box() {
|
||||
// A rule snugly under one text line but enclosed by a drawn cell
|
||||
// box that TILES with vertical neighbors (grid evidence) is a row
|
||||
// ruling of a rect-grid table. Isolated boxes (callout panels) do
|
||||
// not veto — see repeated_snug_underlines_survive_ruling_filter.
|
||||
let mut items = vec![
|
||||
item("one wide cell row", 50.0, 700.0, 300.0, 11.0),
|
||||
item("second wide cell", 50.0, 650.0, 300.0, 11.0),
|
||||
item("third wide cell", 50.0, 600.0, 300.0, 11.0),
|
||||
];
|
||||
let lines = vec![
|
||||
hline(50.0, 350.0, 697.0),
|
||||
hline(50.0, 350.0, 647.0),
|
||||
hline(50.0, 350.0, 597.0),
|
||||
];
|
||||
let boxes = vec![
|
||||
cell_rect(45.0, 690.0, 320.0, 50.0),
|
||||
cell_rect(45.0, 640.0, 320.0, 50.0),
|
||||
cell_rect(45.0, 590.0, 320.0, 50.0),
|
||||
];
|
||||
|
||||
mark_underlined_items(&mut items, &boxes, &lines, 1);
|
||||
|
||||
assert!(items.iter().all(|item| !item.is_underline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_row_spaced_rule_segments_do_not_mark_column_labels() {
|
||||
let mut items = vec![
|
||||
|
||||
+19
-100
@@ -71,18 +71,6 @@ use tounicode::FontCMaps;
|
||||
/// broken font decoding or mojibake.
|
||||
pub const OCR_REASON_SUSPECTED_GARBLED_TEXT: &str = "suspected_garbled_text";
|
||||
|
||||
/// OCR reason: the page is a scanned image (a full-page raster / image-only
|
||||
/// page) with no usable text layer.
|
||||
pub const OCR_REASON_SCANNED: &str = "scanned";
|
||||
|
||||
/// OCR reason: the page has no extractable text and no image to OCR — blank,
|
||||
/// or content the parser cannot reach.
|
||||
pub const OCR_REASON_NO_TEXT: &str = "no_text";
|
||||
|
||||
/// OCR reason: the page's text is drawn as vector outlines (path operators)
|
||||
/// rather than real text operators, so it cannot be extracted as characters.
|
||||
pub const OCR_REASON_VECTOR_TEXT: &str = "vector_text";
|
||||
|
||||
// =========================================================================
|
||||
// Result type
|
||||
// =========================================================================
|
||||
@@ -673,51 +661,18 @@ pub fn extract_text_in_regions_mem(
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
// Exclusive item->region assignment: overlapping layout regions used
|
||||
// to extract shared items into EVERY region they touched (the
|
||||
// 1.5pt inclusion margin makes borders generous), duplicating whole
|
||||
// lines in the final markdown on 21% of bench docs — and downstream
|
||||
// duplicate-handling sometimes dropped the variant holding a
|
||||
// sentence tail, turning duplication into content LOSS. Each item
|
||||
// now belongs to the single region with the largest overlap area;
|
||||
// items are partitioned, never suppressed, so no content can vanish.
|
||||
let all_bounds: Vec<RegionBounds> = regions
|
||||
.iter()
|
||||
.map(|rect| {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
region_bounds(rx1, ry1, rx2, ry2, page_h, coords)
|
||||
})
|
||||
.collect();
|
||||
// Single pass over items: assign each to the best-overlap region and
|
||||
// bucket the clone directly (review: avoid a second O(items x
|
||||
// regions) traversal). `had_candidates` marks regions that touched
|
||||
// at least one item even if every one was assigned elsewhere.
|
||||
let mut region_items: Vec<Vec<TextItem>> = vec![Vec::new(); regions.len()];
|
||||
let mut had_candidates: Vec<bool> = vec![false; regions.len()];
|
||||
if let Some(items) = items {
|
||||
for item in items {
|
||||
let mut best: Option<usize> = None;
|
||||
let mut best_area = 0.0_f32;
|
||||
for (ri, b) in all_bounds.iter().enumerate() {
|
||||
if !region_overlaps_item(item, *b) {
|
||||
continue;
|
||||
}
|
||||
had_candidates[ri] = true;
|
||||
let area = region_item_overlap_area(item, *b);
|
||||
if area > best_area {
|
||||
best_area = area;
|
||||
best = Some(ri);
|
||||
}
|
||||
}
|
||||
if let Some(ri) = best {
|
||||
region_items[ri].push(item.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
for (region_idx, _rect) in regions.iter().enumerate() {
|
||||
let matched: Vec<TextItem> = std::mem::take(&mut region_items[region_idx]);
|
||||
let assigned_count = matched.len();
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
let matched: Vec<TextItem> = match items {
|
||||
Some(items) => items
|
||||
.iter()
|
||||
.filter(|item| region_overlaps_item(item, bounds))
|
||||
.cloned()
|
||||
.collect(),
|
||||
None => Vec::new(),
|
||||
};
|
||||
let has_text_quality_issue = region_items_have_decoding_issue(&matched);
|
||||
let text = collect_text_from_matched_items(matched, adaptive_threshold);
|
||||
let has_cid_issue = is_cid_garbage(&text);
|
||||
@@ -731,21 +686,8 @@ pub fn extract_text_in_regions_mem(
|
||||
// Check per-region text quality instead of blanket page-level
|
||||
// GID rejection. A GID font in a logo elsewhere on the page
|
||||
// shouldn't force GPU OCR for clean text regions.
|
||||
// A region whose ONLY overlapping items were assigned to a
|
||||
// better-overlapping neighbor must not fall back to OCR: the
|
||||
// pixels it would re-read belong to that neighbor, and OCR
|
||||
// would reintroduce the duplication exclusivity removed.
|
||||
// Before exclusive assignment these regions were non-empty
|
||||
// native (no OCR), so this preserves the old OCR load too.
|
||||
// Requires ZERO items assigned HERE: a region whose own
|
||||
// assigned items materialize to empty text (whitespace-only,
|
||||
// collector-filtered) keeps its OCR fallback.
|
||||
let lost_to_neighbor = text.trim().is_empty()
|
||||
&& ocr_reason.is_none()
|
||||
&& assigned_count == 0
|
||||
&& had_candidates[region_idx];
|
||||
let needs_ocr = !lost_to_neighbor
|
||||
&& (ocr_reason.is_some() || text.trim().is_empty() || is_garbage_text(&text));
|
||||
let needs_ocr =
|
||||
ocr_reason.is_some() || text.trim().is_empty() || is_garbage_text(&text);
|
||||
|
||||
page_results.push(RegionText {
|
||||
text,
|
||||
@@ -3261,26 +3203,8 @@ fn region_bounds(
|
||||
}
|
||||
}
|
||||
|
||||
/// Inclusion margin shared by the region/item overlap predicates and the
|
||||
/// exclusive-assignment area score — these MUST stay in sync: an item that
|
||||
/// passes the boolean guard must always have positive overlap area.
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
|
||||
/// Overlap area between an item and region bounds (same margin as the
|
||||
/// boolean test) — the exclusive-assignment score.
|
||||
fn region_item_overlap_area(item: &TextItem, bounds: RegionBounds) -> f32 {
|
||||
let item_x_max = item.x + text_utils::effective_width(item);
|
||||
let item_y_max = item.y + item.height;
|
||||
let x_overlap = (item_x_max.min(bounds.x_max + REGION_MARGIN)
|
||||
- item.x.max(bounds.x_min - REGION_MARGIN))
|
||||
.max(0.0);
|
||||
let y_overlap = (item_y_max.min(bounds.y_max + REGION_MARGIN)
|
||||
- item.y.max(bounds.y_min - REGION_MARGIN))
|
||||
.max(0.0);
|
||||
x_overlap * y_overlap
|
||||
}
|
||||
|
||||
fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let item_x_min = item.x;
|
||||
let item_x_max = item.x + text_utils::effective_width(item);
|
||||
let item_y_min = item.y;
|
||||
@@ -3296,6 +3220,7 @@ fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
|
||||
}
|
||||
|
||||
fn region_overlaps_rect(rect: &PdfRect, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let (x_min, y_min, x_max, y_max) = normalized_rect_edges(rect);
|
||||
ranges_overlap(
|
||||
x_min,
|
||||
@@ -3311,6 +3236,7 @@ fn region_overlaps_rect(rect: &PdfRect, bounds: RegionBounds) -> bool {
|
||||
}
|
||||
|
||||
fn region_overlaps_line(line: &PdfLine, bounds: RegionBounds) -> bool {
|
||||
const REGION_MARGIN: f32 = 1.5;
|
||||
let x_min = line.x1.min(line.x2);
|
||||
let x_max = line.x1.max(line.x2);
|
||||
let y_min = line.y1.min(line.y2);
|
||||
@@ -3539,7 +3465,6 @@ fn process_document(
|
||||
let pages_needing_ocr = detection.pages_needing_ocr;
|
||||
let title = detection.title;
|
||||
let confidence = detection.confidence;
|
||||
let detection_ocr_reasons = detection.ocr_reasons_by_page;
|
||||
|
||||
// DetectOnly → return immediately
|
||||
if options.mode == ProcessMode::DetectOnly {
|
||||
@@ -3549,7 +3474,7 @@ fn process_document(
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons),
|
||||
ocr_reasons_by_page: Vec::new(),
|
||||
title,
|
||||
confidence,
|
||||
layout: LayoutComplexity::default(),
|
||||
@@ -3565,7 +3490,7 @@ fn process_document(
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
ocr_reasons_by_page: page_ocr_reasons_vec(detection_ocr_reasons),
|
||||
ocr_reasons_by_page: Vec::new(),
|
||||
title,
|
||||
confidence,
|
||||
layout: LayoutComplexity::default(),
|
||||
@@ -3831,13 +3756,7 @@ fn process_document(
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
ocr_reasons_by_page: {
|
||||
// Detector reasons (scanned / no_text / vector_text / garbled) merged
|
||||
// with the markdown-stage garbled detection, deduped per page.
|
||||
let mut merged = detection_ocr_reasons;
|
||||
merge_ocr_reasons(&mut merged, text_quality_reasons_by_page);
|
||||
page_ocr_reasons_vec(merged)
|
||||
},
|
||||
ocr_reasons_by_page: page_ocr_reasons_vec(text_quality_reasons_by_page),
|
||||
title,
|
||||
confidence,
|
||||
layout,
|
||||
|
||||
@@ -2367,29 +2367,7 @@ fn detect_merged_cluster_table(
|
||||
/// suitable for rect-backed tables where we already know tabular structure exists
|
||||
/// (no need for anti-paragraph safeguards).
|
||||
fn cluster_x_positions(items: &[(usize, &TextItem)], min_threshold: f32) -> Vec<f32> {
|
||||
// Column edges come from where text STARTS. An item whose left edge hugs
|
||||
// the previous item's right edge on the same line is a continuation run
|
||||
// (style boundary, script change, underline split) — feeding its x-start
|
||||
// in here fabricates a phantom column mid-cell.
|
||||
let mut sorted: Vec<&TextItem> = items.iter().map(|&(_, i)| i).collect();
|
||||
sorted.sort_by(|a, b| a.y.total_cmp(&b.y).then(a.x.total_cmp(&b.x)));
|
||||
let mut x_positions: Vec<f32> = Vec::with_capacity(sorted.len());
|
||||
for (idx, item) in sorted.iter().enumerate() {
|
||||
let is_continuation = idx > 0 && {
|
||||
let prev = sorted[idx - 1];
|
||||
// Style/underline splits leave runs that TOUCH (gap ~0); real
|
||||
// cell boundaries in even the tightest tables keep a visible
|
||||
// gap. 2pt separates the two without eating dense-table columns.
|
||||
// The negative side is bounded too: text overhanging from an
|
||||
// adjacent cell overlaps by far more than italic kerning ever
|
||||
// does, and must still start its own column.
|
||||
let gap = item.x - (prev.x + prev.width);
|
||||
(prev.y - item.y).abs() <= 2.0 && gap < 2.0 && gap > -4.0 && item.x >= prev.x
|
||||
};
|
||||
if !is_continuation {
|
||||
x_positions.push(item.x);
|
||||
}
|
||||
}
|
||||
let mut x_positions: Vec<f32> = items.iter().map(|(_, i)| i.x).collect();
|
||||
x_positions.sort_by(|a, b| a.total_cmp(b));
|
||||
|
||||
if x_positions.is_empty() {
|
||||
|
||||
@@ -1102,7 +1102,6 @@ fn test_pages_needing_ocr_field_accessible() {
|
||||
title: None,
|
||||
ocr_recommended: false,
|
||||
pages_needing_ocr: Vec::new(),
|
||||
ocr_reasons_by_page: std::collections::BTreeMap::new(),
|
||||
};
|
||||
assert!(detection_result.pages_needing_ocr.is_empty());
|
||||
|
||||
|
||||
@@ -72,11 +72,10 @@ Month or shorter period in which tips were received **4** Net tips (lines **1 +
|
||||
|
||||
forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—**Use this form to report tips you receive to your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See **Pub. 531**, Reporting Tip Income, for more information. You can get additional copies of **Pub. 1244**, Employee’s Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676).
|
||||
|
||||
<u>Instructions (continued)</u>
|
||||
**Instructions** *(continued)*
|
||||
|
||||
**Unreported Tips.—**If you received tips of $20 or more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you **must** use Form 1040 and **Form 4137,** Social Security and Medicare Tax on Unreported Tip Income, to report them. You may **not** use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act **cannot** use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—**Get **Pub. 531,** Reporting Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—**If you do not keep a daily record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law.
|
||||
|
||||
### Instructions (continued)
|
||||
|
||||
Use this space to total your tips for the year
|
||||
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
||||(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows: §1.1563-3T Rules for determining stock ownership (temporary). (a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a)|
|
||||
|---|---|---|---|
|
||||
||through (d)(2)(iii).|||
|
||||
||(iv)|Statement|. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include-- (A) A description of each of the controlled groups in which the corporation could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and (B) The following representation: [INSERT NAME AND EMPLOYER IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER OF THE [INSERT DESIGNATION OF GROUP].|
|
||||
||(v)|Election|-- (A) Election filed. An election filed under paragraph (d)(2)(iv) of this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in|
|
||||
(e) [Reserved]. For further guidance, see §1.1563-3T(e)(1). Par. 50. Section 1.1563-3T is added to read as follows:
|
||||
<u>§1.1563-3T Rules for determining stock ownership (temporary)</u>.
|
||||
|
||||
(a) through (d)(2)(iii) [Reserved]. For further guidance, see §1.1563-3(a)
|
||||
through (d)(2)(iii). (iv) <u>Statement</u>. If the application of paragraph (d)(2)(ii) or (iii) of §1.1563-3 does not result in a corporation being treated as a component member of only one controlled group of corporations on a December 31, then such corporation will be treated as a component member of only one such group on such date. Such corporation may elect the group in which it is to be included by including on or with its income tax return a statement entitled, “STATEMENT TO ELECT CONTROLLED GROUP PURSUANT TO §1.1563-3T(d)(2)(iv).” The statement must include--
|
||||
|
||||
(A) A description of each of the controlled groups in which the corporation
|
||||
could be included. The description must include the name and employer identification number of each component member of each such group and the stock ownership of the component members of each such group; and
|
||||
|
||||
(B) The following representation: [INSERT NAME AND EMPLOYER
|
||||
IDENTIFICATION NUMBER OF CORPORATION] ELECTS TO BE TREATED AS A COMPONENT MEMBER OF THE [INSERT DESIGNATION OF GROUP].
|
||||
|
||||
(v) <u>Election</u>-- (A) <u>Election filed</u>. An election filed under paragraph (d)(2)(iv) of
|
||||
this section is irrevocable and effective until paragraph (d)(2)(ii) or (iii) of §1.1563-3 applies or until a change in the stock ownership of the corporation results in
|
||||
|
||||
|termination of membership in the controlled group in which such corporation has||
|
||||
|---|---|
|
||||
@@ -20,7 +29,7 @@
|
||||
Federal income tax return (including any amended return filed on or before the due date (including extensions) of such original return) timely filed on or after May 30,
|
||||
|
||||
2006.
|
||||
(2) <u>Expiration date</u>. The applicability of this section will expire on May 26,
|
||||
(2) Expiration date. The applicability of this section will expire on May 26,
|
||||
2009. Par. 51. Section 1.6012-2 is amended by revising paragraph (c) and adding paragraph (k) to read as follows: <u>§1.6012-2 Corporations required to make returns of income</u>.
|
||||
* * * * *
|
||||
(c) [Reserved]. For further guidance, see §1.6012-2T(c).
|
||||
|
||||
@@ -43,9 +43,9 @@ l
|
||||
|
||||
**Freon** **®** **12 Saturation Properties-Temperature Table**
|
||||
|
||||
|Temp|Pressure||Volume||Density||Enthalpy|||Entropy|Temp|
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
|°C|[kPa]|[m³ Liquid v f|/kg] Vapour v g|[kg/m³ Liquid d f|] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C|
|
||||
|Temp|Pressure||Volume|||Density||Enthalpy|||Entropy|Temp|
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
|°C|[kPa]|[m³ Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m³] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C|
|
||||
|
||||
|-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100|
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
|
||||
Reference in New Issue
Block a user