Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Abimael Martell
|
e482978008 | ||
|
Abimael Martell
|
b88eb24e94 |
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "firecrawl-pdf-inspector",
|
||||
"version": "0.7.5",
|
||||
"version": "0.7.4",
|
||||
"description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
|
||||
@@ -99,26 +99,6 @@ pub struct PageRegionTexts {
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// LaTeX reconstruction result for a single formula region.
|
||||
#[napi(object)]
|
||||
pub struct FormulaLatexResult {
|
||||
/// Reconstructed LaTeX string.
|
||||
pub latex: String,
|
||||
/// The linearized raw text (before LaTeX reconstruction).
|
||||
pub raw_text: String,
|
||||
/// Heuristic confidence in the LaTeX output (0.0–1.0).
|
||||
pub confidence: f64,
|
||||
/// `true` when extraction failed entirely and GPU OCR is needed.
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
/// LaTeX reconstruction results for one page's formula regions.
|
||||
#[napi(object)]
|
||||
pub struct PageFormulaLatexResults {
|
||||
pub page: u32,
|
||||
pub regions: Vec<FormulaLatexResult>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -337,70 +317,6 @@ pub fn extract_tables_in_regions(
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract formula text within bounding-box regions from a PDF.
|
||||
///
|
||||
/// Like `extractTextInRegions` but uses formula-specific quality checks.
|
||||
/// Formula text is legitimately symbol-heavy (Greek letters, math operators)
|
||||
/// so the generic garbage-text check is relaxed. When the text decodes
|
||||
/// cleanly, `needsOcr` is `false` — the caller can skip GPU OCR.
|
||||
///
|
||||
/// Coordinates are PDF points with top-left origin.
|
||||
#[napi]
|
||||
pub fn extract_formulas_in_regions(
|
||||
buffer: Buffer,
|
||||
page_regions: Vec<PageRegions>,
|
||||
) -> Result<Vec<PageRegionTexts>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let regions = parse_page_regions(&page_regions);
|
||||
|
||||
catch_panic("extract_formulas_in_regions", move || {
|
||||
let results = pdf_inspector::extract_formulas_in_regions_mem(&bytes, ®ions)
|
||||
.map_err(|e| to_napi_err(e, "extract_formulas_in_regions"))?;
|
||||
Ok(to_page_region_texts(results))
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract formula text within bounding-box regions and reconstruct LaTeX.
|
||||
///
|
||||
/// Like `extractFormulasInRegions` but additionally reconstructs LaTeX from
|
||||
/// the positioned text items. Each result includes the raw text, reconstructed
|
||||
/// LaTeX, a confidence score, and the `needsOcr` flag.
|
||||
///
|
||||
/// The confidence score (0.0–1.0) indicates how reliable the heuristic LaTeX
|
||||
/// reconstruction is. The caller should use this to decide whether to trust
|
||||
/// the LaTeX or fall back to GPU OCR.
|
||||
///
|
||||
/// Coordinates are PDF points with top-left origin.
|
||||
#[napi]
|
||||
pub fn extract_formulas_in_regions_as_latex(
|
||||
buffer: Buffer,
|
||||
page_regions: Vec<PageRegions>,
|
||||
) -> Result<Vec<PageFormulaLatexResults>> {
|
||||
let bytes: Vec<u8> = buffer.to_vec();
|
||||
let regions = parse_page_regions(&page_regions);
|
||||
|
||||
catch_panic("extract_formulas_in_regions_as_latex", move || {
|
||||
let results = pdf_inspector::extract_formulas_in_regions_as_latex(&bytes, ®ions)
|
||||
.map_err(|e| to_napi_err(e, "extract_formulas_in_regions_as_latex"))?;
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.map(|page_result| PageFormulaLatexResults {
|
||||
page: page_result.page,
|
||||
regions: page_result
|
||||
.regions
|
||||
.into_iter()
|
||||
.map(|r| FormulaLatexResult {
|
||||
latex: r.latex,
|
||||
raw_text: r.raw_text,
|
||||
confidence: r.confidence as f64,
|
||||
needs_ocr: r.needs_ocr,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
|
||||
/// Per-page markdown extraction result.
|
||||
#[napi(object)]
|
||||
pub struct PageMarkdownResult {
|
||||
|
||||
@@ -216,6 +216,7 @@ pub(crate) fn extract_page_text_items(
|
||||
let mut marked_content_stack: Vec<MarkedContentEntry> = Vec::new();
|
||||
let mut suppress_glyph_extraction = false;
|
||||
let mut actual_text_start_tm: Option<[f32; 6]> = None; // text matrix at BDC entry
|
||||
let mut actual_text_glyph_tm: Option<[f32; 6]> = None; // text matrix at first glyph inside BDC
|
||||
/// Get the innermost MCID from the marked content stack.
|
||||
fn current_mcid(stack: &[MarkedContentEntry]) -> Option<i64> {
|
||||
stack.iter().rev().find_map(|e| e.mcid)
|
||||
@@ -349,8 +350,15 @@ pub(crate) fn extract_page_text_items(
|
||||
)
|
||||
})
|
||||
});
|
||||
// ActualText: suppress glyph extraction, just advance text matrix
|
||||
// ActualText: suppress glyph extraction, just advance text matrix.
|
||||
// Capture the FIRST glyph's text matrix as the rendering position
|
||||
// for the ActualText item. Td ops between BDC and the first Tj
|
||||
// may have moved the position to the correct line — the BDC-entry
|
||||
// position (actual_text_start_tm) can be on the previous line.
|
||||
if suppress_glyph_extraction {
|
||||
if actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
}
|
||||
if let Some(w_ts) = w_ts_opt {
|
||||
text_matrix[4] += w_ts * text_matrix[0];
|
||||
text_matrix[5] += w_ts * text_matrix[1];
|
||||
@@ -425,6 +433,10 @@ pub(crate) fn extract_page_text_items(
|
||||
let font_info = font_widths.get(¤t_font);
|
||||
let is_invisible = (text_rendering_mode == 3 && !include_invisible)
|
||||
|| suppress_glyph_extraction;
|
||||
// Capture first-glyph position for ActualText
|
||||
if suppress_glyph_extraction && actual_text_glyph_tm.is_none() {
|
||||
actual_text_glyph_tm = Some(text_matrix);
|
||||
}
|
||||
|
||||
// Compute space threshold based on font metrics when available
|
||||
let space_threshold = if let Some(font_info) = font_info {
|
||||
@@ -700,6 +712,7 @@ pub(crate) fn extract_page_text_items(
|
||||
if actual_text.is_some() {
|
||||
suppress_glyph_extraction = true;
|
||||
actual_text_start_tm = Some(text_matrix);
|
||||
actual_text_glyph_tm = None; // reset — will be captured at first Tj/TJ
|
||||
}
|
||||
marked_content_stack.push(MarkedContentEntry { actual_text, mcid });
|
||||
}
|
||||
@@ -707,8 +720,13 @@ pub(crate) fn extract_page_text_items(
|
||||
// End Marked Content — emit ActualText item with correct width
|
||||
if let Some(entry) = marked_content_stack.pop() {
|
||||
if let Some(at) = entry.actual_text {
|
||||
// Compute width from text matrix advancement during BDC..EMC
|
||||
if let Some(start_tm) = actual_text_start_tm.take() {
|
||||
// Use the first-glyph position (if available) instead of the
|
||||
// BDC-entry position. Td operators between BDC and the first
|
||||
// Tj may have moved the text position to the correct line —
|
||||
// the BDC-entry position can be on the previous line.
|
||||
let glyph_tm = actual_text_glyph_tm.take();
|
||||
let entry_tm = actual_text_start_tm.take();
|
||||
if let Some(start_tm) = glyph_tm.or(entry_tm) {
|
||||
let combined = multiply_matrices(&start_tm, &ctm);
|
||||
if combined[0].abs() >= combined[1].abs() {
|
||||
rotation_votes.horizontal += 1;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,418 +0,0 @@
|
||||
//! Unicode to LaTeX character mapping.
|
||||
//!
|
||||
//! Maps Unicode math symbols, Greek letters, operators, and relations to their
|
||||
//! LaTeX command equivalents. Only includes characters that commonly appear in
|
||||
//! PDF text extraction of mathematical formulas.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Returns a reference to the global Unicode → LaTeX mapping table.
|
||||
pub fn unicode_to_latex_map() -> &'static HashMap<char, &'static str> {
|
||||
static MAP: OnceLock<HashMap<char, &'static str>> = OnceLock::new();
|
||||
MAP.get_or_init(build_map)
|
||||
}
|
||||
|
||||
fn build_map() -> HashMap<char, &'static str> {
|
||||
let entries: &[(char, &str)] = &[
|
||||
// ── Greek lowercase ─────────────────────────────────────────
|
||||
('\u{03B1}', r"\alpha"),
|
||||
('\u{03B2}', r"\beta"),
|
||||
('\u{03B3}', r"\gamma"),
|
||||
('\u{03B4}', r"\delta"),
|
||||
('\u{03B5}', r"\varepsilon"),
|
||||
('\u{03F5}', r"\epsilon"),
|
||||
('\u{03B6}', r"\zeta"),
|
||||
('\u{03B7}', r"\eta"),
|
||||
('\u{03B8}', r"\theta"),
|
||||
('\u{03D1}', r"\vartheta"),
|
||||
('\u{03B9}', r"\iota"),
|
||||
('\u{03BA}', r"\kappa"),
|
||||
('\u{03BB}', r"\lambda"),
|
||||
('\u{03BC}', r"\mu"),
|
||||
('\u{03BD}', r"\nu"),
|
||||
('\u{03BE}', r"\xi"),
|
||||
('\u{03C0}', r"\pi"),
|
||||
('\u{03D6}', r"\varpi"),
|
||||
('\u{03C1}', r"\rho"),
|
||||
('\u{03C2}', r"\varsigma"),
|
||||
('\u{03C3}', r"\sigma"),
|
||||
('\u{03C4}', r"\tau"),
|
||||
('\u{03C5}', r"\upsilon"),
|
||||
('\u{03C6}', r"\varphi"),
|
||||
('\u{03D5}', r"\phi"),
|
||||
('\u{03C7}', r"\chi"),
|
||||
('\u{03C8}', r"\psi"),
|
||||
('\u{03C9}', r"\omega"),
|
||||
// ── Greek uppercase ─────────────────────────────────────────
|
||||
('\u{0393}', r"\Gamma"),
|
||||
('\u{0394}', r"\Delta"),
|
||||
('\u{0398}', r"\Theta"),
|
||||
('\u{039B}', r"\Lambda"),
|
||||
('\u{039E}', r"\Xi"),
|
||||
('\u{03A0}', r"\Pi"),
|
||||
('\u{03A3}', r"\Sigma"),
|
||||
('\u{03A5}', r"\Upsilon"),
|
||||
('\u{03A6}', r"\Phi"),
|
||||
('\u{03A8}', r"\Psi"),
|
||||
('\u{03A9}', r"\Omega"),
|
||||
// ── Large operators ─────────────────────────────────────────
|
||||
('\u{222B}', r"\int"),
|
||||
('\u{222C}', r"\iint"),
|
||||
('\u{222D}', r"\iiint"),
|
||||
('\u{222E}', r"\oint"),
|
||||
('\u{2211}', r"\sum"),
|
||||
('\u{220F}', r"\prod"),
|
||||
('\u{2210}', r"\coprod"),
|
||||
// ── Roots / radicals ────────────────────────────────────────
|
||||
('\u{221A}', r"\sqrt"),
|
||||
// ── Calculus / differential ─────────────────────────────────
|
||||
('\u{2202}', r"\partial"),
|
||||
('\u{2207}', r"\nabla"),
|
||||
// ── Binary operators ────────────────────────────────────────
|
||||
('\u{00B1}', r"\pm"),
|
||||
('\u{2213}', r"\mp"),
|
||||
('\u{00D7}', r"\times"),
|
||||
('\u{00F7}', r"\div"),
|
||||
('\u{2217}', r"\ast"),
|
||||
('\u{22C6}', r"\star"),
|
||||
('\u{00B7}', r"\cdot"),
|
||||
('\u{2219}', r"\bullet"),
|
||||
('\u{2218}', r"\circ"),
|
||||
('\u{2020}', r"\dagger"),
|
||||
('\u{2021}', r"\ddagger"),
|
||||
('\u{2295}', r"\oplus"),
|
||||
('\u{2297}', r"\otimes"),
|
||||
('\u{2227}', r"\wedge"),
|
||||
('\u{2228}', r"\vee"),
|
||||
('\u{2229}', r"\cap"),
|
||||
('\u{222A}', r"\cup"),
|
||||
// ── Relations ───────────────────────────────────────────────
|
||||
('\u{2264}', r"\leq"),
|
||||
('\u{2265}', r"\geq"),
|
||||
('\u{2260}', r"\neq"),
|
||||
('\u{2248}', r"\approx"),
|
||||
('\u{223C}', r"\sim"),
|
||||
('\u{2243}', r"\simeq"),
|
||||
('\u{2261}', r"\equiv"),
|
||||
('\u{226A}', r"\ll"),
|
||||
('\u{226B}', r"\gg"),
|
||||
('\u{221D}', r"\propto"),
|
||||
('\u{2208}', r"\in"),
|
||||
('\u{2209}', r"\notin"),
|
||||
('\u{220B}', r"\ni"),
|
||||
('\u{2282}', r"\subset"),
|
||||
('\u{2283}', r"\supset"),
|
||||
('\u{2286}', r"\subseteq"),
|
||||
('\u{2287}', r"\supseteq"),
|
||||
('\u{22A2}', r"\vdash"),
|
||||
('\u{22A3}', r"\dashv"),
|
||||
('\u{22A4}', r"\top"),
|
||||
('\u{22A5}', r"\bot"),
|
||||
('\u{2225}', r"\parallel"),
|
||||
('\u{22A5}', r"\perp"),
|
||||
// ── Arrows ──────────────────────────────────────────────────
|
||||
('\u{2190}', r"\leftarrow"),
|
||||
('\u{2192}', r"\to"),
|
||||
('\u{2191}', r"\uparrow"),
|
||||
('\u{2193}', r"\downarrow"),
|
||||
('\u{2194}', r"\leftrightarrow"),
|
||||
('\u{21D0}', r"\Leftarrow"),
|
||||
('\u{21D2}', r"\Rightarrow"),
|
||||
('\u{21D4}', r"\Leftrightarrow"),
|
||||
('\u{21A6}', r"\mapsto"),
|
||||
('\u{2197}', r"\nearrow"),
|
||||
('\u{2198}', r"\searrow"),
|
||||
// ── Miscellaneous symbols ───────────────────────────────────
|
||||
('\u{221E}', r"\infty"),
|
||||
('\u{2200}', r"\forall"),
|
||||
('\u{2203}', r"\exists"),
|
||||
('\u{2204}', r"\nexists"),
|
||||
('\u{2205}', r"\emptyset"),
|
||||
('\u{00AC}', r"\neg"),
|
||||
('\u{00B0}', r"^\circ"),
|
||||
('\u{2032}', r"'"), // prime (common in physics: x')
|
||||
('\u{2033}', r"''"), // double prime
|
||||
('\u{210F}', r"\hbar"),
|
||||
('\u{2113}', r"\ell"),
|
||||
('\u{211C}', r"\Re"),
|
||||
('\u{2111}', r"\Im"),
|
||||
('\u{2118}', r"\wp"),
|
||||
('\u{2135}', r"\aleph"),
|
||||
// ── Dots ────────────────────────────────────────────────────
|
||||
('\u{22EF}', r"\cdots"),
|
||||
('\u{22EE}', r"\vdots"),
|
||||
('\u{22F1}', r"\ddots"),
|
||||
('\u{2026}', r"\ldots"),
|
||||
// ── Delimiters / brackets ───────────────────────────────────
|
||||
('\u{27E8}', r"\langle"),
|
||||
('\u{27E9}', r"\rangle"),
|
||||
('\u{2308}', r"\lceil"),
|
||||
('\u{2309}', r"\rceil"),
|
||||
('\u{230A}', r"\lfloor"),
|
||||
('\u{230B}', r"\rfloor"),
|
||||
('\u{2016}', r"\|"),
|
||||
// ── Accents / decorations (as standalone chars) ─────────────
|
||||
('\u{0302}', r"\hat{}"),
|
||||
('\u{0303}', r"\tilde{}"),
|
||||
('\u{0304}', r"\bar{}"),
|
||||
('\u{0307}', r"\dot{}"),
|
||||
('\u{0308}', r"\ddot{}"),
|
||||
('\u{20D7}', r"\vec{}"),
|
||||
// Hat/tilde as standalone characters (sometimes extracted separately)
|
||||
('\u{02C6}', r"\hat{}"),
|
||||
('\u{02DC}', r"\tilde{}"),
|
||||
// ── Subscript/superscript digits (Unicode) ──────────────────
|
||||
('\u{2070}', "^{0}"),
|
||||
('\u{00B9}', "^{1}"),
|
||||
('\u{00B2}', "^{2}"),
|
||||
('\u{00B3}', "^{3}"),
|
||||
('\u{2074}', "^{4}"),
|
||||
('\u{2075}', "^{5}"),
|
||||
('\u{2076}', "^{6}"),
|
||||
('\u{2077}', "^{7}"),
|
||||
('\u{2078}', "^{8}"),
|
||||
('\u{2079}', "^{9}"),
|
||||
('\u{207A}', "^{+}"),
|
||||
('\u{207B}', "^{-}"),
|
||||
('\u{2080}', "_{0}"),
|
||||
('\u{2081}', "_{1}"),
|
||||
('\u{2082}', "_{2}"),
|
||||
('\u{2083}', "_{3}"),
|
||||
('\u{2084}', "_{4}"),
|
||||
('\u{2085}', "_{5}"),
|
||||
('\u{2086}', "_{6}"),
|
||||
('\u{2087}', "_{7}"),
|
||||
('\u{2088}', "_{8}"),
|
||||
('\u{2089}', "_{9}"),
|
||||
('\u{208A}', "_{+}"),
|
||||
('\u{208B}', "_{-}"),
|
||||
// ── Math italic letters (sometimes used in PDF fonts) ───────
|
||||
// These map back to plain ASCII in LaTeX (math mode handles italics)
|
||||
];
|
||||
|
||||
let mut map = HashMap::with_capacity(entries.len());
|
||||
for &(ch, latex) in entries {
|
||||
map.insert(ch, latex);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Convert a single character to its LaTeX representation.
|
||||
///
|
||||
/// Returns `Some(latex_str)` if the character has a known mapping,
|
||||
/// or `None` if it should be kept as-is.
|
||||
pub fn char_to_latex(ch: char) -> Option<&'static str> {
|
||||
unicode_to_latex_map().get(&ch).copied()
|
||||
}
|
||||
|
||||
/// Returns true if a character is a "known math character" — either ASCII
|
||||
/// alphanumeric, basic punctuation used in math, or a mapped Unicode symbol.
|
||||
pub fn is_known_math_char(ch: char) -> bool {
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
return true;
|
||||
}
|
||||
// Common ASCII math characters
|
||||
matches!(
|
||||
ch,
|
||||
'+' | '-'
|
||||
| '*'
|
||||
| '/'
|
||||
| '='
|
||||
| '<'
|
||||
| '>'
|
||||
| '('
|
||||
| ')'
|
||||
| '['
|
||||
| ']'
|
||||
| '{'
|
||||
| '}'
|
||||
| ','
|
||||
| '.'
|
||||
| ':'
|
||||
| ';'
|
||||
| '!'
|
||||
| '|'
|
||||
| '\''
|
||||
| '"'
|
||||
| '^'
|
||||
| '_'
|
||||
| '~'
|
||||
| ' '
|
||||
| '\n'
|
||||
| '\t'
|
||||
) || unicode_to_latex_map().contains_key(&ch)
|
||||
}
|
||||
|
||||
/// Convert a string of text to LaTeX, applying per-character mappings.
|
||||
/// Characters without mappings are left as-is.
|
||||
///
|
||||
/// Returns `(latex_string, fraction_of_chars_that_were_known)`.
|
||||
pub fn text_to_latex_chars(text: &str) -> (String, f32) {
|
||||
let map = unicode_to_latex_map();
|
||||
let mut result = String::with_capacity(text.len() * 2);
|
||||
let mut total_nonws = 0usize;
|
||||
let mut known = 0usize;
|
||||
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
result.push(ch);
|
||||
continue;
|
||||
}
|
||||
total_nonws += 1;
|
||||
|
||||
if let Some(latex) = map.get(&ch) {
|
||||
// Add space before LaTeX commands that start with backslash
|
||||
// to prevent them from merging with preceding text
|
||||
if latex.starts_with('\\') && !result.is_empty() && !result.ends_with(' ') {
|
||||
// Only add space if the last char is alphanumeric (to avoid "x \alpha" but allow "( \alpha")
|
||||
let last = result.chars().last().unwrap();
|
||||
if last.is_alphanumeric() || last == '}' {
|
||||
result.push(' ');
|
||||
}
|
||||
}
|
||||
result.push_str(latex);
|
||||
// Add trailing space after LaTeX commands so next char doesn't merge
|
||||
if latex.starts_with('\\') && !latex.ends_with('}') && !latex.ends_with('\'') {
|
||||
result.push(' ');
|
||||
}
|
||||
known += 1;
|
||||
} else if is_known_math_char(ch) {
|
||||
result.push(ch);
|
||||
known += 1;
|
||||
} else {
|
||||
// Unknown character — keep it but it lowers confidence
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
|
||||
let frac = if total_nonws == 0 {
|
||||
1.0
|
||||
} else {
|
||||
known as f32 / total_nonws as f32
|
||||
};
|
||||
(result, frac)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_greek_lowercase() {
|
||||
assert_eq!(char_to_latex('\u{03B1}'), Some(r"\alpha"));
|
||||
assert_eq!(char_to_latex('\u{03B2}'), Some(r"\beta"));
|
||||
assert_eq!(char_to_latex('\u{03B3}'), Some(r"\gamma"));
|
||||
assert_eq!(char_to_latex('\u{03B4}'), Some(r"\delta"));
|
||||
assert_eq!(char_to_latex('\u{03B5}'), Some(r"\varepsilon"));
|
||||
assert_eq!(char_to_latex('\u{03B6}'), Some(r"\zeta"));
|
||||
assert_eq!(char_to_latex('\u{03B7}'), Some(r"\eta"));
|
||||
assert_eq!(char_to_latex('\u{03B8}'), Some(r"\theta"));
|
||||
assert_eq!(char_to_latex('\u{03D1}'), Some(r"\vartheta"));
|
||||
assert_eq!(char_to_latex('\u{03B9}'), Some(r"\iota"));
|
||||
assert_eq!(char_to_latex('\u{03BA}'), Some(r"\kappa"));
|
||||
assert_eq!(char_to_latex('\u{03BB}'), Some(r"\lambda"));
|
||||
assert_eq!(char_to_latex('\u{03BC}'), Some(r"\mu"));
|
||||
assert_eq!(char_to_latex('\u{03BD}'), Some(r"\nu"));
|
||||
assert_eq!(char_to_latex('\u{03BE}'), Some(r"\xi"));
|
||||
assert_eq!(char_to_latex('\u{03C0}'), Some(r"\pi"));
|
||||
assert_eq!(char_to_latex('\u{03C1}'), Some(r"\rho"));
|
||||
assert_eq!(char_to_latex('\u{03C3}'), Some(r"\sigma"));
|
||||
assert_eq!(char_to_latex('\u{03C4}'), Some(r"\tau"));
|
||||
assert_eq!(char_to_latex('\u{03C9}'), Some(r"\omega"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_greek_uppercase() {
|
||||
assert_eq!(char_to_latex('\u{0393}'), Some(r"\Gamma"));
|
||||
assert_eq!(char_to_latex('\u{0394}'), Some(r"\Delta"));
|
||||
assert_eq!(char_to_latex('\u{0398}'), Some(r"\Theta"));
|
||||
assert_eq!(char_to_latex('\u{039B}'), Some(r"\Lambda"));
|
||||
assert_eq!(char_to_latex('\u{03A3}'), Some(r"\Sigma"));
|
||||
assert_eq!(char_to_latex('\u{03A6}'), Some(r"\Phi"));
|
||||
assert_eq!(char_to_latex('\u{03A9}'), Some(r"\Omega"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operators() {
|
||||
assert_eq!(char_to_latex('\u{222B}'), Some(r"\int"));
|
||||
assert_eq!(char_to_latex('\u{2211}'), Some(r"\sum"));
|
||||
assert_eq!(char_to_latex('\u{220F}'), Some(r"\prod"));
|
||||
assert_eq!(char_to_latex('\u{221A}'), Some(r"\sqrt"));
|
||||
assert_eq!(char_to_latex('\u{2202}'), Some(r"\partial"));
|
||||
assert_eq!(char_to_latex('\u{2207}'), Some(r"\nabla"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_relations() {
|
||||
assert_eq!(char_to_latex('\u{2264}'), Some(r"\leq"));
|
||||
assert_eq!(char_to_latex('\u{2265}'), Some(r"\geq"));
|
||||
assert_eq!(char_to_latex('\u{2260}'), Some(r"\neq"));
|
||||
assert_eq!(char_to_latex('\u{2248}'), Some(r"\approx"));
|
||||
assert_eq!(char_to_latex('\u{223C}'), Some(r"\sim"));
|
||||
assert_eq!(char_to_latex('\u{226A}'), Some(r"\ll"));
|
||||
assert_eq!(char_to_latex('\u{226B}'), Some(r"\gg"));
|
||||
assert_eq!(char_to_latex('\u{221E}'), Some(r"\infty"));
|
||||
assert_eq!(char_to_latex('\u{2208}'), Some(r"\in"));
|
||||
assert_eq!(char_to_latex('\u{2209}'), Some(r"\notin"));
|
||||
assert_eq!(char_to_latex('\u{2282}'), Some(r"\subset"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_misc_symbols() {
|
||||
assert_eq!(char_to_latex('\u{00B1}'), Some(r"\pm"));
|
||||
assert_eq!(char_to_latex('\u{00D7}'), Some(r"\times"));
|
||||
assert_eq!(char_to_latex('\u{00B7}'), Some(r"\cdot"));
|
||||
assert_eq!(char_to_latex('\u{00B0}'), Some(r"^\circ"));
|
||||
assert_eq!(char_to_latex('\u{2192}'), Some(r"\to"));
|
||||
assert_eq!(char_to_latex('\u{21D2}'), Some(r"\Rightarrow"));
|
||||
assert_eq!(char_to_latex('\u{210F}'), Some(r"\hbar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unicode_super_sub_digits() {
|
||||
assert_eq!(char_to_latex('\u{00B2}'), Some("^{2}"));
|
||||
assert_eq!(char_to_latex('\u{00B3}'), Some("^{3}"));
|
||||
assert_eq!(char_to_latex('\u{2082}'), Some("_{2}"));
|
||||
assert_eq!(char_to_latex('\u{2083}'), Some("_{3}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_known_math_char() {
|
||||
// ASCII math
|
||||
assert!(is_known_math_char('+'));
|
||||
assert!(is_known_math_char('='));
|
||||
assert!(is_known_math_char('('));
|
||||
assert!(is_known_math_char('x'));
|
||||
assert!(is_known_math_char('0'));
|
||||
// Mapped Unicode
|
||||
assert!(is_known_math_char('\u{03B1}')); // alpha
|
||||
assert!(is_known_math_char('\u{2264}')); // leq
|
||||
// Unknown
|
||||
assert!(!is_known_math_char('\u{E000}')); // PUA
|
||||
assert!(!is_known_math_char('\u{4E00}')); // CJK
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_to_latex_simple() {
|
||||
let (latex, frac) = text_to_latex_chars("x + y");
|
||||
assert_eq!(latex, "x + y");
|
||||
assert!((frac - 1.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_to_latex_greek() {
|
||||
let (latex, frac) = text_to_latex_chars("αβγ");
|
||||
assert!(latex.contains(r"\alpha"));
|
||||
assert!(latex.contains(r"\beta"));
|
||||
assert!(latex.contains(r"\gamma"));
|
||||
assert!((frac - 1.0).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_to_latex_mixed() {
|
||||
let (latex, frac) = text_to_latex_chars("x ≤ y");
|
||||
assert!(latex.contains(r"\leq"));
|
||||
assert!((frac - 1.0).abs() < 0.01);
|
||||
}
|
||||
}
|
||||
+122
-354
@@ -1,3 +1,9 @@
|
||||
// Rust 1.95 introduced collapsible_match for `if` inside match arms.
|
||||
// The content-stream parsers use this pattern extensively (match on operator
|
||||
// name, then check `in_text_block && !op.operands.is_empty()`). Collapsing
|
||||
// these into match guards would hurt readability. Allow crate-wide.
|
||||
#![allow(clippy::collapsible_match)]
|
||||
|
||||
//! Smart PDF detection and text extraction using lopdf
|
||||
//!
|
||||
//! # Quick start
|
||||
@@ -28,7 +34,6 @@ pub mod python;
|
||||
pub mod adobe_korea1;
|
||||
pub mod detector;
|
||||
pub mod extractor;
|
||||
pub mod formula_latex;
|
||||
pub mod glyph_names;
|
||||
pub mod markdown;
|
||||
pub mod process_mode;
|
||||
@@ -463,98 +468,6 @@ pub struct PageRegionResult {
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// Shared page extraction state for region-based extraction functions.
|
||||
struct RegionExtractionData {
|
||||
items_by_page: HashMap<u32, Vec<TextItem>>,
|
||||
page_heights: HashMap<u32, f32>,
|
||||
#[allow(dead_code)]
|
||||
gid_pages: HashSet<u32>,
|
||||
page_thresholds: HashMap<u32, f32>,
|
||||
rotated_pages: HashSet<u32>,
|
||||
}
|
||||
|
||||
/// Extract text items, page heights, and metadata for the pages needed by region queries.
|
||||
///
|
||||
/// This is the shared boilerplate for `extract_text_in_regions_mem`,
|
||||
/// `extract_tables_in_regions_mem`, and `extract_formulas_in_regions_mem`.
|
||||
fn prepare_region_extraction(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<RegionExtractionData, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let pages = doc.get_pages();
|
||||
|
||||
let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect();
|
||||
|
||||
// Fast mode: skip expensive TrueType font fallback parsing.
|
||||
// Fonts that can't be decoded from ToUnicode alone will produce empty/garbage
|
||||
// text, triggering needs_ocr=true → GPU OCR fallback in the pipeline.
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
let ((mut items, _rects, _lines), has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
if has_gid {
|
||||
gid_pages.insert(*page_num);
|
||||
}
|
||||
if coords_rotated {
|
||||
rotated_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
}
|
||||
|
||||
Ok(RegionExtractionData {
|
||||
items_by_page,
|
||||
page_heights,
|
||||
gid_pages,
|
||||
page_thresholds,
|
||||
rotated_pages,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve per-page coord space and adaptive threshold for a given page.
|
||||
fn page_region_context(
|
||||
data: &RegionExtractionData,
|
||||
page_1idx: u32,
|
||||
) -> (f32, f32, RegionCoordSpace) {
|
||||
let page_h = data.page_heights.get(&page_1idx).copied().unwrap_or(792.0);
|
||||
let adaptive_threshold = data
|
||||
.page_thresholds
|
||||
.get(&page_1idx)
|
||||
.copied()
|
||||
.unwrap_or(0.10);
|
||||
let coords = if data.rotated_pages.contains(&page_1idx) {
|
||||
RegionCoordSpace::Rotated90Ccw
|
||||
} else {
|
||||
RegionCoordSpace::Standard
|
||||
};
|
||||
(page_h, adaptive_threshold, coords)
|
||||
}
|
||||
|
||||
/// Extract text within bounding-box regions from a PDF in memory.
|
||||
///
|
||||
/// This is designed for hybrid OCR pipelines: a layout model detects regions
|
||||
@@ -578,14 +491,70 @@ pub fn extract_text_in_regions_mem(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<Vec<PageRegionResult>, PdfError> {
|
||||
let data = prepare_region_extraction(buffer, page_regions)?;
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let pages = doc.get_pages();
|
||||
|
||||
// Build a set of pages we need to extract (1-indexed for lopdf)
|
||||
let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect();
|
||||
|
||||
// Fast mode: skip expensive TrueType font fallback parsing.
|
||||
// Fonts that can't be decoded from ToUnicode alone will produce empty/garbage
|
||||
// text, triggering needs_ocr=true → GPU OCR fallback in the pipeline.
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||
|
||||
// Extract text items for needed pages only
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get page height from MediaBox for coordinate flip
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
// Extract text items for this page
|
||||
let ((mut items, _rects, _lines), has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
if has_gid {
|
||||
gid_pages.insert(*page_num);
|
||||
}
|
||||
if coords_rotated {
|
||||
rotated_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
}
|
||||
|
||||
// For each page's regions, filter and assemble text
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
|
||||
for (page_0idx, regions) in page_regions {
|
||||
let page_1idx = page_0idx + 1;
|
||||
let items = data.items_by_page.get(&page_1idx);
|
||||
let (page_h, adaptive_threshold, coords) = page_region_context(&data, page_1idx);
|
||||
let items = items_by_page.get(&page_1idx);
|
||||
let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
|
||||
let _page_has_gid = gid_pages.contains(&page_1idx);
|
||||
let adaptive_threshold = page_thresholds.get(&page_1idx).copied().unwrap_or(0.10);
|
||||
let coords = if rotated_pages.contains(&page_1idx) {
|
||||
RegionCoordSpace::Rotated90Ccw
|
||||
} else {
|
||||
RegionCoordSpace::Standard
|
||||
};
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
@@ -639,20 +608,74 @@ pub fn extract_tables_in_regions_mem(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<Vec<PageRegionResult>, PdfError> {
|
||||
let data = prepare_region_extraction(buffer, page_regions)?;
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let pages = doc.get_pages();
|
||||
|
||||
let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect();
|
||||
let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
|
||||
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
|
||||
let mut rotated_pages: HashSet<u32> = HashSet::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
let ((mut items, _rects, _lines), has_gid, coords_rotated) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
let threshold = text_utils::fix_letterspaced_items(&mut items);
|
||||
if threshold > 0.10 {
|
||||
page_thresholds.insert(*page_num, threshold);
|
||||
}
|
||||
if has_gid {
|
||||
gid_pages.insert(*page_num);
|
||||
}
|
||||
if coords_rotated {
|
||||
rotated_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
|
||||
for (page_0idx, regions) in page_regions {
|
||||
let page_1idx = page_0idx + 1;
|
||||
let items = data.items_by_page.get(&page_1idx);
|
||||
let (page_h, _adaptive_threshold, coords) = page_region_context(&data, page_1idx);
|
||||
let items = items_by_page.get(&page_1idx);
|
||||
let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
|
||||
let _page_has_gid = gid_pages.contains(&page_1idx);
|
||||
let coords = if rotated_pages.contains(&page_1idx) {
|
||||
RegionCoordSpace::Rotated90Ccw
|
||||
} else {
|
||||
RegionCoordSpace::Standard
|
||||
};
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
// Note: we intentionally DO NOT bail on page_has_gid here.
|
||||
// The GID flag means some font on the page uses unresolvable
|
||||
// glyph IDs, but that font may only appear in a logo or
|
||||
// header — not in the table region. Instead we let the
|
||||
// per-region text quality checks (is_garbage_text, is_cid_garbage,
|
||||
// detect_encoding_issues) reject based on the actual extracted
|
||||
// content. This avoids rejecting clean tables just because an
|
||||
// unrelated decorative font on the same page is GID-encoded.
|
||||
|
||||
let matched: Vec<TextItem> = match items {
|
||||
Some(items) => {
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
@@ -733,164 +756,6 @@ pub fn extract_tables_in_regions_mem(
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Extract formula text within bounding-box regions from a PDF in memory.
|
||||
///
|
||||
/// Similar to [`extract_text_in_regions_mem`] but uses formula-specific quality
|
||||
/// checks. Formula text is legitimately symbol-heavy (Greek letters, math
|
||||
/// operators, subscripts) so the generic `is_garbage_text` check — which rejects
|
||||
/// text with <50% alphanumeric characters — would false-positive on valid
|
||||
/// formula regions.
|
||||
///
|
||||
/// When the extracted text decodes cleanly, `needs_ocr` is `false` and the
|
||||
/// caller can skip GPU OCR. When extraction fails (empty, PUA-heavy, encoding
|
||||
/// issues), `needs_ocr` is `true` for OCR fallback.
|
||||
pub fn extract_formulas_in_regions_mem(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<Vec<PageRegionResult>, PdfError> {
|
||||
let data = prepare_region_extraction(buffer, page_regions)?;
|
||||
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
|
||||
for (page_0idx, regions) in page_regions {
|
||||
let page_1idx = page_0idx + 1;
|
||||
let items = data.items_by_page.get(&page_1idx);
|
||||
let (page_h, adaptive_threshold, coords) = page_region_context(&data, page_1idx);
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
let text = match items {
|
||||
Some(items) => collect_text_in_region_with_options(
|
||||
items,
|
||||
rx1,
|
||||
ry1,
|
||||
rx2,
|
||||
ry2,
|
||||
page_h,
|
||||
coords,
|
||||
adaptive_threshold,
|
||||
),
|
||||
None => String::new(),
|
||||
};
|
||||
|
||||
// Formula-specific quality checks:
|
||||
// - Skip is_garbage_text (formulas are legitimately symbol-heavy)
|
||||
// - Keep CID/encoding checks (broken font decode is still broken)
|
||||
// - Add PUA check (extensible delimiters that didn't decode)
|
||||
let needs_ocr = text.trim().is_empty()
|
||||
|| is_cid_garbage(&text)
|
||||
|| detect_encoding_issues(&text)
|
||||
|| is_formula_garbage(&text);
|
||||
|
||||
page_results.push(RegionText { text, needs_ocr });
|
||||
}
|
||||
|
||||
results.push(PageRegionResult {
|
||||
page: *page_0idx,
|
||||
regions: page_results,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Extract formula text within bounding-box regions and reconstruct LaTeX.
|
||||
///
|
||||
/// For each formula bbox, this function:
|
||||
/// 1. Gets positioned text items inside the bbox
|
||||
/// 2. Analyzes font sizes and vertical positions to detect sub/superscripts
|
||||
/// 3. Detects simple fractions from vertically stacked text
|
||||
/// 4. Converts Unicode math symbols to LaTeX commands
|
||||
/// 5. Reconstructs structured LaTeX from the positioned items
|
||||
///
|
||||
/// Each result includes a `confidence` score (0.0–1.0) indicating how reliable
|
||||
/// the LaTeX reconstruction is. Low-confidence results should be sent to GPU OCR.
|
||||
///
|
||||
/// The `needs_ocr` flag is set only when extraction fails entirely (empty text,
|
||||
/// encoding garbage), NOT based on confidence — the caller decides the threshold.
|
||||
pub fn extract_formulas_in_regions_as_latex(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<Vec<formula_latex::PageFormulaResult>, PdfError> {
|
||||
let data = prepare_region_extraction(buffer, page_regions)?;
|
||||
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
|
||||
for (page_0idx, regions) in page_regions {
|
||||
let page_1idx = page_0idx + 1;
|
||||
let items = data.items_by_page.get(&page_1idx);
|
||||
let (page_h, _adaptive_threshold, coords) = page_region_context(&data, page_1idx);
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
let result = match items {
|
||||
Some(items) => {
|
||||
// Convert the bbox from top-left origin to the item coordinate space
|
||||
let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
|
||||
|
||||
// Filter items whose center falls inside the bbox
|
||||
let matched = formula_latex::filter_items_in_bbox(
|
||||
items,
|
||||
bounds.x_min,
|
||||
bounds.y_min,
|
||||
bounds.x_max,
|
||||
bounds.y_max,
|
||||
);
|
||||
|
||||
if matched.is_empty() {
|
||||
formula_latex::FormulaResult {
|
||||
latex: String::new(),
|
||||
raw_text: String::new(),
|
||||
confidence: 0.0,
|
||||
needs_ocr: true,
|
||||
confidence_breakdown: Vec::new(),
|
||||
}
|
||||
} else {
|
||||
let (latex, raw_text, confidence, confidence_breakdown) =
|
||||
formula_latex::reconstruct_latex(&matched);
|
||||
|
||||
// Same formula-specific quality checks as extract_formulas_in_regions_mem
|
||||
let needs_ocr = raw_text.trim().is_empty()
|
||||
|| is_cid_garbage(&raw_text)
|
||||
|| detect_encoding_issues(&raw_text)
|
||||
|| is_formula_garbage(&raw_text);
|
||||
|
||||
formula_latex::FormulaResult {
|
||||
latex,
|
||||
raw_text,
|
||||
confidence,
|
||||
needs_ocr,
|
||||
confidence_breakdown,
|
||||
}
|
||||
}
|
||||
}
|
||||
None => formula_latex::FormulaResult {
|
||||
latex: String::new(),
|
||||
raw_text: String::new(),
|
||||
confidence: 0.0,
|
||||
needs_ocr: true,
|
||||
confidence_breakdown: Vec::new(),
|
||||
},
|
||||
};
|
||||
|
||||
page_results.push(result);
|
||||
}
|
||||
|
||||
results.push(formula_latex::PageFormulaResult {
|
||||
page: *page_0idx,
|
||||
regions: page_results,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Get page height in points from MediaBox.
|
||||
fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option<f32> {
|
||||
let page_dict = doc.get_dictionary(page_id).ok()?;
|
||||
@@ -1488,49 +1353,6 @@ fn is_cid_garbage(text: &str) -> bool {
|
||||
high_latin * 5 >= total * 2 && ascii_letters * 3 < total
|
||||
}
|
||||
|
||||
/// Detect formula text that is unlikely to be usable despite passing generic checks.
|
||||
///
|
||||
/// Formula text (Greek letters, math operators, variables) is legitimately
|
||||
/// symbol-heavy, so `is_garbage_text` would false-positive. This check instead
|
||||
/// catches:
|
||||
///
|
||||
/// 1. **Private Use Area (PUA) characters** — TeX extensible delimiter glyphs
|
||||
/// (large brackets from CMEX fonts) often map to PUA U+E000–F8FF when the
|
||||
/// ToUnicode CMap is missing. >10% PUA means significant undecoded content.
|
||||
///
|
||||
/// 2. **Control characters** — C0 controls (U+0000–001F excluding whitespace)
|
||||
/// indicate broken font encoding, not formula content. >30% is rejected.
|
||||
fn is_formula_garbage(text: &str) -> bool {
|
||||
let mut total = 0usize;
|
||||
let mut pua = 0usize;
|
||||
let mut control = 0usize;
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
if ('\u{E000}'..='\u{F8FF}').contains(&ch) {
|
||||
pua += 1;
|
||||
}
|
||||
let cp = ch as u32;
|
||||
if cp < 0x20 {
|
||||
control += 1;
|
||||
}
|
||||
}
|
||||
if total < 3 {
|
||||
return false;
|
||||
}
|
||||
// >10% PUA — significant undecoded extensible delimiters
|
||||
if pua * 10 > total {
|
||||
return true;
|
||||
}
|
||||
// >30% control chars — broken encoding
|
||||
if control * 10 > total * 3 {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Detect markdown tables with suspicious structure that suggest the heuristic
|
||||
/// missed/mangled rows or columns. Returns true when the caller should treat
|
||||
/// the result as `needs_ocr` and fall back to GPU OCR.
|
||||
@@ -2248,58 +2070,4 @@ mod tests {
|
||||
"Valid Japanese text should not be flagged as garbage"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_formula_garbage_accepts_math_text() {
|
||||
// Greek letters, math operators, variables — typical formula text
|
||||
let formula = "Φ(ν) = ∫ ∞ dze −it r1 − iνr2 sinh z";
|
||||
assert!(
|
||||
!is_formula_garbage(formula),
|
||||
"Valid formula text should not be flagged as garbage"
|
||||
);
|
||||
|
||||
// Dense operator text
|
||||
let operators = "α + β − γ × δ ÷ ε ≤ ζ ≥ η ≈ θ ≠ ι ± κ";
|
||||
assert!(
|
||||
!is_formula_garbage(operators),
|
||||
"Math operator text should not be flagged as garbage"
|
||||
);
|
||||
|
||||
// Short formula (e.g. single equation variable)
|
||||
let short = "αβ";
|
||||
assert!(
|
||||
!is_formula_garbage(short),
|
||||
"Short formula text should not be flagged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_formula_garbage_rejects_pua_heavy() {
|
||||
// Simulates extensible delimiters from CMEX fonts mapping to PUA
|
||||
let pua_heavy = "x \u{F8EB} \u{F8EC} \u{F8ED} \u{F8F6} \u{F8F7} \u{F8F8} y";
|
||||
assert!(
|
||||
is_formula_garbage(pua_heavy),
|
||||
"PUA-heavy text should be flagged as formula garbage"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_formula_garbage_rejects_control_chars() {
|
||||
// Control characters indicate broken encoding
|
||||
let control_heavy = "a\x01b\x02c\x03d\x04e\x05f\x06g\x07h\x08i";
|
||||
assert!(
|
||||
is_formula_garbage(control_heavy),
|
||||
"Control-char-heavy text should be flagged as formula garbage"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_formula_garbage_accepts_few_pua() {
|
||||
// A few PUA chars among many valid chars is fine (<10% threshold)
|
||||
let mostly_good = "Φ(ν) = ∫ dze r1 − iνr2 sinh z α β γ δ ε ζ η θ \u{F8EB}";
|
||||
assert!(
|
||||
!is_formula_garbage(mostly_good),
|
||||
"Mostly-good text with rare PUA should pass"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user