Merge branch 'main' into feat/python-bindings
This commit is contained in:
Abimael Martell
+569
-16
@@ -31,6 +31,7 @@ pub mod extractor;
|
||||
pub mod glyph_names;
|
||||
pub mod markdown;
|
||||
pub mod process_mode;
|
||||
pub mod structure_tree;
|
||||
pub mod tables;
|
||||
pub mod text_utils;
|
||||
pub mod tounicode;
|
||||
@@ -48,7 +49,7 @@ pub use process_mode::ProcessMode;
|
||||
pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
|
||||
|
||||
use lopdf::Document;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use tounicode::FontCMaps;
|
||||
|
||||
@@ -266,6 +267,264 @@ pub fn process_pdf_mem_with_config(
|
||||
)
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Region-based text extraction (for hybrid OCR pipelines)
|
||||
// =========================================================================
|
||||
|
||||
/// Lightweight classification result for routing decisions.
|
||||
#[derive(Debug)]
|
||||
pub struct PdfClassification {
|
||||
/// The detected PDF type.
|
||||
pub pdf_type: PdfType,
|
||||
/// Total page count.
|
||||
pub page_count: u32,
|
||||
/// 0-indexed page numbers that need OCR (scanned/image pages).
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Detection confidence score (0.0–1.0).
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
/// Classify a PDF from a memory buffer without extracting text.
|
||||
/// Returns the PDF type and which pages need OCR (~10-50ms).
|
||||
pub fn classify_pdf_mem(buffer: &[u8]) -> Result<PdfClassification, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, page_count) = load_document_from_mem(buffer)?;
|
||||
let detection = detector::detect_from_document(&doc, page_count, &DetectionConfig::default())?;
|
||||
Ok(PdfClassification {
|
||||
pdf_type: detection.pdf_type,
|
||||
page_count,
|
||||
// Convert from 1-indexed to 0-indexed for caller convenience
|
||||
pages_needing_ocr: detection.pages_needing_ocr.iter().map(|&p| p - 1).collect(),
|
||||
confidence: detection.confidence,
|
||||
})
|
||||
}
|
||||
|
||||
/// Result for a single region's text extraction.
|
||||
#[derive(Debug)]
|
||||
pub struct RegionText {
|
||||
/// Extracted text (may be empty if region has no text items).
|
||||
pub text: String,
|
||||
/// `true` when the text should not be trusted and OCR should be used instead.
|
||||
/// Set when: the region is empty, the page uses GID-encoded fonts, or the
|
||||
/// extracted text fails garbage/encoding checks.
|
||||
pub needs_ocr: bool,
|
||||
}
|
||||
|
||||
/// Result for a page's region extractions.
|
||||
#[derive(Debug)]
|
||||
pub struct PageRegionResult {
|
||||
/// 0-indexed page number.
|
||||
pub page: u32,
|
||||
/// Per-region results, parallel to the input regions.
|
||||
pub regions: Vec<RegionText>,
|
||||
}
|
||||
|
||||
/// Extract text within bounding-box regions from a PDF in memory.
|
||||
///
|
||||
/// This is designed for hybrid OCR pipelines: a layout model detects regions
|
||||
/// in a rendered page image, and this function extracts the PDF text that
|
||||
/// falls within each region — avoiding GPU OCR for text-based pages.
|
||||
///
|
||||
/// Each region result includes a `needs_ocr` flag that is set when extraction
|
||||
/// quality is suspect (empty text, GID-encoded fonts, garbage/encoding issues).
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `buffer` — PDF file bytes
|
||||
/// * `page_regions` — list of `(page_number_0indexed, Vec<[x1, y1, x2, y2]>)`.
|
||||
/// Coordinates are in **PDF points** with **top-left origin** (matching typical
|
||||
/// layout model output after coordinate conversion).
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A `Vec<PageRegionResult>` parallel to `page_regions`.
|
||||
pub fn extract_text_in_regions_mem(
|
||||
buffer: &[u8],
|
||||
page_regions: &[(u32, Vec<[f32; 4]>)],
|
||||
) -> Result<Vec<PageRegionResult>, PdfError> {
|
||||
validate_pdf_bytes(buffer)?;
|
||||
let (doc, _page_count) = load_document_from_mem(buffer)?;
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
let pages = doc.get_pages();
|
||||
|
||||
// Build a set of pages we need to extract
|
||||
let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect(); // to 1-indexed
|
||||
|
||||
// Extract text items for needed pages only
|
||||
let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
|
||||
let mut page_heights: HashMap<u32, f32> = HashMap::new();
|
||||
let mut gid_pages: HashSet<u32> = HashSet::new();
|
||||
|
||||
for (page_num, &page_id) in pages.iter() {
|
||||
if !needed_pages.contains(page_num) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get page height from MediaBox for coordinate flip
|
||||
let height = get_page_height(&doc, page_id).unwrap_or(792.0);
|
||||
page_heights.insert(*page_num, height);
|
||||
|
||||
// Extract text items for this page
|
||||
let ((mut items, _rects, _lines), has_gid) =
|
||||
extractor::content_stream::extract_page_text_items(
|
||||
&doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
&font_cmaps,
|
||||
false,
|
||||
)?;
|
||||
text_utils::fix_letterspaced_items(&mut items);
|
||||
if has_gid {
|
||||
gid_pages.insert(*page_num);
|
||||
}
|
||||
items_by_page.insert(*page_num, items);
|
||||
}
|
||||
|
||||
// For each page's regions, filter and assemble text
|
||||
let mut results = Vec::with_capacity(page_regions.len());
|
||||
|
||||
for (page_0idx, regions) in page_regions {
|
||||
let page_1idx = page_0idx + 1;
|
||||
let items = items_by_page.get(&page_1idx);
|
||||
let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
|
||||
let page_has_gid = gid_pages.contains(&page_1idx);
|
||||
|
||||
let mut page_results = Vec::with_capacity(regions.len());
|
||||
|
||||
for rect in regions {
|
||||
let [rx1, ry1, rx2, ry2] = *rect;
|
||||
|
||||
let text = match items {
|
||||
Some(items) => collect_text_in_region(items, rx1, ry1, rx2, ry2, page_h),
|
||||
None => String::new(),
|
||||
};
|
||||
|
||||
let needs_ocr = text.trim().is_empty()
|
||||
|| page_has_gid
|
||||
|| is_garbage_text(&text)
|
||||
|| detect_encoding_issues(&text);
|
||||
|
||||
page_results.push(RegionText { text, needs_ocr });
|
||||
}
|
||||
|
||||
results.push(PageRegionResult {
|
||||
page: *page_0idx,
|
||||
regions: page_results,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Get page height in points from MediaBox.
|
||||
fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option<f32> {
|
||||
let page_dict = doc.get_dictionary(page_id).ok()?;
|
||||
// Try MediaBox directly, then follow reference
|
||||
let media_box = page_dict.get(b"MediaBox").ok()?;
|
||||
let arr = match media_box {
|
||||
lopdf::Object::Array(a) => a,
|
||||
lopdf::Object::Reference(r) => {
|
||||
if let Ok(lopdf::Object::Array(a)) = doc.get_object(*r) {
|
||||
a
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
if arr.len() >= 4 {
|
||||
let y1 = obj_to_f32(&arr[1])?;
|
||||
let y2 = obj_to_f32(&arr[3])?;
|
||||
Some((y2 - y1).abs())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn obj_to_f32(obj: &lopdf::Object) -> Option<f32> {
|
||||
match obj {
|
||||
lopdf::Object::Integer(i) => Some(*i as f32),
|
||||
lopdf::Object::Real(f) => Some(*f),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect text items that fall within a region bbox (top-left origin, PDF points)
|
||||
/// and return them as a single string in reading order.
|
||||
fn collect_text_in_region(
|
||||
items: &[TextItem],
|
||||
rx1: f32,
|
||||
ry1: f32,
|
||||
rx2: f32,
|
||||
ry2: f32,
|
||||
page_height: f32,
|
||||
) -> String {
|
||||
// Convert region from top-left to bottom-left origin
|
||||
let by1 = page_height - ry2; // top-left y2 → bottom-left y1
|
||||
let by2 = page_height - ry1; // top-left y1 → bottom-left y2
|
||||
|
||||
// Collect items whose center falls within the region
|
||||
let mut matched: Vec<&TextItem> = items
|
||||
.iter()
|
||||
.filter(|item| {
|
||||
let cx = item.x + item.width / 2.0;
|
||||
let cy = item.y + item.height / 2.0;
|
||||
cx >= rx1 && cx <= rx2 && cy >= by1 && cy <= by2
|
||||
})
|
||||
.collect();
|
||||
|
||||
if matched.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Sort top→bottom (descending Y in bottom-left coords), then left→right
|
||||
matched.sort_by(|a, b| {
|
||||
let line_threshold = a.font_size.max(b.font_size) * 0.5;
|
||||
let y_diff = b.y - a.y; // descending Y = top to bottom
|
||||
if y_diff.abs() < line_threshold {
|
||||
a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal)
|
||||
} else {
|
||||
y_diff
|
||||
.partial_cmp(&0.0_f32)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
}
|
||||
});
|
||||
|
||||
// Group into lines and join
|
||||
let mut lines: Vec<String> = Vec::new();
|
||||
let mut current_line = String::new();
|
||||
let mut last_y = f32::NAN;
|
||||
let mut last_x_end = 0.0_f32;
|
||||
|
||||
for item in &matched {
|
||||
let line_threshold = item.font_size * 0.5;
|
||||
let same_line = (item.y - last_y).abs() < line_threshold;
|
||||
|
||||
if !same_line && !current_line.is_empty() {
|
||||
lines.push(current_line.clone());
|
||||
current_line.clear();
|
||||
}
|
||||
|
||||
if !current_line.is_empty() {
|
||||
// Insert space if there's a gap between items on the same line
|
||||
let gap = item.x - last_x_end;
|
||||
if gap > item.font_size * 0.15 {
|
||||
current_line.push(' ');
|
||||
}
|
||||
}
|
||||
|
||||
current_line.push_str(&item.text);
|
||||
last_y = item.y;
|
||||
last_x_end = item.x + item.width;
|
||||
}
|
||||
|
||||
if !current_line.is_empty() {
|
||||
lines.push(current_line);
|
||||
}
|
||||
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Internal: single-load document pipeline
|
||||
// =========================================================================
|
||||
@@ -276,20 +535,23 @@ pub fn process_pdf_mem_with_config(
|
||||
/// are combined here, but lopdf loads the full doc in `load()` so we extract
|
||||
/// page count from it directly to avoid the metadata-only round-trip.
|
||||
fn load_document_from_path<P: AsRef<Path>>(path: P) -> Result<(Document, u32), PdfError> {
|
||||
let doc = match Document::load(&path) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_with_password(&path, "")?,
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let page_count = doc.get_pages().len() as u32;
|
||||
Ok((doc, page_count))
|
||||
let buffer = std::fs::read(&path)?;
|
||||
load_document_from_mem(&buffer)
|
||||
}
|
||||
|
||||
/// Load a PDF from a memory buffer.
|
||||
fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
|
||||
let doc = match Document::load_mem(buffer) {
|
||||
// Fix malformed struct element names before parsing. Some PDF generators
|
||||
// write bare names (/S Code) instead of proper PDF names (/S /Code), which
|
||||
// causes lopdf to silently drop the entire object.
|
||||
let fixed = structure_tree::fix_bare_struct_names(buffer);
|
||||
let buf = fixed.as_ref();
|
||||
|
||||
let doc = match Document::load_mem(buf) {
|
||||
Ok(d) => d,
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => Document::load_mem_with_password(buffer, "")?,
|
||||
Err(ref e) if is_encrypted_lopdf_error(e) => {
|
||||
Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))?
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
let page_count = doc.get_pages().len() as u32;
|
||||
@@ -343,7 +605,38 @@ fn process_document(
|
||||
// Step 2 — Extraction (reuses the already-loaded document)
|
||||
let extracted = {
|
||||
let font_cmaps = FontCMaps::from_doc(&doc);
|
||||
extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, options.page_filter.as_ref())
|
||||
let result = extractor::extract_positioned_text_from_doc(
|
||||
&doc,
|
||||
&font_cmaps,
|
||||
options.page_filter.as_ref(),
|
||||
);
|
||||
|
||||
// For Mixed/template PDFs: if normal extraction produces garbage text
|
||||
// (mostly non-alphanumeric), retry with invisible (Tr=3) text included.
|
||||
// This unlocks OCR text layers behind scanned images.
|
||||
if pdf_type == PdfType::Mixed {
|
||||
if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _)| e) {
|
||||
let sample: String = items.iter().take(200).map(|i| i.text.as_str()).collect();
|
||||
if is_garbage_text(&sample) || sample.trim().is_empty() {
|
||||
extractor::extract_positioned_text_include_invisible(
|
||||
&doc,
|
||||
&font_cmaps,
|
||||
options.page_filter.as_ref(),
|
||||
)
|
||||
} else {
|
||||
result
|
||||
}
|
||||
} else {
|
||||
// Normal extraction failed — try invisible as fallback
|
||||
extractor::extract_positioned_text_include_invisible(
|
||||
&doc,
|
||||
&font_cmaps,
|
||||
options.page_filter.as_ref(),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
result
|
||||
}
|
||||
};
|
||||
|
||||
// For Mixed PDFs, extraction failure is non-fatal
|
||||
@@ -353,8 +646,75 @@ fn process_document(
|
||||
Some(extracted?)
|
||||
};
|
||||
|
||||
let (markdown, layout, has_encoding_issues) = match extracted {
|
||||
Some((items, rects, lines)) => {
|
||||
// Parse structure tree for tagged PDFs (reuses the loaded document)
|
||||
let (struct_roles, struct_tables) = structure_tree::StructTree::from_doc(&doc)
|
||||
.map(|tree| {
|
||||
let page_ids = doc.get_pages();
|
||||
let roles = tree.mcid_to_roles(&page_ids);
|
||||
let tables = tree.extract_tables(&page_ids);
|
||||
if !roles.is_empty() {
|
||||
log::debug!(
|
||||
"structure tree: {} pages with MCID roles, {} total MCIDs, {} tagged tables",
|
||||
roles.len(),
|
||||
tree.mcid_count(),
|
||||
tables.len()
|
||||
);
|
||||
}
|
||||
let roles = if roles.is_empty() { None } else { Some(roles) };
|
||||
(roles, tables)
|
||||
})
|
||||
.unwrap_or((None, Vec::new()));
|
||||
|
||||
let (markdown, layout, has_encoding_issues, gid_pages) = match extracted {
|
||||
Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => {
|
||||
// For TextBased PDFs with pages flagged for OCR (Identity-H or
|
||||
// Type3 fonts without ToUnicode), check whether the CID-as-Unicode
|
||||
// passthrough actually produced readable text. If a page's text
|
||||
// is garbage, strip its items so we don't emit mojibake.
|
||||
// Only applies to TextBased — for Mixed PDFs, OCR flags come from
|
||||
// template images rather than font encoding issues.
|
||||
let (items, rects, lines) =
|
||||
if pages_needing_ocr.is_empty() || pdf_type != PdfType::TextBased {
|
||||
(items, rects, lines)
|
||||
} else {
|
||||
let ocr_set: std::collections::HashSet<u32> =
|
||||
pages_needing_ocr.iter().copied().collect();
|
||||
// Collect text per OCR-flagged page and check quality
|
||||
let mut garbage_pages: std::collections::HashSet<u32> =
|
||||
std::collections::HashSet::new();
|
||||
for &pg in &ocr_set {
|
||||
let page_text: String = items
|
||||
.iter()
|
||||
.filter(|i| i.page == pg)
|
||||
.map(|i| i.text.as_str())
|
||||
.collect();
|
||||
if is_cid_garbage(&page_text) {
|
||||
garbage_pages.insert(pg);
|
||||
}
|
||||
}
|
||||
if garbage_pages.is_empty() {
|
||||
(items, rects, lines)
|
||||
} else {
|
||||
log::debug!(
|
||||
"suppressing garbage text from OCR-flagged pages: {:?}",
|
||||
garbage_pages
|
||||
);
|
||||
let items: Vec<_> = items
|
||||
.into_iter()
|
||||
.filter(|i| !garbage_pages.contains(&i.page))
|
||||
.collect();
|
||||
let rects: Vec<_> = rects
|
||||
.into_iter()
|
||||
.filter(|r| !garbage_pages.contains(&r.page))
|
||||
.collect();
|
||||
let lines: Vec<_> = lines
|
||||
.into_iter()
|
||||
.filter(|l| !garbage_pages.contains(&l.page))
|
||||
.collect();
|
||||
(items, rects, lines)
|
||||
}
|
||||
};
|
||||
|
||||
let layout = compute_layout_complexity(&items, &rects, &lines);
|
||||
|
||||
let md = if options.mode == ProcessMode::Analyze {
|
||||
@@ -365,13 +725,91 @@ fn process_document(
|
||||
options.markdown,
|
||||
&rects,
|
||||
&lines,
|
||||
&page_thresholds,
|
||||
struct_roles.as_ref(),
|
||||
&struct_tables,
|
||||
))
|
||||
};
|
||||
|
||||
let enc = md.as_ref().is_some_and(|m| detect_encoding_issues(m));
|
||||
(md, layout, enc)
|
||||
(md, layout, enc, gid_encoded_pages)
|
||||
}
|
||||
None => (None, LayoutComplexity::default(), false),
|
||||
None => (
|
||||
None,
|
||||
LayoutComplexity::default(),
|
||||
false,
|
||||
std::collections::HashSet::new(),
|
||||
),
|
||||
};
|
||||
|
||||
// If the extracted text is predominantly garbage (non-alphanumeric) and
|
||||
// the PDF is image-backed (Mixed/template), upgrade to Scanned — the text
|
||||
// layer comes from a bad OCR pass, and callers should use proper OCR.
|
||||
let (pdf_type, markdown, confidence) =
|
||||
if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) {
|
||||
(PdfType::Scanned, None, 0.95)
|
||||
} else {
|
||||
(pdf_type, markdown, confidence)
|
||||
};
|
||||
|
||||
// If a TextBased PDF produces garbage text, the fonts are undecodable
|
||||
// (e.g. Identity-H without ToUnicode for non-Latin scripts like Cyrillic).
|
||||
// Drop the useless markdown and flag all pages for OCR.
|
||||
let (markdown, has_encoding_issues, force_ocr_all) = if pdf_type == PdfType::TextBased
|
||||
&& markdown.as_ref().is_some_and(|m| is_garbage_text(m))
|
||||
{
|
||||
log::debug!("TextBased PDF has garbage text — flagging all pages for OCR");
|
||||
(None, true, true)
|
||||
} else {
|
||||
(markdown, has_encoding_issues, false)
|
||||
};
|
||||
|
||||
// Add pages with gid-encoded fonts (unresolvable encoding) to OCR list.
|
||||
// When ALL pages have gid-encoded fonts, suppress unreliable markdown.
|
||||
let all_gid = !gid_pages.is_empty() && gid_pages.len() as u32 >= page_count;
|
||||
let mut pages_needing_ocr = pages_needing_ocr;
|
||||
if force_ocr_all {
|
||||
pages_needing_ocr = (1..=page_count).collect();
|
||||
}
|
||||
if !gid_pages.is_empty() {
|
||||
log::debug!("pages with gid-encoded fonts (need OCR): {:?}", gid_pages);
|
||||
for page in gid_pages {
|
||||
if !pages_needing_ocr.contains(&page) {
|
||||
pages_needing_ocr.push(page);
|
||||
}
|
||||
}
|
||||
pages_needing_ocr.sort_unstable();
|
||||
}
|
||||
|
||||
// Detect sparse extraction: when a TEXT-BASED PDF produces very few
|
||||
// characters per page, the text is likely embedded in images/forms
|
||||
// that need OCR. Flag all pages for OCR in this case.
|
||||
// Only check when markdown was actually generated (not in Analyze mode).
|
||||
if pdf_type == PdfType::TextBased
|
||||
&& page_count > 0
|
||||
&& pages_needing_ocr.is_empty()
|
||||
&& markdown.is_some()
|
||||
{
|
||||
let md_len = markdown.as_ref().map_or(0, |m| m.len());
|
||||
let chars_per_page = md_len as f32 / page_count as f32;
|
||||
if chars_per_page < 50.0 && md_len < 500 {
|
||||
log::debug!(
|
||||
"sparse extraction: {:.0} chars/page — recommending OCR for all {} pages",
|
||||
chars_per_page,
|
||||
page_count
|
||||
);
|
||||
pages_needing_ocr = (1..=page_count).collect();
|
||||
}
|
||||
}
|
||||
|
||||
let markdown = if all_gid {
|
||||
log::debug!(
|
||||
"all {} pages have gid-encoded fonts — suppressing markdown output",
|
||||
page_count
|
||||
);
|
||||
None
|
||||
} else {
|
||||
markdown
|
||||
};
|
||||
|
||||
Ok(PdfProcessResult {
|
||||
@@ -427,6 +865,76 @@ fn detect_encoding_issues(markdown: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if extracted text is predominantly garbage (non-alphanumeric).
|
||||
///
|
||||
/// Broken font encodings produce text like "----1-.-.-.___ --.-. .._ I_---."
|
||||
/// where most characters are punctuation/symbols. Real text in any language
|
||||
/// has >50% alphanumeric characters.
|
||||
fn is_garbage_text(markdown: &str) -> bool {
|
||||
let mut alphanum = 0usize;
|
||||
let mut non_alphanum = 0usize;
|
||||
for ch in markdown.chars() {
|
||||
if ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
// Skip markdown syntax chars that we add (not from the PDF)
|
||||
if matches!(ch, '#' | '*' | '|' | '-' | '\n') {
|
||||
continue;
|
||||
}
|
||||
if ch.is_alphanumeric() {
|
||||
alphanum += 1;
|
||||
} else {
|
||||
non_alphanum += 1;
|
||||
}
|
||||
}
|
||||
let total = alphanum + non_alphanum;
|
||||
total >= 50 && alphanum * 2 < total
|
||||
}
|
||||
|
||||
/// Detect garbage from failed CID-to-Unicode mapping on Identity-H fonts.
|
||||
///
|
||||
/// When CID values don't correspond to Unicode codepoints, the raw bytes often
|
||||
/// produce characters in the C1 control range (U+0080–U+009F) or Private Use
|
||||
/// Area, mixed with random Latin Extended characters. Valid text in any
|
||||
/// language almost never contains C1 controls. We also fall back to the
|
||||
/// general `is_garbage_text` check for non-alphanumeric-heavy patterns.
|
||||
fn is_cid_garbage(text: &str) -> bool {
|
||||
if is_garbage_text(text) {
|
||||
return true;
|
||||
}
|
||||
let mut total = 0usize;
|
||||
let mut c1_control = 0usize;
|
||||
let mut high_latin = 0usize;
|
||||
for ch in text.chars() {
|
||||
if ch.is_whitespace() {
|
||||
continue;
|
||||
}
|
||||
total += 1;
|
||||
// C1 control characters (U+0080–U+009F) — almost never in real text
|
||||
if ('\u{0080}'..='\u{009F}').contains(&ch) {
|
||||
c1_control += 1;
|
||||
}
|
||||
// High Latin-1 (U+00A0–U+00FF) — legitimate in Western European text
|
||||
// but when combined with ASCII in CID passthrough, indicates mojibake
|
||||
// from CID values being misinterpreted as Latin-1 characters.
|
||||
if ('\u{00A0}'..='\u{00FF}').contains(&ch) {
|
||||
high_latin += 1;
|
||||
}
|
||||
}
|
||||
if total < 5 {
|
||||
return false;
|
||||
}
|
||||
// If ≥5% of non-whitespace chars are C1 controls, it's garbage
|
||||
if c1_control * 20 >= total {
|
||||
return true;
|
||||
}
|
||||
// If ≥40% of non-whitespace chars are high Latin-1 AND the text has few
|
||||
// ASCII letters, it's likely CID-as-Latin-1 mojibake (Japanese/CJK PDFs
|
||||
// where CID values 0x80-0xFF become accented Latin characters).
|
||||
let ascii_letters = text.chars().filter(|c| c.is_ascii_alphabetic()).count();
|
||||
high_latin * 5 >= total * 2 && ascii_letters * 3 < total
|
||||
}
|
||||
|
||||
/// Analyse extracted items and rects for layout complexity.
|
||||
fn compute_layout_complexity(
|
||||
items: &[types::TextItem],
|
||||
@@ -507,7 +1015,7 @@ fn compute_layout_complexity(
|
||||
|
||||
let mut pages_with_columns: Vec<u32> = Vec::new();
|
||||
for page in seen_pages {
|
||||
let cols = extractor::detect_columns(items, page);
|
||||
let cols = extractor::detect_columns(items, page, pages_with_tables.contains(&page));
|
||||
if cols.len() >= 2 {
|
||||
pages_with_columns.push(page);
|
||||
}
|
||||
@@ -726,4 +1234,49 @@ mod tests {
|
||||
let text = "a$b c$d e$f";
|
||||
assert!(!detect_encoding_issues(text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_garbage_text_detection() {
|
||||
// Simulates garbage output from Identity-H fonts without ToUnicode.
|
||||
// Needs >= 50 non-whitespace chars and < 50% alphanumeric.
|
||||
let garbage = ",&<X ~%5&8-!A ~*(!,-!U (/#!U X ~#/=U 9/%*(!U !( X \
|
||||
(%U-(-/ V %&((8-#&&< *,(6--< %5&8-!( (,(/! #/<5U X \
|
||||
º&( >/5 /5&(#(8-!5 *,(6--( *,%@/-A W";
|
||||
assert!(is_garbage_text(garbage));
|
||||
|
||||
// Normal text should not be garbage
|
||||
let normal = "This is a normal paragraph with words and sentences that contains enough characters to pass the threshold.";
|
||||
assert!(!is_garbage_text(normal));
|
||||
|
||||
// Cyrillic text should not be garbage
|
||||
let cyrillic =
|
||||
"Роботизированные технологии комплексы для производства металлургических предприятий";
|
||||
assert!(!is_garbage_text(cyrillic));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cid_garbage_detection() {
|
||||
// Simulates CID garbage from Identity-H fonts: Latin Extended chars
|
||||
// mixed with C1 control characters (U+0080–U+009F).
|
||||
let cid_garbage = "Ë>íÓ\tý\r\u{0088}æ&Ït\u{0094}äí;\ný;wAL¢©èåD\rü£\
|
||||
qq\u{0096}¶Í Æ\réá; Ô 7G\u{008B}ý;èÕç¢ £ ý;C";
|
||||
assert!(
|
||||
is_cid_garbage(cid_garbage),
|
||||
"CID garbage with C1 controls should be detected"
|
||||
);
|
||||
|
||||
// Valid Korean text (CID-as-Unicode passthrough) should NOT be garbage
|
||||
let korean = "본 가격표는 국내 거주 중인 외국인을 위한 한국어 가격표의 비공식 번역본입니다";
|
||||
assert!(
|
||||
!is_cid_garbage(korean),
|
||||
"Valid Korean text should not be flagged as garbage"
|
||||
);
|
||||
|
||||
// Valid Japanese text should NOT be garbage
|
||||
let japanese = "羽田空港新飛行経路に係る航空機騒音の測定結果";
|
||||
assert!(
|
||||
!is_cid_garbage(japanese),
|
||||
"Valid Japanese text should not be flagged as garbage"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user