feat(detector): Scan all pages with early-exit for reliable classification
Change max_pages_to_sample from 5 to u32::MAX so every page is analyzed. This prevents misclassifying Mixed PDFs as TextBased when scanned pages fall outside the old 5-page sample window. Add early-exit: stop scanning as soon as a non-text page is found, since the PDF can't be purely TextBased. A 492-page mixed PDF exits after 2 pages instead of scanning all 492. Also add title and confidence fields to PdfProcessResult for downstream consumers (NAPI wrapper, feature-flag gating). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
b929bbd92e
commit
f54a296c44
+13
-4
@@ -59,7 +59,7 @@ pub struct DetectionConfig {
|
||||
impl Default for DetectionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_pages_to_sample: 5,
|
||||
max_pages_to_sample: u32::MAX,
|
||||
min_text_ops_per_page: 3,
|
||||
text_page_ratio_threshold: 0.6,
|
||||
}
|
||||
@@ -153,13 +153,14 @@ fn detect_from_document(
|
||||
let mut pages_with_images = 0u32;
|
||||
let mut pages_with_template_images = 0u32;
|
||||
let mut total_text_ops = 0u32;
|
||||
|
||||
// Cache Phase 1 results to avoid re-analyzing sampled pages in Phase 2
|
||||
let mut analysis_cache: HashMap<u32, PageAnalysis> = HashMap::new();
|
||||
let mut pages_actually_sampled = 0u32;
|
||||
|
||||
for page_num in &sample_indices {
|
||||
if let Some(&page_id) = pages.get(page_num) {
|
||||
let analysis = analyze_page_content(doc, page_id);
|
||||
pages_actually_sampled += 1;
|
||||
if analysis.text_operator_count >= config.min_text_ops_per_page {
|
||||
pages_with_text += 1;
|
||||
}
|
||||
@@ -170,11 +171,19 @@ fn detect_from_document(
|
||||
pages_with_template_images += 1;
|
||||
}
|
||||
total_text_ops += analysis.text_operator_count;
|
||||
analysis_cache.insert(*page_num, analysis);
|
||||
analysis_cache.insert(*page_num, analysis.clone());
|
||||
|
||||
// Early exit: if this page is non-text (no text ops but has images),
|
||||
// this PDF won't be purely TextBased. Stop scanning remaining pages.
|
||||
if analysis.text_operator_count < config.min_text_ops_per_page
|
||||
&& (analysis.has_images || analysis.has_template_image)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let pages_sampled = sample_indices.len() as u32;
|
||||
let pages_sampled = pages_actually_sampled;
|
||||
let text_ratio = if pages_sampled > 0 {
|
||||
pages_with_text as f32 / pages_sampled as f32
|
||||
} else {
|
||||
|
||||
+20
@@ -33,6 +33,10 @@ pub struct PdfProcessResult {
|
||||
pub processing_time_ms: u64,
|
||||
/// 1-indexed page numbers that need OCR.
|
||||
pub pages_needing_ocr: Vec<u32>,
|
||||
/// Title from PDF metadata (if available)
|
||||
pub title: Option<String>,
|
||||
/// Detection confidence score (0.0 - 1.0)
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
/// Process a PDF file with smart detection and extraction
|
||||
@@ -51,6 +55,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
||||
let page_count = detection.page_count;
|
||||
let pdf_type = detection.pdf_type;
|
||||
let pages_needing_ocr = detection.pages_needing_ocr;
|
||||
let title = detection.title;
|
||||
let confidence = detection.confidence;
|
||||
|
||||
let result = match pdf_type {
|
||||
PdfType::TextBased => {
|
||||
@@ -65,6 +71,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
PdfType::Scanned | PdfType::ImageBased => {
|
||||
@@ -76,6 +84,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
PdfType::Mixed => {
|
||||
@@ -90,6 +100,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -108,6 +120,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
||||
let page_count = detection.page_count;
|
||||
let pdf_type = detection.pdf_type;
|
||||
let pages_needing_ocr = detection.pages_needing_ocr;
|
||||
let title = detection.title;
|
||||
let confidence = detection.confidence;
|
||||
|
||||
let result = match pdf_type {
|
||||
PdfType::TextBased => {
|
||||
@@ -122,6 +136,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
|
||||
@@ -131,6 +147,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
},
|
||||
PdfType::Mixed => {
|
||||
let items = extractor::extract_text_with_positions_mem(buffer).ok();
|
||||
@@ -143,6 +161,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
||||
page_count,
|
||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||
pages_needing_ocr,
|
||||
title,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -56,7 +56,7 @@ fn make_text_item_with_font(
|
||||
#[test]
|
||||
fn test_detection_config_default() {
|
||||
let config = DetectionConfig::default();
|
||||
assert_eq!(config.max_pages_to_sample, 5);
|
||||
assert_eq!(config.max_pages_to_sample, u32::MAX);
|
||||
assert_eq!(config.min_text_ops_per_page, 3);
|
||||
assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001);
|
||||
}
|
||||
@@ -863,6 +863,8 @@ fn test_pages_needing_ocr_field_accessible() {
|
||||
page_count: 1,
|
||||
processing_time_ms: 0,
|
||||
pages_needing_ocr: vec![1, 3],
|
||||
title: None,
|
||||
confidence: 1.0,
|
||||
};
|
||||
assert_eq!(process_result.pages_needing_ocr, vec![1, 3]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user