feat(detector): Scan all pages with early-exit for reliable classification

Change max_pages_to_sample from 5 to u32::MAX so every page is analyzed.
This prevents misclassifying Mixed PDFs as TextBased when scanned pages
fall outside the old 5-page sample window.

Add early-exit: stop scanning as soon as a non-text page is found, since
the PDF can't be purely TextBased. A 492-page mixed PDF exits after 2
pages instead of scanning all 492.

Also add title and confidence fields to PdfProcessResult for downstream
consumers (NAPI wrapper, feature-flag gating).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-02-16 14:45:07 -08:00
co-authored by Claude Opus 4.6
parent b929bbd92e
commit f54a296c44
3 changed files with 36 additions and 5 deletions
+13 -4
View File
@@ -59,7 +59,7 @@ pub struct DetectionConfig {
impl Default for DetectionConfig { impl Default for DetectionConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {
max_pages_to_sample: 5, max_pages_to_sample: u32::MAX,
min_text_ops_per_page: 3, min_text_ops_per_page: 3,
text_page_ratio_threshold: 0.6, text_page_ratio_threshold: 0.6,
} }
@@ -153,13 +153,14 @@ fn detect_from_document(
let mut pages_with_images = 0u32; let mut pages_with_images = 0u32;
let mut pages_with_template_images = 0u32; let mut pages_with_template_images = 0u32;
let mut total_text_ops = 0u32; let mut total_text_ops = 0u32;
// Cache Phase 1 results to avoid re-analyzing sampled pages in Phase 2 // Cache Phase 1 results to avoid re-analyzing sampled pages in Phase 2
let mut analysis_cache: HashMap<u32, PageAnalysis> = HashMap::new(); let mut analysis_cache: HashMap<u32, PageAnalysis> = HashMap::new();
let mut pages_actually_sampled = 0u32;
for page_num in &sample_indices { for page_num in &sample_indices {
if let Some(&page_id) = pages.get(page_num) { if let Some(&page_id) = pages.get(page_num) {
let analysis = analyze_page_content(doc, page_id); let analysis = analyze_page_content(doc, page_id);
pages_actually_sampled += 1;
if analysis.text_operator_count >= config.min_text_ops_per_page { if analysis.text_operator_count >= config.min_text_ops_per_page {
pages_with_text += 1; pages_with_text += 1;
} }
@@ -170,11 +171,19 @@ fn detect_from_document(
pages_with_template_images += 1; pages_with_template_images += 1;
} }
total_text_ops += analysis.text_operator_count; total_text_ops += analysis.text_operator_count;
analysis_cache.insert(*page_num, analysis); analysis_cache.insert(*page_num, analysis.clone());
// Early exit: if this page is non-text (no text ops but has images),
// this PDF won't be purely TextBased. Stop scanning remaining pages.
if analysis.text_operator_count < config.min_text_ops_per_page
&& (analysis.has_images || analysis.has_template_image)
{
break;
}
} }
} }
let pages_sampled = sample_indices.len() as u32; let pages_sampled = pages_actually_sampled;
let text_ratio = if pages_sampled > 0 { let text_ratio = if pages_sampled > 0 {
pages_with_text as f32 / pages_sampled as f32 pages_with_text as f32 / pages_sampled as f32
} else { } else {
+20
View File
@@ -33,6 +33,10 @@ pub struct PdfProcessResult {
pub processing_time_ms: u64, pub processing_time_ms: u64,
/// 1-indexed page numbers that need OCR. /// 1-indexed page numbers that need OCR.
pub pages_needing_ocr: Vec<u32>, pub pages_needing_ocr: Vec<u32>,
/// Title from PDF metadata (if available)
pub title: Option<String>,
/// Detection confidence score (0.0 - 1.0)
pub confidence: f32,
} }
/// Process a PDF file with smart detection and extraction /// Process a PDF file with smart detection and extraction
@@ -51,6 +55,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
let page_count = detection.page_count; let page_count = detection.page_count;
let pdf_type = detection.pdf_type; let pdf_type = detection.pdf_type;
let pages_needing_ocr = detection.pages_needing_ocr; let pages_needing_ocr = detection.pages_needing_ocr;
let title = detection.title;
let confidence = detection.confidence;
let result = match pdf_type { let result = match pdf_type {
PdfType::TextBased => { PdfType::TextBased => {
@@ -65,6 +71,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
} }
} }
PdfType::Scanned | PdfType::ImageBased => { PdfType::Scanned | PdfType::ImageBased => {
@@ -76,6 +84,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
} }
} }
PdfType::Mixed => { PdfType::Mixed => {
@@ -90,6 +100,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
} }
} }
}; };
@@ -108,6 +120,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
let page_count = detection.page_count; let page_count = detection.page_count;
let pdf_type = detection.pdf_type; let pdf_type = detection.pdf_type;
let pages_needing_ocr = detection.pages_needing_ocr; let pages_needing_ocr = detection.pages_needing_ocr;
let title = detection.title;
let confidence = detection.confidence;
let result = match pdf_type { let result = match pdf_type {
PdfType::TextBased => { PdfType::TextBased => {
@@ -122,6 +136,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
} }
} }
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult { PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
@@ -131,6 +147,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
}, },
PdfType::Mixed => { PdfType::Mixed => {
let items = extractor::extract_text_with_positions_mem(buffer).ok(); let items = extractor::extract_text_with_positions_mem(buffer).ok();
@@ -143,6 +161,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
page_count, page_count,
processing_time_ms: start.elapsed().as_millis() as u64, processing_time_ms: start.elapsed().as_millis() as u64,
pages_needing_ocr, pages_needing_ocr,
title,
confidence,
} }
} }
}; };
+3 -1
View File
@@ -56,7 +56,7 @@ fn make_text_item_with_font(
#[test] #[test]
fn test_detection_config_default() { fn test_detection_config_default() {
let config = DetectionConfig::default(); let config = DetectionConfig::default();
assert_eq!(config.max_pages_to_sample, 5); assert_eq!(config.max_pages_to_sample, u32::MAX);
assert_eq!(config.min_text_ops_per_page, 3); assert_eq!(config.min_text_ops_per_page, 3);
assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001); assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001);
} }
@@ -863,6 +863,8 @@ fn test_pages_needing_ocr_field_accessible() {
page_count: 1, page_count: 1,
processing_time_ms: 0, processing_time_ms: 0,
pages_needing_ocr: vec![1, 3], pages_needing_ocr: vec![1, 3],
title: None,
confidence: 1.0,
}; };
assert_eq!(process_result.pages_needing_ocr, vec![1, 3]); assert_eq!(process_result.pages_needing_ocr, vec![1, 3]);
} }