feat(detector): Scan all pages with early-exit for reliable classification
Change max_pages_to_sample from 5 to u32::MAX so every page is analyzed. This prevents misclassifying Mixed PDFs as TextBased when scanned pages fall outside the old 5-page sample window. Add early-exit: stop scanning as soon as a non-text page is found, since the PDF can't be purely TextBased. A 492-page mixed PDF exits after 2 pages instead of scanning all 492. Also add title and confidence fields to PdfProcessResult for downstream consumers (NAPI wrapper, feature-flag gating). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
b929bbd92e
commit
f54a296c44
+13
-4
@@ -59,7 +59,7 @@ pub struct DetectionConfig {
|
|||||||
impl Default for DetectionConfig {
|
impl Default for DetectionConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
max_pages_to_sample: 5,
|
max_pages_to_sample: u32::MAX,
|
||||||
min_text_ops_per_page: 3,
|
min_text_ops_per_page: 3,
|
||||||
text_page_ratio_threshold: 0.6,
|
text_page_ratio_threshold: 0.6,
|
||||||
}
|
}
|
||||||
@@ -153,13 +153,14 @@ fn detect_from_document(
|
|||||||
let mut pages_with_images = 0u32;
|
let mut pages_with_images = 0u32;
|
||||||
let mut pages_with_template_images = 0u32;
|
let mut pages_with_template_images = 0u32;
|
||||||
let mut total_text_ops = 0u32;
|
let mut total_text_ops = 0u32;
|
||||||
|
|
||||||
// Cache Phase 1 results to avoid re-analyzing sampled pages in Phase 2
|
// Cache Phase 1 results to avoid re-analyzing sampled pages in Phase 2
|
||||||
let mut analysis_cache: HashMap<u32, PageAnalysis> = HashMap::new();
|
let mut analysis_cache: HashMap<u32, PageAnalysis> = HashMap::new();
|
||||||
|
let mut pages_actually_sampled = 0u32;
|
||||||
|
|
||||||
for page_num in &sample_indices {
|
for page_num in &sample_indices {
|
||||||
if let Some(&page_id) = pages.get(page_num) {
|
if let Some(&page_id) = pages.get(page_num) {
|
||||||
let analysis = analyze_page_content(doc, page_id);
|
let analysis = analyze_page_content(doc, page_id);
|
||||||
|
pages_actually_sampled += 1;
|
||||||
if analysis.text_operator_count >= config.min_text_ops_per_page {
|
if analysis.text_operator_count >= config.min_text_ops_per_page {
|
||||||
pages_with_text += 1;
|
pages_with_text += 1;
|
||||||
}
|
}
|
||||||
@@ -170,11 +171,19 @@ fn detect_from_document(
|
|||||||
pages_with_template_images += 1;
|
pages_with_template_images += 1;
|
||||||
}
|
}
|
||||||
total_text_ops += analysis.text_operator_count;
|
total_text_ops += analysis.text_operator_count;
|
||||||
analysis_cache.insert(*page_num, analysis);
|
analysis_cache.insert(*page_num, analysis.clone());
|
||||||
|
|
||||||
|
// Early exit: if this page is non-text (no text ops but has images),
|
||||||
|
// this PDF won't be purely TextBased. Stop scanning remaining pages.
|
||||||
|
if analysis.text_operator_count < config.min_text_ops_per_page
|
||||||
|
&& (analysis.has_images || analysis.has_template_image)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let pages_sampled = sample_indices.len() as u32;
|
let pages_sampled = pages_actually_sampled;
|
||||||
let text_ratio = if pages_sampled > 0 {
|
let text_ratio = if pages_sampled > 0 {
|
||||||
pages_with_text as f32 / pages_sampled as f32
|
pages_with_text as f32 / pages_sampled as f32
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
+20
@@ -33,6 +33,10 @@ pub struct PdfProcessResult {
|
|||||||
pub processing_time_ms: u64,
|
pub processing_time_ms: u64,
|
||||||
/// 1-indexed page numbers that need OCR.
|
/// 1-indexed page numbers that need OCR.
|
||||||
pub pages_needing_ocr: Vec<u32>,
|
pub pages_needing_ocr: Vec<u32>,
|
||||||
|
/// Title from PDF metadata (if available)
|
||||||
|
pub title: Option<String>,
|
||||||
|
/// Detection confidence score (0.0 - 1.0)
|
||||||
|
pub confidence: f32,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process a PDF file with smart detection and extraction
|
/// Process a PDF file with smart detection and extraction
|
||||||
@@ -51,6 +55,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
|||||||
let page_count = detection.page_count;
|
let page_count = detection.page_count;
|
||||||
let pdf_type = detection.pdf_type;
|
let pdf_type = detection.pdf_type;
|
||||||
let pages_needing_ocr = detection.pages_needing_ocr;
|
let pages_needing_ocr = detection.pages_needing_ocr;
|
||||||
|
let title = detection.title;
|
||||||
|
let confidence = detection.confidence;
|
||||||
|
|
||||||
let result = match pdf_type {
|
let result = match pdf_type {
|
||||||
PdfType::TextBased => {
|
PdfType::TextBased => {
|
||||||
@@ -65,6 +71,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PdfType::Scanned | PdfType::ImageBased => {
|
PdfType::Scanned | PdfType::ImageBased => {
|
||||||
@@ -76,6 +84,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PdfType::Mixed => {
|
PdfType::Mixed => {
|
||||||
@@ -90,6 +100,8 @@ pub fn process_pdf<P: AsRef<Path>>(path: P) -> Result<PdfProcessResult, PdfError
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -108,6 +120,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
|||||||
let page_count = detection.page_count;
|
let page_count = detection.page_count;
|
||||||
let pdf_type = detection.pdf_type;
|
let pdf_type = detection.pdf_type;
|
||||||
let pages_needing_ocr = detection.pages_needing_ocr;
|
let pages_needing_ocr = detection.pages_needing_ocr;
|
||||||
|
let title = detection.title;
|
||||||
|
let confidence = detection.confidence;
|
||||||
|
|
||||||
let result = match pdf_type {
|
let result = match pdf_type {
|
||||||
PdfType::TextBased => {
|
PdfType::TextBased => {
|
||||||
@@ -122,6 +136,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
|
PdfType::Scanned | PdfType::ImageBased => PdfProcessResult {
|
||||||
@@ -131,6 +147,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
},
|
},
|
||||||
PdfType::Mixed => {
|
PdfType::Mixed => {
|
||||||
let items = extractor::extract_text_with_positions_mem(buffer).ok();
|
let items = extractor::extract_text_with_positions_mem(buffer).ok();
|
||||||
@@ -143,6 +161,8 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result<PdfProcessResult, PdfError> {
|
|||||||
page_count,
|
page_count,
|
||||||
processing_time_ms: start.elapsed().as_millis() as u64,
|
processing_time_ms: start.elapsed().as_millis() as u64,
|
||||||
pages_needing_ocr,
|
pages_needing_ocr,
|
||||||
|
title,
|
||||||
|
confidence,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ fn make_text_item_with_font(
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_detection_config_default() {
|
fn test_detection_config_default() {
|
||||||
let config = DetectionConfig::default();
|
let config = DetectionConfig::default();
|
||||||
assert_eq!(config.max_pages_to_sample, 5);
|
assert_eq!(config.max_pages_to_sample, u32::MAX);
|
||||||
assert_eq!(config.min_text_ops_per_page, 3);
|
assert_eq!(config.min_text_ops_per_page, 3);
|
||||||
assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001);
|
assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001);
|
||||||
}
|
}
|
||||||
@@ -863,6 +863,8 @@ fn test_pages_needing_ocr_field_accessible() {
|
|||||||
page_count: 1,
|
page_count: 1,
|
||||||
processing_time_ms: 0,
|
processing_time_ms: 0,
|
||||||
pages_needing_ocr: vec![1, 3],
|
pages_needing_ocr: vec![1, 3],
|
||||||
|
title: None,
|
||||||
|
confidence: 1.0,
|
||||||
};
|
};
|
||||||
assert_eq!(process_result.pages_needing_ocr, vec![1, 3]);
|
assert_eq!(process_result.pages_needing_ocr, vec![1, 3]);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user