Compare commits

...
6 changed files with 1226 additions and 23 deletions
+16
View File
@@ -380,6 +380,22 @@ provides the same native-only behavior through the OCR result/provenance
shape; `Force` renders every selected page. Learned layout intentionally
returns an explicit unsupported error in this lightweight pipeline.
For ambiguous mixed pages, `Auto` privately retains clean native fragments
instead of discarding them when OCR is selected. After recognition it compares
script-agnostic text quality, OCR confidence, character overlap, and material
new coverage. Exact native text wins over a duplicate or weak OCR hypothesis;
complementary image-backed text is fused; and pages where both candidates are
weak recommend the hosted document pipeline. Public native-only extraction
continues to suppress pages marked unreliable, and clean text documents pay no
renderer or model-initialization cost.
In `Auto`, pages routed only for suspicious font encoding or vectorized text
first get a bounded positioned-text probe through PDFium. A credible recovered
text layer with sufficient geometric page coverage skips rasterization and
model loading for that page; garbled, partial, or insubstantial recovery
continues through OCR. Recovered tables are reflected in the same document
metadata as tables found by the primary extractor.
The one-call API keeps the most recently used verified OCR engine in process.
Long-lived workers therefore verify the pinned artifacts and build the ONNX
sessions once, then reuse those loaded sessions across documents. The cache is
+20 -3
View File
@@ -459,8 +459,15 @@ pub fn extract_pages_markdown_mem(
buffer: &[u8],
pages: Option<&[u32]>,
) -> Result<PagesExtractionResult, PdfError> {
extract_pages_markdown_mem_impl(buffer, pages, None, &MarkdownOptions::default(), false)
.map(|(result, _)| result)
extract_pages_markdown_mem_impl(
buffer,
pages,
None,
&MarkdownOptions::default(),
false,
false,
)
.map(|(result, _)| result)
}
#[cfg(all(feature = "ocr", not(target_arch = "wasm32")))]
@@ -476,6 +483,7 @@ pub(crate) fn extract_pages_markdown_mem_for_ocr(
password,
markdown_options,
markdown_options.strip_headers_footers,
true,
)
}
@@ -485,6 +493,7 @@ fn extract_pages_markdown_mem_impl(
password: Option<&str>,
markdown_options: &MarkdownOptions,
strip_repeated_headers_footers: bool,
preserve_ocr_candidates: bool,
) -> Result<(PagesExtractionResult, u32), PdfError> {
validate_pdf_bytes(buffer)?;
let (doc, page_count) = load_document_from_mem_with_password(buffer, password)?;
@@ -663,7 +672,15 @@ fn extract_pages_markdown_mem_impl(
results.push(PageMarkdown {
page: page_0idx,
markdown: if needs_ocr { String::new() } else { md },
// The public native extractor continues to suppress unreliable
// text. The OCR orchestrator retains clean partial text
// internally so it can compare/fuse it with OCR before deciding
// what is safe to return.
markdown: if needs_ocr && !preserve_ocr_candidates {
String::new()
} else {
md
},
needs_ocr,
ocr_reason,
});
+401 -5
View File
@@ -6,6 +6,7 @@ use std::time::Instant;
use thiserror::Error;
use crate::markdown::{to_markdown_from_items_with_rects_and_page_count, MarkdownOptions};
use crate::text_quality::{detect_encoding_issues, is_cid_garbage, is_garbage_text};
use crate::types::{ItemType, TextItem};
use crate::PageMarkdown;
@@ -91,6 +92,73 @@ pub struct FusedPages {
pub ocr_time_ms: u64,
}
/// Origin of a trustworthy native-text candidate retained for adaptive OCR.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum NativeCandidateOrigin {
/// Text produced by pdf-inspector's normal native extractor.
Extractor,
/// Positioned text independently recovered through PDFium.
Pdfium,
}
impl NativeCandidateOrigin {
fn description(self) -> &'static str {
match self {
Self::Extractor => "native extraction",
Self::Pdfium => "PDFium native recovery",
}
}
}
#[derive(Debug, Clone, Copy)]
struct TextCandidateQuality {
alphanumeric_chars: usize,
score: f32,
}
/// Clean native text retained while an ambiguous page is compared with OCR.
#[derive(Debug, Clone)]
pub(crate) struct NativeFallbackCandidate {
markdown: String,
quality: TextCandidateQuality,
origin: NativeCandidateOrigin,
}
impl NativeFallbackCandidate {
/// True when an independent native recovery is substantial enough to
/// cancel OCR for recoverable font/vector routing reasons.
pub(crate) fn is_complete_recovery(&self) -> bool {
self.quality.alphanumeric_chars >= 40 && self.quality.score >= 0.68
}
pub(crate) fn markdown(&self) -> &str {
&self.markdown
}
pub(crate) fn is_stronger_than(&self, other: &Self) -> bool {
self.quality.alphanumeric_chars > other.quality.alphanumeric_chars
|| (self.quality.alphanumeric_chars == other.quality.alphanumeric_chars
&& self.quality.score > other.quality.score)
}
}
/// Validates and scores native Markdown for possible post-OCR comparison.
///
/// This intentionally uses script-agnostic evidence. A native candidate only
/// needs to be trustworthy, not necessarily complete: a clean native header
/// can still be fused with an image-backed OCR body.
pub(crate) fn assess_native_candidate(
markdown: String,
origin: NativeCandidateOrigin,
) -> Option<NativeFallbackCandidate> {
let quality = assess_text_candidate(&markdown)?;
(quality.alphanumeric_chars >= 8).then_some(NativeFallbackCandidate {
markdown,
quality,
origin,
})
}
/// Converts positioned OCR spans to Markdown through pdf-inspector's existing
/// deterministic geometry, reading-order, table, and Markdown pipeline.
///
@@ -116,13 +184,46 @@ pub fn ocr_page_to_markdown(
/// OCR replaces pages whose native extraction was already rejected. On clean
/// native pages (for example in `Force` mode), normalized duplicate OCR blocks
/// are removed and only genuinely additional blocks are appended. Pages that
/// needed OCR but still have no credible local result recommend the hosted
/// needed OCR but still have no credible OCR result recommend the hosted
/// document pipeline instead of silently presenting an empty result as final.
pub fn fuse_ocr_pages(
native_pages: &[PageMarkdown],
ocr_run: &OcrRun,
document_page_count: u32,
options: &OcrFusionOptions,
) -> Result<FusedPages, OcrFusionError> {
fuse_ocr_pages_impl(
native_pages,
ocr_run,
document_page_count,
options,
&BTreeMap::new(),
)
}
/// OCR-pipeline fusion with trustworthy partial native candidates.
pub(crate) fn fuse_ocr_pages_adaptive(
native_pages: &[PageMarkdown],
ocr_run: &OcrRun,
document_page_count: u32,
options: &OcrFusionOptions,
native_candidates: &BTreeMap<u32, NativeFallbackCandidate>,
) -> Result<FusedPages, OcrFusionError> {
fuse_ocr_pages_impl(
native_pages,
ocr_run,
document_page_count,
options,
native_candidates,
)
}
fn fuse_ocr_pages_impl(
native_pages: &[PageMarkdown],
ocr_run: &OcrRun,
document_page_count: u32,
options: &OcrFusionOptions,
native_candidates: &BTreeMap<u32, NativeFallbackCandidate>,
) -> Result<FusedPages, OcrFusionError> {
validate_options(options)?;
@@ -178,16 +279,31 @@ pub fn fuse_ocr_pages(
document_page_count,
);
let ocr_markdown = preserve_ocr_line_breaks(&ocr_markdown, local);
let (markdown, source) = if native.markdown.trim().is_empty() || native.needs_ocr {
(ocr_markdown, PageContentSource::Ocr)
let (markdown, source, adaptive_recommends_hosted) = if let Some(candidate) =
native_candidates
.get(&page_number)
.filter(|_| native.needs_ocr)
{
let choice = choose_adaptive_content(
candidate,
&ocr_markdown,
local.ocr.mean_confidence,
options.hosted_recommendation_confidence,
);
warnings.push(choice.warning);
(choice.markdown, choice.source, choice.recommend_hosted)
} else if native.markdown.trim().is_empty() || native.needs_ocr {
(ocr_markdown, PageContentSource::Ocr, false)
} else {
merge_native_and_ocr(&native.markdown, &ocr_markdown)
let (markdown, source) = merge_native_and_ocr(&native.markdown, &ocr_markdown);
(markdown, source, false)
};
let weak_ocr = local
.ocr
.mean_confidence
.is_none_or(|confidence| confidence < options.hosted_recommendation_confidence);
let recommend_hosted = native.needs_ocr && (markdown.trim().is_empty() || weak_ocr);
let recommend_hosted = native.needs_ocr
&& (markdown.trim().is_empty() || weak_ocr || adaptive_recommends_hosted);
if native.needs_ocr && markdown.trim().is_empty() {
warnings.push("OCR produced no usable text".to_string());
}
@@ -244,6 +360,155 @@ pub fn fuse_ocr_pages(
})
}
struct AdaptiveContentChoice {
markdown: String,
source: PageContentSource,
warning: String,
recommend_hosted: bool,
}
fn choose_adaptive_content(
native: &NativeFallbackCandidate,
ocr: &str,
ocr_confidence: Option<f32>,
weak_ocr_threshold: f32,
) -> AdaptiveContentChoice {
let ocr_quality = assess_text_candidate(ocr);
let weak_ocr = ocr_confidence.is_none_or(|confidence| confidence < weak_ocr_threshold);
if weak_ocr || ocr_quality.is_none() {
return AdaptiveContentChoice {
markdown: ensure_trailing_newline(native.markdown()),
source: PageContentSource::Native,
warning: format!(
"kept trustworthy {} because OCR was weak or unusable",
native.origin.description()
),
recommend_hosted: true,
};
}
let ocr_quality = ocr_quality.expect("checked above");
let overlap = content_overlap(native.markdown(), ocr);
let ocr_novel_chars = ocr_quality
.alphanumeric_chars
.saturating_sub(overlap.shared_chars);
let material_novelty =
ocr_novel_chars >= 2 && ocr_novel_chars * 8 >= ocr_quality.alphanumeric_chars.max(1);
let native_substantially_covered =
overlap.shared_chars * 4 >= native.quality.alphanumeric_chars.max(1) * 3;
if native_substantially_covered && !material_novelty {
return AdaptiveContentChoice {
markdown: ensure_trailing_newline(native.markdown()),
source: PageContentSource::Native,
warning: format!(
"kept trustworthy {} because OCR added no material coverage",
native.origin.description()
),
recommend_hosted: false,
};
}
// A shorter, materially lower-quality OCR hypothesis should not displace
// exact native text even when its mean confidence happens to be high.
if ocr_quality.score + 0.12 < native.quality.score
&& ocr_quality.alphanumeric_chars * 10
<= native.quality.alphanumeric_chars.saturating_mul(11)
{
return AdaptiveContentChoice {
markdown: ensure_trailing_newline(native.markdown()),
source: PageContentSource::Native,
warning: format!(
"kept higher-quality {} after comparing OCR",
native.origin.description()
),
recommend_hosted: false,
};
}
let (markdown, source) = merge_native_and_ocr(native.markdown(), ocr);
let warning = match source {
PageContentSource::Native => format!(
"kept trustworthy {} because OCR duplicated its content",
native.origin.description()
),
PageContentSource::Fused => format!(
"fused trustworthy {} with complementary OCR",
native.origin.description()
),
PageContentSource::Ocr => unreachable!("merge never returns OCR-only content"),
};
AdaptiveContentChoice {
markdown,
source,
warning,
recommend_hosted: false,
}
}
#[derive(Debug, Clone, Copy)]
struct ContentOverlap {
shared_chars: usize,
}
fn content_overlap(first: &str, second: &str) -> ContentOverlap {
let mut first_counts = BTreeMap::<char, usize>::new();
for character in normalized_content_chars(first) {
*first_counts.entry(character).or_insert(0) += 1;
}
let mut second_counts = BTreeMap::<char, usize>::new();
for character in normalized_content_chars(second) {
*second_counts.entry(character).or_insert(0) += 1;
}
let shared_chars = first_counts
.iter()
.map(|(character, count)| (*count).min(second_counts.get(character).copied().unwrap_or(0)))
.sum();
ContentOverlap { shared_chars }
}
fn normalized_content_chars(text: &str) -> impl Iterator<Item = char> + '_ {
text.chars()
.flat_map(char::to_lowercase)
.filter(|character| character.is_alphanumeric())
}
fn assess_text_candidate(markdown: &str) -> Option<TextCandidateQuality> {
if markdown.trim().is_empty()
|| is_garbage_text(markdown)
|| is_cid_garbage(markdown)
|| detect_encoding_issues(markdown)
{
return None;
}
let alphanumeric_chars = markdown
.chars()
.filter(|character| character.is_alphanumeric())
.count();
if alphanumeric_chars == 0 {
return None;
}
let visible_chars = markdown
.chars()
.filter(|character| !character.is_whitespace())
.count()
.max(1);
let density = alphanumeric_chars as f32 / visible_chars as f32;
let length_score = (alphanumeric_chars as f32 / 160.0).min(1.0);
let nonempty_lines = markdown
.lines()
.filter(|line| !line.trim().is_empty())
.count()
.max(1);
let line_score = (alphanumeric_chars as f32 / nonempty_lines as f32 / 12.0).min(1.0);
let score = (0.45 + length_score * 0.25 + density * 0.20 + line_score * 0.10).min(1.0);
Some(TextCandidateQuality {
alphanumeric_chars,
score,
})
}
/// Converts recognized line polygons to ordinary PDF-space text items.
fn ocr_text_items(page: &RoutedOcrPage) -> (Vec<TextItem>, usize) {
let mut discarded = 0usize;
@@ -778,6 +1043,10 @@ mod tests {
}
}
fn native_candidate(markdown: &str) -> NativeFallbackCandidate {
assess_native_candidate(markdown.to_string(), NativeCandidateOrigin::Extractor).unwrap()
}
#[test]
fn scanned_page_uses_geometry_ordered_ocr_and_provenance() {
let native = [native(0, "", true)];
@@ -930,6 +1199,133 @@ mod tests {
assert_eq!(result.pages[0].provenance.source, PageContentSource::Native);
}
#[test]
fn adaptive_fallback_keeps_native_text_when_ocr_is_weak() {
let native = [native(0, "", true)];
let run = run(vec![routed_page(
1,
vec![span("Inv0ice total uncertain", 10.0, 0.3)],
Some(0.3),
)]);
let candidates = BTreeMap::from([(
1,
native_candidate("Invoice total: $420.00\nPayment received\n"),
)]);
let result =
fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates)
.unwrap();
assert_eq!(result.pages[0].provenance.source, PageContentSource::Native);
assert_eq!(
result.pages[0].markdown,
"Invoice total: $420.00\nPayment received\n"
);
assert!(result.pages[0].provenance.hosted_recommended);
assert!(result.pages[0].provenance.warnings[0].contains("OCR was weak"));
}
#[test]
fn adaptive_fallback_prefers_exact_native_text_over_duplicate_ocr() {
let native = [native(0, "", true)];
let run = run(vec![routed_page(
1,
vec![span("Invoice total 420.00 Payment received", 10.0, 0.98)],
Some(0.98),
)]);
let candidates = BTreeMap::from([(
1,
native_candidate("Invoice total: $420.00\nPayment received\n"),
)]);
let result =
fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates)
.unwrap();
assert_eq!(result.pages[0].provenance.source, PageContentSource::Native);
assert!(!result.pages[0].provenance.hosted_recommended);
assert_eq!(result.pages[0].markdown.matches("Invoice").count(), 1);
assert!(result.pages[0].provenance.warnings[0].contains("no material coverage"));
}
#[test]
fn adaptive_fallback_fuses_short_novel_ocr_content() {
let native = [native(0, "", true)];
let run = run(vec![routed_page(
1,
vec![span("Status ready 42", 10.0, 0.98)],
Some(0.98),
)]);
let candidates = BTreeMap::from([(1, native_candidate("Status ready\n"))]);
let result =
fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates)
.unwrap();
assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused);
assert!(result.pages[0].markdown.contains("42"));
assert!(!result.pages[0].provenance.hosted_recommended);
}
#[test]
fn adaptive_fallback_recommends_hosted_for_high_confidence_garbage_ocr() {
let native = [native(0, "", true)];
let garbage = "a@@b%%c&&d==e~~".repeat(12);
let run = run(vec![routed_page(
1,
vec![span(&garbage, 10.0, 0.99)],
Some(0.99),
)]);
let candidates = BTreeMap::from([(1, native_candidate("Invoice total 420\n"))]);
let result =
fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates)
.unwrap();
assert_eq!(result.pages[0].provenance.source, PageContentSource::Native);
assert!(result.pages[0].provenance.hosted_recommended);
assert!(!result.pages[0].markdown.contains("@@"));
}
#[test]
fn adaptive_fallback_fuses_native_header_with_scanned_body() {
let native = [native(0, "", true)];
let run = run(vec![routed_page(
1,
vec![
span("Quarterly account report", 10.0, 0.96),
span("March revenue 420 units", 30.0, 0.96),
span("April revenue 510 units", 50.0, 0.96),
],
Some(0.96),
)]);
let candidates = BTreeMap::from([(1, native_candidate("# Quarterly account report\n"))]);
let result =
fuse_ocr_pages_adaptive(&native, &run, 1, &OcrFusionOptions::new(), &candidates)
.unwrap();
assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused);
assert_eq!(result.pages[0].markdown.matches("Quarterly").count(), 1);
assert!(result.pages[0].markdown.contains("March revenue 420 units"));
assert!(result.pages[0].markdown.contains("April revenue 510 units"));
assert!(result.pages[0].provenance.warnings[0].contains("complementary"));
}
#[test]
fn native_candidate_scoring_is_multilingual_and_rejects_garbage() {
assert!(assess_native_candidate(
"請求書 合計金額 4200円\n支払済み\n".to_string(),
NativeCandidateOrigin::Extractor,
)
.is_some());
assert!(assess_native_candidate(
"a@@b%%c&&d==e~~".repeat(12),
NativeCandidateOrigin::Extractor,
)
.is_none());
}
#[test]
fn force_mode_appends_only_additional_ocr_blocks() {
let native = [native(0, "Native title\n", false)];
+203 -1
View File
@@ -2,9 +2,11 @@
use std::path::Path;
use firecrawl_pdfium::{Pdfium, PixelFormat, PixelPoint, RenderConfig};
use firecrawl_pdfium::{PageChar, Pdfium, PixelFormat, PixelPoint, RenderConfig};
use thiserror::Error;
use crate::types::{ItemType, TextItem};
use super::{
PageRenderer, PageTransform, RenderBufferError, RenderOptions, RenderPixelFormat, RenderedPage,
};
@@ -65,6 +67,15 @@ pub struct PdfiumRenderer {
pdfium: Pdfium,
}
/// Positioned native text recovered from one selected PDF page.
#[derive(Debug)]
pub(crate) struct PdfiumTextPage {
pub(crate) page: u32,
pub(crate) page_width: f32,
pub(crate) page_height: f32,
pub(crate) items: Vec<TextItem>,
}
impl PdfiumRenderer {
/// Loads PDFium using `firecrawl-pdfium`'s documented discovery chain.
pub fn load() -> Result<Self, RenderError> {
@@ -100,6 +111,56 @@ impl PdfiumRenderer {
self.render_pages_impl(pdf_bytes, pages, password, options)
}
/// Extracts positioned native text from selected 1-indexed pages.
///
/// This is deliberately separate from rendering: callers can probe a
/// suspicious embedded text layer before paying for rasterization and
/// OCR. A page-level text failure is treated as an unavailable recovery
/// candidate so the caller can continue to its normal OCR fallback.
pub(crate) fn extract_text_pages(
&self,
pdf_bytes: &[u8],
pages: &[u32],
password: Option<&str>,
) -> Result<Vec<PdfiumTextPage>, RenderError> {
const MAX_TEXT_CHARS_PER_PAGE: usize = 250_000;
if pages.is_empty() {
return Ok(Vec::new());
}
if pages.contains(&0) {
return Err(RenderError::InvalidPageNumber);
}
let document = self.pdfium.load_document(pdf_bytes.to_vec(), password)?;
let page_count = document.page_count();
if let Some(&page) = pages.iter().find(|&&page| page as usize > page_count) {
return Err(RenderError::PageOutOfBounds { page, page_count });
}
let mut recovered = Vec::with_capacity(pages.len());
for &page_number in pages {
let page = document.page(page_number as usize - 1)?;
let page_size = page.size();
let text = match page.text_with_limit(MAX_TEXT_CHARS_PER_PAGE) {
Ok(text) => text,
Err(error) => {
log::debug!(
"page {page_number}: positioned native text recovery unavailable: {error}"
);
continue;
}
};
recovered.push(PdfiumTextPage {
page: page_number,
page_width: page_size.width,
page_height: page_size.height,
items: text_chars_to_items(text.chars(), page_number),
});
}
Ok(recovered)
}
fn render_pages_impl(
&self,
pdf_bytes: &[u8],
@@ -145,6 +206,106 @@ impl PdfiumRenderer {
}
}
fn text_chars_to_items(chars: &[PageChar], page: u32) -> Vec<TextItem> {
#[derive(Debug, Clone, Copy)]
struct Bounds {
left: f64,
bottom: f64,
right: f64,
top: f64,
}
fn flush(items: &mut Vec<TextItem>, text: &mut String, bounds: &mut Option<Bounds>, page: u32) {
let Some(bounds) = bounds.take() else {
text.clear();
return;
};
if text.is_empty() {
return;
}
let width = (bounds.right - bounds.left) as f32;
let height = (bounds.top - bounds.bottom) as f32;
let x = bounds.left as f32;
let y = bounds.bottom as f32;
if !x.is_finite()
|| !y.is_finite()
|| !width.is_finite()
|| !height.is_finite()
|| width <= 0.0
|| height <= 0.0
{
text.clear();
return;
}
items.push(TextItem {
text: std::mem::take(text),
x,
y,
width,
height,
font: "PDFium native text".to_string(),
font_size: height.max(1.0),
page,
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: ItemType::Text,
mcid: None,
});
}
let mut items = Vec::new();
let mut text = String::new();
let mut bounds: Option<Bounds> = None;
for character in chars {
let Some(value) = character.unicode else {
flush(&mut items, &mut text, &mut bounds, page);
continue;
};
if value.is_whitespace() {
flush(&mut items, &mut text, &mut bounds, page);
continue;
}
let rect = character.loose_bounds.normalized();
if !rect.left.is_finite()
|| !rect.bottom.is_finite()
|| !rect.right.is_finite()
|| !rect.top.is_finite()
|| rect.width() <= 0.0
|| rect.height() <= 0.0
{
flush(&mut items, &mut text, &mut bounds, page);
continue;
}
text.push(value);
bounds = Some(match bounds {
Some(bounds) => Bounds {
left: bounds.left.min(rect.left),
bottom: bounds.bottom.min(rect.bottom),
right: bounds.right.max(rect.right),
top: bounds.top.max(rect.top),
},
None => Bounds {
left: rect.left,
bottom: rect.bottom,
right: rect.right,
top: rect.top,
},
});
}
flush(&mut items, &mut text, &mut bounds, page);
items.sort_by(|first, second| {
first
.page
.cmp(&second.page)
.then(second.y.total_cmp(&first.y))
.then(first.x.total_cmp(&second.x))
});
items
}
impl PageRenderer for PdfiumRenderer {
type Error = RenderError;
@@ -237,6 +398,17 @@ fn bgr_to_rgb_in_place(
#[cfg(test)]
mod tests {
use super::*;
use firecrawl_pdfium::{PagePoint, PageRect};
fn page_char(value: char, bounds: PageRect) -> PageChar {
PageChar {
unicode: Some(value),
code: value as u32,
bounds,
loose_bounds: bounds,
origin: PagePoint::new(bounds.left, bounds.bottom),
}
}
#[test]
fn bgr_pixels_are_converted_to_rgb_in_place() {
@@ -263,4 +435,34 @@ mod tests {
Err(RenderBufferError::InvalidBufferLength { .. })
));
}
#[test]
fn invalid_character_geometry_splits_text_runs() {
let chars = [
page_char('A', PageRect::new(0.0, 0.0, 8.0, 10.0)),
page_char('X', PageRect::new(10.0, 0.0, 10.0, 10.0)),
page_char('B', PageRect::new(20.0, 0.0, 28.0, 10.0)),
];
let items = text_chars_to_items(&chars, 1);
assert_eq!(
items
.iter()
.map(|item| item.text.as_str())
.collect::<Vec<_>>(),
["A", "B"]
);
}
#[test]
fn coordinates_that_overflow_f32_are_discarded() {
let left = f64::from(f32::MAX) * 2.0;
let chars = [page_char(
'A',
PageRect::new(left, 0.0, left + 1.0e30, 10.0),
)];
assert!(text_chars_to_items(&chars, 1).is_empty());
}
}
+530 -12
View File
@@ -1,20 +1,31 @@
//! One-call native extraction and OCR pipeline.
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Instant;
use thiserror::Error;
use crate::{MarkdownOptions, PageOcrReasons, PdfError};
use crate::text_quality::{
analyze_text_quality, detect_encoding_issues, is_cid_garbage, is_garbage_text,
};
use crate::{
MarkdownOptions, PageOcrReasons, PdfError, OCR_REASON_SUSPECTED_GARBLED_TEXT,
OCR_REASON_VECTOR_TEXT,
};
use super::fusion::{
assess_native_candidate, fuse_ocr_pages_adaptive, NativeCandidateOrigin,
NativeFallbackCandidate,
};
use super::oar::onnx_runtime_library_path;
use super::pdfium::PdfiumTextPage;
use super::{
fuse_ocr_pages, route_ocr_pages, run_ocr_pages, FusedPageMarkdown, HttpModelDownloadError,
HttpModelDownloader, ModelAcquireError, ModelStore, ModelStoreError, OarOcrEngine, OarOcrError,
OcrFusionError, OcrFusionOptions, OcrMode, OcrOptions, OcrRoutingError, OcrRun, OcrRunError,
PdfiumRenderer, RenderError, RenderOptions, PP_OCR_V6_SMALL,
route_ocr_pages, run_ocr_pages, FusedPageMarkdown, HttpModelDownloadError, HttpModelDownloader,
ModelAcquireError, ModelStore, ModelStoreError, OarOcrEngine, OarOcrError, OcrFusionError,
OcrFusionOptions, OcrMode, OcrOptions, OcrRoutingError, OcrRun, OcrRunError, PdfiumRenderer,
RenderError, RenderOptions, PP_OCR_V6_SMALL,
};
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -210,7 +221,7 @@ pub fn process_pdf_with_ocr_mem(
let mut page_markdown_options = options.markdown.clone();
page_markdown_options.include_page_numbers = false;
let (native, page_count) = crate::extract_pages_markdown_mem_for_ocr(
let (mut native, page_count) = crate::extract_pages_markdown_mem_for_ocr(
buffer,
selected_pages_zero_indexed.as_deref(),
options.password.as_deref(),
@@ -223,13 +234,99 @@ pub fn process_pdf_with_ocr_mem(
return Err(OcrPipelineError::InvalidSelectedPage { page: invalid });
}
let routed = route_ocr_pages(
let initially_routed = route_ocr_pages(
options.ocr.mode,
page_count,
&native.pages_needing_ocr,
selected_pages.as_deref(),
)?;
// The OCR extractor retains clean native fragments on pages that still
// require OCR. Remove them from the ordinary native result now so Off mode
// preserves the existing suppression contract, while keeping a private
// candidate for post-OCR comparison and geometry-aware fusion.
let mut native_candidates = BTreeMap::<u32, NativeFallbackCandidate>::new();
for page in &mut native.pages {
if !page.needs_ocr || page.markdown.trim().is_empty() {
continue;
}
let page_number = page.page + 1;
let markdown = std::mem::take(&mut page.markdown);
if may_preserve_extractor_candidate(page_number, &native.ocr_reasons_by_page) {
if let Some(candidate) =
assess_native_candidate(markdown, NativeCandidateOrigin::Extractor)
{
native_candidates.insert(page_number, candidate);
}
}
}
let mut routed = initially_routed.clone();
let mut renderer = None;
let mut recovered_natively = BTreeSet::new();
if options.ocr.mode == OcrMode::Auto {
let complete_recovery_candidates =
native_recovery_candidates(&initially_routed, &native.ocr_reasons_by_page);
let mut native_probe_pages: BTreeSet<u32> =
complete_recovery_candidates.iter().copied().collect();
native_probe_pages.extend(
native_candidates
.keys()
.filter(|page| routed.contains(page))
.copied(),
);
if !native_probe_pages.is_empty() {
let native_renderer = PdfiumRenderer::load()?;
let recovered = native_renderer.extract_text_pages(
buffer,
&native_probe_pages.iter().copied().collect::<Vec<_>>(),
options.password.as_deref(),
)?;
renderer = Some(native_renderer);
for page in recovered {
let Some(markdown) =
credible_native_recovery(&page.items, page_count, &page_markdown_options)
else {
continue;
};
let Some(candidate) =
assess_native_candidate(markdown, NativeCandidateOrigin::Pdfium)
else {
continue;
};
let may_skip_ocr = complete_recovery_candidates.contains(&page.page)
&& candidate.is_complete_recovery()
&& native_recovery_covers_page(&page);
if may_skip_ocr {
let Some(native_page) = native
.pages
.iter_mut()
.find(|entry| entry.page + 1 == page.page)
else {
continue;
};
native_page.markdown = candidate.markdown().to_string();
native_page.needs_ocr = false;
native_candidates.remove(&page.page);
recovered_natively.insert(page.page);
} else {
match native_candidates.entry(page.page) {
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(candidate);
}
std::collections::btree_map::Entry::Occupied(mut entry) => {
if candidate.is_stronger_than(entry.get()) {
entry.insert(candidate);
}
}
}
}
}
routed.retain(|page| !recovered_natively.contains(page));
}
}
let ocr_run = if routed.is_empty() {
OcrRun {
pages: Vec::new(),
@@ -239,7 +336,10 @@ pub fn process_pdf_with_ocr_mem(
} else {
// Resolve the native renderer before any network request so a missing
// PDFium installation cannot trigger a model download it cannot use.
let renderer = PdfiumRenderer::load()?;
let renderer = match renderer {
Some(renderer) => renderer,
None => PdfiumRenderer::load()?,
};
let engine = cached_ocr_engine(&options.ocr)?;
run_ocr_pages(
&renderer,
@@ -256,7 +356,20 @@ pub fn process_pdf_with_ocr_mem(
.markdown(page_markdown_options)
.render_dpi(options.render.dpi)
.hosted_recommendation_confidence(options.hosted_recommendation_confidence);
let fused = fuse_ocr_pages(&native.pages, &ocr_run, page_count, &fusion_options)?;
let mut fused = fuse_ocr_pages_adaptive(
&native.pages,
&ocr_run,
page_count,
&fusion_options,
&native_candidates,
)?;
for page in &mut fused.pages {
if recovered_natively.contains(&page.page) {
page.provenance
.warnings
.push("recovered a credible positioned native text layer before OCR".to_string());
}
}
let pages_recommending_hosted = fused
.pages
.iter()
@@ -265,6 +378,14 @@ pub fn process_pdf_with_ocr_mem(
.collect();
let markdown = assemble_document_markdown(&fused.pages, options.markdown.include_page_numbers);
let mut pages_with_tables = native.pages_with_tables;
for page in &fused.pages {
if markdown_has_table(&page.markdown) && !pages_with_tables.contains(&page.page) {
pages_with_tables.push(page.page);
}
}
pages_with_tables.sort_unstable();
Ok(OcrPdfResult {
markdown,
pages: fused.pages,
@@ -273,9 +394,9 @@ pub fn process_pdf_with_ocr_mem(
pages_routed_to_ocr: routed,
pages_recommending_hosted,
ocr_reasons_by_page: native.ocr_reasons_by_page,
pages_with_tables: native.pages_with_tables,
pages_with_tables: pages_with_tables.clone(),
pages_with_columns: native.pages_with_columns,
is_complex: native.is_complex,
is_complex: native.is_complex || !pages_with_tables.is_empty(),
processing_time_ms: elapsed_ms(started),
render_time_ms: fused.render_time_ms,
ocr_time_ms: fused.ocr_time_ms,
@@ -360,6 +481,239 @@ fn normalized_cache_path(path: PathBuf) -> PathBuf {
absolute
}
fn native_recovery_candidates(routed: &[u32], reasons: &[PageOcrReasons]) -> Vec<u32> {
let reasons_by_page: BTreeMap<u32, &PageOcrReasons> =
reasons.iter().map(|entry| (entry.page, entry)).collect();
routed
.iter()
.copied()
.filter(|page| {
reasons_by_page.get(page).is_some_and(|entry| {
!entry.reasons.is_empty()
&& entry.reasons.iter().all(|reason| {
reason == OCR_REASON_SUSPECTED_GARBLED_TEXT
|| reason == OCR_REASON_VECTOR_TEXT
})
})
})
.collect()
}
fn may_preserve_extractor_candidate(page: u32, reasons: &[PageOcrReasons]) -> bool {
reasons
.iter()
.find(|entry| entry.page == page)
.is_some_and(|entry| {
!entry.reasons.is_empty()
&& !entry
.reasons
.iter()
.any(|reason| reason == OCR_REASON_SUSPECTED_GARBLED_TEXT)
&& entry.reasons.iter().any(|reason| {
reason == crate::OCR_REASON_SCANNED || reason == OCR_REASON_VECTOR_TEXT
})
})
}
fn credible_native_recovery(
items: &[crate::TextItem],
document_page_count: u32,
options: &MarkdownOptions,
) -> Option<String> {
let text = items
.iter()
.map(|item| item.text.as_str())
.collect::<Vec<_>>()
.join(" ");
if is_garbage_text(&text) || is_cid_garbage(&text) {
return None;
}
let markdown = crate::to_markdown_from_items_with_rects_and_page_count(
items.to_vec(),
options.clone(),
&[],
document_page_count,
);
let markdown = remove_duplicate_table_lines(&markdown);
let structured_uniform_ascii = is_uniform_case_structured_ascii(&text, &markdown);
let quality = analyze_text_quality(items);
if !structured_uniform_ascii && (quality.has_encoding_issues || detect_encoding_issues(&text)) {
return None;
}
(!markdown.trim().is_empty()).then_some(markdown)
}
fn native_recovery_covers_page(page: &PdfiumTextPage) -> bool {
const VERTICAL_BANDS: f32 = 6.0;
let width = page.page_width;
let height = page.page_height;
if !width.is_finite() || !height.is_finite() || width <= 0.0 || height <= 0.0 {
return false;
}
let mut min_left = width;
let mut max_right = 0.0_f32;
let mut min_bottom = height;
let mut max_top = 0.0_f32;
let mut positioned_items = 0usize;
let mut occupied_bands = BTreeSet::new();
for item in &page.items {
if !item.text.chars().any(char::is_alphanumeric)
|| !item.x.is_finite()
|| !item.y.is_finite()
|| !item.width.is_finite()
|| !item.height.is_finite()
|| item.width <= 0.0
|| item.height <= 0.0
{
continue;
}
let left = item.x.clamp(0.0, width);
let right = (item.x + item.width).clamp(0.0, width);
let bottom = item.y.clamp(0.0, height);
let top = (item.y + item.height).clamp(0.0, height);
if right <= left || top <= bottom {
continue;
}
positioned_items += 1;
min_left = min_left.min(left);
max_right = max_right.max(right);
min_bottom = min_bottom.min(bottom);
max_top = max_top.max(top);
let center = (bottom + top) * 0.5;
let band = ((center / height) * VERTICAL_BANDS)
.floor()
.clamp(0.0, VERTICAL_BANDS - 1.0) as u8;
occupied_bands.insert(band);
}
positioned_items >= 6
&& (max_right - min_left) / width >= 0.15
&& (max_top - min_bottom) / height >= 0.35
&& occupied_bands.len() >= 3
}
fn is_uniform_case_structured_ascii(text: &str, markdown: &str) -> bool {
if !text.is_ascii() || text.contains('$') || text.chars().any(char::is_control) {
return false;
}
let letters: Vec<_> = text
.chars()
.filter(|character| character.is_ascii_alphabetic())
.collect();
if letters.len() < 200 {
return false;
}
let uniform_case = letters
.iter()
.all(|character| character.is_ascii_uppercase())
|| letters
.iter()
.all(|character| character.is_ascii_lowercase());
if !uniform_case {
return false;
}
if markdown_has_table(markdown) {
return true;
}
let nonempty_lines = markdown
.lines()
.filter(|line| !line.trim().is_empty())
.count();
let visible_chars = text
.chars()
.filter(|character| !character.is_whitespace())
.count()
.max(1);
let structural_chars = text
.chars()
.filter(|character| {
character.is_ascii_digit()
|| matches!(
character,
'{' | '}'
| '['
| ']'
| '('
| ')'
| '<'
| '>'
| '_'
| '='
| '+'
| '*'
| '/'
| '\\'
| '|'
| '&'
| '^'
| '%'
| '#'
| '@'
| '~'
)
})
.count();
nonempty_lines >= 4 && structural_chars * 20 >= visible_chars
}
fn markdown_has_table(markdown: &str) -> bool {
markdown.lines().any(|line| {
let trimmed = line.trim();
trimmed.starts_with('|')
&& trimmed.ends_with('|')
&& trimmed
.split('|')
.filter(|cell| !cell.is_empty())
.all(|cell| !cell.is_empty() && cell.chars().all(|ch| ch == '-'))
})
}
fn remove_duplicate_table_lines(markdown: &str) -> String {
let mut output = String::new();
let mut adjacent_table_row = None;
for line in markdown.lines() {
let trimmed = line.trim();
let is_table_line = trimmed.starts_with('|') && trimmed.ends_with('|');
if is_table_line {
if !trimmed.contains("|---") && trimmed.matches('|').count() >= 4 {
let canonical = canonical_table_text(trimmed);
if !canonical.is_empty() {
adjacent_table_row = Some(canonical);
}
}
} else if trimmed.is_empty() {
// Keep adjacency across the blank line emitted after a table.
} else {
let duplicate = adjacent_table_row
.as_ref()
.is_some_and(|table_row| *table_row == canonical_table_text(trimmed));
adjacent_table_row = None;
if duplicate {
continue;
}
}
output.push_str(line);
output.push('\n');
}
while output.ends_with("\n\n\n") {
output.pop();
}
output
}
fn canonical_table_text(text: &str) -> String {
text.replace('|', " ")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn assemble_document_markdown(pages: &[FusedPageMarkdown], include_page_numbers: bool) -> String {
let mut document = String::new();
for (index, page) in pages.iter().enumerate() {
@@ -427,6 +781,25 @@ pub enum OcrPipelineError {
mod tests {
use super::*;
fn recovery_item(text: &str, x: f32, y: f32, width: f32, height: f32) -> crate::TextItem {
crate::TextItem {
text: text.to_string(),
x,
y,
width,
height,
font: "PDFium native text".to_string(),
font_size: height,
page: 1,
is_bold: false,
is_italic: false,
is_underline: false,
is_strikeout: false,
item_type: crate::types::ItemType::Text,
mcid: None,
}
}
#[test]
fn cache_path_is_stable_when_a_relative_directory_is_created() {
let current = std::fs::canonicalize(std::env::current_dir().unwrap()).unwrap();
@@ -442,6 +815,130 @@ mod tests {
assert_eq!(before, after);
}
#[test]
fn native_recovery_is_limited_to_recoverable_routing_reasons() {
let routed = [1, 2, 3, 4];
let reasons = [
PageOcrReasons {
page: 1,
reasons: vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()],
},
PageOcrReasons {
page: 2,
reasons: vec![crate::OCR_REASON_SCANNED.to_string()],
},
PageOcrReasons {
page: 3,
reasons: vec![OCR_REASON_VECTOR_TEXT.to_string()],
},
PageOcrReasons {
page: 4,
reasons: vec![
OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string(),
crate::OCR_REASON_SCANNED.to_string(),
],
},
];
assert_eq!(native_recovery_candidates(&routed, &reasons), vec![1, 3]);
}
#[test]
fn native_recovery_requires_text_coverage_beyond_a_header() {
let header = PdfiumTextPage {
page: 1,
page_width: 600.0,
page_height: 800.0,
items: (0..8)
.map(|index| recovery_item("HEADER", index as f32 * 60.0, 740.0, 50.0, 12.0))
.collect(),
};
assert!(!native_recovery_covers_page(&header));
let complete = PdfiumTextPage {
page: 1,
page_width: 600.0,
page_height: 800.0,
items: vec![
recovery_item("Top one", 40.0, 700.0, 180.0, 12.0),
recovery_item("Top two", 260.0, 680.0, 180.0, 12.0),
recovery_item("Middle one", 40.0, 400.0, 180.0, 12.0),
recovery_item("Middle two", 260.0, 380.0, 180.0, 12.0),
recovery_item("Bottom one", 40.0, 100.0, 180.0, 12.0),
recovery_item("Bottom two", 260.0, 80.0, 180.0, 12.0),
],
};
assert!(native_recovery_covers_page(&complete));
}
#[test]
fn uniform_case_guard_requires_structured_content() {
let table_text = "STATUS CODE 100 READY ".repeat(20);
let table_markdown = "|STATUS|CODE|\n|---|---|\n|READY|100|\n|READY|200|\n|READY|300|\n";
assert!(is_uniform_case_structured_ascii(
&table_text,
table_markdown
));
let prose = "THIS IS ORDINARY UPPERCASE PROSE WITH NATURAL WORDS ".repeat(20);
let prose_markdown = prose
.split_whitespace()
.collect::<Vec<_>>()
.chunks(8)
.map(|line| line.join(" "))
.collect::<Vec<_>>()
.join("\n");
assert!(!is_uniform_case_structured_ascii(&prose, &prose_markdown));
}
#[test]
fn recovered_markdown_drops_plain_duplicates_of_table_rows() {
let markdown = "|Date|Value|Status|\n|---|---|---|\n|April 1|42|ok|\n\nApril 1 42 ok\n";
assert_eq!(
remove_duplicate_table_lines(markdown),
"|Date|Value|Status|\n|---|---|---|\n|April 1|42|ok|\n\n"
);
}
#[test]
fn recovered_markdown_keeps_nonadjacent_repeated_table_text() {
let markdown = "|Date|Value|Status|\n|---|---|---|\n|April 1|42|ok|\n\nSummary follows.\n\nApril 1 42 ok\n";
assert_eq!(remove_duplicate_table_lines(markdown), markdown);
}
#[test]
fn extractor_candidates_require_clean_partial_content_reasons() {
let reasons = [
PageOcrReasons {
page: 1,
reasons: vec![crate::OCR_REASON_SCANNED.to_string()],
},
PageOcrReasons {
page: 2,
reasons: vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()],
},
PageOcrReasons {
page: 3,
reasons: vec![OCR_REASON_VECTOR_TEXT.to_string()],
},
PageOcrReasons {
page: 4,
reasons: vec![
crate::OCR_REASON_SCANNED.to_string(),
OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string(),
],
},
];
assert!(may_preserve_extractor_candidate(1, &reasons));
assert!(!may_preserve_extractor_candidate(2, &reasons));
assert!(may_preserve_extractor_candidate(3, &reasons));
assert!(!may_preserve_extractor_candidate(4, &reasons));
assert!(!may_preserve_extractor_candidate(5, &reasons));
}
#[test]
fn off_mode_extracts_native_text_without_runtime_side_effects() {
let bytes = std::fs::read("tests/fixtures/thermo-freon12.pdf").unwrap();
@@ -488,9 +985,30 @@ mod tests {
let result = process_pdf_with_ocr_mem(&bytes, OcrPdfOptions::new()).unwrap();
assert!(result.pages_routed_to_ocr.is_empty());
assert!(result.markdown.trim().is_empty());
assert_eq!(result.pages_recommending_hosted, vec![1]);
}
#[test]
fn partial_native_scan_text_is_retained_only_inside_ocr_orchestration() {
let bytes = std::fs::read("tests/fixtures/scan_with_native_header_text.pdf").unwrap();
let public = crate::extract_pages_markdown_mem(&bytes, None).unwrap();
assert!(public.pages[0].needs_ocr);
assert!(public.pages[0].markdown.trim().is_empty());
let (ocr, _) = crate::extract_pages_markdown_mem_for_ocr(
&bytes,
None,
None,
&MarkdownOptions::default(),
)
.unwrap();
assert!(ocr.pages[0].needs_ocr);
assert!(ocr.pages[0]
.markdown
.contains("Order Detail Report by Account"));
}
#[test]
fn password_is_redacted_and_used_for_native_extraction() {
let options = OcrPdfOptions::new().password("secret123");
+56 -2
View File
@@ -2,7 +2,8 @@
#[cfg(feature = "ocr")]
use pdf_inspector::vision::{
process_pdf_with_ocr_mem, ModelDownloadPolicy, OcrPdfOptions, PageContentSource,
process_pdf_with_ocr_mem, ModelDownloadPolicy, OcrPdfOptions, OcrPipelineError,
PageContentSource,
};
use pdf_inspector::vision::{
ModelStore, OarOcrEngine, OcrEngine, OcrMode, OcrOptions, PageTransform, RenderPixelFormat,
@@ -127,7 +128,16 @@ fn complete_ocr_pipeline_routes_and_assembles_a_scanned_fixture() {
assert_eq!(result.pages_routed_to_ocr, vec![1]);
assert!(!result.markdown.trim().is_empty());
assert_eq!(result.pages[0].provenance.source, PageContentSource::Ocr);
assert!(result
.markdown
.contains("Order Date Item Code Description Status Unit Cost\n\n03/14/2024"));
assert!(result.markdown.contains("$482,110.40\n\n05/02/2024"));
assert_eq!(result.pages[0].provenance.source, PageContentSource::Fused);
assert!(result.pages[0]
.provenance
.warnings
.iter()
.any(|warning| warning.contains("complementary OCR")));
assert_eq!(
result.pages[0].provenance.ocr_model.as_ref().unwrap().name,
PP_OCR_V6_SMALL.id
@@ -135,6 +145,50 @@ fn complete_ocr_pipeline_routes_and_assembles_a_scanned_fixture() {
assert_eq!(repeated.markdown, result.markdown);
}
#[cfg(all(feature = "ocr", feature = "render-pdfium"))]
#[test]
fn auto_recovers_credible_native_text_before_loading_ocr_models() {
let Some(_renderer) = load_renderer() else {
return;
};
let bytes = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
let ocr = OcrOptions::new()
.mode(OcrMode::Auto)
.model_directory("/models/must-not-be-read")
.model_downloads(ModelDownloadPolicy::Offline);
let result = process_pdf_with_ocr_mem(&bytes, OcrPdfOptions::new().ocr(ocr)).unwrap();
assert_eq!(result.pages_recommended_for_ocr, vec![1]);
assert!(result.pages_routed_to_ocr.is_empty());
assert!(result.markdown.contains("羽田空港新飛行経路"));
assert!(result.markdown.contains("|4月30日|有|81.0|"));
assert!(result.markdown.contains("※1 最大騒音レベル"));
assert!(result.pages_with_tables.contains(&1));
assert_eq!(result.pages[0].provenance.source, PageContentSource::Native);
assert!(result.pages[0].provenance.ocr_model.is_none());
}
#[cfg(all(feature = "ocr", feature = "render-pdfium"))]
#[test]
fn auto_rejects_garbled_native_recovery_and_continues_to_ocr() {
let Some(_renderer) = load_renderer() else {
return;
};
let bytes = std::fs::read("tests/fixtures/shifted_cipher_tounicode.pdf").unwrap();
let ocr = OcrOptions::new()
.mode(OcrMode::Auto)
.model_directory("/models/must-not-be-read")
.model_downloads(ModelDownloadPolicy::Offline);
let error = process_pdf_with_ocr_mem(&bytes, OcrPdfOptions::new().ocr(ocr)).unwrap_err();
assert!(matches!(
error,
OcrPipelineError::ModelAcquire(_) | OcrPipelineError::ModelStore(_)
));
}
fn recognize(
model_directory: &std::ffi::OsStr,
pages: &[RenderedPage],