diff --git a/src/detector.rs b/src/detector.rs index 6f9ea47..4c37817 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -1659,7 +1659,15 @@ fn hex_val(b: u8) -> Option { /// Standard page: 612x792 points (US Letter) = ~485,000 sq points /// At 2x resolution that's ~1.9M pixels, so we use 250K pixels as threshold /// (accounting for varying DPI and page sizes) -fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { +/// Returns `(has_images, total_image_area, has_template_image)` for a page. +/// `has_template_image` means a single large (>50% page coverage) +/// background image — the signal `classify_pdf`/`detect_pdf_type` uses to +/// route a page to OCR regardless of any incidental native text drawn over +/// it. Exposed at crate visibility so extraction-side per-page `needs_ocr` +/// computation (`extract_pages_markdown_mem`) can consult the same signal +/// instead of maintaining its own, independent notion of "needs OCR" that +/// can silently disagree with detection — see #227. +pub(crate) fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { // Threshold: image covering roughly half a page at 150+ DPI // 612 * 792 / 2 * (150/72)^2 ≈ 1M pixels, but we'll be conservative const TEMPLATE_IMAGE_THRESHOLD: u64 = 500_000; // 500K pixels @@ -1741,6 +1749,62 @@ fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u64, bool) { (has_images, total_area, has_template_image) } +/// Computes both `(needs_ocr_for_template_image, has_vector_text)` for a +/// page from a single shared `analyze_page_content` pass — that call +/// decompresses and scans every content stream (page + XObjects) plus +/// image coverage, so `extract_pages_markdown_mem` must not invoke it +/// twice per page (once per signal) the way `detect_from_document` avoids +/// by caching its per-page `PageAnalysis`. +/// +/// `needs_ocr_for_template_image` is true when a page's template image +/// should be treated as a scan needing OCR — a single full-page background +/// image with little/no real text — rather than a text page that happens +/// to carry a watermark, letterhead, or figure. Mirrors the two distinct +/// signals classification uses to route a template-image page to OCR: +/// +/// 1. `looks_like_scan`: image_count <= 1, few text operators (<50), and +/// low alphanumeric diversity in raw string operands (unless decodable +/// CID/ToUnicode fonts explain that away) — the gate used for +/// `pages_with_template_images` and Mixed-type per-page routing. +/// 2. Insufficient real text volume, using `DetectionConfig::default()`'s +/// `min_text_ops_per_page` (3) — the same threshold Mixed-type per-page +/// routing applies via `text_operator_count < config.min_text_ops_per_page +/// && has_images` (simplified here since a template image implies +/// `has_images`). Deliberately *not* the higher `effective_min_ops` +/// floor (`min_text_ops_per_page.max(10)`) that whole-document +/// `PdfType::ImageBased`/`Scanned` classification uses for +/// `pages_with_text` — that's a cross-page aggregate decision this +/// per-page function has no way to replicate exactly, and the lower +/// per-page threshold is the one a single page's own signals can +/// actually agree with. +/// +/// `has_vector_text` is true when a page has vector-outlined text (glyphs +/// drawn as paths rather than shown via text-showing operators) — +/// `detect_from_document`'s Mixed-type per-page routing always sends +/// these pages to OCR, independent of any template-image check, since +/// outlined glyphs can't be extracted as text at all. +/// +/// Exposed at crate visibility so `extract_pages_markdown_mem` can apply +/// the same gates classification needs elsewhere instead of treating the +/// raw signals alone as sufficient — see #227/#231. +pub(crate) fn page_ocr_signals(doc: &Document, page_id: ObjectId) -> (bool, bool) { + let analysis = analyze_page_content(doc, page_id); + + let needs_ocr_for_template_image = if !analysis.has_template_image { + false + } else { + let alphanum_low = analysis.unique_alphanum_chars < 10 + && !(analysis.has_decodable_text_fonts && analysis.text_operator_count >= 10); + let looks_like_scan = + analysis.image_count <= 1 && analysis.text_operator_count < 50 && alphanum_low; + let insufficient_text = + analysis.text_operator_count < DetectionConfig::default().min_text_ops_per_page; + looks_like_scan || insufficient_text + }; + + (needs_ocr_for_template_image, analysis.has_vector_text) +} + /// Recursively collect image dimensions from XObject resources, /// including images nested inside Form XObjects. fn collect_images_from_resources( diff --git a/src/lib.rs b/src/lib.rs index 9df9066..769cda4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -516,6 +516,7 @@ pub fn extract_pages_markdown_mem( let mut results = Vec::with_capacity(pages_slice.len()); let mut pages_needing_ocr = Vec::new(); let mut ocr_reasons_by_page = BTreeMap::new(); + let lopdf_pages = doc.get_pages(); for &page_0idx in pages_slice { // Out-of-range pages → empty + needs_ocr @@ -549,6 +550,25 @@ pub fn extract_pages_markdown_mem( let has_gid = gid_pages.contains(&page_1idx); let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx); + // A page can extract cleanly (no decoding issues, non-empty text) + // while still being fundamentally a scan: a full-page raster with + // a little genuine native text drawn over it (a header, a stamp, a + // cover-sheet annotation). Text-quality signals alone can't see + // that — consult the same "large background image" signal + // classify_pdf/detect_pdf_type already uses, so the two APIs can't + // silently disagree on whether a page needs OCR. See #227. + // Also covers vector-outlined text (glyphs drawn as paths, not + // shown via a text-showing operator): a hybrid page with real + // embedded-font body text elsewhere would otherwise still extract + // non-empty, non-garbled markdown and miss OCR routing entirely. + // detect_from_document's Mixed-type per-page routing always sends + // these pages to OCR; mirror that here too. Both signals share one + // analyze_page_content pass — see page_ocr_signals's doc comment. + let (has_template_image, has_vector_text) = lopdf_pages + .get(&page_1idx) + .map(|&page_id| detector::page_ocr_signals(&doc, page_id)) + .unwrap_or((false, false)); + // Build markdown with document-wide font stats let options = MarkdownOptions { base_font_size: Some(font_stats.most_common_size), @@ -586,10 +606,20 @@ pub fn extract_pages_markdown_mem( OCR_REASON_SUSPECTED_GARBLED_TEXT, ); } + if has_template_image { + add_ocr_reason(&mut ocr_reasons_by_page, page_1idx, OCR_REASON_SCANNED); + } + if has_vector_text { + add_ocr_reason(&mut ocr_reasons_by_page, page_1idx, OCR_REASON_VECTOR_TEXT); + } let ocr_reason = page_ocr_reason(&ocr_reasons_by_page, page_1idx); - let needs_ocr = - ocr_reason.is_some() || md.trim().is_empty() || has_gid || is_garbage_text(&md); + let needs_ocr = ocr_reason.is_some() + || md.trim().is_empty() + || has_gid + || is_garbage_text(&md) + || has_template_image + || has_vector_text; if needs_ocr { pages_needing_ocr.push(page_1idx); diff --git a/tests/fixtures/scan_with_native_header_text.pdf b/tests/fixtures/scan_with_native_header_text.pdf new file mode 100644 index 0000000..c82100a --- /dev/null +++ b/tests/fixtures/scan_with_native_header_text.pdf @@ -0,0 +1,79 @@ +%PDF-1.3 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 1584 /Length 12524 /Subtype /Image + /Type /XObject /Width 1224 +>> +stream +Gb"-Vq6O&^R5j]N,!c5I[00Q-A#Kh+b%<3UHeAQ17._*h\O)g[//_qHLm'es.)Uj9M(pkA?H*-L!CoF/&D+F-**.h`:kV(d.%:M7p\EZESncU9DAo9^qgMGoCO-a1!.BOIA?pL>JI59%[pYUIMac==l.sG@+A4][:eZ)VIGj;9+U=n>4F'H+k]lWA^Ut=k/T:RKBPRBYsB>i9.Dr8%\HE52:-l6<3bO42u5P*)TCY,`&jH_QmarlKdcZX_>\2'a"[N4F$@'Pb3$6hU=QDX-5I(L;,['\9BjQe\m*`NELY[bQap%3,$;l`[3VDn>21_hSbhX?:B^REbQPDgkVh>?`jjGpNFjn4bq3]^lUR3g@@bVRQjj`0>#rTn/]]CsroMd'doe?NN(A6m/HgigYC,;_(\.UM.ffX#Db;?V3I(%$A2Jun^Fe*9:S7@Jg[Ee"U[Zf-,arl@e>_rfmk#lqddeCl?)RJ,ao3nk&l^i0uK4aH5D7m^Kt0mBC$:$:T6\baC8"ba;9&C*rN"9.p_r5.j_^[U%TA;d[(^VX]cn7j=j(b"YT)P:'-YSB1ZOP/3LO>Q$aMS%ABaQmEEZE\!%&p5M#f88=/*4#VQcpd3/P`dD@.+O<%-N#4G[?[VB4LCP=[o*uTki&AFHNP#l(!;Mj*56\B;n(tpJ56\B.eZ(r5hXTHLmp1(dJ"e/](BZXn,&q1;l8WfB7_cSkDAUoVEU8=0sJXODnhhRVS\4f:S1rl;\PX!?XLQpp[7*=ai]c4Idk/Y[9nf3IbmAC#.80oVDo!0,ldoFzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!!#8Ak0BOP\om>04*Q.SZcndkU\O6@Vk9Opk007BUIUC!g=d,`7Vphf/^nL]SimMl%B#b?>4Qk*hRuusH"GmIIH*3-`@[E0?f%YSRjb@A[Am#p4p(utkh!#qCTkEP9l+\$[]QU2M_6aT>Aq2T\ac+GCE@s"iQ?#ceWl)=m+ARc/L&IM*BKuJ-VTnHo]X[m*dK"Tn(+UF;c?W*-"kpO3,kP)^-2bffsBkIA]r4boc6G`8jQJtjoICit-MLbj7*^"[$b0N6Fgn6rc1;+lER6>n?g@pUhm^W;rqkGi\euYiLVl%^&qs:Z1N(uP/e0n[KiItLECl-$+9q-V0iuamiCo^X@rlIrbo!R%MHGBkB(KrTj?i8n"8bqu(5G'cHd]Qlf,/oS4DZ0G)?+P,,ffe5)k007fh033?55F9'=FRfZ^4Srh`?,FWbD-CHi?+Y8cZ=HPVr:62hCt5Oc%$G*+!6CR[T]4gUB]3j,ZF=jH<0aAiJUpo>61hP@ct)p3cc)\"iHTRr;VP_$/A*6LcNdFQh*,gjIq#f>Y2:8n3si9g_:cK+C/5[a4<[cCI$eNZ8pnE&9'dY[#DmNZIHB]mK*\o(MnBT8t9dK!o@?U3O;"s8Fh#4L8SmI%(C$P//QJ>e..]kK`1Tq&Mkds#9uYJA6DmpEd;[+ruTsAGl5moB4G\[uHB,P9s#H]H&qKgnToo)K4Xn19X$4]WuD/[LYLi5SQ+phL,=uk@]ej^[9[YA!.C'QM-<6n*&/o;2Eg\Oa6WM=%A;mSMki\Xg_KCDB4R0]KZ5ZZdikY3rZ-ZX!4+)3-!psDnH[#[r1$UX]r9q:n'@[dRQHap6jaHrqbq&J,Jh]]6*;cQoZ8#ABa"^ro\bTO&"ThHL%]P:\N)'2fHQd#o1#f[n/sfVjqd"/>:iS^RIbL*sVcGn5lfeTcm=3?G1ZUF3ZcrQ5YS$d%`cC-U@.IZ$#n'Xi^9t-s0pIq&\DRh'Vfjr4lgUAiFbjQ[;r5VYe(]XQoYWIU?9n@)7F<6\ZlNf_kR:a,V0E1]*E,h:T#E9E,c+VbXAlG=FK.g_e.OnVe,/:.u"SNK*>Z[7fr^-/4^RSo:rXldj_(#WS">p$:4r>NU,@8VLFR!!(s3;s!HUI0]b!+oqZDzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!!!"L5@.i0A'^`0HhZqKYOgo$X3gG0dnd/rfX>YOf%&:[(bhUYT0D@kSM=VDo`M(tT2;_M3HO?Tkg?06[W]TF&_ja2X)c4`Vj\X)d[E%/*e*nCE^u<&WSHY%H2$-=LCP=C9,KEp8?uO_2ZJj6:_LaZ?")oSQ^O96KKe1=G5qX$lY1*KlJ&f:SiqFp*3W4G\T28l`[@!`V2b6a@gpqLhaH7_Ep$3E4PO%`#k2u!4].Q/=P'WW'4R\o_ZPeub]$ET4nqhKU7P>ISM;eu`0)0lkD`.)IO<(CZ_Be0'=g[9\9[].o?TY9qBa@T2t*8XcC$@p_@&$0:cq1WO0VS34Y0oLQ.qRi91E6.Y%g/U9I0%-!W^"p,*Q(L4ErQ_c_Jji?+Zk%O%,gAVbY@qqA3LdEQT`.,W0^OlJ/rAOX!ARgS]o^g1/e!'pnQ6k005KAP+d3DGtSk]M(4\Z<*hF8Kh!K,bF!L,"9YP*jGf=A\:LCb4Sd[^B=;*)B'M<2dEUs7A2+cMYb#5pL\TV37UV'Na+HV1RSt(]9,5@*)8T_Zmt*31cI?`T%gYgf=ULc4.W,$8"!NQWMc=+YJk:YEkBb?aiI-#OSNnr(boW:4AIQQNQ>?+Y9Vk=1\oF4bLTAas#f%jj5_Mq;'>\hhVk,?\)YoGou7aGcElB&UZmge9!a8cSkd7O5b'PM@0OI^`t?W`=bI9CC3&7k\,W0FA1'/83M09bbU+)3[1c\STs)b:gUmBiZC\Ci4.YeZ7;[0>8:F6;4b+j2Nq3Lo_aTXHE\J'k]/\H:H[!Mj*KqRG?,(jeAX?\gc;j;-/@=3h?*e-;'HVSaU"I0))CnJt\a&e45@5gg?KFE#FOEJj+7;#<>a(#?\2sZjnAfe7O[j`ik`P[&,O)'%Q7Z>;\!T4!C/Pq+7U5Z--2O9#K/RZ)S8"Tfs2mglWlZosr8JO-\G5'AYJ'Y`oUaus&L4Hgieim$k+u!UkZ'XH%+D+MYGnLXW9W^7,B&I.mG#+j33g8DrT'/b;2CLWU$+7Cb*6G>i\ac(l[o^3W;TVYo?l\bdra$VEBi5P,an'#!!%Kn4M;%$!5K[D?3pT3zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!:Ue;`f(dbl4]mHrqo#e[pqo3E9UZT@C"82?[F`hgiHhL^QCg#J4_Lr4cNum,`B2:l^_eH$>Q5O?2o&X`B[)k.RjH2t/p$:53>IUK,VNQ"S9Ug&)At4>Hkt,=/S2i`YoVemo*AM$A[VabSbqd\4$m7@^W`?+ro]\/>^jq7+[F\_HmKVgmN/JH'9$1GOG.mS">EeVBrR3top!cpZ_7+:92YK/nMAbU%S!R+RkCa(1nh0%>G+%Oa=p:PbHma"#e9_!uQCY,`&jH<0q"A%BUHKh9,=0F5oO:NHi6cNYWMc=kgj%GR?'0G$Ej:)lqCZ'bPaj8]VW<8Q8H:8AX;Ep9i5($NU\rngcOCdL.+9^\D*ELJFKsmY>IHq.e##i(rpc%FE'+*78.9"&AD"uJA)sH"GOOB[Y*[.l(A@4S+2LH*)CrDso#hOc4rfjfp2BshCtPscbEDaa0l#4:gMaj?Z*?$/)nAb-T.XH_\A#\kA5.NCW/7>FaH7]E5'H9p"J65%rV,3ai.;>/G5pf&J,85i:I\dGZ=V,l`Ms7&)cU88&L0b-o[#sfo\Fl_]osg>:?sF&''7En6E,GWGq6?,!.[9U2)R8G[r7>E`ufeYaX*9ATt.]o\T+rrjiYl>4"&:EJ,]7t$Pu9@a,V0ogkGUL2r>TFe&nJ+>]M4$m367*o?9!`cBW9/"elt^?U_m0>H"E/l63,i9pi5$?jC`"s?9N#s"\`[kc\1n=&_^hq';R301?J3]PODOB4n%JH+s6pQpYTr&Qg<@s-!DJt@/X!O*[:ICBr'j,P*!ogKrr&#p02iN3m<.#d(CQ;5n\IYH8E!(!*#T2p'bsK>%NRUsgUAgS`4l38=2.An/"[I9cJ([bIiqa_K_]mD[fc\]59FlNSNO(mYJjSaf3a%+.Ar]AC=T?FmB*kS>5TXPnR_K:<&]KZ5ZZc=(K0=%\f0>&uFs5dVeV5^FCeeQk@$p/Ig`(W;4$]GM[m'G"j\"`0Zb"0[GDnPU0]\U/ACF&J)Q4s&``ugAZd\TUsUF`s&iTdJI^?1l"i`+gKHZrXrr@(TkhI^r7>YTp%"dsWdZigAZ5:[JO?(ZTBko4NM+Xq%VT$rVYaBX&alBU.)8Lo]U@:B%GS*q_h\mL55X]Qs8&r!qN([OO4gjPs7_$CG:&p@"WfdNYZ%GpO]j(^])Vd,@hjLnpOAeT/cP`Zn[C^R>L_$H&;aC<.Q1qYMA8]oF)uD4@q4,n?15N)mZKI'nRLWEr8A1bqXod;d%NfkrUeR_f0XiYmbOmCX#O'6N"A)?rU55#?;#05-@eY?p?^Ir*(mu'=?L*!/b'``PcoqG9B5Kr\atq.8"q>EnAQ2N7+t`uL890D1aei9;@J_98%&1lUA2CGp_nEhIU7:G!!%Kto$?%GEm$%S.3=2Q3,d]gIA,b8=D#gH0k9@lT3l$0I%00A;LE/oJd"Mr$q\m&(aRP=b2_Oba>`G@]6Eh*^K4cj2%k[b!E?LJjiYDl#=jMTBN\ikSir8W1D8:VB_o>(pY9gcA5AVZjd)3gPUcKSV*`?A/\?,hZ?u?6NZC2.^U^q)rVF\i].@;fEr;`qhL"^No-$gW(@^XJ)=2?^%'MbEa`<%?/0fdA)nD"NO)3iJ3+3M2UsQ-Aj?H,]J=?Zf[q$T;0)]uMCYhX3ENr>EG*B5RQc.G9he=b>[LiFr"Zc*N@Y%fL88o&+C2/CUIjMgTKjTL#hCR?i[kn%:G@<8t@@DG1#J8YP6`j19C!Q7lSd8ScN_WG_;$54UT,nWT+8pOE5;Hf+^,dO#XO^_Y$&g-eZm0,8+bjZ^M.!1!/uj@=n"p[:=ejN-DmdnEJHWKBrqH=PhjDr)HC[VXXF^W"^%;_+A/M\e%,?72@mEkd)%lNSi;OJ[G!j.DDpm^Qckd\Yi#0E;(QI,hs.]Fk2Ezzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!8tA!ba@pu%KQP0rc[\b]D'GuYO(#QDnJP"C0%EGMSOmS1nAT"5kXQTj,V26U+gB6XJo-r7cD`b+8tl9J":9h=6u+t:!,EO[;/k)6'4L:?bapN%CQ6RhgTo^@q15#,TRC93Y^i)l`\'0.%b&3lQ]X6kbF35Y^#]tqYL%%l<@Ea,C^pf$m%G(6Ou>%g1dJ@HgeY3-Vl@Zs-3-Jj]V]c!%4S>VIo`!m8[KGU94^4#mQ61!dh*cHZmf])Q$`8nbr8DBRl>9&KF,VWo7s@4rp$PfNZL?p:bgWp%X%-j0mWDrZ?2V"!G&&H1h]^K(-V??;d]DC>hX2JW`@/^Y\?g1;q'8u?QFifjhCW[MaGa]4_#5gee/uFI.4t%r5X<2%pj/%<:&m1kF<2)qn`^oS7pg&nftVXe)l2d*f.Z<++/&,q<)CFNM?js/;V/QO$/fqG,Am2`//EFnOC1I\ohetDV_%+rUnc,rqPMN=LrTgldi2]71u)8br++CEloiCGOJ:Ab+3/?q?eSq!!#j/r2')4J,]%p*dXd3nqc>AJHM*JPnV&+5AP"KG4"!`gc0e^RdQ;Bp$1'X\VkRb]J*=n1qU(A,]^j4hi/-&lHS*"V@a30BNK&p']6:mbm^l9%De5?!,PBftB2"=ET?tjC:ZY"`"NqoHu]!m1,n!-6clQrKVc,c2XPbUfWju,a&cqZ14]:EpYL78P0X!_s8?<^^3re%6bb\BlIDr!E`t8?k0+]!B#NOqNg\BR2)S+"4Ls6NrpK7SmaFrr*AV.rqPDdnmlQ"83;J=[b?#ii^hBlShc.#-bIH]%O6;*41o#i[ViN0B28e]NA67[g*',6t,H7Z,u_j,Bl]H=%4[H@Ddh_N?\ND1M4l/q=E&9e&M2NeTfs51U?[d877ZSV(E8eZ#bGSV(Q\"Jfl=/4OU-3ksFRIqp;!M!9[r5WqqT@t'!!#ilr1@.`maK/:qQODGP4)IIS2dcYj2T*YG4"!0p\V,'iela8q*W\S06Z-`1/+<(]r%Z=^jlIYp%@Q.m)h2gr-'HMAfE[HN!?tBpV,aT>EXae^P:fhdt'+-If&Nl(a=n9i=CG5o:Q%Fq8&dnD/F.NN$[)=G)Wf/>ISLlc+EV1d6$g01V_/`o&\&Ap$*7o%N[`Wp$1(]Y\JC*\UTO:;lendstream +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject << +/FormXob.533677ade4d1b19a29d18580c2f91e53 3 0 R +>> +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (anonymous) /CreationDate (D:20260803113641+00'00') /Creator (anonymous) /Keywords () /ModDate (D:20260803113641+00'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (unspecified) /Title (untitled) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 200 +>> +stream +GapXO4V*,u'LhcqME@g?o(kZ:W'MaDUZp\DZm6Qpg>Lo([h-bVmVm48aWom2B[1Jqp(lDO5TI3;a\9PmENnfSMSuBX2LK5FD&qVT%;htq.(!l^Q]>6)%6';>3fFM570?j~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000092 00000 n +0000000199 00000 n +0000012917 00000 n +0000013173 00000 n +0000013241 00000 n +0000013502 00000 n +0000013561 00000 n +trailer +<< +/ID +[<20c50f8e50c432c58efb66712e9b1aa9><20c50f8e50c432c58efb66712e9b1aa9>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +13851 +%%EOF diff --git a/tests/fixtures/text_page_with_watermark_image.pdf b/tests/fixtures/text_page_with_watermark_image.pdf new file mode 100644 index 0000000..e561e12 Binary files /dev/null and b/tests/fixtures/text_page_with_watermark_image.pdf differ diff --git a/tests/fixtures/vector_outlined_text_with_caption.pdf b/tests/fixtures/vector_outlined_text_with_caption.pdf new file mode 100644 index 0000000..4152b19 --- /dev/null +++ b/tests/fixtures/vector_outlined_text_with_caption.pdf @@ -0,0 +1,1239 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [4 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj +4 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >> +endobj +5 0 obj +<< /Length 8585 >> +stream +50.0 700 m +55.0 710 l +60.0 700 l +58.0 690 c +h +f +S +s +53.0 700 m +58.0 710 l +63.0 700 l +61.0 690 c +h +f +S +s +56.0 700 m +61.0 710 l +66.0 700 l +64.0 690 c +h +f +S +s +59.0 700 m +64.0 710 l +69.0 700 l +67.0 690 c +h +f +S +s +62.0 700 m +67.0 710 l +72.0 700 l +70.0 690 c +h +f +S +s +65.0 700 m +70.0 710 l +75.0 700 l +73.0 690 c +h +f +S +s +68.0 700 m +73.0 710 l +78.0 700 l +76.0 690 c +h +f +S +s +71.0 700 m +76.0 710 l +81.0 700 l +79.0 690 c +h +f +S +s +74.0 700 m +79.0 710 l +84.0 700 l +82.0 690 c +h +f +S +s +77.0 700 m +82.0 710 l +87.0 700 l +85.0 690 c +h +f +S +s +80.0 700 m +85.0 710 l +90.0 700 l +88.0 690 c +h +f +S +s +83.0 700 m +88.0 710 l +93.0 700 l +91.0 690 c +h +f +S +s +86.0 700 m +91.0 710 l +96.0 700 l +94.0 690 c +h +f +S +s +89.0 700 m +94.0 710 l +99.0 700 l +97.0 690 c +h +f +S +s +92.0 700 m +97.0 710 l +102.0 700 l +100.0 690 c +h +f +S +s +95.0 700 m +100.0 710 l +105.0 700 l +103.0 690 c +h +f +S +s +98.0 700 m +103.0 710 l +108.0 700 l +106.0 690 c +h +f +S +s +101.0 700 m +106.0 710 l +111.0 700 l +109.0 690 c +h +f +S +s +104.0 700 m +109.0 710 l +114.0 700 l +112.0 690 c +h +f +S +s +107.0 700 m +112.0 710 l +117.0 700 l +115.0 690 c +h +f +S +s +110.0 700 m +115.0 710 l +120.0 700 l +118.0 690 c +h +f +S +s +113.0 700 m +118.0 710 l +123.0 700 l +121.0 690 c +h +f +S +s +116.0 700 m +121.0 710 l +126.0 700 l +124.0 690 c +h +f +S +s +119.0 700 m +124.0 710 l +129.0 700 l +127.0 690 c +h +f +S +s +122.0 700 m +127.0 710 l +132.0 700 l +130.0 690 c +h +f +S +s +125.0 700 m +130.0 710 l +135.0 700 l +133.0 690 c +h +f +S +s +128.0 700 m +133.0 710 l +138.0 700 l +136.0 690 c +h +f +S +s +131.0 700 m +136.0 710 l +141.0 700 l +139.0 690 c +h +f +S +s +134.0 700 m +139.0 710 l +144.0 700 l +142.0 690 c +h +f +S +s +137.0 700 m +142.0 710 l +147.0 700 l +145.0 690 c +h +f +S +s +140.0 700 m +145.0 710 l +150.0 700 l +148.0 690 c +h +f +S +s +143.0 700 m +148.0 710 l +153.0 700 l +151.0 690 c +h +f +S +s +146.0 700 m +151.0 710 l +156.0 700 l +154.0 690 c +h +f +S +s +149.0 700 m +154.0 710 l +159.0 700 l +157.0 690 c +h +f +S +s +152.0 700 m +157.0 710 l +162.0 700 l +160.0 690 c +h +f +S +s +155.0 700 m +160.0 710 l +165.0 700 l +163.0 690 c +h +f +S +s +158.0 700 m +163.0 710 l +168.0 700 l +166.0 690 c +h +f +S +s +161.0 700 m +166.0 710 l +171.0 700 l +169.0 690 c +h +f +S +s +164.0 700 m +169.0 710 l +174.0 700 l +172.0 690 c +h +f +S +s +167.0 700 m +172.0 710 l +177.0 700 l +175.0 690 c +h +f +S +s +170.0 700 m +175.0 710 l +180.0 700 l +178.0 690 c +h +f +S +s +173.0 700 m +178.0 710 l +183.0 700 l +181.0 690 c +h +f +S +s +176.0 700 m +181.0 710 l +186.0 700 l +184.0 690 c +h +f +S +s +179.0 700 m +184.0 710 l +189.0 700 l +187.0 690 c +h +f +S +s +182.0 700 m +187.0 710 l +192.0 700 l +190.0 690 c +h +f +S +s +185.0 700 m +190.0 710 l +195.0 700 l +193.0 690 c +h +f +S +s +188.0 700 m +193.0 710 l +198.0 700 l +196.0 690 c +h +f +S +s +191.0 700 m +196.0 710 l +201.0 700 l +199.0 690 c +h +f +S +s +194.0 700 m +199.0 710 l +204.0 700 l +202.0 690 c +h +f +S +s +197.0 700 m +202.0 710 l +207.0 700 l +205.0 690 c +h +f +S +s +200.0 700 m +205.0 710 l +210.0 700 l +208.0 690 c +h +f +S +s +203.0 700 m +208.0 710 l +213.0 700 l +211.0 690 c +h +f +S +s +206.0 700 m +211.0 710 l +216.0 700 l +214.0 690 c +h +f +S +s +209.0 700 m +214.0 710 l +219.0 700 l +217.0 690 c +h +f +S +s +212.0 700 m +217.0 710 l +222.0 700 l +220.0 690 c +h +f +S +s +215.0 700 m +220.0 710 l +225.0 700 l +223.0 690 c +h +f +S +s +218.0 700 m +223.0 710 l +228.0 700 l +226.0 690 c +h +f +S +s +221.0 700 m +226.0 710 l +231.0 700 l +229.0 690 c +h +f +S +s +224.0 700 m +229.0 710 l +234.0 700 l +232.0 690 c +h +f +S +s +227.0 700 m +232.0 710 l +237.0 700 l +235.0 690 c +h +f +S +s +230.0 700 m +235.0 710 l +240.0 700 l +238.0 690 c +h +f +S +s +233.0 700 m +238.0 710 l +243.0 700 l +241.0 690 c +h +f +S +s +236.0 700 m +241.0 710 l +246.0 700 l +244.0 690 c +h +f +S +s +239.0 700 m +244.0 710 l +249.0 700 l +247.0 690 c +h +f +S +s +242.0 700 m +247.0 710 l +252.0 700 l +250.0 690 c +h +f +S +s +245.0 700 m +250.0 710 l +255.0 700 l +253.0 690 c +h +f +S +s +248.0 700 m +253.0 710 l +258.0 700 l +256.0 690 c +h +f +S +s +251.0 700 m +256.0 710 l +261.0 700 l +259.0 690 c +h +f +S +s +254.0 700 m +259.0 710 l +264.0 700 l +262.0 690 c +h +f +S +s +257.0 700 m +262.0 710 l +267.0 700 l +265.0 690 c +h +f +S +s +260.0 700 m +265.0 710 l +270.0 700 l +268.0 690 c +h +f +S +s +263.0 700 m +268.0 710 l +273.0 700 l +271.0 690 c +h +f +S +s +266.0 700 m +271.0 710 l +276.0 700 l +274.0 690 c +h +f +S +s +269.0 700 m +274.0 710 l +279.0 700 l +277.0 690 c +h +f +S +s +272.0 700 m +277.0 710 l +282.0 700 l +280.0 690 c +h +f +S +s +275.0 700 m +280.0 710 l +285.0 700 l +283.0 690 c +h +f +S +s +278.0 700 m +283.0 710 l +288.0 700 l +286.0 690 c +h +f +S +s +281.0 700 m +286.0 710 l +291.0 700 l +289.0 690 c +h +f +S +s +284.0 700 m +289.0 710 l +294.0 700 l +292.0 690 c +h +f +S +s +287.0 700 m +292.0 710 l +297.0 700 l +295.0 690 c +h +f +S +s +290.0 700 m +295.0 710 l +300.0 700 l +298.0 690 c +h +f +S +s +293.0 700 m +298.0 710 l +303.0 700 l +301.0 690 c +h +f +S +s +296.0 700 m +301.0 710 l +306.0 700 l +304.0 690 c +h +f +S +s +299.0 700 m +304.0 710 l +309.0 700 l +307.0 690 c +h +f +S +s +302.0 700 m +307.0 710 l +312.0 700 l +310.0 690 c +h +f +S +s +305.0 700 m +310.0 710 l +315.0 700 l +313.0 690 c +h +f +S +s +308.0 700 m +313.0 710 l +318.0 700 l +316.0 690 c +h +f +S +s +311.0 700 m +316.0 710 l +321.0 700 l +319.0 690 c +h +f +S +s +314.0 700 m +319.0 710 l +324.0 700 l +322.0 690 c +h +f +S +s +317.0 700 m +322.0 710 l +327.0 700 l +325.0 690 c +h +f +S +s +320.0 700 m +325.0 710 l +330.0 700 l +328.0 690 c +h +f +S +s +323.0 700 m +328.0 710 l +333.0 700 l +331.0 690 c +h +f +S +s +326.0 700 m +331.0 710 l +336.0 700 l +334.0 690 c +h +f +S +s +329.0 700 m +334.0 710 l +339.0 700 l +337.0 690 c +h +f +S +s +332.0 700 m +337.0 710 l +342.0 700 l +340.0 690 c +h +f +S +s +335.0 700 m +340.0 710 l +345.0 700 l +343.0 690 c +h +f +S +s +338.0 700 m +343.0 710 l +348.0 700 l +346.0 690 c +h +f +S +s +341.0 700 m +346.0 710 l +351.0 700 l +349.0 690 c +h +f +S +s +344.0 700 m +349.0 710 l +354.0 700 l +352.0 690 c +h +f +S +s +347.0 700 m +352.0 710 l +357.0 700 l +355.0 690 c +h +f +S +s +350.0 700 m +355.0 710 l +360.0 700 l +358.0 690 c +h +f +S +s +353.0 700 m +358.0 710 l +363.0 700 l +361.0 690 c +h +f +S +s +356.0 700 m +361.0 710 l +366.0 700 l +364.0 690 c +h +f +S +s +359.0 700 m +364.0 710 l +369.0 700 l +367.0 690 c +h +f +S +s +362.0 700 m +367.0 710 l +372.0 700 l +370.0 690 c +h +f +S +s +365.0 700 m +370.0 710 l +375.0 700 l +373.0 690 c +h +f +S +s +368.0 700 m +373.0 710 l +378.0 700 l +376.0 690 c +h +f +S +s +371.0 700 m +376.0 710 l +381.0 700 l +379.0 690 c +h +f +S +s +374.0 700 m +379.0 710 l +384.0 700 l +382.0 690 c +h +f +S +s +377.0 700 m +382.0 710 l +387.0 700 l +385.0 690 c +h +f +S +s +380.0 700 m +385.0 710 l +390.0 700 l +388.0 690 c +h +f +S +s +383.0 700 m +388.0 710 l +393.0 700 l +391.0 690 c +h +f +S +s +386.0 700 m +391.0 710 l +396.0 700 l +394.0 690 c +h +f +S +s +389.0 700 m +394.0 710 l +399.0 700 l +397.0 690 c +h +f +S +s +392.0 700 m +397.0 710 l +402.0 700 l +400.0 690 c +h +f +S +s +395.0 700 m +400.0 710 l +405.0 700 l +403.0 690 c +h +f +S +s +398.0 700 m +403.0 710 l +408.0 700 l +406.0 690 c +h +f +S +s +401.0 700 m +406.0 710 l +411.0 700 l +409.0 690 c +h +f +S +s +404.0 700 m +409.0 710 l +414.0 700 l +412.0 690 c +h +f +S +s +407.0 700 m +412.0 710 l +417.0 700 l +415.0 690 c +h +f +S +s +410.0 700 m +415.0 710 l +420.0 700 l +418.0 690 c +h +f +S +s +413.0 700 m +418.0 710 l +423.0 700 l +421.0 690 c +h +f +S +s +416.0 700 m +421.0 710 l +426.0 700 l +424.0 690 c +h +f +S +s +419.0 700 m +424.0 710 l +429.0 700 l +427.0 690 c +h +f +S +s +422.0 700 m +427.0 710 l +432.0 700 l +430.0 690 c +h +f +S +s +425.0 700 m +430.0 710 l +435.0 700 l +433.0 690 c +h +f +S +s +428.0 700 m +433.0 710 l +438.0 700 l +436.0 690 c +h +f +S +s +431.0 700 m +436.0 710 l +441.0 700 l +439.0 690 c +h +f +S +s +434.0 700 m +439.0 710 l +444.0 700 l +442.0 690 c +h +f +S +s +437.0 700 m +442.0 710 l +447.0 700 l +445.0 690 c +h +f +S +s +440.0 700 m +445.0 710 l +450.0 700 l +448.0 690 c +h +f +S +s +443.0 700 m +448.0 710 l +453.0 700 l +451.0 690 c +h +f +S +s +446.0 700 m +451.0 710 l +456.0 700 l +454.0 690 c +h +f +S +s +449.0 700 m +454.0 710 l +459.0 700 l +457.0 690 c +h +f +S +s +452.0 700 m +457.0 710 l +462.0 700 l +460.0 690 c +h +f +S +s +455.0 700 m +460.0 710 l +465.0 700 l +463.0 690 c +h +f +S +s +458.0 700 m +463.0 710 l +468.0 700 l +466.0 690 c +h +f +S +s +461.0 700 m +466.0 710 l +471.0 700 l +469.0 690 c +h +f +S +s +464.0 700 m +469.0 710 l +474.0 700 l +472.0 690 c +h +f +S +s +467.0 700 m +472.0 710 l +477.0 700 l +475.0 690 c +h +f +S +s +470.0 700 m +475.0 710 l +480.0 700 l +478.0 690 c +h +f +S +s +473.0 700 m +478.0 710 l +483.0 700 l +481.0 690 c +h +f +S +s +476.0 700 m +481.0 710 l +486.0 700 l +484.0 690 c +h +f +S +s +479.0 700 m +484.0 710 l +489.0 700 l +487.0 690 c +h +f +S +s +482.0 700 m +487.0 710 l +492.0 700 l +490.0 690 c +h +f +S +s +485.0 700 m +490.0 710 l +495.0 700 l +493.0 690 c +h +f +S +s +488.0 700 m +493.0 710 l +498.0 700 l +496.0 690 c +h +f +S +s +491.0 700 m +496.0 710 l +501.0 700 l +499.0 690 c +h +f +S +s +494.0 700 m +499.0 710 l +504.0 700 l +502.0 690 c +h +f +S +s +497.0 700 m +502.0 710 l +507.0 700 l +505.0 690 c +h +f +S +s +BT +/F1 10 Tf +1 0 0 1 50 50 Tm (Fig 1 Fig 1 Fig 1 Fig 1) Tj +1 0 0 1 50 62 Tm (Fig 1 Fig 1 Fig 1 Fig 1) Tj +1 0 0 1 50 74 Tm (Fig 1 Fig 1 Fig 1 Fig 1) Tj +1 0 0 1 50 86 Tm (Fig 1 Fig 1 Fig 1 Fig 1) Tj +1 0 0 1 50 98 Tm (Fig 1 Fig 1 Fig 1 Fig 1) Tj +ET +endstream +endobj +xref +0 6 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000185 00000 n +0000000311 00000 n +trailer +<< /Size 6 /Root 1 0 R >> +startxref +8948 +%%EOF \ No newline at end of file diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index a0b1e5f..3cfc06e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3926,6 +3926,65 @@ fn encrypted_pdf_decrypts_with_correct_password() { ); } +/// Regression for the #231 review finding: `extract_pages_markdown`'s +/// `has_template_image` check must be gated the same way +/// `classify_pdf`/`detect_pdf_type` gates it (image_count <= 1, few text +/// ops, low alphanumeric diversity) — not treated as sufficient on its +/// own. The fixture is a real text page with substantial, richly varied +/// body text (>=50 Tj ops) drawn over a full-bleed background image +/// (e.g. letterhead/watermark). Before the fix, has_template_image alone +/// forced needs_ocr=true and discarded the page's clean markdown; now the +/// page must extract normally. +#[test] +fn test_extract_pages_markdown_does_not_ocr_text_page_with_watermark_image() { + let buf = std::fs::read("tests/fixtures/text_page_with_watermark_image.pdf").unwrap(); + + let ext = extract_pages_markdown_mem(&buf, None).expect("fixture should extract"); + let page = &ext.pages[0]; + assert!( + !page.needs_ocr, + "a text page with substantial real text should not be routed to OCR \ + just because it has a background image" + ); + assert!( + page.markdown.contains("watermark"), + "expected the page's real body text to be preserved, got: {:?}", + page.markdown + ); +} + +/// Regression for the #231 review finding: `extract_pages_markdown` never +/// checked `has_vector_text` at all, even though `detect_from_document`'s +/// Mixed-type per-page routing always sends vector-outlined-text pages to +/// OCR (outlined glyphs can't be extracted as text). A page with massive +/// path ops (outlined decorative text) plus a short genuine caption would +/// extract that caption cleanly — non-empty, non-garbled — so the +/// existing empty/garbage-text checks alone couldn't catch it. +#[test] +fn test_extract_pages_markdown_ocrs_page_with_vector_outlined_text() { + let buf = std::fs::read("tests/fixtures/vector_outlined_text_with_caption.pdf").unwrap(); + + let cls = pdf_inspector::detector::detect_pdf_type_mem(&buf).expect("fixture should classify"); + assert!( + cls.pages_needing_ocr.contains(&1), + "classify_pdf should flag page 1 as needing OCR (vector-outlined text), got: {:?}", + cls.pages_needing_ocr + ); + + let ext = extract_pages_markdown_mem(&buf, None).expect("fixture should extract"); + let page = &ext.pages[0]; + assert!( + page.needs_ocr, + "extract_pages_markdown must agree with classify_pdf that this page needs OCR" + ); + assert!( + page.markdown.is_empty(), + "a page flagged needs_ocr must not return markdown as if extraction were \ + trustworthy, got: {:?}", + page.markdown + ); +} + #[test] fn pdf_options_debug_redacts_password() { let opts = PdfOptions::new().password("secret123"); @@ -3936,3 +3995,37 @@ fn pdf_options_debug_redacts_password() { ); assert!(dbg.contains("REDACTED"), "expected redaction marker: {dbg}"); } + +/// Regression for #227: `extract_pages_markdown`'s per-page `needs_ocr` +/// must agree with `classify_pdf`/`detect_pdf_type` on the same page. The +/// fixture is a full-page raster "scan" with a single line of genuine +/// native text drawn over it (a header) — the native text extracts +/// perfectly cleanly (no decoding issues, non-empty), so a needs_ocr +/// computation based on text-quality signals alone says `false`, while +/// detection correctly sees a dominant background image and says the page +/// needs OCR. Both must now agree it needs OCR, and the markdown must not +/// be returned as if the extraction were trustworthy. +#[test] +fn test_extract_pages_markdown_agrees_with_classify_on_scan_with_native_header() { + let buf = std::fs::read("tests/fixtures/scan_with_native_header_text.pdf").unwrap(); + + let cls = pdf_inspector::detector::detect_pdf_type_mem(&buf).expect("fixture should classify"); + assert!( + cls.pages_needing_ocr.contains(&1), + "classify_pdf should flag page 1 as needing OCR (image-dominated), got: {:?}", + cls.pages_needing_ocr + ); + + let ext = extract_pages_markdown_mem(&buf, None).expect("fixture should extract"); + let page = &ext.pages[0]; + assert!( + page.needs_ocr, + "extract_pages_markdown must agree with classify_pdf that this page needs OCR" + ); + assert!( + page.markdown.is_empty(), + "a page flagged needs_ocr must not return markdown as if extraction were \ + trustworthy, got: {:?}", + page.markdown + ); +}