From 43c9a813888a087a6eda3757b95b42c5fd4efbbf Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Mon, 23 Feb 2026 08:59:19 -0800 Subject: [PATCH] Lower dollar-as-space threshold to catch chest-wall PDF pattern The chest-wall PDF uses $ as separator but also has trailing $ after spaces, giving only 13.7% letter-dollar-letter ratio (594 of 4332). Add absolute count threshold (>20) alongside the ratio check so both concentrated and dispersed substitution patterns are caught. Co-Authored-By: Claude Opus 4.6 --- src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 07e4b09..e0bcf6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -404,8 +404,10 @@ pub fn process_pdf_mem_with_config( /// Two heuristics: /// 1. **U+FFFD**: Any replacement character indicates decode failures. /// 2. **Dollar-as-space**: Pattern like `Word$Word$Word` where `$` is used as a -/// word separator due to broken ToUnicode CMaps. Triggers when >50% of `$` -/// characters appear between letters AND the count exceeds 10. +/// word separator due to broken ToUnicode CMaps. Triggers when either: +/// - More than 50% of `$` are between letters (clear substitution pattern), OR +/// - More than 20 letter-dollar-letter occurrences (even if some `$` are also +/// used as trailing/leading separators, 20+ is far beyond normal financial text). fn detect_encoding_issues(markdown: &str) -> bool { // Heuristic 1: U+FFFD replacement characters if markdown.contains('\u{FFFD}') { @@ -425,7 +427,7 @@ fn detect_encoding_issues(markdown: &str) -> bool { letter_dollar_letter += 1; } } - if letter_dollar_letter * 2 > total_dollars { + if letter_dollar_letter > 20 || letter_dollar_letter * 2 > total_dollars { return true; } }