From b084769fda1886e26f7ecd178a86b39047f3376e Mon Sep 17 00:00:00 2001
From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com>
Date: Wed, 15 Jul 2026 17:18:16 -0700
Subject: [PATCH] feat(markdown): add fidelity output profile (#168)

---
 README.md                   |  3 +++
 docs/rust-api.md            |  2 +-
 src/bin/pdf2md.rs           |  7 +++++++
 src/lib.rs                  |  1 +
 src/markdown/mod.rs         | 17 +++++++++++++++++
 src/markdown/postprocess.rs | 26 +++++++++++++++++++++++---
 tests/integration_tests.rs  |  3 +++
 7 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 15f3bfe..1fcc743 100644
--- a/README.md
+++ b/README.md
@@ -119,6 +119,9 @@ pdf2md document.pdf --items-json
 # Raw markdown only (no headers)
 pdf2md document.pdf --raw
 
+# Token-efficient output (collapses long dot leaders and similar source padding)
+pdf2md document.pdf --compact
+
 # Insert page break markers (<!-- Page N -->)
 pdf2md document.pdf --pages
 
diff --git a/docs/rust-api.md b/docs/rust-api.md
index eb072bb..fc443a6 100644
--- a/docs/rust-api.md
+++ b/docs/rust-api.md
@@ -37,7 +37,7 @@ For the latest unreleased changes, use the git dependency instead:
 pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector" }
 ```
 
-The crate also ships CLI binaries — `pdf2md` (PDF → Markdown, with `--json`, `--pages`, `--select-pages`) and `detect-pdf` (classification, with `--analyze --json`):
+The crate also ships CLI binaries — `pdf2md` (PDF → Markdown, with `--json`, `--pages`, `--select-pages`, and the opt-in token-saving `--compact` profile) and `detect-pdf` (classification, with `--analyze --json`):
 
 ```bash
 cargo install pdf-inspector
diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs
index 52f26ed..aaa0886 100644
--- a/src/bin/pdf2md.rs
+++ b/src/bin/pdf2md.rs
@@ -206,6 +206,9 @@ fn main() {
         eprintln!("  --json              Output result as JSON");
         eprintln!("  --items-json        Output positioned TextItem JSON");
         eprintln!("  --raw               Output only markdown (no headers)");
+        eprintln!(
+            "  --compact           Collapse token-heavy source formatting such as dot leaders"
+        );
         eprintln!("  --pages             Insert page break markers (<!-- Page N -->)");
         eprintln!("  --select-pages N    Only process specified pages (e.g. 1,3,5-10)");
         eprintln!("  --password PW       Password for an encrypted PDF");
@@ -218,6 +221,7 @@ fn main() {
     let json_output = args.iter().any(|a| a == "--json");
     let items_json_output = args.iter().any(|a| a == "--items-json");
     let raw_output = args.iter().any(|a| a == "--raw");
+    let compact_output = args.iter().any(|a| a == "--compact");
     let page_numbers = args.iter().any(|a| a == "--pages");
     let detect_only = args.iter().any(|a| a == "--detect-only");
     let analyze = args.iter().any(|a| a == "--analyze");
@@ -276,6 +280,9 @@ fn main() {
     };
 
     let mut options = PdfOptions::new().mode(process_mode);
+    if compact_output {
+        options.markdown.profile = pdf_inspector::MarkdownProfile::Compact;
+    }
     options.markdown.include_page_numbers = page_numbers;
     if let Some(pages) = page_filter {
         options.page_filter = Some(pages);
diff --git a/src/lib.rs b/src/lib.rs
index c428602..db3f93a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -54,6 +54,7 @@ pub use extractor::{
 };
 pub use markdown::{
     to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions,
+    MarkdownProfile,
 };
 pub use process_mode::ProcessMode;
 pub use types::{LayoutComplexity, PdfLine, PdfRect, TextItem};
diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs
index 33e7600..26c48e9 100644
--- a/src/markdown/mod.rs
+++ b/src/markdown/mod.rs
@@ -500,9 +500,25 @@ pub(crate) fn filter_lines_to_band(
         .collect()
 }
 
+/// Output policy for Markdown post-processing.
+///
+/// [`MarkdownProfile::Fidelity`] preserves source characters wherever possible.
+/// [`MarkdownProfile::Compact`] enables optional token-saving rewrites that may
+/// be useful for agent context windows but are not byte-faithful to the PDF.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum MarkdownProfile {
+    /// Preserve source text fidelity. This is the default.
+    #[default]
+    Fidelity,
+    /// Prefer token-efficient output, including collapsing long dot leaders.
+    Compact,
+}
+
 /// Options for markdown conversion
 #[derive(Debug, Clone)]
 pub struct MarkdownOptions {
+    /// Source-fidelity versus token-efficient post-processing policy.
+    pub profile: MarkdownProfile,
     /// Detect headers by font size
     pub detect_headers: bool,
     /// Detect list items
@@ -536,6 +552,7 @@ pub struct MarkdownOptions {
 impl Default for MarkdownOptions {
     fn default() -> Self {
         Self {
+            profile: MarkdownProfile::default(),
             detect_headers: true,
             detect_lists: true,
             detect_code: true,
diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs
index 5891c1e..c74091a 100644
--- a/src/markdown/postprocess.rs
+++ b/src/markdown/postprocess.rs
@@ -2,12 +2,15 @@
 
 use regex::Regex;
 
-use super::MarkdownOptions;
+use super::{MarkdownOptions, MarkdownProfile};
 
 /// Clean up markdown output with post-processing
 pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> String {
-    // Collapse dot leaders (e.g. TOC entries: "Introduction...............................1")
-    text = collapse_dot_leaders(&text);
+    if options.profile == MarkdownProfile::Compact {
+        // Dot-leader collapse saves tokens but changes source text, so it is
+        // reserved for the explicit compact profile.
+        text = collapse_dot_leaders(&text);
+    }
 
     // Fix hyphenation first (before other processing)
     if options.fix_hyphenation {
@@ -355,6 +358,23 @@ fn format_urls(text: &str) -> String {
 mod tests {
     use super::*;
 
+    #[test]
+    fn fidelity_profile_preserves_dot_leaders() {
+        let input = "Introduction............................1".to_string();
+        let result = clean_markdown(input.clone(), &MarkdownOptions::default());
+        assert_eq!(result, format!("{input}\n"));
+    }
+
+    #[test]
+    fn compact_profile_collapses_dot_leaders() {
+        let input = "Introduction............................1".to_string();
+        let options = MarkdownOptions {
+            profile: MarkdownProfile::Compact,
+            ..MarkdownOptions::default()
+        };
+        assert_eq!(clean_markdown(input, &options), "Introduction ... 1\n");
+    }
+
     // --- collapse_dot_leaders ---
 
     #[test]
diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs
index c5fd3fe..86cd03d 100644
--- a/tests/integration_tests.rs
+++ b/tests/integration_tests.rs
@@ -337,6 +337,7 @@ fn test_group_into_lines_sorting_by_x() {
 #[test]
 fn test_markdown_options_default() {
     let opts = MarkdownOptions::default();
+    assert_eq!(opts.profile, pdf_inspector::MarkdownProfile::Fidelity);
     assert!(opts.detect_headers);
     assert!(opts.detect_lists);
     assert!(opts.detect_code);
@@ -346,6 +347,7 @@ fn test_markdown_options_default() {
 #[test]
 fn test_markdown_options_custom() {
     let opts = MarkdownOptions {
+        profile: pdf_inspector::MarkdownProfile::Compact,
         detect_headers: false,
         detect_lists: true,
         detect_code: false,
@@ -361,6 +363,7 @@ fn test_markdown_options_custom() {
         ..Default::default()
     };
     assert!(!opts.detect_headers);
+    assert_eq!(opts.profile, pdf_inspector::MarkdownProfile::Compact);
     assert!(opts.detect_lists);
     assert!(!opts.detect_code);
     assert_eq!(opts.base_font_size, Some(14.0));