chore: bump npm version to 0.3.5

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
fix: stop counting Do operators as images in content stream scanner (#25 )
2026-04-08 16:44:45 -07:00 · 2026-04-08 16:41:40 -07:00 · 2026-04-08 10:45:51 -07:00 · 2026-04-04 21:58:05 -07:00 · 2026-04-04 12:32:31 -07:00 · 2026-04-04 12:19:38 -07:00
22 changed files with 1149 additions and 215 deletions
@@ -17,7 +17,7 @@ crate-type = ["lib", "cdylib"]
 pyo3 = { version = "0.25", features = ["extension-module"], optional = true }

 # PDF parsing
-lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "052674053814a9f4897af94f0b8e46a545c9b329", features = ["rayon"] }
+lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "7a05512d831415b1f2b1ce522391d6beab8a1284", features = ["rayon"] }

 # Error handling
 thiserror = "2.0"
@@ -55,5 +55,3 @@ path = "src/bin/detect_pdf.rs"
 [[bin]]
 name = "dump_ops"
 path = "src/bin/dump_ops.rs"
-
-
@@ -16,6 +16,23 @@ Built by [Firecrawl](https://firecrawl.dev) to handle text-based PDFs locally in
 - **Single document load** — The document is parsed once and shared between detection and extraction, avoiding redundant I/O.
 - **Lightweight** — Pure Rust, no ML models, no external services. Single dependency on `lopdf` for PDF parsing.

+## Benchmark
+
+Evaluated on the [opendataloader-bench](https://github.com/opendataloader-project/opendataloader-bench) corpus (200 PDFs). Only direct text extraction engines are shown — no OCR, no ML models. Scores are 0-1, higher is better.
+
+| Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
+|---|---|---|---|---|---|
+| pdf-inspector | 0.78 | 0.87 | 0.59 | 0.57 | 4s |
+| opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
+| pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
+| markitdown | 0.58 | 0.88 | 0.00 | 0.00 | 8s |
+
+For context, engines that use OCR/ML (docling, marker, mineru) score 0.83-0.88 overall but take 2-180 minutes on the same corpus.
+
+**Where we do well:** Speed (fastest of all engines), reading order, table detection vs other direct-text tools.
+
+**Where we lag:** Heading detection trails opendataloader — many PDFs use bold text at body font size for headings, or headings that are only slightly larger than body text. Table detection trails OCR-based engines that can see visual table structure.
+
 ## Quick start

 ### Python
@@ -129,12 +129,6 @@ version = "3.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"

-[[package]]
-name = "bytecount"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
-
 [[package]]
 name = "cbc"
 version = "0.1.2"
@@ -679,7 +673,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 [[package]]
 name = "lopdf"
 version = "0.40.0"
-source = "git+https://github.com/J-F-Liu/lopdf?rev=052674053814a9f4897af94f0b8e46a545c9b329#052674053814a9f4897af94f0b8e46a545c9b329"
+source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
 dependencies = [
 "aes",
 "bitflags",
@@ -695,7 +689,6 @@ dependencies = [
 "log",
 "md-5",
 "nom",
- "nom_locate",
 "rand",
 "rangemap",
 "rayon",
@@ -807,17 +800,6 @@ dependencies = [
 "memchr",
 ]

-[[package]]
-name = "nom_locate"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"
-dependencies = [
- "bytecount",
- "memchr",
- "nom",
-]
-
 [[package]]
 name = "num-conv"
 version = "0.2.1"
@@ -1,6 +1,6 @@
 {
  "name": "firecrawl-pdf-inspector",
-  "version": "0.3.1",
+  "version": "0.3.5",
  "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
  "main": "index.js",
  "types": "index.d.ts",
@@ -730,7 +730,7 @@ fn scan_content_for_text_operators(
    unique_chars: &mut HashSet<u8>,
 ) -> (u32, u32, u32, u32) {
    let mut text_ops = 0u32;
-    let mut image_count = 0u32;
+    let image_count = 0u32;
    let mut path_ops = 0u32;
    let mut font_changes = 0u32;

@@ -771,14 +771,10 @@ fn scan_content_for_text_operators(
            }
        }

-        // Look for 'Do' operator (XObject/image placement)
-        if b == b'D'
-            && i + 1 < content.len()
-            && content[i + 1] == b'o'
-            && (i + 2 >= content.len() || content[i + 2].is_ascii_whitespace())
-        {
-            image_count += 1;
-        }
+        // Note: We do NOT count 'Do' operators here because Do invokes any
+        // XObject — including Form XObjects that contain text.  Actual image
+        // detection is handled by scan_xobjects_in_resources (checks Subtype)
+        // and analyze_page_images (measures pixel area).

        // Count path construction/painting operators.
        // Single-byte: m (moveto), l (lineto), c (curveto), h (closepath),
@@ -1185,23 +1181,24 @@ mod tests {
        // H, e, l, o = 4 unique
        assert!(uchars.len() >= 4);

-        // Content with Do (image)
+        // Content with Do (XObject invocation — not counted as image here;
+        // actual image detection is handled by scan_xobjects_in_resources)
        uchars.clear();
        let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q";
        let (ops3, imgs3, _, _) = scan_content_for_text_operators(content3, &mut uchars);
        assert_eq!(ops3, 0);
-        assert_eq!(imgs3, 1);
+        assert_eq!(imgs3, 0);
    }

    #[test]
    fn test_image_dominated_detection() {
-        // Simulate a page with many Do operators and minimal text
+        // Do operators are no longer counted as images by scan_content_for_text_operators.
+        // Image-dominated detection now relies on scan_xobjects_in_resources which
+        // checks XObject Subtype. Here we verify that Do operators don't inflate image_count.
        let mut content = Vec::new();
-        // Add 50 Do operators (image-heavy)
        for i in 0..50 {
            content.extend_from_slice(format!("/Im{i} Do\n").as_bytes());
        }
-        // Add a few text operators with only a bullet char
        content.extend_from_slice(b"BT (x) Tj ET\n");
        content.extend_from_slice(b"BT (x) Tj ET\n");
        content.extend_from_slice(b"BT (x) Tj ET\n");
@@ -1209,15 +1206,8 @@ mod tests {
        let mut uchars = HashSet::new();
        let (ops, imgs, _, _) = scan_content_for_text_operators(&content, &mut uchars);
        assert_eq!(ops, 3);
-        assert_eq!(imgs, 50);
-        // Only 'x' unique char
+        assert_eq!(imgs, 0); // Do operators are not counted here
        assert_eq!(uchars.len(), 1);
-
-        // This should be image-dominated: 50 > 10 && 50 > 3*3=9
-        let is_image_dominated = imgs > 10 && imgs > ops * 3;
-        assert!(is_image_dominated);
-        // And fails unique char threshold
-        assert!(uchars.len() < 5);
    }

    #[test]
@@ -1227,12 +1217,9 @@ mod tests {
        let mut uchars = HashSet::new();
        let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars);
        assert_eq!(ops, 1);
-        assert_eq!(imgs, 2);
-        // Many unique chars from the sentence
+        assert_eq!(imgs, 0); // Do operators not counted here
+                             // Many unique chars from the sentence
        assert!(uchars.len() >= 5);
-        // Not image-dominated: 2 > 10 fails
-        let is_image_dominated = imgs > 10 && imgs > ops * 3;
-        assert!(!is_image_dominated);
    }

    #[test]
@@ -82,7 +82,7 @@ pub(crate) fn extract_page_text_items(
    page_num: u32,
    font_cmaps: &FontCMaps,
    include_invisible: bool,
-) -> Result<(PageExtraction, bool), PdfError> {
+) -> Result<(PageExtraction, bool, bool), PdfError> {
    use lopdf::content::Content;

    let mut items = Vec::new();
@@ -182,7 +182,7 @@ pub(crate) fn extract_page_text_items(
            content.operations.len(),
            MAX_OPERATIONS
        );
-        return Ok(((Vec::new(), Vec::new(), Vec::new()), false));
+        return Ok(((Vec::new(), Vec::new(), Vec::new()), false, false));
    }

    // Graphics state tracking
@@ -1009,11 +1009,12 @@ pub(crate) fn extract_page_text_items(
    // Some PDFs embed landscape content in portrait pages using a rotated text
    // matrix (e.g. [0, b, -b, 0, tx, ty] for 90° CCW).  The layout engine
    // assumes x=horizontal, y=vertical — so we swap coordinates to match.
-    let (items, rects, lines) = correct_rotated_page(items, rects, lines, &rotation_votes);
+    let (items, rects, lines, coords_rotated) =
+        correct_rotated_page(items, rects, lines, &rotation_votes);

    let items = super::merge_text_items(items);
    let items = super::merge_subscript_items(items);
-    Ok(((items, rects, lines), has_gid_fonts))
+    Ok(((items, rects, lines), has_gid_fonts, coords_rotated))
 }

 /// Counts of text operators with horizontal vs rotated combined matrices.
@@ -1030,9 +1031,9 @@ fn correct_rotated_page(
    mut rects: Vec<PdfRect>,
    mut lines: Vec<PdfLine>,
    votes: &RotationVotes,
-) -> (Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>) {
+) -> (Vec<TextItem>, Vec<PdfRect>, Vec<PdfLine>, bool) {
    if items.len() < 2 {
-        return (items, rects, lines);
+        return (items, rects, lines, false);
    }

    // Use the combined-matrix direction votes collected during extraction.
@@ -1041,7 +1042,7 @@ fn correct_rotated_page(
    let total_votes = votes.horizontal + votes.rotated;
    if total_votes == 0 || votes.rotated * 3 < total_votes * 2 {
        // Less than ~67% of text operators are rotated → not a rotated page
-        return (items, rects, lines);
+        return (items, rects, lines, false);
    }

    log::debug!(
@@ -1092,7 +1093,7 @@ fn correct_rotated_page(
        line.y2 = new_y2;
    }

-    (items, rects, lines)
+    (items, rects, lines, true)
 }

 /// Remove near-duplicate rects (same coordinates within 0.5 pt tolerance).
@@ -1228,7 +1229,7 @@ mod tests {

        let font_cmaps = FontCMaps::from_doc(&doc);
        let result = extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap();
-        let ((items, rects, lines), _has_gid) = result;
+        let ((items, rects, lines), _has_gid, _coords_rotated) = result;
        assert!(items.is_empty());
        assert!(rects.is_empty());
        assert!(lines.is_empty());
@@ -35,6 +35,7 @@ pub(crate) fn detect_columns(
    if page_items.is_empty() {
        return vec![];
    }
+    debug!("page {}: detect_columns: {} items", page, page_items.len());

    // Find page bounds
    let x_min = page_items.iter().map(|i| i.x).fold(f32::INFINITY, f32::min);
@@ -163,10 +164,17 @@ pub(crate) fn detect_columns(
                }
            }
        }
+        // Try XY-cut fallback before giving up
+        if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
+            return columns;
+        }
        return vec![ColumnRegion { x_min, x_max }];
    }

-    return validate_and_build_columns(
+    // Try center-based assignment first (handles asymmetric layouts / sidebars
+    // better than edge-based). Fall back to edge-based if center produces
+    // a degenerate split (one side empty).
+    let result = validate_and_build_columns(
        &valleys,
        &page_items,
        x_min,
@@ -175,8 +183,177 @@ pub(crate) fn detect_columns(
        MIN_ITEMS_PER_COLUMN,
        MIN_VERTICAL_SPAN_RATIO,
        page,
-        false, // edge-based assignment for absolute valleys
+        true, // center-based assignment
    );
+    if result.len() > 1 {
+        return result;
+    }
+    let result = validate_and_build_columns(
+        &valleys,
+        &page_items,
+        x_min,
+        BIN_WIDTH,
+        x_max,
+        MIN_ITEMS_PER_COLUMN,
+        MIN_VERTICAL_SPAN_RATIO,
+        page,
+        false, // edge-based fallback
+    );
+    if result.len() > 1 {
+        return result;
+    }
+
+    // Fallback: XY-cut style gap detection.  When the histogram finds no
+    // clear valleys (common with asymmetric/sidebar layouts), look for the
+    // largest horizontal gap between item edges.  This is a simplified
+    // single-level XY-cut inspired by opendataloader's XY-Cut++ algorithm.
+    if page_items.len() >= 20 && !page_has_table {
+        if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
+            return columns;
+        }
+    }
+
+    vec![ColumnRegion { x_min, x_max }]
+}
+
+/// Simplified single-level XY-cut: find the largest horizontal gap between
+/// item right-edges and left-edges.  If the gap is wide enough and both sides
+/// have sufficient items with vertical overlap, split into two columns.
+///
+/// Inspired by opendataloader's XY-Cut++ algorithm but without full recursion.
+/// Handles asymmetric layouts (sidebars) that the histogram misses because
+/// the narrow column has too few items to register in the occupancy profile.
+fn try_xy_cut_split(
+    page_items: &[&TextItem],
+    page_x_min: f32,
+    page_x_max: f32,
+    page: u32,
+) -> Option<Vec<ColumnRegion>> {
+    const MIN_GAP: f32 = 15.0; // minimum gap to consider a split
+    const MIN_ITEMS_MAJOR: usize = 10; // major column must have ≥10 items
+    const MIN_ITEMS_MINOR: usize = 3; // minor column (sidebar) must have ≥3
+
+    let page_width = page_x_max - page_x_min;
+    if page_width < 200.0 {
+        return None;
+    }
+
+    // Collect all item edges: (right_edge, left_edge) pairs sorted by right_edge
+    // The gap between one item's right edge and the next item's left edge
+    // reveals column gutters.
+    let mut edges: Vec<(f32, f32)> = page_items
+        .iter()
+        .map(|i| (i.x, i.x + effective_width(i)))
+        .collect();
+    edges.sort_by(|a, b| a.0.total_cmp(&b.0));
+
+    // Find the largest gap between consecutive items (by left edge).
+    // Use a sweep: sort left edges, find max gap between sorted right edges
+    // of items to the left and left edges of items to the right.
+    let mut left_edges: Vec<f32> = page_items.iter().map(|i| i.x).collect();
+    left_edges.sort_by(|a, b| a.total_cmp(b));
+
+    // Build prefix max of right edges (for items sorted by left edge)
+    let mut sorted_by_left: Vec<(f32, f32)> = page_items
+        .iter()
+        .map(|i| (i.x, i.x + effective_width(i)))
+        .collect();
+    sorted_by_left.sort_by(|a, b| a.0.total_cmp(&b.0));
+
+    let mut best_gap = 0.0f32;
+    let mut best_split = 0.0f32;
+    let mut max_right_so_far = f32::NEG_INFINITY;
+
+    for i in 0..sorted_by_left.len() - 1 {
+        let (_, right) = sorted_by_left[i];
+        max_right_so_far = max_right_so_far.max(right);
+
+        let (next_left, _) = sorted_by_left[i + 1];
+        let gap = next_left - max_right_so_far;
+        if gap > best_gap {
+            best_gap = gap;
+            best_split = (max_right_so_far + next_left) / 2.0;
+        }
+    }
+
+    if best_gap < MIN_GAP {
+        return None;
+    }
+
+    // Don't split at page margins (within 10% of edges)
+    let margin = page_width * 0.10;
+    if best_split - page_x_min < margin || page_x_max - best_split < margin {
+        return None;
+    }
+
+    // Count items on each side
+    let left_count = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 <= best_split)
+        .count();
+    let right_count = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 > best_split)
+        .count();
+
+    let (minor, major) = if left_count <= right_count {
+        (left_count, right_count)
+    } else {
+        (right_count, left_count)
+    };
+
+    if major < MIN_ITEMS_MAJOR || minor < MIN_ITEMS_MINOR {
+        return None;
+    }
+
+    // Check vertical overlap — both sides should span a meaningful Y range
+    let left_items: Vec<&&TextItem> = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 <= best_split)
+        .collect();
+    let right_items: Vec<&&TextItem> = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 > best_split)
+        .collect();
+
+    let l_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
+    let l_y_max = left_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::NEG_INFINITY, f32::max);
+    let r_y_min = right_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::INFINITY, f32::min);
+    let r_y_max = right_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::NEG_INFINITY, f32::max);
+
+    let overlap_min = l_y_min.max(r_y_min);
+    let overlap_max = l_y_max.min(r_y_max);
+    let overlap = (overlap_max - overlap_min).max(0.0);
+    let y_range = (l_y_max.max(r_y_max) - l_y_min.min(r_y_min)).max(1.0);
+
+    if overlap / y_range < 0.20 {
+        return None;
+    }
+
+    debug!(
+        "page {}: XY-cut split at x={:.1} (gap={:.1}pt, left={}, right={})",
+        page, best_split, best_gap, left_count, right_count
+    );
+
+    Some(vec![
+        ColumnRegion {
+            x_min: page_x_min,
+            x_max: best_split,
+        },
+        ColumnRegion {
+            x_min: best_split,
+            x_max: page_x_max,
+        },
+    ])
 }

 /// Check whether each proposed column contains paragraph-like content.
@@ -505,7 +682,15 @@ fn validate_and_build_columns(
            })
            .collect();

-        if left_items.len() < min_items || right_items.len() < min_items {
+        // Require both sides to have items. Symmetric layout needs min_items
+        // on each side. Asymmetric layouts (sidebars) are accepted when the
+        // dominant side has ≥ min_items and the smaller side has ≥ 3 items.
+        let (smaller, larger) = if left_items.len() <= right_items.len() {
+            (left_items.len(), right_items.len())
+        } else {
+            (right_items.len(), left_items.len())
+        };
+        if larger < min_items || smaller < 3 {
            continue;
        }

@@ -188,7 +188,7 @@ fn extract_positioned_text_impl(
                continue;
            }
        }
-        let ((mut items, rects, lines), has_gid_fonts) =
+        let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) =
            extract_page_text_items(doc, page_id, *page_num, font_cmaps, include_invisible)?;
        if has_gid_fonts {
            gid_encoded_pages.insert(*page_num);
@@ -344,16 +344,22 @@ pub fn extract_text_in_regions_mem(
 ) -> Result<Vec<PageRegionResult>, PdfError> {
    validate_pdf_bytes(buffer)?;
    let (doc, _page_count) = load_document_from_mem(buffer)?;
-    let font_cmaps = FontCMaps::from_doc(&doc);
    let pages = doc.get_pages();

-    // Build a set of pages we need to extract
-    let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect(); // to 1-indexed
+    // Build a set of pages we need to extract (1-indexed for lopdf)
+    let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect();
+
+    // Fast mode: skip expensive TrueType font fallback parsing.
+    // Fonts that can't be decoded from ToUnicode alone will produce empty/garbage
+    // text, triggering needs_ocr=true → GPU OCR fallback in the pipeline.
+    let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));

    // Extract text items for needed pages only
    let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
    let mut page_heights: HashMap<u32, f32> = HashMap::new();
    let mut gid_pages: HashSet<u32> = HashSet::new();
+    let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
+    let mut rotated_pages: HashSet<u32> = HashSet::new();

    for (page_num, &page_id) in pages.iter() {
        if !needed_pages.contains(page_num) {
@@ -365,7 +371,7 @@ pub fn extract_text_in_regions_mem(
        page_heights.insert(*page_num, height);

        // Extract text items for this page
-        let ((mut items, _rects, _lines), has_gid) =
+        let ((mut items, _rects, _lines), has_gid, coords_rotated) =
            extractor::content_stream::extract_page_text_items(
                &doc,
                page_id,
@@ -373,10 +379,16 @@ pub fn extract_text_in_regions_mem(
                &font_cmaps,
                false,
            )?;
-        text_utils::fix_letterspaced_items(&mut items);
+        let threshold = text_utils::fix_letterspaced_items(&mut items);
+        if threshold > 0.10 {
+            page_thresholds.insert(*page_num, threshold);
+        }
        if has_gid {
            gid_pages.insert(*page_num);
        }
+        if coords_rotated {
+            rotated_pages.insert(*page_num);
+        }
        items_by_page.insert(*page_num, items);
    }

@@ -388,6 +400,12 @@ pub fn extract_text_in_regions_mem(
        let items = items_by_page.get(&page_1idx);
        let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
        let page_has_gid = gid_pages.contains(&page_1idx);
+        let adaptive_threshold = page_thresholds.get(&page_1idx).copied().unwrap_or(0.10);
+        let coords = if rotated_pages.contains(&page_1idx) {
+            RegionCoordSpace::Rotated90Ccw
+        } else {
+            RegionCoordSpace::Standard
+        };

        let mut page_results = Vec::with_capacity(regions.len());

@@ -395,13 +413,23 @@ pub fn extract_text_in_regions_mem(
            let [rx1, ry1, rx2, ry2] = *rect;

            let text = match items {
-                Some(items) => collect_text_in_region(items, rx1, ry1, rx2, ry2, page_h),
+                Some(items) => collect_text_in_region_with_options(
+                    items,
+                    rx1,
+                    ry1,
+                    rx2,
+                    ry2,
+                    page_h,
+                    coords,
+                    adaptive_threshold,
+                ),
                None => String::new(),
            };

            let needs_ocr = text.trim().is_empty()
                || page_has_gid
                || is_garbage_text(&text)
+                || is_cid_garbage(&text)
                || detect_encoding_issues(&text);

            page_results.push(RegionText { text, needs_ocr });
@@ -449,9 +477,23 @@ fn obj_to_f32(obj: &lopdf::Object) -> Option<f32> {
    }
 }

+#[derive(Clone, Copy)]
+enum RegionCoordSpace {
+    Standard,
+    Rotated90Ccw,
+}
+
+#[derive(Clone, Copy)]
+struct RegionBounds {
+    x_min: f32,
+    y_min: f32,
+    x_max: f32,
+    y_max: f32,
+}
+
 /// Collect text items that fall within a region bbox (top-left origin, PDF points)
 /// and return them as a single string in reading order.
-fn collect_text_in_region(
+pub fn collect_text_in_region(
    items: &[TextItem],
    rx1: f32,
    ry1: f32,
@@ -459,83 +501,134 @@ fn collect_text_in_region(
    ry2: f32,
    page_height: f32,
 ) -> String {
-    // Convert region from top-left to bottom-left origin
-    let by1 = page_height - ry2; // top-left y2 → bottom-left y1
-    let by2 = page_height - ry1; // top-left y1 → bottom-left y2
+    collect_text_in_region_with_options(
+        items,
+        rx1,
+        ry1,
+        rx2,
+        ry2,
+        page_height,
+        infer_region_coord_space(items),
+        0.10,
+    )
+}

-    // Collect items whose center falls within the region
-    let mut matched: Vec<&TextItem> = items
+#[allow(clippy::too_many_arguments)]
+fn collect_text_in_region_with_options(
+    items: &[TextItem],
+    rx1: f32,
+    ry1: f32,
+    rx2: f32,
+    ry2: f32,
+    page_height: f32,
+    coord_space: RegionCoordSpace,
+    adaptive_threshold: f32,
+) -> String {
+    let bounds = region_bounds(rx1, ry1, rx2, ry2, page_height, coord_space);
+    let matched: Vec<TextItem> = items
        .iter()
-        .filter(|item| {
-            let cx = item.x + item.width / 2.0;
-            let cy = item.y + item.height / 2.0;
-            cx >= rx1 && cx <= rx2 && cy >= by1 && cy <= by2
-        })
+        .filter(|item| region_overlaps_item(item, bounds))
+        .cloned()
        .collect();
-
    if matched.is_empty() {
        return String::new();
    }

-    // Sort top→bottom (descending Y in bottom-left coords), then left→right.
-    // Uses total_cmp to avoid panics on NaN values from bogus font metrics.
-    matched.sort_by(|a, b| {
-        let fs_a = if a.font_size.is_finite() {
-            a.font_size
+    // Simple extraction: the caller (fire-pdf) already handles reading order
+    // and column splitting via the layout model. We just need to sort items
+    // top-to-bottom, left-to-right and group into lines.
+    let mut sorted = matched;
+    sorted.sort_by(|a, b| b.y.total_cmp(&a.y).then(a.x.total_cmp(&b.x)));
+
+    let y_tolerance = 3.0;
+    let mut lines: Vec<extractor::TextLine> = Vec::new();
+
+    for item in sorted {
+        let should_merge = lines.last().is_some_and(|last_line: &extractor::TextLine| {
+            last_line.page == item.page && (last_line.y - item.y).abs() < y_tolerance
+        });
+        if should_merge {
+            lines.last_mut().unwrap().items.push(item);
        } else {
-            0.0
-        };
-        let fs_b = if b.font_size.is_finite() {
-            b.font_size
-        } else {
-            0.0
-        };
-        let line_threshold = fs_a.max(fs_b) * 0.5;
-        let ay = if a.y.is_finite() { a.y } else { 0.0 };
-        let by = if b.y.is_finite() { b.y } else { 0.0 };
-        let y_diff = by - ay; // descending Y = top to bottom
-        if y_diff.abs() < line_threshold {
-            let ax = if a.x.is_finite() { a.x } else { 0.0 };
-            let bx = if b.x.is_finite() { b.x } else { 0.0 };
-            ax.total_cmp(&bx)
-        } else {
-            by.total_cmp(&ay)
+            let y = item.y;
+            let page = item.page;
+            lines.push(extractor::TextLine {
+                items: vec![item],
+                y,
+                page,
+                adaptive_threshold,
+            });
        }
-    });
-
-    // Group into lines and join
-    let mut lines: Vec<String> = Vec::new();
-    let mut current_line = String::new();
-    let mut last_y = f32::NAN;
-    let mut last_x_end = 0.0_f32;
-
-    for item in &matched {
-        let line_threshold = item.font_size * 0.5;
-        let same_line = (item.y - last_y).abs() < line_threshold;
-
-        if !same_line && !current_line.is_empty() {
-            lines.push(current_line.clone());
-            current_line.clear();
-        }
-
-        if !current_line.is_empty() {
-            // Insert space if there's a gap between items on the same line
-            let gap = item.x - last_x_end;
-            if gap > item.font_size * 0.15 {
-                current_line.push(' ');
-            }
-        }
-
-        current_line.push_str(&item.text);
-        last_y = item.y;
-        last_x_end = item.x + item.width;
    }

-    if !current_line.is_empty() {
-        lines.push(current_line);
+    // Sort items within each line by X position
+    for line in &mut lines {
+        text_utils::sort_line_items(&mut line.items);
    }

-    lines.join("\n")
+    lines
+        .into_iter()
+        .map(|line| line.text())
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+fn infer_region_coord_space(items: &[TextItem]) -> RegionCoordSpace {
+    // Rotated-page normalization currently maps y = -old_x, so most text items
+    // land at negative Y. Use this to keep `collect_text_in_region` behavior
+    // compatible for direct callers that do not have extractor metadata.
+    let negative_y = items.iter().filter(|item| item.y < 0.0).count();
+    if !items.is_empty() && negative_y * 2 >= items.len() {
+        RegionCoordSpace::Rotated90Ccw
+    } else {
+        RegionCoordSpace::Standard
+    }
+}
+
+fn region_bounds(
+    rx1: f32,
+    ry1: f32,
+    rx2: f32,
+    ry2: f32,
+    page_height: f32,
+    coord_space: RegionCoordSpace,
+) -> RegionBounds {
+    let tx_min = rx1.min(rx2);
+    let tx_max = rx1.max(rx2);
+    let ty_min = ry1.min(ry2);
+    let ty_max = ry1.max(ry2);
+    let by_min = page_height - ty_max;
+    let by_max = page_height - ty_min;
+    match coord_space {
+        RegionCoordSpace::Standard => RegionBounds {
+            x_min: tx_min,
+            y_min: by_min,
+            x_max: tx_max,
+            y_max: by_max,
+        },
+        RegionCoordSpace::Rotated90Ccw => RegionBounds {
+            x_min: by_min,
+            x_max: by_max,
+            y_min: -tx_max,
+            y_max: -tx_min,
+        },
+    }
+}
+
+fn region_overlaps_item(item: &TextItem, bounds: RegionBounds) -> bool {
+    const REGION_MARGIN: f32 = 1.5;
+    let item_x_min = item.x;
+    let item_x_max = item.x + text_utils::effective_width(item);
+    let item_y_min = item.y;
+    let item_y_max = item.y + item.height;
+
+    let x_overlap = (item_x_max.min(bounds.x_max + REGION_MARGIN)
+        - item_x_min.max(bounds.x_min - REGION_MARGIN))
+    .max(0.0);
+    let y_overlap = (item_y_max.min(bounds.y_max + REGION_MARGIN)
+        - item_y_min.max(bounds.y_min - REGION_MARGIN))
+    .max(0.0);
+    x_overlap > 0.0 && y_overlap > 0.0
 }

 // =========================================================================
@@ -8,6 +8,24 @@ use log::debug;
 /// Font statistics for a document
 pub(crate) struct FontStats {
    pub(crate) most_common_size: f32,
+    /// Font size frequency distribution (size_key → line count).
+    /// Used for rarity-based heading detection.
+    pub(crate) size_counts: HashMap<i32, usize>,
+    /// Total number of lines counted.
+    pub(crate) total_lines: usize,
+}
+
+/// Compute how rare a font size is in the document (0.0 = most common, 1.0 = unique).
+/// Mirrors opendataloader's font rarity boosting approach: heading fonts appear on
+/// far fewer lines than body text, so their percentile rank is high.
+pub(crate) fn font_size_rarity(font_size: f32, stats: &FontStats) -> f32 {
+    if stats.total_lines == 0 {
+        return 0.0;
+    }
+    let key = (font_size * 10.0) as i32;
+    let count = stats.size_counts.get(&key).copied().unwrap_or(0);
+    // Rarity = 1 - (frequency ratio). A size used on 1/100 lines has rarity ~0.99.
+    1.0 - (count as f32 / stats.total_lines as f32)
 }

 /// Calculate font stats directly from items (before grouping into lines)
@@ -21,6 +39,8 @@ pub(crate) fn calculate_font_stats_from_items(items: &[TextItem]) -> FontStats {
        }
    }

+    let total_lines = size_counts.values().sum();
+
    // Break ties by preferring the smaller font size for deterministic output
    let most_common_size = size_counts
        .iter()
@@ -30,7 +50,11 @@ pub(crate) fn calculate_font_stats_from_items(items: &[TextItem]) -> FontStats {
        .map(|(size, _)| *size as f32 / 10.0)
        .unwrap_or(12.0);

-    FontStats { most_common_size }
+    FontStats {
+        most_common_size,
+        size_counts,
+        total_lines,
+    }
 }

 /// Calculate font stats from grouped lines
@@ -48,6 +72,8 @@ pub(crate) fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
        }
    }

+    let total_lines = size_counts.values().sum();
+
    // Break ties by preferring the smaller font size for deterministic output
    let most_common_size = size_counts
        .iter()
@@ -57,7 +83,23 @@ pub(crate) fn calculate_font_stats(lines: &[TextLine]) -> FontStats {
        .map(|(size, _)| *size as f32 / 10.0)
        .unwrap_or(12.0);

-    FontStats { most_common_size }
+    FontStats {
+        most_common_size,
+        size_counts,
+        total_lines,
+    }
+}
+
+/// Determine the heading level for a bold-only line that didn't meet the font-size
+/// threshold.  These are common in academic papers where section headings are bold
+/// at the same size as body text.
+///
+/// Returns a level below the lowest font-size tier (or H2 when no tiers exist).
+pub(crate) fn bold_heading_level(heading_tiers: &[f32]) -> usize {
+    let level = heading_tiers.len() + 1;
+    // Clamp to 1..=6 — if no font-size tiers, bold headings become H2
+    // (H1 is reserved for titles which are typically larger)
+    level.clamp(2, 6)
 }

 /// Detect TOC-style lines that contain dot leaders (e.g., "Section Name .... 42").
@@ -4,13 +4,11 @@
 pub(crate) fn is_caption_line(text: &str) -> bool {
    let trimmed = text.trim();

-    // Common caption prefixes in multiple languages
-    let caption_prefixes = [
-        "Figure ",
+    // Caption prefixes that always match (always followed by identifiers)
+    let always_prefixes = [
        "Figura ",
        "Fig. ",
        "Fig ",
-        "Table ",
        "Tabela ",
        "Source:",
        "Fonte:",
@@ -27,17 +25,39 @@ pub(crate) fn is_caption_line(text: &str) -> bool {
        "Photo ",
        "Foto ",
    ];
-
-    // Check if line starts with a caption prefix
-    for prefix in &caption_prefixes {
+    for prefix in &always_prefixes {
        if trimmed.starts_with(prefix) {
            return true;
        }
    }

-    // Check case-insensitive patterns
+    // "Figure" and "Table" need a digit/reference after them to distinguish
+    // captions ("Table 1", "Figure 3.2") from headings ("Table of Contents")
+    for prefix in ["Figure ", "Table "] {
+        if let Some(rest) = trimmed.strip_prefix(prefix) {
+            if rest
+                .trim_start()
+                .starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
+            {
+                return true;
+            }
+        }
+    }
+
+    // Check case-insensitive patterns — require digit or punctuation after
+    // prefix to avoid matching "Table of Contents" or "Figure drawing" etc.
    let lower = trimmed.to_lowercase();
-    if lower.starts_with("figure ") || lower.starts_with("table ") || lower.starts_with("source:") {
+    for pfx in ["figure ", "table "] {
+        if let Some(rest) = lower.strip_prefix(pfx) {
+            if rest
+                .trim_start()
+                .starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
+            {
+                return true;
+            }
+        }
+    }
+    if lower.starts_with("source:") {
        return true;
    }

@@ -1,19 +1,101 @@
 //! Core line-to-markdown conversion loop with table/image interleaving.

-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};

 use crate::structure_tree::StructRole;
 use crate::types::TextLine;

 use super::analysis::{
-    calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold, detect_header_level,
-    has_dot_leaders,
+    bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
+    detect_header_level, font_size_rarity, has_dot_leaders,
 };
 use super::classify::{format_list_item, is_caption_line, is_list_item, is_monospace_font};
 use super::postprocess::clean_markdown;
 use super::preprocess::{merge_drop_caps, merge_heading_lines};
 use super::MarkdownOptions;

+/// Pre-scan lines to find "isolated" ones: short lines with paragraph breaks both
+/// before and after.  These are heading candidates even at body font size — common
+/// in academic papers ("Acknowledgements", "B.3 Prompt Engineering").
+fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32) -> HashSet<usize> {
+    let mut set = HashSet::new();
+    for i in 0..lines.len() {
+        let line = &lines[i];
+        let plain = line.text();
+        let trimmed = plain.trim();
+        let word_count = trimmed.split_whitespace().count();
+        if !(1..=6).contains(&word_count) || trimmed.len() <= 3 {
+            continue;
+        }
+        let font_size = line.items.first().map(|it| it.font_size).unwrap_or(0.0);
+        if font_size < base_size * 0.95 {
+            continue;
+        }
+        if is_list_item(trimmed) || is_caption_line(trimmed) {
+            continue;
+        }
+
+        // Reject lines that look like wrapped paragraph text:
+        // ends with hyphen, comma, preposition, or lowercase continuation
+        let last_char = trimmed.chars().last().unwrap_or(' ');
+        if last_char == '-' || last_char == ',' || last_char == ';' {
+            continue;
+        }
+        // Last word is a common continuation word → wrapped paragraph
+        let last_word = trimmed.split_whitespace().last().unwrap_or("");
+        let continuation_words = [
+            "the", "a", "an", "and", "or", "of", "in", "to", "for", "with", "by", "on", "at",
+            "from", "as", "is", "are", "was", "were", "be", "that", "this", "their", "its", "our",
+            "your", "has", "have", "had", "not",
+        ];
+        if continuation_words.contains(&last_word.to_lowercase().as_str()) {
+            continue;
+        }
+
+        // Paragraph break BEFORE
+        let break_before = if i == 0 {
+            true
+        } else {
+            let prev = &lines[i - 1];
+            prev.page != line.page || (prev.y - line.y).abs() > para_threshold
+        };
+
+        // Paragraph break AFTER
+        let break_after = if i + 1 >= lines.len() {
+            true
+        } else {
+            let next = &lines[i + 1];
+            next.page != line.page || (line.y - next.y).abs() > para_threshold
+        };
+
+        if !break_before || !break_after {
+            continue;
+        }
+
+        set.insert(i);
+    }
+
+    // Density guard: if too many lines on a page are "isolated", they're
+    // all paragraph lines in a multi-column layout, not headings.  Real
+    // headings are rare — at most ~20% of lines on a page.
+    let mut page_line_counts: HashMap<u32, (usize, usize)> = HashMap::new(); // (total, isolated)
+    for (i, line) in lines.iter().enumerate() {
+        let entry = page_line_counts.entry(line.page).or_insert((0, 0));
+        entry.0 += 1;
+        if set.contains(&i) {
+            entry.1 += 1;
+        }
+    }
+    for (&page, &(total, isolated)) in &page_line_counts {
+        if total > 0 && isolated as f32 / total as f32 > 0.25 {
+            // Too many isolated lines on this page — remove them all
+            set.retain(|&i| lines[i].page != page);
+        }
+    }
+
+    set
+}
+
 /// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
 ///
 /// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
@@ -256,6 +338,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
    // threshold and cause every line to be treated as a paragraph break.
    let para_threshold = compute_paragraph_threshold(&lines, base_size);

+    // Pre-scan: identify isolated lines (paragraph break before AND after).
+    // These are heading candidates even without bold/large font — common in
+    // academic papers where section titles like "Acknowledgements" sit alone
+    // between paragraphs at body font size. Inspired by opendataloader's
+    // lookahead in HeadingProcessor (prevNode/nextNode context).
+    let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
+
    let mut output = String::new();
    let mut current_page = 0u32;
    let mut prev_y = f32::MAX;
@@ -277,7 +366,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
    all_content_pages.sort();
    all_content_pages.dedup();

-    for line in lines {
+    for (line_idx, line) in lines.iter().enumerate() {
        // Page break
        if line.page != current_page {
            // Flush current page's remaining tables and images
@@ -405,7 +494,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(

        // Detect figure/table captions and source citations
        // These should be on their own line followed by a paragraph break
-        let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(&line, roles));
+        let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(line, roles));

        // Determine if this line is code (struct-tree or font-based) for block accumulation
        let is_code_line = struct_role
@@ -443,7 +532,42 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
            && plain_trimmed.split_whitespace().count() <= 15
        {
            let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
-            detect_header_level(line_font_size, base_size, &heading_tiers)
+            detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
+                // Rarity-based heading detection (inspired by opendataloader).
+                // Heading probability scoring with lookahead context.
+                // Score = rarity * 0.5 + bold * 0.3 + standalone * 0.2
+                //       + isolated * 0.3 (paragraph break before AND after)
+                // Only consider lines at or above body font size.
+                if line_font_size < base_size * 0.95 {
+                    return None;
+                }
+                let word_count = plain_trimmed.split_whitespace().count();
+                if !(1..=15).contains(&word_count) {
+                    return None;
+                }
+                let rarity = font_size_rarity(line_font_size, &font_stats);
+                let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
+                let standalone = !in_paragraph;
+                let isolated = isolated_lines.contains(&line_idx);
+
+                let score = rarity * 0.5
+                    + if all_bold { 0.3 } else { 0.0 }
+                    + if standalone { 0.2 } else { 0.0 }
+                    + if isolated { 0.3 } else { 0.0 };
+
+                // Require standalone + at least one strong signal.
+                // Non-bold, non-isolated lines need very high rarity (≥0.97)
+                // to avoid classifying ordinary body text as headings in
+                // multi-column layouts where column switches break
+                // paragraph continuity and minor font-size variation
+                // inflates rarity scores.
+                let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
+                if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
+                    Some(bold_heading_level(&heading_tiers))
+                } else {
+                    None
+                }
+            })
        } else {
            None
        };
@@ -626,6 +750,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
    // Compute the typical line spacing for paragraph break detection
    let para_threshold = compute_paragraph_threshold(&lines, base_size);

+    let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
+
    let mut output = String::new();
    let mut current_page = 0u32;
    let mut prev_y = f32::MAX;
@@ -634,7 +760,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
    let mut last_list_x: Option<f32> = None;
    let mut prev_had_dot_leaders = false;

-    for line in lines {
+    for (line_idx, line) in lines.iter().enumerate() {
        // Page break
        if line.page != current_page {
            if current_page > 0 {
@@ -699,7 +825,27 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
        {
            let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
            if let Some(header_level) =
-                detect_header_level(line_font_size, base_size, &heading_tiers)
+                detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
+                    if line_font_size < base_size * 0.95 {
+                        return None;
+                    }
+                    let word_count = plain_trimmed.split_whitespace().count();
+                    if !(1..=15).contains(&word_count) {
+                        return None;
+                    }
+                    let rarity = font_size_rarity(line_font_size, &font_stats);
+                    let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
+                    let standalone = !in_paragraph;
+                    let isolated = isolated_lines.contains(&line_idx);
+                    let score = rarity * 0.5
+                        + if all_bold { 0.3 } else { 0.0 }
+                        + if standalone { 0.2 } else { 0.0 }
+                        + if isolated { 0.3 } else { 0.0 };
+                    if score >= 0.5 && standalone && word_count >= 2 {
+                        return Some(bold_heading_level(&heading_tiers));
+                    }
+                    None
+                })
            {
                if in_paragraph {
                    output.push_str("\n\n");
@@ -1016,6 +1162,62 @@ mod tests {
        );
    }

+    #[test]
+    fn test_rarity_heading_requires_strong_signal() {
+        // Simulate a two-column academic paper where body text lines become
+        // "standalone" due to column switches.  Body text at the same font
+        // size as most of the document should NOT be classified as headings
+        // just because of moderate rarity + standalone.
+        //
+        // Regression: previously, lines with rarity ~0.62 and standalone=true
+        // scored 0.51 (>=0.5 threshold), producing hundreds of false ## headings.
+
+        // Create many body-text lines at font_size=10.9 (most common)
+        let mut lines = Vec::new();
+        for i in 0..20 {
+            let mut item = make_item("This is ordinary body text in a paragraph.", 1, None);
+            item.font_size = 10.9;
+            item.y = 700.0 - i as f32 * 14.0;
+            lines.push(make_line(vec![item]));
+        }
+        // A few lines at a slightly different size (simulating column B text)
+        for i in 0..10 {
+            let mut item = make_item("Another body text line from the second column.", 1, None);
+            item.font_size = 11.0; // slightly different → non-zero rarity
+            item.y = 700.0 - i as f32 * 14.0;
+            item.x = 320.0; // right column
+            lines.push(make_line(vec![item]));
+        }
+        // One genuine bold heading
+        let mut heading_item = make_item("3 Philosophical Perspectives", 1, None);
+        heading_item.font_size = 10.9;
+        heading_item.is_bold = true;
+        heading_item.y = 200.0;
+        lines.push(make_line(vec![heading_item]));
+
+        let md = to_markdown_from_lines_with_tables_and_images(
+            lines,
+            MarkdownOptions::default(),
+            HashMap::new(),
+            HashMap::new(),
+            &std::collections::HashSet::new(),
+            None,
+        );
+
+        // The bold heading should be detected
+        assert!(
+            md.contains("## 3 Philosophical Perspectives"),
+            "Bold heading should be detected: {md}"
+        );
+
+        // Body text lines should NOT be headings
+        let heading_count = md.lines().filter(|l| l.starts_with("##")).count();
+        assert!(
+            heading_count <= 2,
+            "Expected at most 2 headings but found {heading_count} in:\n{md}"
+        );
+    }
+
    #[test]
    fn test_struct_role_code_multiline_accumulation() {
        let mut line1 = make_item("fn main() {", 1, Some(0));
@@ -601,6 +601,15 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
        let group = page_groups.get(&page).unwrap();
        let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();

+        // Detect columns early — on multi-column pages, the merged-band retry
+        // should skip body-font heuristic table detection (which mistakes column
+        // text for tables). Individual band heuristic detection is left enabled
+        // because bands are scoped to single columns.
+        let page_has_columns = {
+            let cols = crate::extractor::detect_columns(&page_items, page, false);
+            cols.len() >= 2
+        };
+
        // Check for side-by-side layout (e.g. two tables placed left and right)
        let mut bands = split_side_by_side(&page_items);
        // Fallback: use rect hint regions to detect side-by-side layout
@@ -873,10 +882,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
                run_heuristic(&unclaimed_items, &unclaimed_map, 6);
            }

-            // 4. Column-based table detection: last resort for borderless tabular
-            //    layouts (e.g. exam/reference grids) when ALL structural methods
-            //    found nothing. Only runs when no rects/lines exist (truly borderless)
-            //    and no other detection method found tables in this band.
+            // 4. Column-based table detection for borderless tabular layouts.
            let band_has_tables = band_items.iter().enumerate().any(|(idx, _)| {
                band_index_map
                    .get(idx)
@@ -903,6 +909,65 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
            }
        }

+        // 5. Thin-rect border synthesis: last resort for PDFs that draw table
+        //    borders as thin filled rectangles (common in spreadsheet exports).
+        //    Only runs when ALL other methods found nothing on this page.
+        if !page_tables.contains_key(&page) {
+            let page_rects: Vec<&crate::types::PdfRect> =
+                rects.iter().filter(|r| r.page == page).collect();
+            let mut synth_lines: Vec<crate::types::PdfLine> = Vec::new();
+            for r in &page_rects {
+                let (mut w, mut h) = (r.width, r.height);
+                let (mut x, mut y) = (r.x, r.y);
+                if w < 0.0 {
+                    x += w;
+                    w = -w;
+                }
+                if h < 0.0 {
+                    y += h;
+                    h = -h;
+                }
+                if h < 2.0 && w >= 10.0 {
+                    let mid_y = y + h / 2.0;
+                    synth_lines.push(crate::types::PdfLine {
+                        x1: x,
+                        y1: mid_y,
+                        x2: x + w,
+                        y2: mid_y,
+                        page,
+                    });
+                } else if w < 2.0 && h >= 10.0 {
+                    let mid_x = x + w / 2.0;
+                    synth_lines.push(crate::types::PdfLine {
+                        x1: mid_x,
+                        y1: y,
+                        x2: mid_x,
+                        y2: y + h,
+                        page,
+                    });
+                }
+            }
+            if synth_lines.len() >= 10 {
+                let page_text: Vec<TextItem> = text_items
+                    .iter()
+                    .filter(|i| i.page == page)
+                    .cloned()
+                    .collect();
+                let line_tables = detect_tables_from_lines(&page_text, &synth_lines, page);
+                for table in &line_tables {
+                    for &idx in &table.item_indices {
+                        table_items.insert(idx);
+                    }
+                    let table_y = table.rows.first().copied().unwrap_or(0.0);
+                    let table_md = table_to_markdown(table);
+                    page_tables
+                        .entry(page)
+                        .or_default()
+                        .push((table_y, table_md));
+                }
+            }
+        }
+
        // Merged-band retry: if we split into bands but found no tables in
        // any band, retry heuristic detection with all items as a single band.
        // This catches borderless tables whose text-column alignment was
@@ -915,7 +980,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines(
                band_items.len(),
                was_split
            );
-            let heuristic_tables = detect_tables(band_items, base_size, false);
+            let heuristic_tables = detect_tables(band_items, base_size, page_has_columns);
            for table in &heuristic_tables {
                for &idx in &table.item_indices {
                    if let Some(&page_idx) = band_index_map.get(idx) {
@@ -248,21 +248,34 @@ fn convert_text_items(items: Vec<crate::TextItem>) -> Vec<PyTextItem> {
        .collect()
 }

-fn parse_page_regions(page_regions: Vec<(u32, Vec<Vec<f64>>)>) -> Vec<(u32, Vec<[f32; 4]>)> {
+fn parse_page_regions(
+    page_regions: Vec<(u32, Vec<Vec<f64>>)>,
+) -> PyResult<Vec<(u32, Vec<[f32; 4]>)>> {
    page_regions
        .into_iter()
        .map(|(page, regions)| {
-            let bboxes: Vec<[f32; 4]> = regions
-                .iter()
-                .map(|r| {
-                    if r.len() != 4 {
-                        [0.0, 0.0, 0.0, 0.0]
-                    } else {
-                        [r[0] as f32, r[1] as f32, r[2] as f32, r[3] as f32]
-                    }
-                })
-                .collect();
-            (page, bboxes)
+            let mut bboxes: Vec<[f32; 4]> = Vec::with_capacity(regions.len());
+            for (idx, region) in regions.into_iter().enumerate() {
+                if region.len() != 4 {
+                    return Err(PyValueError::new_err(format!(
+                        "Invalid region at page {page}, index {idx}: expected [x1, y1, x2, y2], got {} values",
+                        region.len()
+                    )));
+                }
+                let [x1, y1, x2, y2] = [region[0], region[1], region[2], region[3]];
+                if !(x1.is_finite() && y1.is_finite() && x2.is_finite() && y2.is_finite()) {
+                    return Err(PyValueError::new_err(format!(
+                        "Invalid region at page {page}, index {idx}: coordinates must be finite numbers"
+                    )));
+                }
+                if x2 < x1 || y2 < y1 {
+                    return Err(PyValueError::new_err(format!(
+                        "Invalid region at page {page}, index {idx}: expected x2>=x1 and y2>=y1, got [{x1}, {y1}, {x2}, {y2}]"
+                    )));
+                }
+                bboxes.push([x1 as f32, y1 as f32, x2 as f32, y2 as f32]);
+            }
+            Ok((page, bboxes))
        })
        .collect()
 }
@@ -424,7 +437,7 @@ fn extract_text_in_regions_bytes(
    data: &[u8],
    page_regions: Vec<(u32, Vec<Vec<f64>>)>,
 ) -> PyResult<Vec<PyPageRegionTexts>> {
-    let regions = parse_page_regions(page_regions);
+    let regions = parse_page_regions(page_regions)?;
    let results = crate::extract_text_in_regions_mem(data, &regions).map_err(to_py_err)?;
    Ok(convert_region_results(results))
 }
@@ -219,7 +219,7 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo
            body_font_low,
            body_font_high,
        );
-        if body_candidates.len() >= 9 {
+        if body_candidates.len() >= 6 {
            let regions = find_table_regions_strict(&body_candidates);
            log::debug!("body-font: {} strict regions found", regions.len());

@@ -241,7 +241,7 @@ pub fn detect_tables(items: &[TextItem], base_font_size: f32, skip_body_font: bo
                    body_candidates.len()
                );

-                if region_items.len() < 9 {
+                if region_items.len() < 6 {
                    continue;
                }

@@ -801,7 +801,25 @@ fn has_table_like_content(cells: &[Vec<String>], mode: TableDetectionMode) -> bo
    // Bypass content check for wide tables (3+ columns) — text-only tables
    // (category lists, program descriptions) are legitimate if they passed
    // all structural validations (alignment, consistency, not key-value).
-    pct_data > min_pct || num_cols >= 3
+    // Also bypass for 2-column body-font tables with short cells (avg ≤40 chars),
+    // which are likely definition/category lists, not paragraph text.
+    if pct_data > min_pct || num_cols >= 3 {
+        return true;
+    }
+    if num_cols == 2 && matches!(mode, TableDetectionMode::BodyFont) {
+        let non_empty: Vec<usize> = cells
+            .iter()
+            .skip(1)
+            .flat_map(|row| row.iter())
+            .filter(|c| !c.trim().is_empty())
+            .map(|c| c.trim().len())
+            .collect();
+        if !non_empty.is_empty() {
+            let avg_len = non_empty.iter().sum::<usize>() / non_empty.len();
+            return avg_len <= 25;
+        }
+    }
+    false
 }

 /// Check if a cell value looks like table data
@@ -243,9 +243,11 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
                .map(|s| (s - mean_spacing).powi(2))
                .sum::<f32>()
                / spacings.len() as f32;
-            let cv = variance.sqrt() / mean_spacing; // coefficient of variation
-                                                     // CV < 0.05 means nearly identical spacing — chart grid
-            if cv < 0.05 {
+            let cv = variance.sqrt() / mean_spacing;
+            // CV < 0.02 means nearly identical spacing — likely chart grid.
+            // Spreadsheet-exported tables often have uniform rows (CV 0.03-0.05),
+            // so we use a tighter threshold to avoid false negatives.
+            if cv < 0.02 {
                return Vec::new();
            }
        }
@@ -1771,15 +1771,49 @@ impl FontCMaps {
    /// Iterates every page, collects fonts (including Form XObject fonts),
    /// and parses any `/ToUnicode` streams via lopdf's decompression.
    pub fn from_doc(doc: &Document) -> Self {
+        Self::from_doc_pages(doc, None)
+    }
+
+    /// Build FontCMaps for specific pages only. Pass `None` for all pages.
+    pub fn from_doc_pages(doc: &Document, page_filter: Option<&HashSet<u32>>) -> Self {
+        Self::from_doc_pages_inner(doc, page_filter, false)
+    }
+
+    /// Build FontCMaps in fast mode: skip expensive TrueType font fallback
+    /// parsing. Fonts that can't be decoded from their ToUnicode CMap alone
+    /// will be missing, causing text extraction to produce empty/garbage text
+    /// which triggers `needs_ocr` fallback. This is ideal for hybrid OCR
+    /// pipelines where GPU OCR is always available as a fallback.
+    pub fn from_doc_pages_fast(doc: &Document, page_filter: Option<&HashSet<u32>>) -> Self {
+        Self::from_doc_pages_inner(doc, page_filter, true)
+    }
+
+    fn from_doc_pages_inner(
+        doc: &Document,
+        page_filter: Option<&HashSet<u32>>,
+        skip_truetype_fallback: bool,
+    ) -> Self {
        let mut by_obj_num: HashMap<u32, CMapEntry> = HashMap::new();

-        for (_page_num, &page_id) in doc.get_pages().iter() {
+        for (page_num, &page_id) in doc.get_pages().iter() {
+            if let Some(filter) = page_filter {
+                if !filter.contains(page_num) {
+                    continue;
+                }
+            }
            // Page-level fonts (includes inherited parent resources)
            let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
-            Self::collect_cmaps_from_fonts(&fonts, doc, &mut by_obj_num);
+            Self::collect_cmaps_from_fonts_inner(
+                &fonts,
+                doc,
+                &mut by_obj_num,
+                skip_truetype_fallback,
+            );

-            // Fonts inside Form XObjects referenced by this page
-            Self::collect_cmaps_from_xobjects(doc, page_id, &mut by_obj_num);
+            if !skip_truetype_fallback {
+                // Fonts inside Form XObjects referenced by this page
+                Self::collect_cmaps_from_xobjects(doc, page_id, &mut by_obj_num);
+            }
        }

        FontCMaps { by_obj_num }
@@ -1792,6 +1826,15 @@ impl FontCMaps {
        fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
        doc: &Document,
        by_obj_num: &mut HashMap<u32, CMapEntry>,
+    ) {
+        Self::collect_cmaps_from_fonts_inner(fonts, doc, by_obj_num, false);
+    }
+
+    fn collect_cmaps_from_fonts_inner(
+        fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
+        doc: &Document,
+        by_obj_num: &mut HashMap<u32, CMapEntry>,
+        skip_truetype_fallback: bool,
    ) {
        // First pass: collect ToUnicode CMaps
        for font_dict in fonts.values() {
@@ -1825,13 +1868,32 @@ impl FontCMaps {
                );
                let (mut primary, mut remapped) =
                    try_remap_subset_cmap(cmap, font_dict, doc, obj_num);
-                let mut fallback = build_fallback_tounicode_from_encoding(font_dict, doc)
-                    .or_else(|| build_fallback_cmap_for_type0(font_dict, doc))
-                    .or_else(|| build_fallback_cmap_for_simple(font_dict, doc));

-                // If the ToUnicode map is extremely sparse, prefer the fallback
-                // (often a better mapping for Symbol/Wingdings/Arabic CID fonts).
+                // Only build expensive fallbacks when the primary CMap is sparse.
+                // build_fallback_cmap_for_type0 can take seconds on large embedded
+                // TrueType fonts (decompressing + parsing 100K+ byte font files).
+                // Skip entirely when the primary CMap is sufficient.
                let primary_entries = primary.char_map.len() + primary.ranges.len();
+                let mut fallback = if primary_entries < 10 && !skip_truetype_fallback {
+                    // Try cheap fallback first; only attempt expensive TrueType
+                    // parsing if cheap fallbacks don't yield results.
+                    let cheap = build_fallback_tounicode_from_encoding(font_dict, doc)
+                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc));
+                    if cheap.is_some() {
+                        cheap
+                    } else {
+                        build_fallback_cmap_for_type0(font_dict, doc)
+                    }
+                } else if primary_entries < 10 {
+                    // Fast mode: only try cheap fallbacks, skip TrueType parsing.
+                    // Regions using this font will get needs_ocr=true.
+                    build_fallback_tounicode_from_encoding(font_dict, doc)
+                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))
+                } else {
+                    // Primary is rich enough; only try the cheap encoding fallback
+                    build_fallback_tounicode_from_encoding(font_dict, doc)
+                };
+
                if primary_entries < 10 {
                    if let Some(fb) = fallback.take() {
                        debug!(
@@ -1852,8 +1914,12 @@ impl FontCMaps {
                );
            } else {
                // ToUnicode present but parse failed; try fallbacks to avoid empty decoding.
-                let fallback = build_fallback_cmap_for_type0(font_dict, doc)
-                    .or_else(|| build_fallback_cmap_for_simple(font_dict, doc));
+                let fallback = if skip_truetype_fallback {
+                    build_fallback_cmap_for_simple(font_dict, doc)
+                } else {
+                    build_fallback_cmap_for_type0(font_dict, doc)
+                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))
+                };
                if let Some(fb) = fallback {
                    debug!(
                        "ToUnicode CMap obj={} parse failed; using fallback (entries={})",
@@ -1874,6 +1940,10 @@ impl FontCMaps {

        // Second pass: Identity-H/V fonts without ToUnicode
        // Try: (1) embedded TrueType/OpenType cmap, (2) predefined CID→Unicode mapping
+        // Skip entirely in fast mode — these fonts require expensive TrueType parsing.
+        if skip_truetype_fallback {
+            return;
+        }
        for font_dict in fonts.values() {
            if font_dict.get(b"ToUnicode").is_ok() {
                continue;
@@ -4,9 +4,11 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
 use pdf_inspector::extractor::group_into_lines;
 use pdf_inspector::types::TextLine;
 use pdf_inspector::{
-    detect_pdf_type, extract_text, extract_text_with_positions, process_pdf_with_options,
-    to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
+    detect_pdf_type, extract_text, extract_text_in_regions_mem, extract_text_with_positions,
+    process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
+    PdfType, TextItem,
 };
+use std::collections::HashSet;

 // Helper to create test TextItems
 fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
@@ -1107,3 +1109,238 @@ fn test_rotated_table_layout_correction() {
        "District data should be in a markdown table row"
    );
 }
+
+// =========================================================================
+// extract_text_in_regions_mem tests
+// =========================================================================
+
+/// Build full-page region args for `page_count` pages.
+/// Uses a generously large bbox (1200x1200) to capture any page size.
+fn full_page_regions(page_count: u32) -> Vec<(u32, Vec<[f32; 4]>)> {
+    (0..page_count)
+        .map(|p| (p, vec![[0.0, 0.0, 1200.0, 1200.0]]))
+        .collect()
+}
+
+/// Normalize text for comparison: lowercase, strip non-alphanumeric, split into words.
+fn normalize_words(text: &str) -> HashSet<String> {
+    text.split(|c: char| !c.is_alphanumeric())
+        .map(|w| w.to_lowercase())
+        .filter(|w| w.len() > 3)
+        .collect()
+}
+
+/// Fraction of normalized words in `a` that also appear in `b`.
+fn word_overlap_ratio(a: &str, b: &str) -> f64 {
+    let words_a = normalize_words(a);
+    if words_a.is_empty() {
+        return if normalize_words(b).is_empty() {
+            1.0
+        } else {
+            0.0
+        };
+    }
+    let words_b = normalize_words(b);
+    let overlap = words_a.intersection(&words_b).count();
+    overlap as f64 / words_a.len() as f64
+}
+
+#[test]
+fn test_extract_regions_mem_basic_text_pdf() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = process_pdf_mem(&buf).unwrap();
+    let page_count = result.page_count;
+
+    let regions = extract_text_in_regions_mem(&buf, &full_page_regions(page_count)).unwrap();
+    assert_eq!(regions.len(), page_count as usize);
+
+    // Each result should have exactly 1 region (we passed one per page)
+    for r in &regions {
+        assert_eq!(r.regions.len(), 1);
+    }
+
+    // First page should have non-empty text
+    let first = &regions[0].regions[0];
+    assert!(!first.text.trim().is_empty(), "First page should have text");
+    assert_eq!(regions[0].page, 0);
+}
+
+#[test]
+fn test_extract_regions_mem_identity_h_needs_ocr() {
+    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
+    let regions =
+        extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+    assert_eq!(regions.len(), 1);
+    assert!(
+        regions[0].regions[0].needs_ocr,
+        "Identity-H font without ToUnicode should trigger needs_ocr"
+    );
+}
+
+#[test]
+fn test_extract_regions_mem_multiple_regions_per_page() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let regions = extract_text_in_regions_mem(
+        &buf,
+        &[(
+            0,
+            vec![
+                [0.0, 0.0, 300.0, 100.0],   // small top-left
+                [0.0, 0.0, 1200.0, 1200.0], // full page
+            ],
+        )],
+    )
+    .unwrap();
+
+    assert_eq!(regions.len(), 1);
+    assert_eq!(regions[0].regions.len(), 2);
+
+    let small_len = regions[0].regions[0].text.len();
+    let full_len = regions[0].regions[1].text.len();
+    assert!(
+        full_len >= small_len,
+        "Full-page region ({full_len}) should have at least as much text as small region ({small_len})"
+    );
+}
+
+#[test]
+fn test_extract_regions_mem_nonexistent_page() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let regions =
+        extract_text_in_regions_mem(&buf, &[(9999, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+    assert_eq!(regions.len(), 1);
+    assert!(
+        regions[0].regions[0].needs_ocr,
+        "Nonexistent page should trigger needs_ocr"
+    );
+}
+
+#[test]
+fn test_extract_regions_mem_empty_region() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let regions = extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 0.0, 0.0]])]).unwrap();
+    assert_eq!(regions.len(), 1);
+    assert!(
+        regions[0].regions[0].needs_ocr,
+        "Zero-area region should trigger needs_ocr"
+    );
+}
+
+#[test]
+fn test_extract_regions_mem_not_a_pdf() {
+    let result = extract_text_in_regions_mem(b"not a pdf", &[(0, vec![[0.0, 0.0, 100.0, 100.0]])]);
+    assert!(result.is_err(), "Non-PDF input should return an error");
+}
+
+#[test]
+fn test_extract_regions_mem_rotated_page_not_false_empty() {
+    let buf = std::fs::read("tests/fixtures/tnagriculture_06_12.pdf").unwrap();
+    let regions =
+        extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+    assert_eq!(regions.len(), 1);
+    assert_eq!(regions[0].regions.len(), 1);
+    let region = &regions[0].regions[0];
+    assert!(
+        !region.text.trim().is_empty(),
+        "Rotated page full-region extraction should not be empty"
+    );
+    assert!(
+        !region.needs_ocr,
+        "Rotated page with native text should not be flagged for OCR fallback"
+    );
+    assert!(
+        region
+            .text
+            .contains("DISTRICT WISE PRODUCTION OF SPICES AND CONDIMENTS"),
+        "Expected known title from rotated fixture in extracted region text"
+    );
+}
+
+#[test]
+fn test_collect_text_in_region_keeps_partial_overlap_items() {
+    let item = make_text_item("EdgeWord", 100.0, 700.0, 12.0, 1);
+    // Region intersects only the left edge of the item. Center x=124 falls
+    // outside x=[95,120], so center-only containment would drop it.
+    let text = pdf_inspector::collect_text_in_region(&[item], 95.0, 80.0, 120.0, 110.0, 800.0);
+    assert!(
+        text.contains("EdgeWord"),
+        "Partially overlapping items should be retained in region extraction"
+    );
+}
+
+#[test]
+fn test_collect_text_in_region_uses_rtl_sorting() {
+    let items = vec![
+        make_text_item("بكم", 240.0, 700.0, 12.0, 1),
+        make_text_item("مرحبا", 300.0, 700.0, 12.0, 1),
+    ];
+    let text = pdf_inspector::collect_text_in_region(&items, 0.0, 0.0, 600.0, 800.0, 800.0);
+    assert_eq!(
+        text, "مرحبا بكم",
+        "Region path should reuse RTL-aware line sorting"
+    );
+}
+
+// =========================================================================
+// Fast vs normal extraction comparison
+// =========================================================================
+
+/// For each text-based fixture PDF, compare `extract_text_in_regions_mem` (fast path)
+/// against `process_pdf_mem` (normal path). If the fast path claims needs_ocr=false
+/// for a page, verify the extracted text has meaningful overlap with the normal
+/// markdown output — catching silent quality regressions.
+#[test]
+fn test_extract_regions_fast_vs_normal_comparison() {
+    let fixtures = [
+        "tests/fixtures/nexo-price-en.pdf",
+        "tests/fixtures/td9264.pdf",
+        "tests/fixtures/p1244-1996.pdf",
+        "tests/fixtures/real-estate-pricing.pdf",
+        "tests/fixtures/2013-app2.pdf",
+        "tests/fixtures/firecrawl_docs_tagged.pdf",
+        "tests/fixtures/thermo-freon12.pdf",
+    ];
+
+    for fixture in &fixtures {
+        let buf = std::fs::read(fixture).unwrap();
+        let normal = process_pdf_mem(&buf).unwrap();
+        let normal_md = normal.markdown.as_deref().unwrap_or("");
+        let page_count = normal.page_count;
+        let ocr_pages: HashSet<u32> = normal.pages_needing_ocr.iter().copied().collect();
+
+        let regions = extract_text_in_regions_mem(&buf, &full_page_regions(page_count)).unwrap();
+
+        assert_eq!(
+            regions.len(),
+            page_count as usize,
+            "{fixture}: result count should match page count"
+        );
+
+        for pr in &regions {
+            let region = &pr.regions[0];
+            if !region.needs_ocr && !region.text.trim().is_empty() {
+                // Fast path claims this text is trustworthy.
+                // Check that its words appear in the normal markdown output.
+                let overlap = word_overlap_ratio(&region.text, normal_md);
+                assert!(
+                    overlap >= 0.3,
+                    "{fixture} page {}: fast path says needs_ocr=false but only {:.0}% word \
+                     overlap with normal extraction (threshold 30%). \
+                     Fast text sample: {:?}",
+                    pr.page,
+                    overlap * 100.0,
+                    &region.text[..region.text.len().min(200)],
+                );
+            }
+
+            // If fast path flags needs_ocr but normal path didn't, that's overly
+            // conservative but not a bug — just worth knowing.
+            if region.needs_ocr && !ocr_pages.contains(&(pr.page + 1)) {
+                eprintln!(
+                    "INFO: {fixture} page {}: fast path says needs_ocr=true but normal path extracted fine (conservative, not a bug)",
+                    pr.page,
+                );
+            }
+        }
+    }
+}
@@ -8,7 +8,9 @@ Department of the Treasury **Internal Revenue Service**

 # and Report to Employer

-**This publication contains:** **Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer
+### This publication contains:
+
+**Form 4070A, Employee’s Daily Record of** Tips **Form 4070, Employee’s Report of Tips to** Employer

 For the period

@@ -74,7 +76,7 @@ forms simpler, we would be happy to hear from you. You can write to the Tax Form

 **Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law.

-**Instructions (continued)**
+### Instructions (continued)

 Use this space to total your tips for the year

@@ -6,9 +6,7 @@

 8 4 Z E L L / L U R I E R E A L E S T A T E C E N T E R

-**Table I: Cap rate correlations**
-
-**Cap Rate Correlation With:*** **BBB Corp** **10-Year Bond Yield S&P Dividend** **Treasury (10-15 yr) Yield** Multifamily 0.187 0.771 0.068 Industrial-0.221 0.748-0.307 CBD Office-0.449 0.694-0.458 Retail-0.181 0.649-02.58
+**Table I: Cap rate correlations** **Cap Rate Correlation With:*** **BBB Corp** **10-Year Bond Yield S&P Dividend** **Treasury (10-15 yr) Yield** Multifamily 0.187 0.771 0.068 Industrial-0.221 0.748-0.307 CBD Office-0.449 0.694-0.458 Retail-0.181 0.649-02.58

 * Based on 25 years of data for the 10-yrT & S&P DivYld; and 14 years for BBB.
 **Figure 1:** NCREIF cap rates vs. 10-yearTreasury
@@ -34,9 +32,7 @@ R E V I E W 8 5

 1982 1986 1990 1994 1998 2002 2006

-**Table II: Correlationsofspreadsbypropertytype**
-
-**Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office**
+**Table II: Correlationsofspreadsbypropertytype** **Correlation of Cap Rate Spreads Over Treasury** **Multifamily Industrial CBD Office**

 ||Multifamily|Industrial|CBD Office|
 |---|---|---|---|
@@ -1,8 +1,8 @@
-**Technical Information**
+##### Technical Information

 ## l T-12 SI

-DuPont Fluorochemicals
+##### DuPont Fluorochemicals

 #### Thermodynamic Properties

@@ -20,25 +20,22 @@ Tables of the thermodynamic **Units** properties of R-12 have been developed and

 S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume
 A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter
-H = Enthalpy (kJ/kg)
+##### H = Enthalpy (kJ/kg)

-S = Entropy (kJ/kg.K)
+##### S = Entropy (kJ/kg.K)

-**Physical Properties**
+##### Physical Properties

-Chemical Formula CCl2F2
+|Chemical Formula|CCl2F2|
+|---|---|
+|Molecular mass|120.91|
+|Boiling Point At one atmosphere|-29.75°C|
+|Critical Temperature|111.97°C|
+|Critical Pressure|4136 kPa|
+|Critical Density|565.0 kg/m|
+|Critical Volume|0.0018 m|

-Molecular mass 120.91
-
-Boiling Point-29.75°C At one atmosphere
-
-Critical Temperature 111.97°C
-
-Critical Pressure 4136 kPa
-
-3 Critical Density 565.0 kg/m
-
-Critical Volume 0.0018 m /kg
+/kg

 l

@@ -262,6 +262,13 @@ class TestExtractTextInRegions:
        assert results[0].page == 0
        assert results[1].page == 1

+    def test_malformed_region_raises_value_error(self):
+        with pytest.raises(ValueError, match="Invalid region"):
+            pdf_inspector.extract_text_in_regions(
+                fixture_path("thermo-freon12.pdf"),
+                [(0, [[0.0, 0.0, 600.0]])],
+            )
+

 # ---------------------------------------------------------------------------
 # Error handling
Author	SHA1	Message	Date
Abimael MartellandClaude Opus 4.6	2455f1437b	chore: bump npm version to 0.3.5 Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled Details Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled Details Publish npm package / Publish to npm (push) Has been cancelled Details Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-08 16:44:45 -07:00
Abimael MartellandClaude Opus 4.6	640cdaaa13	fix: stop counting Do operators as images in content stream scanner (#25 ) Do invokes any XObject (Form or Image), but scan_content_for_text_operators was counting every Do as an image. PDFs with Form XObjects (e.g. ACS publisher watermark pages) were misclassified as ImageBased because the inflated image_count raised the min text ops threshold above the actual text operator count. Image detection is already correctly handled by scan_xobjects_in_resources (checks Subtype) and analyze_page_images (measures pixel area). Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-08 16:41:40 -07:00
Abimael MartellandClaude Opus 4.6	be313cdb81	fix: require strong signal for rarity-based heading detection (#24 ) In multi-column PDFs, column switches break paragraph continuity, making body text lines appear "standalone". Combined with moderate font-size rarity from minor size variation between columns, this caused hundreds of false heading classifications (e.g. 281 false ## headings on a single academic paper). Non-bold, non-isolated lines now require very high rarity (≥0.97) and short word count (≤8) to qualify as headings via the rarity path. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-08 10:45:51 -07:00
Abimael MartellandClaude Opus 4.6	6a9ff170dc	fix: simplify region text extraction to trust layout model ordering Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled Details Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled Details Publish npm package / Publish to npm (push) Has been cancelled Details When fire-pdf sends pre-segmented bboxes from the layout model, pdf-inspector no longer runs column detection, stream-order heuristics, or newspaper/tabular mode detection within the region. These heuristics conflict with the layout model's decisions and cause wrong reading order. Region extraction now simply: Y-sorts items, groups into lines, and sorts within each line by X position. The heavy heuristics remain available for standalone full-page extraction. Eval showed pure OCR (0.2875 NED) beating native+heuristics (0.2916) across all categories, especially multi-column (-0.08) and newspaper (-0.16). This change should close that gap. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 21:58:05 -07:00
Abimael MartellandClaude Opus 4.6	0db9863919	feat: lookahead-based isolated line heading detection Pre-scan lines to identify "isolated" ones — short lines (1-6 words) with paragraph breaks both before AND after. These are heading candidates even at body font size, common in academic papers ("Acknowledgements", "Limitations", "B.3 Prompt Engineering"). Inspired by opendataloader's HeadingProcessor which passes prevNode and nextNode context to the heading probability scorer. The isolated signal (+0.3) combines with rarity/bold/standalone signals. A per-page density guard prevents false positives on multi-column pages where many lines appear isolated. Continuation word detection (ending in "the", "and", etc.) filters wrapped paragraph lines. MHS=0 docs: 18→13. MHS-S +0.004. No regressions. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 12:32:31 -07:00
Abimael MartellandClaude Opus 4.6	14154ee5ee	docs: update benchmark scores Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 12:19:38 -07:00
Abimael MartellandClaude Opus 4.6	10dd7e2881	feat: XY-cut fallback for column detection on asymmetric layouts When the histogram-based column detector finds no valleys (common with sidebar/asymmetric layouts), fall back to a simplified XY-cut: find the largest horizontal gap between item edges and split there if both sides have enough items with vertical overlap. Inspired by opendataloader's XY-Cut++ algorithm but implemented as a single-level fallback rather than full recursive segmentation. Doc 156: NID 0.545→0.966, Doc 157: NID 0.564→0.962. NID-S +0.007, TEDS-S +0.066 across 200 docs. No regressions. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 12:11:45 -07:00
Abimael MartellandClaude Opus 4.6	cf7e6b895d	Bump napi package version to 0.3.3 Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled Details Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled Details Publish npm package / Publish to npm (push) Has been cancelled Details Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 11:57:32 -07:00
Abimael MartellandClaude Opus 4.6	af34bbb6b5	feat: synthesize lines from thin filled rects as last-resort table detection PDFs that draw table borders as thin filled rectangles (< 2pt, common in spreadsheet exports) were invisible to both rect-based and line-based table detectors. Now converts these thin rects to PdfLine objects and runs line-based detection, but ONLY as a last resort after all other methods (rect, line, heuristic, column-based) found nothing. This avoids the regression from the earlier attempt which ran synthesis at step 2, preempting the heuristic detector on PDFs where it worked better. Also relaxes uniform row spacing threshold (CV 0.05→0.02) to accept spreadsheet-exported tables with even row heights. Benchmark: TEDS 0.519→0.586 (+0.067), overall +0.006, no regressions. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 11:44:11 -07:00
Abimael MartellandClaude Opus 4.6	dbc2de3e9b	revert: undo table splitting changes that caused TEDS regression Reverts commits `937311c`, `1dcb0c6`, `0300e96`, `999f9a2`. The thin-rect-to-line synthesis and stacked table splitting improved extraction for specific government PDFs but caused -0.05 TEDS regression on the benchmark by preempting the heuristic detector with worse line-based grids. These features need more targeted guards before re-enabling. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 11:20:05 -07:00
Abimael MartellandClaude Opus 4.6	999f9a2f2d	feat: split stacked tables at rows without vertical borders Rows that sit between horizontal rules but lack vertical border coverage are not table cells — they're freestanding text (e.g. "Note: The cutoff mark is out of 120"). These rows now split the grid into separate sub-tables, with the unbounded text emitted as plain text between them. Single-cell "tables" (from the split) render as plain text instead of a degenerate 1x1 markdown table. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 10:24:28 -07:00
Abimael MartellandClaude Opus 4.6	0300e96b7d	fix: don't merge column header rows as continuations Rows with 3+ short-valued cells (avg ≤10 chars) and an empty first cell are column headers (e.g. "UR \| SC \| ST \| OBC \| EWS"), not text overflow from the previous row. Prevents them from being merged into the preceding section title row. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 02:24:52 -07:00
Abimael MartellandClaude Opus 4.6	1dcb0c6feb	fix: improve table row splitting for stacked sub-tables Three fixes for better table extraction from spreadsheet-exported PDFs: 1. Convert thin filled rects (< 2pt) to PdfLine objects before line-based table detection. Many PDFs draw table borders as narrow filled rectangles instead of stroked paths — these were invisible to the line detector. 2. Relax uniform row spacing rejection (CV 0.05 → 0.02). Spreadsheet exports have very even row heights that were being rejected as "chart grids". 3. Fix continuation row merging: don't merge rows where the only non-first cell content is a long label (section headers like "Category No. 03"). Don't merge first-cell-only rows with long text ("Note: ..."). Also adds multi-Y row splitting in line-based detection and column-aware table detection skipping for multi-column pages. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 02:20:42 -07:00
Abimael MartellandClaude Opus 4.6	937311ce27	feat: convert thin filled rects to lines for table border detection Many PDFs (especially spreadsheet exports) draw table borders as thin filled rectangles (height/width < 2pt) instead of stroked paths. These were invisible to our line-based table detector since only stroke operations produced PdfLine objects. Now synthesizes PdfLine from thin rects before line-based detection, enabling table detection on border-drawn PDFs like government forms. Also relaxes the uniform row spacing rejection threshold (CV 0.05→0.02) to accept spreadsheet-exported tables with even row heights. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 02:12:05 -07:00
Abimael MartellandClaude Opus 4.6	9c463b9c95	fix: prevent multi-column text from being misdetected as tables On pages where column detection finds 2+ columns, skip body-font heuristic table detection in the merged-band retry path. This prevents sidebar/two-column prose from being formatted as markdown tables. The fix is targeted: per-band heuristic detection still runs (bands are scoped to single columns), so real tables within columns are still detected. Only the merged-band retry (which sees all items across columns) is gated. Also relaxes column validation to accept asymmetric layouts (sidebars) where one side has fewer items, and tries center-based item assignment before edge-based to improve column splitting for asymmetric layouts. Benchmark: NID 0.865→0.869, NID-S 0.798→0.805, overall +0.002. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 02:01:24 -07:00
Abimael MartellandClaude Opus 4.6	1c48a014f7	docs: update benchmark MHS score after rarity-based detection Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:38:21 -07:00
Abimael MartellandClaude Opus 4.6	6e5abd0b48	feat: rarity-based heading detection inspired by opendataloader Replace ad-hoc bold/ratio heading checks with a unified scoring system based on font size rarity. For each line, compute: score = font_rarity * 0.5 + bold * 0.3 + standalone * 0.2 Font rarity measures how infrequently a font size appears across the document — heading fonts are rare while body text is common. This approach (from opendataloader's ModeWeightStatistics) naturally adapts to each document's font distribution instead of relying on fixed thresholds. Guards: require font_size >= 0.95 * base_size (no small-font headings), word_count >= 3, and standalone (paragraph break before). Benchmark improvement: MHS 0.56→0.58, MHS-S 0.66→0.70, overall +0.003. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:38:03 -07:00
Abimael MartellandClaude Opus 4.6	b7ec80097b	chore: update lopdf to latest commit 7a05512d831415b1f2b1ce522391d6beab8a1284 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:27:32 -07:00
Abimael MartellandClaude Opus 4.6	895e47b094	docs: add opendataloader-bench results to README Compare pdf-inspector against other direct text extraction engines (no OCR/ML) on the opendataloader-bench corpus. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:25:08 -07:00
Abimael MartellandClaude Opus 4.6	2f9390e4bf	feat: detect slightly-larger-than-body text as headings Lines with font size 1.10-1.20x body text that are standalone and short (1-8 words) are promoted to headings. This catches academic paper headings where the font is only ~10% larger than body text, below the previous 1.2x threshold. Also syncs the simpler to_markdown_from_lines path to match the table-aware path (removes stale colon exclusion). Benchmark improvement: MHS 0.54→0.56, overall 0.761→0.766. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:18:31 -07:00
Abimael MartellandClaude Opus 4.6	b25a122655	fix: require digit after Table/Figure prefix in caption detection Caption detection was incorrectly classifying "Table of Contents" as a caption because it starts with "Table ". Now "Table" and "Figure" prefixes require a digit, parenthesis, or hash after them — matching actual captions like "Table 1", "Figure 3.2" but not titles. Also removes debug logging left from previous iteration. Benchmark improvement: MHS 0.52→0.54, overall 0.757→0.761. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:13:17 -07:00
Abimael MartellandClaude Opus 4.6	103995c200	feat: remove colon exclusion from bold heading detection The colon exclusion was preventing legitimate headings like "Steps for Using the Microscope:" and "Changing objectives:" from being detected. The single edge case it was protecting (chart sub-headers) is less impactful than the many headings it was blocking. Benchmark improvement: MHS 0.51→0.52, MHS-S 0.61→0.62. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:09:19 -07:00
Abimael MartellandClaude Opus 4.6	28823bd849	feat: improve heuristic table detection for small body-font tables - Lower minimum item count for body-font table candidates from 9 to 6, allowing small 2-3 row tables to be detected. - Allow 2-column body-font tables with short cells (avg ≤25 chars) to bypass the "table-like content" validation. This catches text-only definition/category tables (e.g., species lists) without false-positiving on 2-column paragraph text (which has longer cells). Benchmark improvement: TEDS 0.498→0.519, overall 0.750→0.754. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 01:07:08 -07:00
Abimael MartellandClaude Opus 4.6	830b8d955b	feat: detect bold-only lines as section headings Bold lines at body font size that are standalone (preceded by a paragraph break) and have ≥3 words are promoted to headings. This catches the common pattern in academic/technical PDFs where section headings use bold text at the same size as body text. Guards against false positives: minimum word count, colon-ending exclusion (labels like "Table I:"). Benchmark improvement: MHS 0.37→0.50, overall 0.71→0.75. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-04 00:56:18 -07:00
Abimael Martell	f664f3056d	fix: harden hybrid OCR region extraction path (#23 ) Align region filtering with rotated-page coordinate rewrites, switch region text assembly to the shared line-grouping pipeline, and retain edge-overlap text to avoid false empty regions that incorrectly trigger OCR fallback. Also make Python region inputs fail fast with clear ValueError messages for malformed boxes. Made-with: Cursor	2026-04-03 22:48:13 -07:00
Abimael Martell	ebb01ab6a3	Merge pull request #22 from firecrawl/perf/skip-truetype-fallback-in-region-extract Publish npm package / Build aarch64-apple-darwin (push) Has been cancelled Details Publish npm package / Build x86_64-unknown-linux-gnu (push) Has been cancelled Details Publish npm package / Publish to npm (push) Has been cancelled Details perf: skip TrueType fallback in extract_text_in_regions (22,000x faster font parsing)	2026-04-02 18:14:05 -07:00
Abimael MartellandClaude Opus 4.6	53eff94147	Bump napi package version to 0.3.2 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>	2026-04-02 18:13:20 -07:00
Abimael MartellandClaude Opus 4.6	30244078d8	fix: sort panic and missing CID garbage check in extract_text_in_regions_mem Two bugs in collect_text_in_region / extract_text_in_regions_mem: 1. The threshold-based sort comparator in collect_text_in_region was not transitive, causing Rust's sort to panic on certain PDFs. Replaced with strict total_cmp ordering — the line-grouping phase already handles fuzzy Y matching via threshold. 2. The needs_ocr check was missing is_cid_garbage, so Identity-H fonts with CID garbage (C1 control chars, high Latin mojibake) could pass all quality checks and be served as real text with needs_ocr=false. Also adds 7 integration tests for extract_text_in_regions_mem (previously had zero coverage): basic extraction, Identity-H needs_ocr, multiple regions, nonexistent page, empty region, invalid input, and a fast-vs-normal comparison test across all text-based fixtures. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>	2026-04-02 18:13:11 -07:00
Abimael MartellandClaude Opus 4.6	57673ebb69	perf: skip expensive TrueType fallback in extract_text_in_regions FontCMaps::from_doc can spend 4.5+ seconds decompressing and parsing large embedded TrueType fonts for CID fonts with sparse ToUnicode CMaps. For extract_text_in_regions (hybrid OCR pipeline), this is unnecessary — fonts that can't be decoded cheaply will produce empty/garbage text, triggering needs_ocr=true and GPU OCR fallback. Changes: - Add FontCMaps::from_doc_pages_fast() that skips TrueType font fallback parsing (build_fallback_cmap_for_type0) and Identity-H/V second pass entirely - Add FontCMaps::from_doc_pages() for filtered page sets - extract_text_in_regions_mem uses fast mode - Restructure fallback chain: try cheap fallbacks first, only attempt expensive TrueType parsing when needed and not in fast mode Benchmark on nihms-1771367.pdf (19-page chemistry paper): - FontCMaps fast: 201µs - FontCMaps slow: 4.47s - 22,000x speedup on font parsing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>	2026-04-02 18:13:11 -07:00