fix TSR cell text assignment for overlapping bboxes

Made-with: Cursor

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.6.3",
+  "version": "1.6.0",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/bin/detect_pdf.rs

@@ -1,12 +1,8 @@
 //! CLI tool for detecting PDF type (text-based vs scanned)
 
-use pdf_inspector::{
-    detect_pdf_type, detector::estimate_page_count_from_bytes, process_pdf_with_options,
-    PdfOptions, PdfType, ProcessMode,
-};
+use pdf_inspector::{detect_pdf_type, process_pdf_with_options, PdfOptions, PdfType, ProcessMode};
 use std::env;
 use std::fmt::Write;
-use std::fs;
 use std::process;
 use std::time::Instant;
 
@@ -68,32 +64,6 @@ fn pdf_type_str(pdf_type: &PdfType) -> &'static str {
     }
 }
 
-fn page_count_hint(pdf_path: &str) -> Option<u32> {
-    fs::read(pdf_path)
-        .ok()
-        .map(|bytes| estimate_page_count_from_bytes(&bytes))
-        .filter(|&count| count > 0)
-}
-
-fn print_error(e: &pdf_inspector::PdfError, pdf_path: &str, json_output: bool) {
-    if json_output {
-        if let Some(count) = page_count_hint(pdf_path) {
-            println!(
-                r#"{{"error":"{}","page_count_hint":{}}}"#,
-                json_escape(&e.to_string()),
-                count
-            );
-        } else {
-            println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
-        }
-    } else {
-        eprintln!("Error: {}", e);
-        if let Some(count) = page_count_hint(pdf_path) {
-            eprintln!("Page count hint: {}", count);
-        }
-    }
-}
-
 fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
     match process_pdf_with_options(pdf_path, PdfOptions::new().mode(ProcessMode::Analyze)) {
         Ok(result) => {
@@ -165,7 +135,11 @@ fn run_analyze(pdf_path: &str, json_output: bool, start: Instant) {
             }
         }
         Err(e) => {
-            print_error(&e, pdf_path, json_output);
+            if json_output {
+                println!(r#"{{"error":"{}"}}"#, e);
+            } else {
+                eprintln!("Error: {}", e);
+            }
             process::exit(1);
         }
     }
@@ -262,7 +236,11 @@ fn run_detect_only(pdf_path: &str, json_output: bool, start: Instant) {
             }
         }
         Err(e) => {
-            print_error(&e, pdf_path, json_output);
+            if json_output {
+                println!(r#"{{"error":"{}"}}"#, e);
+            } else {
+                eprintln!("Error: {}", e);
+            }
             process::exit(1);
         }
     }

src/detector.rs

@@ -97,9 +97,26 @@ pub fn detect_pdf_type_with_config<P: AsRef<Path>>(
 ) -> Result<PdfTypeResult, PdfError> {
     crate::validate_pdf_file(&path)?;
 
-    let (doc, page_count) = crate::load_document_from_path(&path)?;
+    // First, load metadata only (fast operation)
+    let metadata = match Document::load_metadata(&path) {
+        Ok(m) => m,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_metadata_with_password(&path, "")?
+        }
+        Err(e) => return Err(e.into()),
+    };
 
-    detect_from_document(&doc, page_count, &config)
+    // Then load the full document for content inspection
+    // We use filtered loading to skip heavy objects we don't need
+    let doc = match Document::load(&path) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_with_password(&path, "")?
+        }
+        Err(e) => return Err(e.into()),
+    };
+
+    detect_from_document(&doc, metadata.page_count, &config)
 }
 
 /// Detect PDF type from memory buffer
@@ -114,64 +131,25 @@ pub fn detect_pdf_type_mem_with_config(
 ) -> Result<PdfTypeResult, PdfError> {
     crate::validate_pdf_bytes(buffer)?;
 
-    let (doc, page_count) = crate::load_document_from_mem(buffer)?;
-
-    detect_from_document(&doc, page_count, &config)
-}
-
-/// Heuristic page-count fallback for malformed PDFs that cannot be parsed.
-///
-/// This scans raw bytes for page dictionaries (`/Type /Page`) while excluding
-/// the page tree node (`/Type /Pages`). It is intended as a low-confidence hint
-/// for diagnostics; parsed page-tree counts remain authoritative.
-pub fn estimate_page_count_from_bytes(buffer: &[u8]) -> u32 {
-    let mut count = 0u32;
-    let mut pos = 0usize;
-
-    while let Some(rel_idx) = find_bytes(&buffer[pos..], b"/Type") {
-        let mut value_pos = pos + rel_idx + b"/Type".len();
-        value_pos = skip_pdf_whitespace(buffer, value_pos);
-
-        if buffer.get(value_pos) == Some(&b'/') {
-            let name_start = value_pos + 1;
-            let name_end = name_start + b"Page".len();
-            if name_end <= buffer.len()
-                && &buffer[name_start..name_end] == b"Page"
-                && buffer
-                    .get(name_end)
-                    .is_none_or(|b| is_pdf_name_delimiter(*b))
-            {
-                count += 1;
-            }
+    // Load metadata first (fast)
+    let metadata = match Document::load_metadata_mem(buffer) {
+        Ok(m) => m,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_metadata_mem_with_password(buffer, "")?
         }
+        Err(e) => return Err(e.into()),
+    };
 
-        pos += rel_idx + b"/Type".len();
-    }
+    // Load document for inspection
+    let doc = match Document::load_mem(buffer) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
+        }
+        Err(e) => return Err(e.into()),
+    };
 
-    count
-}
-
-fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
-    haystack.windows(needle.len()).position(|w| w == needle)
-}
-
-fn skip_pdf_whitespace(buffer: &[u8], mut pos: usize) -> usize {
-    while pos < buffer.len() && is_pdf_whitespace(buffer[pos]) {
-        pos += 1;
-    }
-    pos
-}
-
-fn is_pdf_whitespace(byte: u8) -> bool {
-    matches!(byte, b'\0' | b'\t' | b'\n' | 0x0C | b'\r' | b' ')
-}
-
-fn is_pdf_name_delimiter(byte: u8) -> bool {
-    is_pdf_whitespace(byte)
-        || matches!(
-            byte,
-            b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
-        )
+    detect_from_document(&doc, metadata.page_count, &config)
 }
 
 /// Detection logic on a pre-loaded document.

src/extractor/mod.rs

@@ -36,14 +36,26 @@ pub(crate) use layout::ColumnRegion;
 /// Extract text from PDF file as plain string
 pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
     crate::validate_pdf_file(&path)?;
-    let (doc, _) = crate::load_document_from_path(&path)?;
+    let doc = match Document::load(&path) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_with_password(&path, "")?
+        }
+        Err(e) => return Err(e.into()),
+    };
     extract_text_from_doc(&doc)
 }
 
 /// Extract text from PDF memory buffer
 pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
     crate::validate_pdf_bytes(buffer)?;
-    let (doc, _) = crate::load_document_from_mem(buffer)?;
+    let doc = match Document::load_mem(buffer) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
+        }
+        Err(e) => return Err(e.into()),
+    };
     extract_text_from_doc(&doc)
 }
 
@@ -79,7 +91,13 @@ pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
     page_filter: Option<&HashSet<u32>>,
 ) -> Result<PageExtraction, PdfError> {
     crate::validate_pdf_file(&path)?;
-    let (doc, _) = crate::load_document_from_path(&path)?;
+    let doc = match Document::load(&path) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_with_password(&path, "")?
+        }
+        Err(e) => return Err(e.into()),
+    };
     let font_cmaps = FontCMaps::from_doc(&doc);
     let (extraction, _thresholds, _gid_pages) =
         extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;
@@ -106,7 +124,13 @@ pub(crate) fn extract_text_with_positions_mem_and_rects(
     page_filter: Option<&HashSet<u32>>,
 ) -> Result<PageExtraction, PdfError> {
     crate::validate_pdf_bytes(buffer)?;
-    let (doc, _) = crate::load_document_from_mem(buffer)?;
+    let doc = match Document::load_mem(buffer) {
+        Ok(d) => d,
+        Err(ref e) if crate::is_encrypted_lopdf_error(e) => {
+            Document::load_mem_with_options(buffer, lopdf::LoadOptions::with_password(""))?
+        }
+        Err(e) => return Err(e.into()),
+    };
     let font_cmaps = FontCMaps::from_doc(&doc);
     let (extraction, _thresholds, _gid_pages) =
         extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?;

src/lib.rs

@@ -930,188 +930,21 @@ pub fn extract_tables_with_structure_cells_mem(
         }
 
         normalize_cell_bands(&mut cells);
-
-        // Stage 1: strict text fill — each cell gets the items whose centers
-        // fall inside its (normalized) bbox or whose >=60% overlap rule fires.
-        // Track which item indices any cell claimed so the orphan pass below
-        // doesn't double-assign.
-        let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
         for cell in &mut cells {
             let [x1, y1, x2, y2] = cell.page_pt_bbox;
-            let bounds = region_bounds(x1, y1, x2, y2, page_h, coords);
-            let mut matched: Vec<TextItem> = Vec::new();
-            for (i, item) in items.iter().enumerate() {
-                if tsr_region_contains_item(item, bounds) {
-                    claimed.insert(i);
-                    matched.push(item.clone());
-                }
-            }
+            let raw =
+                collect_text_in_tsr_cell(items, x1, y1, x2, y2, page_h, coords, adaptive_threshold);
             // Markdown cells must be one line — collapse line breaks produced
             // by the line-grouping pass.
-            cell.text = collect_text_from_matched_items(matched, adaptive_threshold)
-                .replace(['\n', '\r'], " ");
+            cell.text = raw.replace(['\n', '\r'], " ");
         }
 
-        // Stage 2: orphan assignment — text items that didn't land in any
-        // cell during stage 1 get assigned to their nearest *empty* cell,
-        // clamped by a plausibility cap derived from cell geometry.
-        //
-        // This recovers two failure modes left by `normalize_cell_bands`:
-        //   (a) header text positioned to the LEFT of a column whose band
-        //       was derived from data cells centered farther right, so the
-        //       header text falls outside the clamped band; and
-        //   (b) local SLANet row drift where a cell's bbox sits slightly
-        //       above/below its target text item, so the strict rules miss.
-        // Empty-cell-only is the safety net: a cell already filled by stage 1
-        // is never overwritten or augmented, so the cell-bleed case PR #62
-        // closed cannot regress.
-        tsr_assign_orphan_items(items, &mut cells, &claimed, page_h, coords);
-
         results.push(cells);
     }
 
     Ok(results)
 }
 
-/// Compute plausibility caps for the orphan-assignment pass. Returns
-/// `(cap_x, cap_y)` — the maximum x/y distance from a text item's center
-/// to a candidate empty cell's bbox before the candidate is rejected.
-///
-/// Caps are derived from cell geometry so they scale with the table:
-/// dense small-row tables get a tight cap, looser tables get more slack.
-/// Floor values guard against degenerate single-cell tables collapsing
-/// the cap to zero.
-fn tsr_assignment_caps(cells: &[tables::StructuredCell]) -> (f32, f32) {
-    let mut widths: Vec<f32> = Vec::with_capacity(cells.len());
-    let mut heights: Vec<f32> = Vec::with_capacity(cells.len());
-    for cell in cells {
-        let [x1, y1, x2, y2] = cell.page_pt_bbox;
-        let w = (x2 - x1).abs();
-        let h = (y2 - y1).abs();
-        if w > 0.0 && h > 0.0 {
-            widths.push(w);
-            heights.push(h);
-        }
-    }
-    if widths.is_empty() {
-        return (0.0, 0.0);
-    }
-    widths.sort_by(|a, b| a.total_cmp(b));
-    heights.sort_by(|a, b| a.total_cmp(b));
-    let median_w = widths[widths.len() / 2];
-    let median_h = heights[heights.len() / 2];
-    // Floor values: even on a dense table, a 5pt floor handles small
-    // pixel-level bbox jitter without being so loose that we'd cross
-    // into a neighboring row/column. Symmetric in both axes.
-    let cap_x = median_w.max(5.0);
-    let cap_y = median_h.max(5.0);
-    (cap_x, cap_y)
-}
-
-/// For each text item that wasn't claimed by any cell during stage 1,
-/// find the nearest *empty* cell within `(cap_x, cap_y)` of the item's
-/// center and append the item's text to that cell. Cells that already
-/// have content are skipped — stage 2 only fills, never augments.
-///
-/// Distance is point-to-rect: 0 if the item center is inside the cell's
-/// bbox, else the axis-aligned gap to the nearest edge. Both x-gap and
-/// y-gap must be within their respective caps for a candidate to qualify;
-/// among qualifying candidates, the smallest combined euclidean distance
-/// wins.
-fn tsr_assign_orphan_items(
-    items: &[TextItem],
-    cells: &mut [tables::StructuredCell],
-    claimed: &std::collections::HashSet<usize>,
-    page_height: f32,
-    coord_space: RegionCoordSpace,
-) {
-    if cells.is_empty() {
-        return;
-    }
-    let (cap_x, cap_y) = tsr_assignment_caps(cells);
-    if cap_x <= 0.0 || cap_y <= 0.0 {
-        return;
-    }
-    // Y-tolerance for "same line as a previous orphan" — multi-token branch
-    // names like "Blue Valley Parkway" are 3 separate text items and should
-    // all stack into the same cell. But two orphans on different rows of
-    // the PDF (different y values) targeting the same empty cell should
-    // NOT merge — that produces the "Mitchell Woonsocket" / "Shawnee Blue
-    // Valley Parkway" run-on cells. Half a row of slack is conservative.
-    let y_tolerance = (cap_y * 0.5).max(3.0);
-
-    // Pre-compute each empty cell's region bounds so we don't re-flip
-    // page coordinates per orphan-candidate pair.
-    let cell_bounds: Vec<Option<RegionBounds>> = cells
-        .iter()
-        .map(|cell| {
-            if !cell.text.is_empty() {
-                return None;
-            }
-            let [x1, y1, x2, y2] = cell.page_pt_bbox;
-            if x1 >= x2 || y1 >= y2 {
-                return None;
-            }
-            Some(region_bounds(x1, y1, x2, y2, page_height, coord_space))
-        })
-        .collect();
-
-    // Track the y-center of the FIRST orphan that landed in each cell so
-    // subsequent orphans only stack if they're on the same line.
-    let mut stage2_first_y: std::collections::HashMap<usize, f32> =
-        std::collections::HashMap::new();
-
-    for (i, item) in items.iter().enumerate() {
-        if claimed.contains(&i) {
-            continue;
-        }
-        let item_w = text_utils::effective_width(item);
-        if item.text.trim().is_empty() {
-            continue;
-        }
-        let cx = item.x + item_w * 0.5;
-        let cy = item.y + item.height * 0.5;
-
-        let mut best: Option<(usize, f32)> = None;
-        for (ci, bounds_opt) in cell_bounds.iter().enumerate() {
-            let Some(bounds) = bounds_opt else {
-                continue;
-            };
-            // If a previous orphan already landed in this cell, only let a
-            // new orphan join if it's on the same line. Cross-line orphans
-            // need to look elsewhere (next-nearest empty cell).
-            if let Some(&first_y) = stage2_first_y.get(&ci) {
-                if (first_y - cy).abs() > y_tolerance {
-                    continue;
-                }
-            }
-            let dx = (bounds.x_min - cx).max(0.0).max(cx - bounds.x_max);
-            let dy = (bounds.y_min - cy).max(0.0).max(cy - bounds.y_max);
-            if dx > cap_x || dy > cap_y {
-                continue;
-            }
-            let dist_sq = dx * dx + dy * dy;
-            if best.is_none_or(|(_, d)| dist_sq < d) {
-                best = Some((ci, dist_sq));
-            }
-        }
-
-        if let Some((ci, _)) = best {
-            // Append, preserving stage 1's content. Same-line orphans
-            // stack to support multi-token text (e.g. "Blue Valley
-            // Parkway"); cross-line orphans are filtered out above.
-            let trimmed = item.text.trim();
-            if cells[ci].text.is_empty() {
-                cells[ci].text = trimmed.to_string();
-            } else {
-                cells[ci].text.push(' ');
-                cells[ci].text.push_str(trimmed);
-            }
-            stage2_first_y.entry(ci).or_insert(cy);
-        }
-    }
-}
-
 /// Extract markdown tables using externally-supplied structure recovery.
 ///
 /// Convenience wrapper around [`extract_tables_with_structure_cells_mem`]
@@ -1225,7 +1058,6 @@ fn collect_text_in_region_with_options(
     collect_text_from_matched_items(matched, adaptive_threshold)
 }
 
-#[cfg(test)]
 #[allow(clippy::too_many_arguments)]
 fn collect_text_in_tsr_cell(
     items: &[TextItem],
@@ -1381,133 +1213,30 @@ fn tsr_region_contains_item(item: &TextItem, bounds: RegionBounds) -> bool {
 /// `Document::load_metadata` for page count + `Document::load` for content
 /// are combined here, but lopdf loads the full doc in `load()` so we extract
 /// page count from it directly to avoid the metadata-only round-trip.
-pub(crate) fn load_document_from_path<P: AsRef<Path>>(
-    path: P,
-) -> Result<(Document, u32), PdfError> {
+fn load_document_from_path<P: AsRef<Path>>(path: P) -> Result<(Document, u32), PdfError> {
     let buffer = std::fs::read(&path)?;
     load_document_from_mem(&buffer)
 }
 
 /// Load a PDF from a memory buffer.
-pub(crate) fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
+fn load_document_from_mem(buffer: &[u8]) -> Result<(Document, u32), PdfError> {
     // Fix malformed struct element names before parsing. Some PDF generators
     // write bare names (/S Code) instead of proper PDF names (/S /Code), which
     // causes lopdf to silently drop the entire object.
     let fixed = structure_tree::fix_bare_struct_names(buffer);
     let buf = fixed.as_ref();
 
-    let doc = match load_document_bytes(buf) {
-        Ok(doc) => doc,
-        Err(first_err) => {
-            for repaired in repair_pdf_container_candidates(buf) {
-                match load_document_bytes(&repaired) {
-                    Ok(doc) => {
-                        log::debug!("loaded PDF after repairing malformed container bytes");
-                        let page_count = doc.get_pages().len() as u32;
-                        return Ok((doc, page_count));
-                    }
-                    Err(e) => {
-                        if is_encrypted_lopdf_error(&e) {
-                            return Err(e.into());
-                        }
-                    }
-                }
-            }
-            return Err(first_err.into());
+    let doc = match Document::load_mem(buf) {
+        Ok(d) => d,
+        Err(ref e) if is_encrypted_lopdf_error(e) => {
+            Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))?
         }
+        Err(e) => return Err(e.into()),
     };
     let page_count = doc.get_pages().len() as u32;
     Ok((doc, page_count))
 }
 
-fn load_document_bytes(buf: &[u8]) -> Result<Document, lopdf::Error> {
-    match Document::load_mem(buf) {
-        Ok(doc) => Ok(doc),
-        Err(ref e) if is_encrypted_lopdf_error(e) => {
-            Document::load_mem_with_options(buf, lopdf::LoadOptions::with_password(""))
-        }
-        Err(e) => Err(e),
-    }
-}
-
-fn repair_pdf_container_candidates(buf: &[u8]) -> Vec<Vec<u8>> {
-    let mut candidates = Vec::new();
-
-    add_repair_candidate(&mut candidates, append_missing_eof_marker(buf), buf);
-
-    let stripped = strip_leading_pdf_container_bytes(buf);
-    if let Some(stripped_buf) = stripped.as_deref() {
-        add_repair_candidate(&mut candidates, Some(stripped_buf.to_vec()), buf);
-        add_repair_candidate(
-            &mut candidates,
-            append_missing_eof_marker(stripped_buf),
-            buf,
-        );
-    }
-
-    candidates
-}
-
-fn add_repair_candidate(
-    candidates: &mut Vec<Vec<u8>>,
-    candidate: Option<Vec<u8>>,
-    original: &[u8],
-) {
-    let Some(candidate) = candidate else {
-        return;
-    };
-    if candidate.as_slice() == original {
-        return;
-    }
-    if candidates.iter().any(|existing| existing == &candidate) {
-        return;
-    }
-    candidates.push(candidate);
-}
-
-fn append_missing_eof_marker(buf: &[u8]) -> Option<Vec<u8>> {
-    if contains_recent_eof_marker(buf) {
-        return None;
-    }
-
-    let mut end = buf.len();
-    while end > 0 && buf[end - 1].is_ascii_whitespace() {
-        end -= 1;
-    }
-
-    if !buf[..end].ends_with(b"%%EO") {
-        return None;
-    }
-
-    let mut repaired = Vec::with_capacity(end + 2);
-    repaired.extend_from_slice(&buf[..end]);
-    repaired.extend_from_slice(b"F\n");
-    Some(repaired)
-}
-
-fn contains_recent_eof_marker(buf: &[u8]) -> bool {
-    let start = buf.len().saturating_sub(1024);
-    buf[start..].windows(b"%%EOF".len()).any(|w| w == b"%%EOF")
-}
-
-fn strip_leading_pdf_container_bytes(buf: &[u8]) -> Option<Vec<u8>> {
-    let mut start = if buf.starts_with(&[0xEF, 0xBB, 0xBF]) {
-        3
-    } else {
-        0
-    };
-
-    while start < buf.len() && buf[start].is_ascii_whitespace() {
-        start += 1;
-    }
-
-    if start > 0 && buf[start..].starts_with(b"%PDF-") {
-        Some(buf[start..].to_vec())
-    } else {
-        None
-    }
-}
-
 /// Core processing pipeline operating on a pre-loaded document.
 fn process_document(
     doc: Document,
@@ -2720,388 +2449,4 @@ mod tests {
         assert!(!cells[2].text.contains("Branch Name"));
         assert!(!cells[2].text.contains("Boardwalk"));
     }
-
-    #[test]
-    fn tsr_assignment_caps_uses_median_geometry() {
-        use crate::tables::StructuredCell;
-        let cells = vec![
-            StructuredCell {
-                row: 0,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [0.0, 0.0, 100.0, 20.0], // 100x20
-            },
-            StructuredCell {
-                row: 0,
-                col: 1,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [100.0, 0.0, 200.0, 20.0],
-            },
-            StructuredCell {
-                row: 1,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [0.0, 20.0, 100.0, 40.0],
-            },
-        ];
-        let (cap_x, cap_y) = tsr_assignment_caps(&cells);
-        assert_eq!(cap_x, 100.0);
-        assert_eq!(cap_y, 20.0);
-    }
-
-    #[test]
-    fn tsr_assignment_caps_floor_protects_degenerate_input() {
-        use crate::tables::StructuredCell;
-        let cells = vec![StructuredCell {
-            row: 0,
-            col: 0,
-            rowspan: 1,
-            colspan: 1,
-            is_header: false,
-            text: String::new(),
-            page_pt_bbox: [0.0, 0.0, 1.0, 1.0],
-        }];
-        let (cap_x, cap_y) = tsr_assignment_caps(&cells);
-        assert_eq!(cap_x, 5.0);
-        assert_eq!(cap_y, 5.0);
-    }
-
-    #[test]
-    fn stage2_recovers_left_aligned_header_text_outside_data_band() {
-        // Symptom A reproduction: the column band derived from data-cell
-        // centers ends up too far right, so header text positioned at the
-        // left of the column falls outside the band and stage 1's strict
-        // membership rejects it. Stage 2 should re-attach by proximity.
-        //
-        // Item coords are bottom-left native; cell page_pt_bbox is top-left.
-        // page_height=200 so a top-left bbox y=[88, 100] flips to native y
-        // bounds [100, 112]; an item at native y=104 (center 108) lands in.
-        use crate::tables::StructuredCell;
-        let items = vec![
-            // Header text — centered in row 0 (native y=104, center 108) but
-            // at the LEFT of the column (x=175, far left of the [410, 700]
-            // data-derived band).
-            test_item("Address", 175.0, 104.0, 50.0, 8.0),
-            // Data row 1 — fits its cell.
-            test_item("205 W Oak St", 420.0, 84.0, 100.0, 8.0),
-            // Data row 2 — fits its cell.
-            test_item("155 E Boardwalk Dr", 420.0, 64.0, 100.0, 8.0),
-        ];
-        // Cells AFTER normalize_cell_bands would have run — col 0 band
-        // shifted right by data-cell centers, header cell now excludes
-        // the "Address" text at center x=200.
-        let mut cells = vec![
-            StructuredCell {
-                row: 0,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: true,
-                text: String::new(),
-                page_pt_bbox: [410.0, 88.0, 700.0, 100.0],
-            },
-            StructuredCell {
-                row: 1,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [410.0, 108.0, 700.0, 116.0],
-            },
-            StructuredCell {
-                row: 2,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [410.0, 128.0, 700.0, 136.0],
-            },
-        ];
-        let page_h = 200.0;
-
-        // Stage 1 mimic — fill cells via the strict rule, track claimed.
-        let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
-        for cell in &mut cells {
-            let [x1, y1, x2, y2] = cell.page_pt_bbox;
-            let bounds = region_bounds(x1, y1, x2, y2, page_h, RegionCoordSpace::Standard);
-            let mut matched: Vec<TextItem> = Vec::new();
-            for (i, item) in items.iter().enumerate() {
-                if tsr_region_contains_item(item, bounds) {
-                    claimed.insert(i);
-                    matched.push(item.clone());
-                }
-            }
-            cell.text = collect_text_from_matched_items(matched, 0.10).replace(['\n', '\r'], " ");
-        }
-        // Header is empty after stage 1 (Address fell outside col 0 band).
-        assert_eq!(cells[0].text, "", "header should be empty after stage 1");
-        // Data rows already populated.
-        assert!(
-            cells[1].text.contains("Oak"),
-            "data row 1 should contain Oak: got {:?}",
-            cells[1].text
-        );
-        assert!(
-            cells[2].text.contains("Boardwalk"),
-            "data row 2 should contain Boardwalk: got {:?}",
-            cells[2].text
-        );
-
-        // Stage 2 should fill the orphan "Address" into the empty header.
-        tsr_assign_orphan_items(
-            &items,
-            &mut cells,
-            &claimed,
-            page_h,
-            RegionCoordSpace::Standard,
-        );
-        assert_eq!(cells[0].text, "Address");
-        // Data rows must NOT have been augmented (already filled by stage 1).
-        assert!(!cells[1].text.contains("Address"));
-        assert!(!cells[2].text.contains("Address"));
-    }
-
-    #[test]
-    fn stage2_recovers_y_shifted_col0_in_consecutive_rows() {
-        // Symptom B reproduction: a stretch of rows where col 0 cell bboxes
-        // sit just above the actual branch-name text. After stage 1 those
-        // cells are empty; stage 2 should pull the orphan items in by
-        // y-proximity.
-        //
-        // page_height=800. Cells are 14pt tall in top-left; flipped native
-        // bounds are [240,254], [220,234], [200,214]. Items sit ~1pt below
-        // each cell's native y range (still within ~1pt of the edge), so
-        // both center-containment and 60% overlap fail in stage 1.
-        use crate::tables::StructuredCell;
-        let items = vec![
-            // Bellevue: native y=235, center 239 — just below row 0's
-            // cell native bottom (240). Closer to row 0 than row 1.
-            test_item("Bellevue", 30.0, 235.0, 45.0, 8.0),
-            // Glenwood: native y=215, center 219 — just below row 1.
-            test_item("Glenwood", 30.0, 215.0, 45.0, 8.0),
-            // Metro Crossing: native y=195, center 199 — just below row 2.
-            test_item("Metro Crossing", 30.0, 195.0, 70.0, 8.0),
-        ];
-        let mut cells = vec![
-            StructuredCell {
-                row: 0,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [10.0, 546.0, 200.0, 560.0],
-            },
-            StructuredCell {
-                row: 1,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [10.0, 566.0, 200.0, 580.0],
-            },
-            StructuredCell {
-                row: 2,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [10.0, 586.0, 200.0, 600.0],
-            },
-        ];
-        let page_h = 800.0;
-
-        let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
-        for cell in &mut cells {
-            let [x1, y1, x2, y2] = cell.page_pt_bbox;
-            let bounds = region_bounds(x1, y1, x2, y2, page_h, RegionCoordSpace::Standard);
-            let mut matched: Vec<TextItem> = Vec::new();
-            for (i, item) in items.iter().enumerate() {
-                if tsr_region_contains_item(item, bounds) {
-                    claimed.insert(i);
-                    matched.push(item.clone());
-                }
-            }
-            cell.text = collect_text_from_matched_items(matched, 0.10).replace(['\n', '\r'], " ");
-        }
-        // All three cells empty after stage 1 (text falls just below each).
-        for c in &cells {
-            assert!(
-                c.text.is_empty(),
-                "stage 1 should leave all cells empty: {:?}",
-                c
-            );
-        }
-
-        tsr_assign_orphan_items(
-            &items,
-            &mut cells,
-            &claimed,
-            page_h,
-            RegionCoordSpace::Standard,
-        );
-        assert_eq!(cells[0].text, "Bellevue");
-        assert_eq!(cells[1].text, "Glenwood");
-        assert_eq!(cells[2].text, "Metro Crossing");
-    }
-
-    #[test]
-    fn stage2_rejects_cross_line_stacking_into_same_cell() {
-        // Two orphans on different rows of the PDF, both equidistant from
-        // the same empty cell. Without the same-line guard they'd both stack
-        // into that cell ("Shawnee Blue Valley Parkway" run-on); the guard
-        // keeps the first orphan and routes the second to the next-nearest
-        // empty cell on its own line.
-        use crate::tables::StructuredCell;
-        // page_h=200. Two empty cells:
-        //   cell X (row 0): top-left y=[100, 110], native [90, 100]
-        //   cell Y (row 1): top-left y=[112, 122], native [78, 88]
-        let mut cells = vec![
-            StructuredCell {
-                row: 0,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [10.0, 100.0, 100.0, 110.0],
-            },
-            StructuredCell {
-                row: 1,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [10.0, 112.0, 100.0, 122.0],
-            },
-        ];
-        // Two orphans, different rows of the PDF (y differs by 14pt = a
-        // full row), both 2pt outside their target cell — both within
-        // cap_y, both equidistant-ish to cell X. Without the same-line
-        // guard they'd both land in X.
-        //   "Shawnee" should belong to cell X (row 0) — center y=98 is
-        //   2pt below X's native min=100.
-        //   "BlueValley" should belong to cell Y (row 1) — center y=84
-        //   is 4pt above Y's native max=88.
-        let items = vec![
-            // Shawnee orphan — closer to X (dy=2) than Y (dy=6 from native min=78).
-            test_item("Shawnee", 30.0, 94.0, 50.0, 8.0),
-            // BlueValley orphan — closer to Y (dy=4) than X (dy=8 from native max=100).
-            test_item("BlueValley", 30.0, 80.0, 60.0, 8.0),
-        ];
-        let claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
-
-        tsr_assign_orphan_items(
-            &items,
-            &mut cells,
-            &claimed,
-            200.0,
-            RegionCoordSpace::Standard,
-        );
-        assert_eq!(cells[0].text, "Shawnee");
-        assert_eq!(cells[1].text, "BlueValley");
-        assert!(!cells[0].text.contains("BlueValley"));
-        assert!(!cells[1].text.contains("Shawnee"));
-    }
-
-    #[test]
-    fn stage2_allows_same_line_orphans_to_stack_into_one_cell() {
-        // Multi-token branch names like "Blue Valley Parkway" are 3 PDF
-        // text items at the SAME y-coordinate. They should all stack into
-        // the cell their row's branch-name belongs to, not get split
-        // across rows by the cross-line guard.
-        use crate::tables::StructuredCell;
-        let mut cells = vec![StructuredCell {
-            row: 0,
-            col: 0,
-            rowspan: 1,
-            colspan: 1,
-            is_header: false,
-            text: String::new(),
-            page_pt_bbox: [10.0, 100.0, 200.0, 110.0],
-        }];
-        // Three same-line items, all 2pt below the cell's native bottom.
-        let items = vec![
-            test_item("Blue", 30.0, 94.0, 25.0, 8.0),
-            test_item("Valley", 60.0, 94.0, 35.0, 8.0),
-            test_item("Parkway", 100.0, 94.0, 45.0, 8.0),
-        ];
-        let claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
-
-        tsr_assign_orphan_items(
-            &items,
-            &mut cells,
-            &claimed,
-            200.0,
-            RegionCoordSpace::Standard,
-        );
-        // All three same-line orphans stacked into the single empty cell.
-        assert_eq!(cells[0].text, "Blue Valley Parkway");
-    }
-
-    #[test]
-    fn stage2_does_not_overwrite_filled_cells_or_admit_far_orphans() {
-        // Stage 2 must only fill EMPTY cells (preserves stage 1's strict
-        // behavior on bleed cases) and must reject orphans that fall far
-        // outside any cell (prevents pulling a figure title into a table).
-        use crate::tables::StructuredCell;
-        let items = vec![
-            test_item("Real", 50.0, 100.0, 30.0, 8.0),
-            // Far orphan — at native y=20 (page bottom edge) on a page where
-            // the table sits around native y=92..104 (top-left y=96..108).
-            // y-distance to nearest cell is ~70pt, far exceeding the ~12pt
-            // cap from median row height.
-            test_item("FigureTitle", 50.0, 20.0, 60.0, 8.0),
-        ];
-        let mut cells = vec![
-            StructuredCell {
-                row: 0,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: String::new(),
-                page_pt_bbox: [40.0, 96.0, 100.0, 108.0],
-            },
-            StructuredCell {
-                row: 1,
-                col: 0,
-                rowspan: 1,
-                colspan: 1,
-                is_header: false,
-                text: "Pre-filled".to_string(),
-                page_pt_bbox: [40.0, 116.0, 100.0, 128.0],
-            },
-        ];
-        let mut claimed: std::collections::HashSet<usize> = std::collections::HashSet::new();
-        // Pretend "Real" got claimed by a different cell (won't be re-assigned).
-        // Don't claim "FigureTitle" — it's the far orphan.
-        claimed.insert(0);
-
-        tsr_assign_orphan_items(
-            &items,
-            &mut cells,
-            &claimed,
-            200.0,
-            RegionCoordSpace::Standard,
-        );
-        // Empty cell stayed empty (orphan was too far).
-        assert_eq!(cells[0].text, "");
-        // Pre-filled cell was not touched.
-        assert_eq!(cells[1].text, "Pre-filled");
-    }
 }

tests/integration_tests.rs

@@ -1,6 +1,6 @@
 //! Integration tests for pdf-to-markdown library
 
-use pdf_inspector::detector::{estimate_page_count_from_bytes, DetectionConfig, ScanStrategy};
+use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
 use pdf_inspector::extractor::group_into_lines;
 use pdf_inspector::types::TextLine;
 use pdf_inspector::{
@@ -11,83 +11,6 @@ use pdf_inspector::{
 };
 use std::collections::HashSet;
 
-fn make_minimal_text_pdf() -> Vec<u8> {
-    let mut pdf = b"%PDF-1.4\n".to_vec();
-    let mut offsets = vec![0usize];
-
-    fn add_object(pdf: &mut Vec<u8>, offsets: &mut Vec<usize>, id: usize, body: &str) {
-        offsets.push(pdf.len());
-        pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes());
-        pdf.extend_from_slice(body.as_bytes());
-        pdf.extend_from_slice(b"\nendobj\n");
-    }
-
-    add_object(
-        &mut pdf,
-        &mut offsets,
-        1,
-        "<< /Type /Catalog /Pages 2 0 R >>",
-    );
-    add_object(
-        &mut pdf,
-        &mut offsets,
-        2,
-        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
-    );
-    add_object(
-        &mut pdf,
-        &mut offsets,
-        3,
-        "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
-    );
-
-    let content = "BT /F1 12 Tf 100 700 Td (Hello World) Tj 0 -14 Td (Second Line) Tj 0 -14 Td (Third Line) Tj ET";
-    add_object(
-        &mut pdf,
-        &mut offsets,
-        4,
-        &format!(
-            "<< /Length {} >>\nstream\n{}\nendstream",
-            content.len(),
-            content
-        ),
-    );
-    add_object(
-        &mut pdf,
-        &mut offsets,
-        5,
-        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
-    );
-
-    let xref_start = pdf.len();
-    pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes());
-    pdf.extend_from_slice(b"0000000000 65535 f \n");
-    for offset in offsets.iter().skip(1) {
-        pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
-    }
-    pdf.extend_from_slice(
-        format!(
-            "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF",
-            offsets.len(),
-            xref_start
-        )
-        .as_bytes(),
-    );
-
-    pdf
-}
-
-fn truncate_eof_marker(mut pdf: Vec<u8>) -> Vec<u8> {
-    assert!(pdf.ends_with(b"%%EOF"));
-    pdf.pop();
-    pdf
-}
-
-fn add_leading_tab(mut pdf: Vec<u8>) -> Vec<u8> {
-    pdf.insert(0, b'\t');
-    pdf
-}
-
 // Helper to create test TextItems
 fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
     use pdf_inspector::types::ItemType;
@@ -903,73 +826,6 @@ fn test_bom_prefixed_pdf_header_not_rejected() {
     }
 }
 
-#[test]
-fn test_process_pdf_mem_repairs_truncated_eof_marker() {
-    let pdf = truncate_eof_marker(make_minimal_text_pdf());
-
-    let result = process_pdf_mem(&pdf).expect("truncated %%EO marker should be repaired");
-
-    assert_eq!(result.pdf_type, PdfType::TextBased);
-    assert_eq!(result.page_count, 1);
-    assert!(
-        result
-            .markdown
-            .as_deref()
-            .unwrap_or_default()
-            .contains("Hello World"),
-        "repaired PDF should still extract text"
-    );
-}
-
-#[test]
-fn test_process_pdf_mem_repairs_leading_tab_and_truncated_eof() {
-    let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
-
-    let result = process_pdf_mem(&pdf).expect("leading whitespace + %%EO should be repaired");
-
-    assert_eq!(result.pdf_type, PdfType::TextBased);
-    assert_eq!(result.page_count, 1);
-    assert!(
-        result
-            .markdown
-            .as_deref()
-            .unwrap_or_default()
-            .contains("Hello World"),
-        "repaired PDF should still extract text"
-    );
-}
-
-#[test]
-fn test_detect_pdf_type_repairs_container_from_path() {
-    let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
-    let dir = tempfile::tempdir().unwrap();
-    let path = dir.path().join("broken-container.pdf");
-    std::fs::write(&path, pdf).unwrap();
-
-    let result = detect_pdf_type(&path).expect("detector should use shared repair loader");
-
-    assert_eq!(result.pdf_type, PdfType::TextBased);
-    assert_eq!(result.page_count, 1);
-    assert_eq!(result.pages_with_text, 1);
-}
-
-#[test]
-fn test_extract_text_mem_uses_container_repair() {
-    let pdf = truncate_eof_marker(make_minimal_text_pdf());
-
-    let text = pdf_inspector::extractor::extract_text_mem(&pdf)
-        .expect("plain text extraction should use shared repair loader");
-
-    assert!(text.contains("Hello World"));
-}
-
-#[test]
-fn test_estimate_page_count_from_bytes_excludes_pages_tree() {
-    let pdf = add_leading_tab(truncate_eof_marker(make_minimal_text_pdf()));
-
-    assert_eq!(estimate_page_count_from_bytes(&pdf), 1);
-}
-
 #[test]
 fn test_not_a_pdf_detect_pdf_type_mem() {
     // Verify detect_pdf_type_mem is also guarded