Bump version from 1.8.9 to 1.8.10

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.8.12",
+  "version": "1.8.10",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

src/lib.rs

@@ -772,10 +772,6 @@ pub fn extract_tables_in_regions_mem(
                 })
                 .unwrap_or_default();
 
-            // Total length of text the page extractor saw inside this
-            // region, used by the captured-fragment guard below.
-            let region_text_chars: usize = matched.iter().map(|i| i.text.chars().count()).sum();
-
             let evaluate = |t: &tables::Table| -> Option<String> {
                 let md = tables::table_to_markdown(t);
                 let trimmed = md.trim();
@@ -787,24 +783,10 @@ pub fn extract_tables_in_regions_mem(
                     || detect_encoding_issues(&md)
                     || looks_like_partial_table_ex(&md, true)
                 {
-                    return None;
+                    None
+                } else {
+                    Some(md)
                 }
-                // Reject extractions that only captured a small fraction
-                // of the text actually in the region. Two recurring
-                // failure shapes this catches:
-                //   - "header-only": detector found the column-header band
-                //     cleanly but missed every data row below (financial
-                //     statements with multi-line column headers + many
-                //     data rows are the dominant case).
-                //   - "sparse": detector returned a couple of fragmentary
-                //     cells even though the region has many lines of text.
-                // The region floor (200 chars) keeps short legitimate
-                // tables (timestamps, units, axis labels) from being
-                // rejected as partial.
-                if captured_only_a_fragment(&md, region_text_chars) {
-                    return None;
-                }
-                Some(md)
             };
 
             let mut accepted_md: Option<String> = None;
@@ -3626,28 +3608,6 @@ fn is_cid_garbage(text: &str) -> bool {
 /// anymore, only "can we extract it correctly?". Paragraph and duplicate-
 /// header checks stay, since those indicate genuine extraction quality
 /// issues regardless of how the region was identified.
-/// Return true when the captured table markdown represents only a small
-/// fraction of the text the page extractor actually saw inside the
-/// region — typically a header-only band or a sparse fragment where
-/// the detector found valid grid structure but missed most of the
-/// data rows below.
-///
-/// Tuned at a 25% floor: tables that captured at least a quarter of
-/// the region's text are treated as complete-enough. Below 25%, the
-/// caller falls back to `needs_ocr = true` so GLM-OCR can take over.
-/// The 200-char region floor keeps short legitimate tables (units,
-/// axis labels, single-row stat blocks) from being mis-flagged.
-fn captured_only_a_fragment(markdown: &str, region_text_chars: usize) -> bool {
-    if region_text_chars <= 200 {
-        return false;
-    }
-    let captured_text_chars: usize = markdown
-        .chars()
-        .filter(|c| !matches!(c, '|' | '-' | '\n'))
-        .count();
-    captured_text_chars * 4 < region_text_chars
-}
-
 fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
     let lines: Vec<&str> = markdown.lines().filter(|l| l.starts_with('|')).collect();
     if lines.len() < 2 {
@@ -3801,57 +3761,6 @@ fn looks_like_partial_table(markdown: &str) -> bool {
     looks_like_partial_table_ex(markdown, false)
 }
 
-#[cfg(test)]
-mod captured_only_a_fragment_tests {
-    use super::captured_only_a_fragment;
-
-    #[test]
-    fn small_region_skips_check() {
-        // Short legitimate tables (axis labels, unit blocks) shouldn't be
-        // flagged even when the captured markdown is tiny.
-        let md = "|Year|Value|\n|---|---|\n|2024|10|";
-        assert!(!captured_only_a_fragment(md, 50));
-    }
-
-    #[test]
-    fn full_table_passes() {
-        // Captured markdown matches the region text — full extraction.
-        let md =
-            "|Name|Year|Country|\n|---|---|---|\n|Alice|2020|US|\n|Bob|2021|UK|\n|Carol|2019|FR|";
-        // Region had ~50 chars of text (rough estimate of just the data words).
-        assert!(!captured_only_a_fragment(md, 50));
-        // Even a much larger region matched by the markdown content passes.
-        assert!(!captured_only_a_fragment(md, md.len()));
-    }
-
-    #[test]
-    fn header_only_extraction_rejected() {
-        // Captured the column-header band (~30 chars) while the region
-        // actually has many rows of data (~1500 chars).
-        let md = "|Description|Year|Amount|\n|---|---|---|";
-        assert!(captured_only_a_fragment(md, 1500));
-    }
-
-    #[test]
-    fn sparse_fragment_rejected() {
-        // A couple of fragment cells captured from a content-rich region.
-        let md = "|percent|for|\n|---|---|\n|sites|15|";
-        assert!(captured_only_a_fragment(md, 2000));
-    }
-
-    #[test]
-    fn boundary_at_25_percent_floor() {
-        // Right at the 25% line: 250 captured chars of 1000 region chars.
-        // The check rejects when captured*4 < region, so 250*4=1000 is NOT
-        // less than 1000 — boundary is treated as acceptable.
-        let md = "x".repeat(250);
-        assert!(!captured_only_a_fragment(&md, 1000));
-        // Just under 25%: 249*4=996 < 1000 — flagged.
-        let md_under = "x".repeat(249);
-        assert!(captured_only_a_fragment(&md_under, 1000));
-    }
-}
-
 #[cfg(test)]
 mod looks_like_partial_table_tests {
     use super::{looks_like_partial_table, looks_like_partial_table_ex};

src/tables/detect_lines.rs

@@ -4,74 +4,11 @@
 //! gridlines.  Many IRS forms and government PDFs use these instead of
 //! `re` (rectangle) operators.
 
-use std::collections::HashSet;
-
 use crate::tables::Table;
 use crate::types::{PdfLine, TextItem};
 
 use super::detect_rects::{assign_items_to_grid, snap_edges};
 
-/// Derive column edges from the x-endpoints of horizontal-rule
-/// segments when no vertical lines were drawn.
-///
-/// Catalog and archival-finding-aid tables are commonly drawn with
-/// per-row horizontal rules broken into N segments (one segment per
-/// cell), with no vertical dividers at all. The segment break points
-/// (e.g. `[50, 127], [127, 485], [485, 562]` per row) implicitly
-/// encode the column boundaries.
-///
-/// Returns column edges if ≥3 distinct x-positions each show up as a
-/// segment endpoint on ≥50% of the unique horizontal-line rows.
-/// Returns `None` otherwise — decorative rules with varying widths
-/// shouldn't be mistaken for a table.
-fn derive_columns_from_horizontal_segments(horizontals: &[(f32, f32, f32)]) -> Option<Vec<f32>> {
-    if horizontals.len() < 3 {
-        return None;
-    }
-
-    let mut endpoints: Vec<f32> = Vec::with_capacity(horizontals.len() * 2);
-    for &(_, x_min, x_max) in horizontals {
-        endpoints.push(x_min);
-        endpoints.push(x_max);
-    }
-    let clusters = snap_edges(&endpoints, 5.0);
-    if clusters.len() < 3 {
-        return None;
-    }
-
-    // Bucket y-values to count unique rows. Tolerance ~0.1pt (×10
-    // rounding) tolerates the snap_edges 3pt clustering used later
-    // for row edges.
-    let unique_rows: HashSet<i32> = horizontals
-        .iter()
-        .map(|&(y, _, _)| (y * 10.0).round() as i32)
-        .collect();
-    if unique_rows.len() < 2 {
-        return None;
-    }
-    let min_rows = (unique_rows.len() as f32 * 0.5).ceil() as usize;
-
-    let qualifying: Vec<f32> = clusters
-        .iter()
-        .copied()
-        .filter(|&cluster_x| {
-            let rows_touched: HashSet<i32> = horizontals
-                .iter()
-                .filter(|&&(_, x_min, x_max)| {
-                    (x_min - cluster_x).abs() < 5.0 || (x_max - cluster_x).abs() < 5.0
-                })
-                .map(|&(y, _, _)| (y * 10.0).round() as i32)
-                .collect();
-            rows_touched.len() >= min_rows
-        })
-        .collect();
-
-    if qualifying.len() < 3 {
-        return None;
-    }
-    Some(qualifying)
-}
-
 /// Detect tables from line segments on a given page.
 ///
 /// Lines are classified as horizontal or vertical, snapped into grid edges,
@@ -115,50 +52,25 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
         // Diagonal lines are ignored
     }
 
-    if horizontals.len() < 3 {
+    if horizontals.len() < 3 || verticals.len() < 2 {
         return Vec::new();
     }
 
-    // If no/very-few vertical lines are drawn, try to derive column edges
-    // from the x-endpoints of the horizontal-rule segments. Catalog and
-    // archival-finding-aid layouts commonly draw each row's horizontal
-    // rule as N segments (one per cell), with no vertical dividers at
-    // all — the segment break points encode the column boundaries.
-    let implicit_col_edges: Option<Vec<f32>> = if verticals.len() < 2 {
-        derive_columns_from_horizontal_segments(&horizontals)
-    } else {
-        None
-    };
-    if verticals.len() < 2 && implicit_col_edges.is_none() {
-        return Vec::new();
-    }
-    let cols_from_segments = implicit_col_edges.is_some();
-
     log::debug!(
-        "detect_lines p{}: {} horiz, {} vert lines (of {} total on page){}",
+        "detect_lines p{}: {} horiz, {} vert lines (of {} total on page)",
         page,
         horizontals.len(),
         verticals.len(),
-        page_lines.len(),
-        if cols_from_segments {
-            " — columns from horizontal segments"
-        } else {
-            ""
-        }
+        page_lines.len()
     );
 
     // Snap Y-values of horizontal lines → row edges
     let h_ys: Vec<f32> = horizontals.iter().map(|(y, _, _)| *y).collect();
     let row_edges = snap_edges(&h_ys, 3.0);
 
-    // Column edges from drawn verticals when present, else from the
-    // horizontal-segment endpoints derived above.
-    let col_edges = if let Some(c) = implicit_col_edges {
-        c
-    } else {
-        let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
-        snap_edges(&v_xs, 3.0)
-    };
+    // Snap X-values of vertical lines → column edges
+    let v_xs: Vec<f32> = verticals.iter().map(|(x, _, _)| *x).collect();
+    let col_edges = snap_edges(&v_xs, 3.0);
 
     log::debug!(
         "detect_lines p{}: {} row edges, {} col edges after snap",
@@ -240,33 +152,24 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32
 
     // Validate vertical lines: at least 2 should span a meaningful height.
     // Full spanning (>30%) is ideal, but accept many shorter lines (>10%)
-    // for tables with partial column separators. Skipped entirely when
-    // columns came from horizontal-segment endpoints — there are no
-    // vertical lines to validate against, and the segment-endpoint
-    // consistency check in `derive_columns_from_horizontal_segments`
-    // is the equivalent guard.
-    let spanning_v = if cols_from_segments {
-        0
-    } else {
-        let s = verticals
-            .iter()
-            .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
-            .count();
-        let p = verticals
-            .iter()
-            .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
-            .count();
-        if s < 2 && p < 4 {
-            log::debug!(
-                "detect_lines p{}: rejected — {} spanning + {} partial V lines",
-                page,
-                s,
-                p
-            );
-            return Vec::new();
-        }
-        s
-    };
+    // for tables with partial column separators.
+    let spanning_v = verticals
+        .iter()
+        .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.3)
+        .count();
+    let partial_v = verticals
+        .iter()
+        .filter(|(_, y_min, y_max)| (y_max - y_min) > table_height * 0.10)
+        .count();
+    if spanning_v < 2 && partial_v < 4 {
+        log::debug!(
+            "detect_lines p{}: rejected — {} spanning + {} partial V lines",
+            page,
+            spanning_v,
+            partial_v
+        );
+        return Vec::new();
+    }
 
     // Row edges need to be in descending order (top of page = higher Y first)
     let mut row_edges_desc = row_edges;
@@ -513,66 +416,6 @@ mod tests {
         assert!(tables.is_empty());
     }
 
-    #[test]
-    fn test_horizontal_segments_only_implicit_columns_accepted() {
-        // Catalog/finding-aid pattern: each row's horizontal rule is
-        // drawn as 3 segments at consistent x-endpoints (50, 127, 485,
-        // 562), with no vertical lines anywhere. The segment break
-        // points must be inferred as column edges.
-        let mut lines = Vec::new();
-        // Slightly uneven row spacing so the chart-gridline rejector
-        // (CV < 0.02) doesn't fire.
-        let row_ys = [80.0_f32, 145.0, 215.0, 280.0, 350.0, 415.0, 485.0];
-        for &y in &row_ys {
-            lines.push(make_hline(y, 50.0, 127.0, 1));
-            lines.push(make_hline(y, 127.0, 485.0, 1));
-            lines.push(make_hline(y, 485.0, 562.0, 1));
-        }
-        // Populate every cell so capture / density checks pass.
-        let mut items = Vec::new();
-        for w in row_ys.windows(2) {
-            let row_y = (w[0] + w[1]) / 2.0;
-            items.push(make_item("id", 80.0, row_y, 1));
-            items.push(make_item("description here", 200.0, row_y, 1));
-            items.push(make_item("date", 510.0, row_y, 1));
-        }
-        let tables = detect_tables_from_lines(&items, &lines, 1);
-        assert_eq!(
-            tables.len(),
-            1,
-            "horizontal-segment-only grid should be accepted"
-        );
-        let t = &tables[0];
-        assert!(
-            t.cells.len() >= 4,
-            "expected ≥4 rows, got {}",
-            t.cells.len()
-        );
-        assert_eq!(t.cells[0].len(), 3, "expected 3 columns");
-    }
-
-    #[test]
-    fn test_horizontal_segments_with_inconsistent_endpoints_rejected() {
-        // Decorative rules of varying widths shouldn't be detected as a
-        // table — each line has its own x-endpoints, no consistent
-        // column boundary survives the 50%-of-rows threshold.
-        let lines = vec![
-            make_hline(100.0, 50.0, 150.0, 1),
-            make_hline(200.0, 50.0, 220.0, 1),
-            make_hline(300.0, 50.0, 310.0, 1),
-            make_hline(400.0, 50.0, 470.0, 1),
-        ];
-        let items = vec![
-            make_item("decorative", 100.0, 150.0, 1),
-            make_item("text", 100.0, 250.0, 1),
-        ];
-        let tables = detect_tables_from_lines(&items, &lines, 1);
-        assert!(
-            tables.is_empty(),
-            "varying-width decorative rules should not be detected"
-        );
-    }
-
     #[test]
     fn test_page_spanning_bare_frame_rejected() {
         // Just an outer A4-sized rectangle: 2 horizontals + 2 verticals.