extractPagesMarkdown: return classification metadata (0.7.0) (#32)

Combine per-page markdown extraction with layout classification into a
single parse. extractPagesMarkdown now returns PagesExtractionResult with
pages_with_tables, pages_with_columns, pages_needing_ocr, and is_complex
alongside the per-page markdown — eliminating redundant PDF parses for
callers that need both.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

README.md

@@ -22,7 +22,7 @@ Evaluated on the [opendataloader-bench](https://github.com/opendataloader-projec
 
 | Engine | Overall | Reading Order (NID) | Tables (TEDS) | Headings (MHS) | Speed (200 docs) |
 |---|---|---|---|---|---|
-| pdf-inspector | 0.77 | 0.87 | 0.52 | 0.58 | 4s |
+| pdf-inspector | 0.78 | 0.87 | 0.59 | 0.57 | 4s |
 | opendataloader | 0.84 | 0.91 | 0.49 | 0.74 | 11s |
 | pymupdf4llm | 0.73 | 0.89 | 0.40 | 0.41 | 18s |
 | markitdown | 0.58 | 0.88 | 0.00 | 0.00 | 8s |

napi/Cargo.lock

@@ -129,12 +129,6 @@ version = "3.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
 
-[[package]]
-name = "bytecount"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
-
 [[package]]
 name = "cbc"
 version = "0.1.2"
@@ -679,7 +673,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 [[package]]
 name = "lopdf"
 version = "0.40.0"
-source = "git+https://github.com/J-F-Liu/lopdf?rev=052674053814a9f4897af94f0b8e46a545c9b329#052674053814a9f4897af94f0b8e46a545c9b329"
+source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284"
 dependencies = [
  "aes",
  "bitflags",
@@ -695,7 +689,6 @@ dependencies = [
  "log",
  "md-5",
  "nom",
- "nom_locate",
  "rand",
  "rangemap",
  "rayon",
@@ -807,17 +800,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "nom_locate"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"
-dependencies = [
- "bytecount",
- "memchr",
- "nom",
-]
-
 [[package]]
 name = "num-conv"
 version = "0.2.1"

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl-pdf-inspector",
-  "version": "0.3.3",
+  "version": "0.7.0",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -5,6 +5,28 @@ use napi_derive::napi;
 use std::collections::HashSet;
 use std::panic;
 
+// ---------------------------------------------------------------------------
+// Enums
+// ---------------------------------------------------------------------------
+
+/// PDF document type classification.
+#[napi(string_enum)]
+pub enum PdfType {
+    TextBased,
+    Scanned,
+    ImageBased,
+    Mixed,
+}
+
+/// Type of a positioned text item.
+#[napi(string_enum)]
+pub enum ItemType {
+    Text,
+    Image,
+    Link,
+    FormField,
+}
+
 // ---------------------------------------------------------------------------
 // Result types
 // ---------------------------------------------------------------------------
@@ -12,7 +34,7 @@ use std::panic;
 /// Full PDF processing result with markdown and metadata.
 #[napi(object)]
 pub struct PdfResult {
-    pub pdf_type: String,
+    pub pdf_type: PdfType,
     pub markdown: Option<String>,
     pub page_count: u32,
     pub processing_time_ms: u32,
@@ -29,7 +51,7 @@ pub struct PdfResult {
 /// Lightweight PDF classification result.
 #[napi(object)]
 pub struct PdfClassification {
-    pub pdf_type: String,
+    pub pdf_type: PdfType,
     pub page_count: u32,
     /// 0-indexed page numbers that need OCR.
     pub pages_needing_ocr: Vec<u32>,
@@ -49,7 +71,9 @@ pub struct TextItem {
     pub page: u32,
     pub is_bold: bool,
     pub is_italic: bool,
-    pub item_type: String,
+    pub item_type: ItemType,
+    /// URL for link items, `None` for other types.
+    pub link_url: Option<String>,
 }
 
 /// A page's regions for text extraction: (page_index_0based, bboxes).
@@ -79,18 +103,18 @@ pub struct PageRegionTexts {
 // Helpers
 // ---------------------------------------------------------------------------
 
-fn pdf_type_string(t: pdf_inspector::PdfType) -> String {
+fn convert_pdf_type(t: pdf_inspector::PdfType) -> PdfType {
     match t {
-        pdf_inspector::PdfType::TextBased => "TextBased".to_string(),
-        pdf_inspector::PdfType::Scanned => "Scanned".to_string(),
-        pdf_inspector::PdfType::ImageBased => "ImageBased".to_string(),
-        pdf_inspector::PdfType::Mixed => "Mixed".to_string(),
+        pdf_inspector::PdfType::TextBased => PdfType::TextBased,
+        pdf_inspector::PdfType::Scanned => PdfType::Scanned,
+        pdf_inspector::PdfType::ImageBased => PdfType::ImageBased,
+        pdf_inspector::PdfType::Mixed => PdfType::Mixed,
     }
 }
 
 fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
     PdfResult {
-        pdf_type: pdf_type_string(r.pdf_type),
+        pdf_type: convert_pdf_type(r.pdf_type),
         markdown: r.markdown,
         page_count: r.page_count,
         processing_time_ms: r.processing_time_ms as u32,
@@ -104,12 +128,12 @@ fn to_napi_result(r: pdf_inspector::PdfProcessResult) -> PdfResult {
     }
 }
 
-fn item_type_string(t: &pdf_inspector::types::ItemType) -> String {
+fn convert_item_type(t: &pdf_inspector::types::ItemType) -> (ItemType, Option<String>) {
     match t {
-        pdf_inspector::types::ItemType::Text => "text".into(),
-        pdf_inspector::types::ItemType::Image => "image".into(),
-        pdf_inspector::types::ItemType::Link(url) => format!("link:{url}"),
-        pdf_inspector::types::ItemType::FormField => "form_field".into(),
+        pdf_inspector::types::ItemType::Text => (ItemType::Text, None),
+        pdf_inspector::types::ItemType::Image => (ItemType::Image, None),
+        pdf_inspector::types::ItemType::Link(url) => (ItemType::Link, Some(url.clone())),
+        pdf_inspector::types::ItemType::FormField => (ItemType::FormField, None),
     }
 }
 
@@ -181,7 +205,7 @@ pub fn classify_pdf(buffer: Buffer) -> Result<PdfClassification> {
         let result =
             pdf_inspector::classify_pdf_mem(&bytes).map_err(|e| to_napi_err(e, "classify_pdf"))?;
         Ok(PdfClassification {
-            pdf_type: pdf_type_string(result.pdf_type),
+            pdf_type: convert_pdf_type(result.pdf_type),
             page_count: result.page_count,
             pages_needing_ocr: result.pages_needing_ocr,
             confidence: result.confidence as f64,
@@ -222,18 +246,22 @@ pub fn extract_text_with_positions(
 
         Ok(items
             .into_iter()
-            .map(|item| TextItem {
-                text: item.text,
-                x: item.x as f64,
-                y: item.y as f64,
-                width: item.width as f64,
-                height: item.height as f64,
-                font: item.font,
-                font_size: item.font_size as f64,
-                page: item.page,
-                is_bold: item.is_bold,
-                is_italic: item.is_italic,
-                item_type: item_type_string(&item.item_type),
+            .map(|item| {
+                let (item_type, link_url) = convert_item_type(&item.item_type);
+                TextItem {
+                    text: item.text,
+                    x: item.x as f64,
+                    y: item.y as f64,
+                    width: item.width as f64,
+                    height: item.height as f64,
+                    font: item.font,
+                    font_size: item.font_size as f64,
+                    page: item.page,
+                    is_bold: item.is_bold,
+                    is_italic: item.is_italic,
+                    item_type,
+                    link_url,
+                }
             })
             .collect())
     })
@@ -255,7 +283,101 @@ pub fn extract_text_in_regions(
     page_regions: Vec<PageRegions>,
 ) -> Result<Vec<PageRegionTexts>> {
     let bytes: Vec<u8> = buffer.to_vec();
-    let regions: Vec<(u32, Vec<[f32; 4]>)> = page_regions
+    let regions = parse_page_regions(&page_regions);
+
+    catch_panic("extract_text_in_regions", move || {
+        let results = pdf_inspector::extract_text_in_regions_mem(&bytes, &regions)
+            .map_err(|e| to_napi_err(e, "extract_text_in_regions"))?;
+        Ok(to_page_region_texts(results))
+    })
+}
+
+/// Extract markdown tables within bounding-box regions from a PDF.
+///
+/// Like `extractTextInRegions` but runs table detection on items within each
+/// region and returns markdown pipe-tables instead of flat text.
+///
+/// When table structure is detected, `text` contains a markdown pipe-table and
+/// `needsOcr` is `false`. When no table is found, `text` is empty and
+/// `needsOcr` is `true` so the caller can fall back to GPU OCR.
+///
+/// Coordinates are PDF points with top-left origin.
+#[napi]
+pub fn extract_tables_in_regions(
+    buffer: Buffer,
+    page_regions: Vec<PageRegions>,
+) -> Result<Vec<PageRegionTexts>> {
+    let bytes: Vec<u8> = buffer.to_vec();
+    let regions = parse_page_regions(&page_regions);
+
+    catch_panic("extract_tables_in_regions", move || {
+        let results = pdf_inspector::extract_tables_in_regions_mem(&bytes, &regions)
+            .map_err(|e| to_napi_err(e, "extract_tables_in_regions"))?;
+        Ok(to_page_region_texts(results))
+    })
+}
+
+/// Per-page markdown extraction result.
+#[napi(object)]
+pub struct PageMarkdownResult {
+    /// 0-indexed page number.
+    pub page: u32,
+    /// Formatted markdown for this page.
+    pub markdown: String,
+    /// `true` when text on this page is unreliable.
+    pub needs_ocr: bool,
+}
+
+/// Combined per-page markdown extraction and layout classification result.
+#[napi(object)]
+pub struct PagesExtractionResult {
+    /// Per-page markdown results.
+    pub pages: Vec<PageMarkdownResult>,
+    /// 1-indexed pages where tables were detected.
+    pub pages_with_tables: Vec<u32>,
+    /// 1-indexed pages where multi-column layout was detected.
+    pub pages_with_columns: Vec<u32>,
+    /// 1-indexed pages that need OCR (scanned/image-based).
+    pub pages_needing_ocr: Vec<u32>,
+    /// True if any page has tables or columns.
+    pub is_complex: bool,
+}
+
+/// Extract formatted markdown for specific pages of a PDF, with layout
+/// classification metadata.
+///
+/// Returns per-page markdown and classification data (tables, columns,
+/// OCR needs) from a single parse. Font statistics are computed from the
+/// full document so header detection is consistent across pages.
+#[napi]
+pub fn extract_pages_markdown(
+    buffer: Buffer,
+    pages: Vec<u32>,
+) -> Result<PagesExtractionResult> {
+    let bytes: Vec<u8> = buffer.to_vec();
+    catch_panic("extract_pages_markdown", move || {
+        let result = pdf_inspector::extract_pages_markdown_mem(&bytes, &pages)
+            .map_err(|e| to_napi_err(e, "extract_pages_markdown"))?;
+        Ok(PagesExtractionResult {
+            pages: result
+                .pages
+                .into_iter()
+                .map(|r| PageMarkdownResult {
+                    page: r.page,
+                    markdown: r.markdown,
+                    needs_ocr: r.needs_ocr,
+                })
+                .collect(),
+            pages_with_tables: result.pages_with_tables,
+            pages_with_columns: result.pages_with_columns,
+            pages_needing_ocr: result.pages_needing_ocr,
+            is_complex: result.is_complex,
+        })
+    })
+}
+
+fn parse_page_regions(page_regions: &[PageRegions]) -> Vec<(u32, Vec<[f32; 4]>)> {
+    page_regions
         .iter()
         .map(|pr| {
             let bboxes: Vec<[f32; 4]> = pr
@@ -271,25 +393,22 @@ pub fn extract_text_in_regions(
                 .collect();
             (pr.page, bboxes)
         })
-        .collect();
+        .collect()
+}
 
-    catch_panic("extract_text_in_regions", move || {
-        let results = pdf_inspector::extract_text_in_regions_mem(&bytes, &regions)
-            .map_err(|e| to_napi_err(e, "extract_text_in_regions"))?;
-
-        Ok(results
-            .into_iter()
-            .map(|page_result| PageRegionTexts {
-                page: page_result.page,
-                regions: page_result
-                    .regions
-                    .into_iter()
-                    .map(|r| RegionText {
-                        text: r.text,
-                        needs_ocr: r.needs_ocr,
-                    })
-                    .collect(),
-            })
-            .collect())
-    })
+fn to_page_region_texts(results: Vec<pdf_inspector::PageRegionResult>) -> Vec<PageRegionTexts> {
+    results
+        .into_iter()
+        .map(|page_result| PageRegionTexts {
+            page: page_result.page,
+            regions: page_result
+                .regions
+                .into_iter()
+                .map(|r| RegionText {
+                    text: r.text,
+                    needs_ocr: r.needs_ocr,
+                })
+                .collect(),
+        })
+        .collect()
 }

src/detector.rs

@@ -598,7 +598,15 @@ fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool {
         if font_dict.get(b"ToUnicode").is_ok() {
             continue;
         }
-        // Identity-H/V without ToUnicode — flag it
+
+        // Check if fallback decoding paths can handle this font.
+        // The extraction pipeline tries: TrueType cmap → CIDSystemInfo → passthrough.
+        // If any of these would succeed, the font is decodable — don't flag it.
+        if identity_h_font_has_fallback(font_dict, doc) {
+            continue;
+        }
+
+        // Identity-H/V without ToUnicode and no fallback — flag it
         log::debug!(
             "page has Identity-H/V font without ToUnicode: {:?}",
             font_dict
@@ -612,6 +620,102 @@ fn page_has_identity_h_no_tounicode(doc: &Document, page_id: ObjectId) -> bool {
     false
 }
 
+/// Check whether an Identity-H font without ToUnicode can still be decoded
+/// via one of the extraction pipeline's fallback paths.
+fn identity_h_font_has_fallback(font_dict: &lopdf::Dictionary, doc: &Document) -> bool {
+    let desc_fonts_obj = match font_dict.get(b"DescendantFonts").ok() {
+        Some(obj) => obj,
+        None => return false,
+    };
+    let desc_fonts = match desc_fonts_obj {
+        Object::Array(arr) => arr,
+        Object::Reference(r) => match doc.get_object(*r) {
+            Ok(Object::Array(arr)) => arr,
+            _ => return false,
+        },
+        _ => return false,
+    };
+    if desc_fonts.is_empty() {
+        return false;
+    }
+    let cid_font_dict = match &desc_fonts[0] {
+        Object::Reference(r) => match doc.get_dictionary(*r) {
+            Ok(d) => d,
+            _ => return false,
+        },
+        Object::Dictionary(d) => d,
+        _ => return false,
+    };
+
+    // Fallback 1: W array CIDs look like Unicode codepoints → passthrough works.
+    // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H where CID = Unicode.
+    if crate::tounicode::cid_values_look_like_unicode(cid_font_dict) {
+        return true;
+    }
+
+    // Fallback 2: Embedded TrueType/OpenType font has a usable cmap table.
+    if let Some(font_descriptor) = cid_font_dict
+        .get(b"FontDescriptor")
+        .ok()
+        .and_then(|o| match o {
+            Object::Reference(r) => doc.get_dictionary(*r).ok(),
+            Object::Dictionary(d) => Some(d),
+            _ => None,
+        })
+    {
+        let font_file_ref = font_descriptor
+            .get(b"FontFile2")
+            .ok()
+            .and_then(|o| o.as_reference().ok())
+            .or_else(|| {
+                font_descriptor
+                    .get(b"FontFile3")
+                    .ok()
+                    .and_then(|o| o.as_reference().ok())
+            });
+        if let Some(ff_ref) = font_file_ref {
+            if embedded_font_has_cmap(doc, ff_ref) {
+                return true;
+            }
+        }
+    }
+
+    false
+}
+
+/// Quick check whether an embedded TrueType/OpenType font has a cmap table
+/// that can map GIDs to Unicode codepoints.
+fn embedded_font_has_cmap(doc: &Document, font_ref: lopdf::ObjectId) -> bool {
+    let stream = match doc.get_object(font_ref).and_then(Object::as_stream) {
+        Ok(s) => s,
+        Err(_) => return false,
+    };
+    let data = match stream.decompressed_content() {
+        Ok(d) => d,
+        Err(_) => return false,
+    };
+    let face = match ttf_parser::Face::parse(&data, 0) {
+        Ok(f) => f,
+        Err(_) => return false,
+    };
+    // Check that the font has a cmap table with at least some Unicode mappings
+    if let Some(cmap) = face.tables().cmap {
+        for subtable in cmap.subtables {
+            if subtable.is_unicode()
+                || (subtable.platform_id == ttf_parser::PlatformId::Windows
+                    && subtable.encoding_id == 0)
+            {
+                let mut count = 0u32;
+                subtable.codepoints(|_| count += 1);
+                if count > 0 {
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
+
 /// Returns true if every font on the page is Type3 (no normal text fonts).
 /// Type3 fonts render glyphs as custom drawings/bitmaps. Without a ToUnicode
 /// CMap, character codes can't be mapped to Unicode — the page needs OCR.
@@ -730,7 +834,7 @@ fn scan_content_for_text_operators(
     unique_chars: &mut HashSet<u8>,
 ) -> (u32, u32, u32, u32) {
     let mut text_ops = 0u32;
-    let mut image_count = 0u32;
+    let image_count = 0u32;
     let mut path_ops = 0u32;
     let mut font_changes = 0u32;
 
@@ -771,14 +875,10 @@ fn scan_content_for_text_operators(
             }
         }
 
-        // Look for 'Do' operator (XObject/image placement)
-        if b == b'D'
-            && i + 1 < content.len()
-            && content[i + 1] == b'o'
-            && (i + 2 >= content.len() || content[i + 2].is_ascii_whitespace())
-        {
-            image_count += 1;
-        }
+        // Note: We do NOT count 'Do' operators here because Do invokes any
+        // XObject — including Form XObjects that contain text.  Actual image
+        // detection is handled by scan_xobjects_in_resources (checks Subtype)
+        // and analyze_page_images (measures pixel area).
 
         // Count path construction/painting operators.
         // Single-byte: m (moveto), l (lineto), c (curveto), h (closepath),
@@ -1185,23 +1285,24 @@ mod tests {
         // H, e, l, o = 4 unique
         assert!(uchars.len() >= 4);
 
-        // Content with Do (image)
+        // Content with Do (XObject invocation — not counted as image here;
+        // actual image detection is handled by scan_xobjects_in_resources)
         uchars.clear();
         let content3 = b"q 100 0 0 100 50 700 cm /Img1 Do Q";
         let (ops3, imgs3, _, _) = scan_content_for_text_operators(content3, &mut uchars);
         assert_eq!(ops3, 0);
-        assert_eq!(imgs3, 1);
+        assert_eq!(imgs3, 0);
     }
 
     #[test]
     fn test_image_dominated_detection() {
-        // Simulate a page with many Do operators and minimal text
+        // Do operators are no longer counted as images by scan_content_for_text_operators.
+        // Image-dominated detection now relies on scan_xobjects_in_resources which
+        // checks XObject Subtype. Here we verify that Do operators don't inflate image_count.
         let mut content = Vec::new();
-        // Add 50 Do operators (image-heavy)
         for i in 0..50 {
             content.extend_from_slice(format!("/Im{i} Do\n").as_bytes());
         }
-        // Add a few text operators with only a bullet char
         content.extend_from_slice(b"BT (x) Tj ET\n");
         content.extend_from_slice(b"BT (x) Tj ET\n");
         content.extend_from_slice(b"BT (x) Tj ET\n");
@@ -1209,15 +1310,8 @@ mod tests {
         let mut uchars = HashSet::new();
         let (ops, imgs, _, _) = scan_content_for_text_operators(&content, &mut uchars);
         assert_eq!(ops, 3);
-        assert_eq!(imgs, 50);
-        // Only 'x' unique char
+        assert_eq!(imgs, 0); // Do operators are not counted here
         assert_eq!(uchars.len(), 1);
-
-        // This should be image-dominated: 50 > 10 && 50 > 3*3=9
-        let is_image_dominated = imgs > 10 && imgs > ops * 3;
-        assert!(is_image_dominated);
-        // And fails unique char threshold
-        assert!(uchars.len() < 5);
     }
 
     #[test]
@@ -1227,12 +1321,9 @@ mod tests {
         let mut uchars = HashSet::new();
         let (ops, imgs, _, _) = scan_content_for_text_operators(content, &mut uchars);
         assert_eq!(ops, 1);
-        assert_eq!(imgs, 2);
-        // Many unique chars from the sentence
+        assert_eq!(imgs, 0); // Do operators not counted here
+                             // Many unique chars from the sentence
         assert!(uchars.len() >= 5);
-        // Not image-dominated: 2 > 10 fails
-        let is_image_dominated = imgs > 10 && imgs > ops * 3;
-        assert!(!is_image_dominated);
     }
 
     #[test]
@@ -1396,6 +1487,123 @@ mod tests {
         assert!(!page_has_identity_h_no_tounicode(&doc, page_id));
     }
 
+    #[test]
+    fn test_identity_h_with_unicode_cids_not_flagged() {
+        // Type0 Identity-H font without ToUnicode but with W array CIDs
+        // that look like Unicode codepoints (e.g. from Chromium/wkhtmltopdf).
+        // The CID-as-Unicode passthrough can decode these — don't flag.
+        use lopdf::dictionary;
+        let mut doc = Document::with_version("1.4");
+        let pages_id = doc.new_object_id();
+        let page_id = doc.new_object_id();
+        // CIDFont with W array containing Unicode-range CIDs (>= 0x41)
+        let cid_font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"CIDFontType2".to_vec()),
+            "W" => Object::Array(vec![
+                Object::Integer(0x41),  // CID 65 = 'A'
+                Object::Array(vec![
+                    Object::Integer(600), Object::Integer(600), Object::Integer(600),
+                ]),
+                Object::Integer(0x61),  // CID 97 = 'a'
+                Object::Array(vec![
+                    Object::Integer(500), Object::Integer(500), Object::Integer(500),
+                ]),
+            ]),
+        });
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"Type0".to_vec()),
+            "BaseFont" => Object::Name(b"ABCDEF+ArialMT".to_vec()),
+            "Encoding" => Object::Name(b"Identity-H".to_vec()),
+            "DescendantFonts" => Object::Array(vec![Object::Reference(cid_font_id)]),
+        });
+        let resources = dictionary! {
+            "Font" => dictionary! {
+                "F1" => Object::Reference(font_id),
+            },
+        };
+        doc.objects.insert(
+            page_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Page",
+                "Parent" => Object::Reference(pages_id),
+                "Resources" => resources,
+            }),
+        );
+        doc.objects.insert(
+            pages_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Pages",
+                "Kids" => vec![Object::Reference(page_id)],
+                "Count" => Object::Integer(1),
+            }),
+        );
+        assert!(
+            !page_has_identity_h_no_tounicode(&doc, page_id),
+            "Should NOT flag: W array CIDs look like Unicode, passthrough works"
+        );
+    }
+
+    #[test]
+    fn test_identity_h_with_low_gid_cids_still_flagged() {
+        // Type0 Identity-H font without ToUnicode and W array CIDs
+        // that are low GID values (subset font, no cmap). These can't
+        // be decoded — should still be flagged.
+        use lopdf::dictionary;
+        let mut doc = Document::with_version("1.4");
+        let pages_id = doc.new_object_id();
+        let page_id = doc.new_object_id();
+        // CIDFont with W array containing low GID values (< 0x41)
+        let cid_font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"CIDFontType2".to_vec()),
+            "W" => Object::Array(vec![
+                Object::Integer(3),  // Low GID
+                Object::Array(vec![
+                    Object::Integer(600), Object::Integer(600), Object::Integer(600),
+                    Object::Integer(600), Object::Integer(600),
+                ]),
+                Object::Integer(10),  // Still low
+                Object::Array(vec![
+                    Object::Integer(500), Object::Integer(500), Object::Integer(500),
+                ]),
+            ]),
+        });
+        let font_id = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => Object::Name(b"Type0".to_vec()),
+            "BaseFont" => Object::Name(b"GPBCHP+TimesNewRoman".to_vec()),
+            "Encoding" => Object::Name(b"Identity-H".to_vec()),
+            "DescendantFonts" => Object::Array(vec![Object::Reference(cid_font_id)]),
+        });
+        let resources = dictionary! {
+            "Font" => dictionary! {
+                "F1" => Object::Reference(font_id),
+            },
+        };
+        doc.objects.insert(
+            page_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Page",
+                "Parent" => Object::Reference(pages_id),
+                "Resources" => resources,
+            }),
+        );
+        doc.objects.insert(
+            pages_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Pages",
+                "Kids" => vec![Object::Reference(page_id)],
+                "Count" => Object::Integer(1),
+            }),
+        );
+        assert!(
+            page_has_identity_h_no_tounicode(&doc, page_id),
+            "Should flag: low GID CIDs, no cmap, no passthrough"
+        );
+    }
+
     #[test]
     fn test_scan_content_counts_tf_operators() {
         let mut uchars = HashSet::new();

src/extractor/layout.rs

@@ -164,6 +164,10 @@ pub(crate) fn detect_columns(
                 }
             }
         }
+        // Try XY-cut fallback before giving up
+        if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
+            return columns;
+        }
         return vec![ColumnRegion { x_min, x_max }];
     }
 
@@ -184,7 +188,7 @@ pub(crate) fn detect_columns(
     if result.len() > 1 {
         return result;
     }
-    return validate_and_build_columns(
+    let result = validate_and_build_columns(
         &valleys,
         &page_items,
         x_min,
@@ -195,6 +199,161 @@ pub(crate) fn detect_columns(
         page,
         false, // edge-based fallback
     );
+    if result.len() > 1 {
+        return result;
+    }
+
+    // Fallback: XY-cut style gap detection.  When the histogram finds no
+    // clear valleys (common with asymmetric/sidebar layouts), look for the
+    // largest horizontal gap between item edges.  This is a simplified
+    // single-level XY-cut inspired by opendataloader's XY-Cut++ algorithm.
+    if page_items.len() >= 20 && !page_has_table {
+        if let Some(columns) = try_xy_cut_split(&page_items, x_min, x_max, page) {
+            return columns;
+        }
+    }
+
+    vec![ColumnRegion { x_min, x_max }]
+}
+
+/// Simplified single-level XY-cut: find the largest horizontal gap between
+/// item right-edges and left-edges.  If the gap is wide enough and both sides
+/// have sufficient items with vertical overlap, split into two columns.
+///
+/// Inspired by opendataloader's XY-Cut++ algorithm but without full recursion.
+/// Handles asymmetric layouts (sidebars) that the histogram misses because
+/// the narrow column has too few items to register in the occupancy profile.
+fn try_xy_cut_split(
+    page_items: &[&TextItem],
+    page_x_min: f32,
+    page_x_max: f32,
+    page: u32,
+) -> Option<Vec<ColumnRegion>> {
+    const MIN_GAP: f32 = 15.0; // minimum gap to consider a split
+    const MIN_ITEMS_MAJOR: usize = 10; // major column must have ≥10 items
+    const MIN_ITEMS_MINOR: usize = 3; // minor column (sidebar) must have ≥3
+
+    let page_width = page_x_max - page_x_min;
+    if page_width < 200.0 {
+        return None;
+    }
+
+    // Collect all item edges: (right_edge, left_edge) pairs sorted by right_edge
+    // The gap between one item's right edge and the next item's left edge
+    // reveals column gutters.
+    let mut edges: Vec<(f32, f32)> = page_items
+        .iter()
+        .map(|i| (i.x, i.x + effective_width(i)))
+        .collect();
+    edges.sort_by(|a, b| a.0.total_cmp(&b.0));
+
+    // Find the largest gap between consecutive items (by left edge).
+    // Use a sweep: sort left edges, find max gap between sorted right edges
+    // of items to the left and left edges of items to the right.
+    let mut left_edges: Vec<f32> = page_items.iter().map(|i| i.x).collect();
+    left_edges.sort_by(|a, b| a.total_cmp(b));
+
+    // Build prefix max of right edges (for items sorted by left edge)
+    let mut sorted_by_left: Vec<(f32, f32)> = page_items
+        .iter()
+        .map(|i| (i.x, i.x + effective_width(i)))
+        .collect();
+    sorted_by_left.sort_by(|a, b| a.0.total_cmp(&b.0));
+
+    let mut best_gap = 0.0f32;
+    let mut best_split = 0.0f32;
+    let mut max_right_so_far = f32::NEG_INFINITY;
+
+    for i in 0..sorted_by_left.len() - 1 {
+        let (_, right) = sorted_by_left[i];
+        max_right_so_far = max_right_so_far.max(right);
+
+        let (next_left, _) = sorted_by_left[i + 1];
+        let gap = next_left - max_right_so_far;
+        if gap > best_gap {
+            best_gap = gap;
+            best_split = (max_right_so_far + next_left) / 2.0;
+        }
+    }
+
+    if best_gap < MIN_GAP {
+        return None;
+    }
+
+    // Don't split at page margins (within 10% of edges)
+    let margin = page_width * 0.10;
+    if best_split - page_x_min < margin || page_x_max - best_split < margin {
+        return None;
+    }
+
+    // Count items on each side
+    let left_count = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 <= best_split)
+        .count();
+    let right_count = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 > best_split)
+        .count();
+
+    let (minor, major) = if left_count <= right_count {
+        (left_count, right_count)
+    } else {
+        (right_count, left_count)
+    };
+
+    if major < MIN_ITEMS_MAJOR || minor < MIN_ITEMS_MINOR {
+        return None;
+    }
+
+    // Check vertical overlap — both sides should span a meaningful Y range
+    let left_items: Vec<&&TextItem> = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 <= best_split)
+        .collect();
+    let right_items: Vec<&&TextItem> = page_items
+        .iter()
+        .filter(|i| i.x + effective_width(i) / 2.0 > best_split)
+        .collect();
+
+    let l_y_min = left_items.iter().map(|i| i.y).fold(f32::INFINITY, f32::min);
+    let l_y_max = left_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::NEG_INFINITY, f32::max);
+    let r_y_min = right_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::INFINITY, f32::min);
+    let r_y_max = right_items
+        .iter()
+        .map(|i| i.y)
+        .fold(f32::NEG_INFINITY, f32::max);
+
+    let overlap_min = l_y_min.max(r_y_min);
+    let overlap_max = l_y_max.min(r_y_max);
+    let overlap = (overlap_max - overlap_min).max(0.0);
+    let y_range = (l_y_max.max(r_y_max) - l_y_min.min(r_y_min)).max(1.0);
+
+    if overlap / y_range < 0.20 {
+        return None;
+    }
+
+    debug!(
+        "page {}: XY-cut split at x={:.1} (gap={:.1}pt, left={}, right={})",
+        page, best_split, best_gap, left_count, right_count
+    );
+
+    Some(vec![
+        ColumnRegion {
+            x_min: page_x_min,
+            x_max: best_split,
+        },
+        ColumnRegion {
+            x_min: best_split,
+            x_max: page_x_max,
+        },
+    ])
 }
 
 /// Check whether each proposed column contains paragraph-like content.

src/lib.rs

@@ -299,6 +299,149 @@ pub fn classify_pdf_mem(buffer: &[u8]) -> Result<PdfClassification, PdfError> {
     })
 }
 
+// =========================================================================
+// Per-page markdown extraction
+// =========================================================================
+
+/// Per-page markdown extraction result.
+#[derive(Debug)]
+pub struct PageMarkdown {
+    /// 0-indexed page number.
+    pub page: u32,
+    /// Formatted markdown for this page.
+    pub markdown: String,
+    /// `true` when text on this page is unreliable (GID-encoded fonts,
+    /// encoding issues, garbage text, or empty extraction).
+    pub needs_ocr: bool,
+}
+
+/// Combined per-page markdown extraction and layout classification result.
+#[derive(Debug)]
+pub struct PagesExtractionResult {
+    /// Per-page markdown results.
+    pub pages: Vec<PageMarkdown>,
+    /// 1-indexed pages where tables were detected.
+    pub pages_with_tables: Vec<u32>,
+    /// 1-indexed pages where multi-column layout was detected.
+    pub pages_with_columns: Vec<u32>,
+    /// 1-indexed pages that need OCR (scanned/image-based).
+    pub pages_needing_ocr: Vec<u32>,
+    /// True if any page has tables or columns.
+    pub is_complex: bool,
+}
+
+/// Extract formatted markdown for specific pages of a PDF, with layout
+/// classification metadata.
+///
+/// Unlike [`process_pdf_mem`] which returns one concatenated markdown string,
+/// this returns per-page markdown so callers can mix direct extraction
+/// (for simple text pages) with GPU OCR (for complex/scanned pages).
+///
+/// Font statistics are computed from the full document so header
+/// detection thresholds are consistent regardless of which pages are
+/// requested. Per-page `needs_ocr` is set when the page has GID-encoded
+/// fonts, encoding issues, or garbage text.
+///
+/// Layout complexity (tables, columns) is computed from the full document
+/// at near-zero cost since the items/rects/lines are already in memory.
+pub fn extract_pages_markdown_mem(
+    buffer: &[u8],
+    pages: &[u32],
+) -> Result<PagesExtractionResult, PdfError> {
+    validate_pdf_bytes(buffer)?;
+    let (doc, page_count) = load_document_from_mem(buffer)?;
+    let font_cmaps = FontCMaps::from_doc(&doc);
+
+    // Extract ALL pages to get accurate, document-wide font stats.
+    let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) =
+        extractor::extract_positioned_text_from_doc(&doc, &font_cmaps, None)?;
+
+    // Compute layout complexity from full document (near-zero cost).
+    let complexity = compute_layout_complexity(&all_items, &all_rects, &all_lines);
+
+    // Compute font stats from full document (cross-page consistency).
+    let font_stats = markdown::analysis::calculate_font_stats_from_items(&all_items);
+
+    let mut results = Vec::with_capacity(pages.len());
+    let mut pages_needing_ocr = Vec::new();
+
+    for &page_0idx in pages {
+        // Out-of-range pages → empty + needs_ocr
+        if page_0idx >= page_count {
+            pages_needing_ocr.push(page_0idx + 1);
+            results.push(PageMarkdown {
+                page: page_0idx,
+                markdown: String::new(),
+                needs_ocr: true,
+            });
+            continue;
+        }
+
+        let page_1idx = page_0idx + 1;
+
+        // Filter items/rects for this page only
+        let page_items: Vec<TextItem> = all_items
+            .iter()
+            .filter(|i| i.page == page_1idx)
+            .cloned()
+            .collect();
+
+        let page_rects: Vec<PdfRect> = all_rects
+            .iter()
+            .filter(|r| r.page == page_1idx)
+            .cloned()
+            .collect();
+
+        let has_gid = gid_pages.contains(&page_1idx);
+
+        // Build markdown with document-wide font stats
+        let options = MarkdownOptions {
+            base_font_size: Some(font_stats.most_common_size),
+            include_page_numbers: false,
+            strip_headers_footers: false,
+            ..MarkdownOptions::default()
+        };
+
+        let md = markdown::to_markdown_from_items_with_rects_and_lines(
+            page_items,
+            options,
+            &page_rects,
+            &[],
+            &page_thresholds,
+            None,
+            &[],
+        );
+
+        let needs_ocr = md.trim().is_empty()
+            || has_gid
+            || is_garbage_text(&md)
+            || is_cid_garbage(&md)
+            || detect_encoding_issues(&md);
+
+        if needs_ocr {
+            pages_needing_ocr.push(page_1idx);
+        }
+
+        results.push(PageMarkdown {
+            page: page_0idx,
+            markdown: if needs_ocr { String::new() } else { md },
+            needs_ocr,
+        });
+    }
+
+    Ok(PagesExtractionResult {
+        pages: results,
+        pages_with_tables: complexity.pages_with_tables,
+        pages_with_columns: complexity.pages_with_columns,
+        pages_needing_ocr,
+        is_complex: complexity.is_complex,
+    })
+}
+
+// =========================================================================
+// Region-based text extraction (for hybrid OCR pipelines)
+// =========================================================================
+
 /// Result for a single region's text extraction.
 #[derive(Debug)]
 pub struct RegionText {
@@ -399,7 +542,7 @@ pub fn extract_text_in_regions_mem(
         let page_1idx = page_0idx + 1;
         let items = items_by_page.get(&page_1idx);
         let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
-        let page_has_gid = gid_pages.contains(&page_1idx);
+        let _page_has_gid = gid_pages.contains(&page_1idx);
         let adaptive_threshold = page_thresholds.get(&page_1idx).copied().unwrap_or(0.10);
         let coords = if rotated_pages.contains(&page_1idx) {
             RegionCoordSpace::Rotated90Ccw
@@ -426,8 +569,10 @@ pub fn extract_text_in_regions_mem(
                 None => String::new(),
             };
 
+            // Check per-region text quality instead of blanket page-level
+            // GID rejection. A GID font in a logo elsewhere on the page
+            // shouldn't force GPU OCR for clean text regions.
             let needs_ocr = text.trim().is_empty()
-                || page_has_gid
                 || is_garbage_text(&text)
                 || is_cid_garbage(&text)
                 || detect_encoding_issues(&text);
@@ -444,6 +589,167 @@ pub fn extract_text_in_regions_mem(
     Ok(results)
 }
 
+/// Extract tables within bounding-box regions from a PDF in memory.
+///
+/// Similar to [`extract_text_in_regions_mem`] but runs table detection on items
+/// within each region and returns markdown pipe-tables instead of flat text.
+///
+/// When table structure is detected, `text` contains a markdown pipe-table and
+/// `needs_ocr` is `false`. When no table is found (too few items, poor alignment,
+/// GID fonts, etc.), `text` is empty and `needs_ocr` is `true` so the caller can
+/// fall back to GPU OCR.
+pub fn extract_tables_in_regions_mem(
+    buffer: &[u8],
+    page_regions: &[(u32, Vec<[f32; 4]>)],
+) -> Result<Vec<PageRegionResult>, PdfError> {
+    validate_pdf_bytes(buffer)?;
+    let (doc, _page_count) = load_document_from_mem(buffer)?;
+    let pages = doc.get_pages();
+
+    let needed_pages: HashSet<u32> = page_regions.iter().map(|(p, _)| p + 1).collect();
+    let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages));
+
+    let mut items_by_page: HashMap<u32, Vec<TextItem>> = HashMap::new();
+    let mut page_heights: HashMap<u32, f32> = HashMap::new();
+    let mut gid_pages: HashSet<u32> = HashSet::new();
+    let mut page_thresholds: HashMap<u32, f32> = HashMap::new();
+    let mut rotated_pages: HashSet<u32> = HashSet::new();
+
+    for (page_num, &page_id) in pages.iter() {
+        if !needed_pages.contains(page_num) {
+            continue;
+        }
+        let height = get_page_height(&doc, page_id).unwrap_or(792.0);
+        page_heights.insert(*page_num, height);
+
+        let ((mut items, _rects, _lines), has_gid, coords_rotated) =
+            extractor::content_stream::extract_page_text_items(
+                &doc,
+                page_id,
+                *page_num,
+                &font_cmaps,
+                false,
+            )?;
+        let threshold = text_utils::fix_letterspaced_items(&mut items);
+        if threshold > 0.10 {
+            page_thresholds.insert(*page_num, threshold);
+        }
+        if has_gid {
+            gid_pages.insert(*page_num);
+        }
+        if coords_rotated {
+            rotated_pages.insert(*page_num);
+        }
+        items_by_page.insert(*page_num, items);
+    }
+
+    let mut results = Vec::with_capacity(page_regions.len());
+
+    for (page_0idx, regions) in page_regions {
+        let page_1idx = page_0idx + 1;
+        let items = items_by_page.get(&page_1idx);
+        let page_h = page_heights.get(&page_1idx).copied().unwrap_or(792.0);
+        let _page_has_gid = gid_pages.contains(&page_1idx);
+        let coords = if rotated_pages.contains(&page_1idx) {
+            RegionCoordSpace::Rotated90Ccw
+        } else {
+            RegionCoordSpace::Standard
+        };
+
+        let mut page_results = Vec::with_capacity(regions.len());
+
+        for rect in regions {
+            let [rx1, ry1, rx2, ry2] = *rect;
+
+            // Note: we intentionally DO NOT bail on page_has_gid here.
+            // The GID flag means some font on the page uses unresolvable
+            // glyph IDs, but that font may only appear in a logo or
+            // header — not in the table region. Instead we let the
+            // per-region text quality checks (is_garbage_text, is_cid_garbage,
+            // detect_encoding_issues) reject based on the actual extracted
+            // content. This avoids rejecting clean tables just because an
+            // unrelated decorative font on the same page is GID-encoded.
+
+            let matched: Vec<TextItem> = match items {
+                Some(items) => {
+                    let bounds = region_bounds(rx1, ry1, rx2, ry2, page_h, coords);
+                    items
+                        .iter()
+                        .filter(|item| region_overlaps_item(item, bounds))
+                        .cloned()
+                        .collect()
+                }
+                None => Vec::new(),
+            };
+
+            if matched.is_empty() {
+                page_results.push(RegionText {
+                    text: String::new(),
+                    needs_ocr: true,
+                });
+                continue;
+            }
+
+            // Compute base_font_size as most common font size in the region
+            let base_font_size = {
+                let mut freq: HashMap<i32, usize> = HashMap::new();
+                for item in &matched {
+                    *freq.entry((item.font_size * 10.0) as i32).or_default() += 1;
+                }
+                freq.into_iter()
+                    .max_by_key(|(_, count)| *count)
+                    .map(|(size, _)| size as f32 / 10.0)
+                    .unwrap_or(12.0)
+            };
+
+            // Run heuristic table detection; skip_body_font = false since
+            // the layout model already identified this region as a table.
+            let detected = tables::detect_tables(&matched, base_font_size, false);
+
+            if let Some(table) = detected.into_iter().next() {
+                let md = tables::table_to_markdown(&table);
+                if md.trim().is_empty() {
+                    page_results.push(RegionText {
+                        text: String::new(),
+                        needs_ocr: true,
+                    });
+                } else {
+                    // needs_ocr fires on any of:
+                    //   - garbage text (non-alphanumeric heavy)
+                    //   - CID/Latin-1 mojibake
+                    //   - encoding issues (U+FFFD, dollar-as-space)
+                    //   - structural giveaways that the table is partial /
+                    //     mis-detected (numeric "header", empty header cells,
+                    //     duplicate header cells). Caught GLM-OCR-as-baseline
+                    //     scoring 0 TEDS on real prod tables in eval.
+                    // Layout model already identified this region as a table,
+                    // so use relaxed partial-table checks (layout_assisted=true).
+                    let needs_ocr = is_garbage_text(&md)
+                        || is_cid_garbage(&md)
+                        || detect_encoding_issues(&md)
+                        || looks_like_partial_table_ex(&md, true);
+                    page_results.push(RegionText {
+                        text: if needs_ocr { String::new() } else { md },
+                        needs_ocr,
+                    });
+                }
+            } else {
+                page_results.push(RegionText {
+                    text: String::new(),
+                    needs_ocr: true,
+                });
+            }
+        }
+
+        results.push(PageRegionResult {
+            page: *page_0idx,
+            regions: page_results,
+        });
+    }
+
+    Ok(results)
+}
+
 /// Get page height in points from MediaBox.
 fn get_page_height(doc: &Document, page_id: lopdf::ObjectId) -> Option<f32> {
     let page_dict = doc.get_dictionary(page_id).ok()?;
@@ -525,9 +831,6 @@ fn collect_text_in_region_with_options(
     adaptive_threshold: f32,
 ) -> String {
     let bounds = region_bounds(rx1, ry1, rx2, ry2, page_height, coord_space);
-    let Some(page) = items.first().map(|item| item.page) else {
-        return String::new();
-    };
     let matched: Vec<TextItem> = items
         .iter()
         .filter(|item| region_overlaps_item(item, bounds))
@@ -536,11 +839,39 @@ fn collect_text_in_region_with_options(
     if matched.is_empty() {
         return String::new();
     }
-    let mut thresholds = HashMap::new();
-    if adaptive_threshold > 0.10 {
-        thresholds.insert(page, adaptive_threshold);
+
+    // Simple extraction: the caller (fire-pdf) already handles reading order
+    // and column splitting via the layout model. We just need to sort items
+    // top-to-bottom, left-to-right and group into lines.
+    let mut sorted = matched;
+    sorted.sort_by(|a, b| b.y.total_cmp(&a.y).then(a.x.total_cmp(&b.x)));
+
+    let y_tolerance = 3.0;
+    let mut lines: Vec<extractor::TextLine> = Vec::new();
+
+    for item in sorted {
+        let should_merge = lines.last().is_some_and(|last_line: &extractor::TextLine| {
+            last_line.page == item.page && (last_line.y - item.y).abs() < y_tolerance
+        });
+        if should_merge {
+            lines.last_mut().unwrap().items.push(item);
+        } else {
+            let y = item.y;
+            let page = item.page;
+            lines.push(extractor::TextLine {
+                items: vec![item],
+                y,
+                page,
+                adaptive_threshold,
+            });
+        }
     }
-    let lines = extractor::group_into_lines_with_thresholds(matched, &thresholds, &HashSet::new());
+
+    // Sort items within each line by X position
+    for line in &mut lines {
+        text_utils::sort_line_items(&mut line.items);
+    }
+
     lines
         .into_iter()
         .map(|line| line.text())
@@ -1016,6 +1347,379 @@ fn is_cid_garbage(text: &str) -> bool {
     high_latin * 5 >= total * 2 && ascii_letters * 3 < total
 }
 
+/// Detect markdown tables with suspicious structure that suggest the heuristic
+/// missed/mangled rows or columns. Returns true when the caller should treat
+/// the result as `needs_ocr` and fall back to GPU OCR.
+///
+/// Catches three failure modes observed in production:
+///
+/// 1. **Header row looks like a data row** — first row starts with a numeric
+///    value (e.g. `|2|...`), suggesting we missed the actual header above it.
+///    Real headers almost never start with a bare number.
+///
+/// 2. **Header has empty cells in a multi-column table** — e.g.
+///    `|Position||Administration|Administration|` (3+ cols, ≥1 empty cell).
+///    Indicates poor column boundary detection.
+///
+/// 3. **Header has duplicate non-empty cells** in a multi-column table —
+///    e.g. `Administration|Administration` appearing as adjacent cells means
+///    we collapsed multi-line headers wrong.
+///
+/// Conservative by design: a few false positives (perfectly fine tables flagged)
+/// just mean we run GPU OCR which is the existing safe path.
+/// When `layout_assisted` is true (the layout model identified this region
+/// as a table), we relax boundary-detection heuristics (numeric header,
+/// empty header cells, sparse first data row) because the layout model
+/// already gave us the table bbox — we're not guessing "is this a table?"
+/// anymore, only "can we extract it correctly?". Paragraph and duplicate-
+/// header checks stay, since those indicate genuine extraction quality
+/// issues regardless of how the region was identified.
+fn looks_like_partial_table_ex(markdown: &str, layout_assisted: bool) -> bool {
+    let lines: Vec<&str> = markdown.lines().filter(|l| l.starts_with('|')).collect();
+    if lines.len() < 2 {
+        return false;
+    }
+    // Header is the first pipe-line; separator is the second
+    let header_line = lines[0];
+    let separator_line = lines.get(1).copied().unwrap_or("");
+    let is_separator = |l: &str| l.chars().all(|c| matches!(c, '|' | '-' | ' '));
+    if !is_separator(separator_line) {
+        // No separator after the first line — not a well-formed pipe-table.
+        // table_to_markdown always emits one when it returns content, so this
+        // shouldn't happen in practice. If it does, fall through to OCR.
+        return true;
+    }
+
+    // Parse header cells: split on '|', drop the leading/trailing empty pieces
+    let cells: Vec<&str> = header_line.split('|').map(|s| s.trim()).collect::<Vec<_>>();
+    // The first and last items are always empty (string starts and ends with '|')
+    if cells.len() < 3 {
+        return false;
+    }
+    let header_cells: Vec<&str> = cells[1..cells.len() - 1].to_vec();
+    let n_cols = header_cells.len();
+    if n_cols < 2 {
+        // Single-column tables are usually lists/keys, not tables. Keep them
+        // (caller can decide), but multi-column header checks below don't
+        // apply.
+        return false;
+    }
+
+    // Failure mode 1: header starts with a bare number (likely we missed
+    // the real header row above). Skip when layout-assisted — the layout
+    // model's bbox includes the real header; a numeric first cell (e.g.,
+    // a year "2024") is legitimate.
+    if !layout_assisted {
+        if let Some(first) = header_cells.first() {
+            let trimmed = first.trim();
+            if !trimmed.is_empty() && trimmed.chars().all(|c| c.is_ascii_digit()) {
+                return true;
+            }
+        }
+    }
+
+    // Failure mode 2: header has empty cells in a multi-column table.
+    // When layout-assisted, allow up to 1 empty header cell (common in
+    // tables with merged/spanning header cells that we can't represent).
+    let empty_count = header_cells.iter().filter(|c| c.is_empty()).count();
+    if layout_assisted {
+        // Reject only if >1 empty header cell (2+ means serious boundary issue)
+        if n_cols >= 3 && empty_count >= 2 {
+            return true;
+        }
+    } else if n_cols >= 3 && empty_count >= 1 {
+        return true;
+    }
+
+    // Failure mode 3: header has duplicate non-empty cells
+    let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
+    for cell in &header_cells {
+        if cell.is_empty() {
+            continue;
+        }
+        if !seen.insert(cell) {
+            return true;
+        }
+    }
+
+    // Failure mode 4: first data row has many empty cells in a multi-column
+    // table. Real tables rarely have a leading row with most cells blank;
+    // when this happens it usually means the heuristic split a multi-row
+    // header (e.g. "Position\nAdministration (1986-1992) | Administration
+    // (1992-1998)") into a single-row header + a sparse data row.
+    if let Some(first_data_line) = lines.get(2) {
+        let data_cells: Vec<&str> = first_data_line
+            .split('|')
+            .map(|s| s.trim())
+            .collect::<Vec<_>>();
+        if data_cells.len() >= 3 {
+            let data_inner = &data_cells[1..data_cells.len() - 1];
+            let empty_data = data_inner.iter().filter(|c| c.is_empty()).count();
+            // ≥3 cols, and significant portion of cells in the first data
+            // row are empty → likely we mis-split a multi-row header.
+            // When layout-assisted, relax from 33% to 50% — the bbox is
+            // more reliable, and real tables with one sparse first row
+            // (totals, subtotals) are common.
+            let threshold = if layout_assisted { 2 } else { 3 };
+            if n_cols >= 3 && empty_data * threshold >= n_cols {
+                return true;
+            }
+        }
+    }
+
+    // Failure mode 5: cells flow as continuation paragraph (text wrapping
+    // mistaken for column structure). When a paragraph of prose gets mis-
+    // detected as a multi-column table, cells in the same column tend to
+    // start with lowercase letters or punctuation (continuation), not
+    // capital letters / digits (new entries). Real tables almost never
+    // have most data cells starting lowercase.
+    //
+    // Signal: ≥2 cols, ≥4 data rows, and ≥60% of non-empty data cells
+    // start with a lowercase letter or continuation punctuation.
+    let data_rows: Vec<Vec<&str>> = lines
+        .iter()
+        .skip(2) // header + separator
+        .map(|l| {
+            let parts: Vec<&str> = l.split('|').map(|s| s.trim()).collect();
+            if parts.len() >= 3 {
+                parts[1..parts.len() - 1].to_vec()
+            } else {
+                Vec::new()
+            }
+        })
+        .filter(|cells| !cells.is_empty())
+        .collect();
+
+    if n_cols >= 2 && data_rows.len() >= 4 {
+        let mut continuation = 0;
+        let mut total = 0;
+        for row in &data_rows {
+            for cell in row {
+                let trimmed = cell.trim();
+                if trimmed.is_empty() {
+                    continue;
+                }
+                total += 1;
+                let first = trimmed.chars().next().unwrap();
+                // Continuation indicators: lowercase letter, common
+                // mid-sentence punctuation, closing quote
+                if first.is_lowercase()
+                    || matches!(first, ',' | '.' | ';' | ')' | '"' | '\'' | '”' | '’')
+                {
+                    continuation += 1;
+                }
+            }
+        }
+        if total > 0 && continuation * 5 >= total * 3 {
+            // ≥60% of cells look like sentence continuations → paragraph
+            // misread as table.
+            return true;
+        }
+    }
+
+    false
+}
+
+/// Original strict validation (no layout assistance). Used by tests and
+/// full-page extraction paths that don't have layout model assistance.
+#[cfg(test)]
+fn looks_like_partial_table(markdown: &str) -> bool {
+    looks_like_partial_table_ex(markdown, false)
+}
+
+#[cfg(test)]
+mod looks_like_partial_table_tests {
+    use super::{looks_like_partial_table, looks_like_partial_table_ex};
+
+    #[test]
+    fn good_table_passes() {
+        let md = "|Name|Year|Country|\n|---|---|---|\n|Alice|2020|US|\n|Bob|2021|UK|";
+        assert!(
+            !looks_like_partial_table(md),
+            "should not flag well-formed table"
+        );
+    }
+
+    #[test]
+    fn header_starting_with_number_is_partial() {
+        // Heuristic missed the actual header row above
+        let md = "|2|Cambodian Women for Peace|9,835|\n|---|---|---|\n|3|Association|711|";
+        assert!(looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn header_with_empty_cells_in_3col_is_partial() {
+        // Empty cell in 3+ column header → bad column detection
+        let md =
+            "|Position||Administration|Administration|\n|---|---|---|---|\n|Senate|24|8.3|16.7|";
+        assert!(looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn header_with_duplicate_cells_is_partial() {
+        // Duplicate "Administration" → collapsed multi-line header wrong
+        let md =
+            "|Position|Administration|Administration|Notes|\n|---|---|---|---|\n|Senate|24|16|x|";
+        assert!(looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn two_column_with_one_empty_cell_passes() {
+        // Many real two-column tables have key-only rows; don't penalise.
+        let md = "|Key||\n|---|---|\n|Alice|123|\n|Bob|456|";
+        // Header "Key|" has one empty cell but only 2 cols total — keep it.
+        assert!(!looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn single_column_table_is_kept() {
+        // Single-column "tables" are common (lists). Caller can decide; we
+        // don't second-guess based on column count alone.
+        let md = "|Item|\n|---|\n|First|\n|Second|";
+        assert!(!looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn no_table_at_all_returns_true() {
+        // table_to_markdown should never produce this, but defensive — if
+        // there's no separator, treat as not-a-table.
+        let md = "Just some text\nWith multiple lines";
+        // No lines start with '|' so we return false (no header to inspect).
+        assert!(!looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn first_data_row_with_many_empty_cells_is_partial() {
+        // Multi-row header collapsed to single-row → first "data row" has
+        // most cells empty (the actual sub-header values).
+        let md = "|Government|No. of Seats|Aquino|Ramos|\n|---|---|---|---|\n|Position|||(1986-1992)|\n|Senate|24|8.3|16.7|";
+        assert!(looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn first_data_row_with_one_empty_cell_in_4col_passes() {
+        // Real data rows can have one empty cell (e.g. missing value);
+        // only flag when ≥1/3 of cells are empty.
+        let md = "|A|B|C|D|\n|---|---|---|---|\n|x|y||z|\n|p|q|r|s|";
+        assert!(!looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn paragraph_misread_as_two_column_table_is_partial() {
+        // Real production failure: text-wrapped paragraph mis-detected as
+        // 2-col table. Each cell continues the previous one as prose.
+        let md = "|Approval is needed from the|Acquisitions of|\n\
+                  |---|---|\n\
+                  |Treasurer if the acquisition|residential and|\n\
+                  |constitutes a \"significant|agricultural|\n\
+                  |action,\" including acquiring an|land by foreign|\n\
+                  |interest in different types of|persons must be|\n\
+                  |land where the monetary|reported to the|";
+        assert!(looks_like_partial_table(md));
+    }
+
+    #[test]
+    fn real_multi_word_table_is_kept() {
+        // Real table with multi-word entries — cells start with capital
+        // letters / proper nouns, NOT lowercase continuations.
+        let md = "|Country|Capital|Notes|\n\
+                  |---|---|---|\n\
+                  |United States|Washington DC|Federal capital|\n\
+                  |United Kingdom|London|City of London is a separate|\n\
+                  |France|Paris|Île-de-France region|\n\
+                  |Germany|Berlin|Reunified 1990|\n\
+                  |Spain|Madrid|Largest city in Spain|";
+        assert!(!looks_like_partial_table(md));
+    }
+
+    // --- layout_assisted relaxation tests ---
+
+    #[test]
+    fn numeric_header_accepted_when_layout_assisted() {
+        // Year as first header cell is valid when layout model gave us the bbox.
+        let md = "|2024|Revenue|Growth|\n|---|---|---|\n|Q1|1.2M|5%|\n|Q2|1.4M|8%|";
+        assert!(
+            looks_like_partial_table(md),
+            "strict mode rejects numeric header"
+        );
+        assert!(
+            !looks_like_partial_table_ex(md, true),
+            "layout-assisted should accept"
+        );
+    }
+
+    #[test]
+    fn one_empty_header_accepted_when_layout_assisted() {
+        // Common in merged-header tables: one spanning cell leaves a gap.
+        let md = "|Position||Senate|House|\n|---|---|---|---|\n|Chair|1|2|3|\n|Vice|4|5|6|";
+        assert!(
+            looks_like_partial_table(md),
+            "strict rejects 1 empty header"
+        );
+        assert!(
+            !looks_like_partial_table_ex(md, true),
+            "layout-assisted allows 1 empty"
+        );
+    }
+
+    #[test]
+    fn two_empty_headers_still_rejected_when_layout_assisted() {
+        // 2+ empty headers is still bad even with layout assistance.
+        let md = "|A|||D|\n|---|---|---|---|\n|x|y|z|w|";
+        assert!(
+            looks_like_partial_table_ex(md, true),
+            "2 empty headers rejected even layout-assisted"
+        );
+    }
+
+    #[test]
+    fn sparse_first_row_relaxed_when_layout_assisted() {
+        // 1/4 empty = 25%, below strict 33% threshold but accepted by layout-assisted 50%.
+        let md = "|A|B|C|D|\n|---|---|---|---|\n|x||y|z|\n|p|q|r|s|";
+        assert!(!looks_like_partial_table(md), "strict: 25% empty is OK");
+        // 2/4 = 50%, strict would flag (2*3>=4), relaxed threshold (2*2>=4) would also flag.
+        let md2 = "|A|B|C|D|\n|---|---|---|---|\n|||y|z|\n|p|q|r|s|";
+        assert!(looks_like_partial_table(md2), "strict: 50% empty flagged");
+        assert!(
+            looks_like_partial_table_ex(md2, true),
+            "layout-assisted: 50% also flagged"
+        );
+        // 2/6 = 33%, strict flags (2*3>=6), relaxed does not (2*2<6)
+        let md3 = "|A|B|C|D|E|F|\n|---|---|---|---|---|---|\n|x|||y|z|w|\n|a|b|c|d|e|f|";
+        assert!(looks_like_partial_table(md3), "strict: 33% flagged");
+        assert!(
+            !looks_like_partial_table_ex(md3, true),
+            "layout-assisted: 33% accepted"
+        );
+    }
+
+    #[test]
+    fn paragraph_still_rejected_when_layout_assisted() {
+        // Paragraph detection is not relaxed — it's a genuine extraction issue.
+        let md = "|Approval is needed from the|Acquisitions of|\n\
+                  |---|---|\n\
+                  |Treasurer if the acquisition|residential and|\n\
+                  |constitutes a \"significant|agricultural|\n\
+                  |action,\" including acquiring an|land by foreign|\n\
+                  |interest in different types of|persons must be|\n\
+                  |land where the monetary|reported to the|";
+        assert!(
+            looks_like_partial_table_ex(md, true),
+            "paragraph rejection stays strict"
+        );
+    }
+
+    #[test]
+    fn duplicate_headers_still_rejected_when_layout_assisted() {
+        let md =
+            "|Position|Administration|Administration|Notes|\n|---|---|---|---|\n|Senate|24|16|x|";
+        assert!(
+            looks_like_partial_table_ex(md, true),
+            "duplicate headers rejected even layout-assisted"
+        );
+    }
+}
+
 /// Analyse extracted items and rects for layout complexity.
 fn compute_layout_complexity(
     items: &[types::TextItem],

src/markdown/convert.rs

@@ -1,6 +1,6 @@
 //! Core line-to-markdown conversion loop with table/image interleaving.
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 
 use crate::structure_tree::StructRole;
 use crate::types::TextLine;
@@ -14,6 +14,139 @@ use super::postprocess::clean_markdown;
 use super::preprocess::{merge_drop_caps, merge_heading_lines};
 use super::MarkdownOptions;
 
+/// Pre-scan struct heading tags to find levels that are overused — i.e., tagged on
+/// so many lines that they clearly represent body text, not real headings.
+/// Returns the set of heading levels (1–6) that should be suppressed.
+///
+/// Some PDFs (e.g. British Academy grant guidance) tag every numbered paragraph
+/// line as H2, producing hundreds of false headings. We detect this by checking
+/// if any heading level accounts for >25% of tagged lines.
+fn detect_overused_struct_heading_levels(
+    lines: &[TextLine],
+    struct_roles: Option<
+        &std::collections::HashMap<u32, std::collections::HashMap<i64, StructRole>>,
+    >,
+) -> HashSet<usize> {
+    let mut overused = HashSet::new();
+    let Some(roles) = struct_roles else {
+        return overused;
+    };
+
+    let mut level_counts: HashMap<usize, usize> = HashMap::new();
+    let mut total = 0usize;
+
+    for line in lines {
+        if let Some(role) = resolve_line_struct_role(line, roles) {
+            total += 1;
+            if let Some(level) = struct_role_heading_level(&role) {
+                *level_counts.entry(level).or_insert(0) += 1;
+            }
+        }
+    }
+
+    if total < 20 {
+        return overused;
+    }
+
+    for (&level, &count) in &level_counts {
+        let ratio = count as f32 / total as f32;
+        if ratio > 0.15 {
+            log::debug!(
+                "struct heading H{} overused: {}/{} lines ({:.0}%), suppressing",
+                level,
+                count,
+                total,
+                ratio * 100.0
+            );
+            overused.insert(level);
+        }
+    }
+
+    overused
+}
+
+/// Pre-scan lines to find "isolated" ones: short lines with paragraph breaks both
+/// before and after.  These are heading candidates even at body font size — common
+/// in academic papers ("Acknowledgements", "B.3 Prompt Engineering").
+fn find_isolated_lines(lines: &[TextLine], base_size: f32, para_threshold: f32) -> HashSet<usize> {
+    let mut set = HashSet::new();
+    for i in 0..lines.len() {
+        let line = &lines[i];
+        let plain = line.text();
+        let trimmed = plain.trim();
+        let word_count = trimmed.split_whitespace().count();
+        if !(1..=6).contains(&word_count) || trimmed.len() <= 3 {
+            continue;
+        }
+        let font_size = line.items.first().map(|it| it.font_size).unwrap_or(0.0);
+        if font_size < base_size * 0.95 {
+            continue;
+        }
+        if is_list_item(trimmed) || is_caption_line(trimmed) {
+            continue;
+        }
+
+        // Reject lines that look like wrapped paragraph text:
+        // ends with hyphen, comma, preposition, or lowercase continuation
+        let last_char = trimmed.chars().last().unwrap_or(' ');
+        if last_char == '-' || last_char == ',' || last_char == ';' {
+            continue;
+        }
+        // Last word is a common continuation word → wrapped paragraph
+        let last_word = trimmed.split_whitespace().last().unwrap_or("");
+        let continuation_words = [
+            "the", "a", "an", "and", "or", "of", "in", "to", "for", "with", "by", "on", "at",
+            "from", "as", "is", "are", "was", "were", "be", "that", "this", "their", "its", "our",
+            "your", "has", "have", "had", "not",
+        ];
+        if continuation_words.contains(&last_word.to_lowercase().as_str()) {
+            continue;
+        }
+
+        // Paragraph break BEFORE
+        let break_before = if i == 0 {
+            true
+        } else {
+            let prev = &lines[i - 1];
+            prev.page != line.page || (prev.y - line.y).abs() > para_threshold
+        };
+
+        // Paragraph break AFTER
+        let break_after = if i + 1 >= lines.len() {
+            true
+        } else {
+            let next = &lines[i + 1];
+            next.page != line.page || (line.y - next.y).abs() > para_threshold
+        };
+
+        if !break_before || !break_after {
+            continue;
+        }
+
+        set.insert(i);
+    }
+
+    // Density guard: if too many lines on a page are "isolated", they're
+    // all paragraph lines in a multi-column layout, not headings.  Real
+    // headings are rare — at most ~20% of lines on a page.
+    let mut page_line_counts: HashMap<u32, (usize, usize)> = HashMap::new(); // (total, isolated)
+    for (i, line) in lines.iter().enumerate() {
+        let entry = page_line_counts.entry(line.page).or_insert((0, 0));
+        entry.0 += 1;
+        if set.contains(&i) {
+            entry.1 += 1;
+        }
+    }
+    for (&page, &(total, isolated)) in &page_line_counts {
+        if total > 0 && isolated as f32 / total as f32 > 0.25 {
+            // Too many isolated lines on this page — remove them all
+            set.retain(|&i| lines[i].page != page);
+        }
+    }
+
+    set
+}
+
 /// Resolve the dominant structure role for a text line by looking up its items' MCIDs.
 ///
 /// Returns the first non-container role found (skipping Document/Part/Sect/Div/NonStruct/Span).
@@ -256,6 +389,16 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
     // threshold and cause every line to be treated as a paragraph break.
     let para_threshold = compute_paragraph_threshold(&lines, base_size);
 
+    // Pre-scan: identify isolated lines (paragraph break before AND after).
+    // These are heading candidates even without bold/large font — common in
+    // academic papers where section titles like "Acknowledgements" sit alone
+    // between paragraphs at body font size. Inspired by opendataloader's
+    // lookahead in HeadingProcessor (prevNode/nextNode context).
+    let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
+
+    // Detect struct heading levels that are overused (body text mistagged as headings)
+    let overused_heading_levels = detect_overused_struct_heading_levels(&lines, struct_roles);
+
     let mut output = String::new();
     let mut current_page = 0u32;
     let mut prev_y = f32::MAX;
@@ -277,7 +420,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
     all_content_pages.sort();
     all_content_pages.dedup();
 
-    for line in lines {
+    for (line_idx, line) in lines.iter().enumerate() {
         // Page break
         if line.page != current_page {
             // Flush current page's remaining tables and images
@@ -405,7 +548,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
 
         // Detect figure/table captions and source citations
         // These should be on their own line followed by a paragraph break
-        let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(&line, roles));
+        let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(line, roles));
 
         // Determine if this line is code (struct-tree or font-based) for block accumulation
         let is_code_line = struct_role
@@ -437,7 +580,10 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
         // Structure roles ADD headings (e.g. same-size text tagged H2) but do NOT
         // suppress headings that the font heuristic would detect (some tagged PDFs
         // mark obvious headings as P or Span).
-        let struct_heading = struct_role.as_ref().and_then(struct_role_heading_level);
+        let struct_heading = struct_role
+            .as_ref()
+            .and_then(struct_role_heading_level)
+            .filter(|level| !overused_heading_levels.contains(level));
         let heuristic_heading = if options.detect_headers
             && plain_trimmed.len() > 3
             && plain_trimmed.split_whitespace().count() <= 15
@@ -445,8 +591,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
             let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
             detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
                 // Rarity-based heading detection (inspired by opendataloader).
-                // Score = font_rarity * 0.5 + bold * 0.3 + standalone * 0.2
-                // Lines scoring above threshold are promoted to headings.
+                // Heading probability scoring with lookahead context.
+                // Score = rarity * 0.5 + bold * 0.3 + standalone * 0.2
+                //       + isolated * 0.3 (paragraph break before AND after)
                 // Only consider lines at or above body font size.
                 if line_font_size < base_size * 0.95 {
                     return None;
@@ -458,13 +605,21 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
                 let rarity = font_size_rarity(line_font_size, &font_stats);
                 let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
                 let standalone = !in_paragraph;
+                let isolated = isolated_lines.contains(&line_idx);
 
                 let score = rarity * 0.5
                     + if all_bold { 0.3 } else { 0.0 }
-                    + if standalone { 0.2 } else { 0.0 };
+                    + if standalone { 0.2 } else { 0.0 }
+                    + if isolated { 0.3 } else { 0.0 };
 
-                // Require standalone + at least one other signal
-                if score >= 0.5 && standalone && word_count >= 3 {
+                // Require standalone + at least one strong signal.
+                // Non-bold, non-isolated lines need very high rarity (≥0.97)
+                // to avoid classifying ordinary body text as headings in
+                // multi-column layouts where column switches break
+                // paragraph continuity and minor font-size variation
+                // inflates rarity scores.
+                let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
+                if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
                     Some(bold_heading_level(&heading_tiers))
                 } else {
                     None
@@ -652,6 +807,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
     // Compute the typical line spacing for paragraph break detection
     let para_threshold = compute_paragraph_threshold(&lines, base_size);
 
+    let isolated_lines = find_isolated_lines(&lines, base_size, para_threshold);
+
     let mut output = String::new();
     let mut current_page = 0u32;
     let mut prev_y = f32::MAX;
@@ -660,7 +817,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
     let mut last_list_x: Option<f32> = None;
     let mut prev_had_dot_leaders = false;
 
-    for line in lines {
+    for (line_idx, line) in lines.iter().enumerate() {
         // Page break
         if line.page != current_page {
             if current_page > 0 {
@@ -736,10 +893,12 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
                     let rarity = font_size_rarity(line_font_size, &font_stats);
                     let all_bold = !line.items.is_empty() && line.items.iter().all(|i| i.is_bold);
                     let standalone = !in_paragraph;
+                    let isolated = isolated_lines.contains(&line_idx);
                     let score = rarity * 0.5
                         + if all_bold { 0.3 } else { 0.0 }
-                        + if standalone { 0.2 } else { 0.0 };
-                    if score >= 0.5 && standalone && word_count >= 3 {
+                        + if standalone { 0.2 } else { 0.0 }
+                        + if isolated { 0.3 } else { 0.0 };
+                    if score >= 0.5 && standalone && word_count >= 2 {
                         return Some(bold_heading_level(&heading_tiers));
                     }
                     None
@@ -1060,6 +1219,62 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_rarity_heading_requires_strong_signal() {
+        // Simulate a two-column academic paper where body text lines become
+        // "standalone" due to column switches.  Body text at the same font
+        // size as most of the document should NOT be classified as headings
+        // just because of moderate rarity + standalone.
+        //
+        // Regression: previously, lines with rarity ~0.62 and standalone=true
+        // scored 0.51 (>=0.5 threshold), producing hundreds of false ## headings.
+
+        // Create many body-text lines at font_size=10.9 (most common)
+        let mut lines = Vec::new();
+        for i in 0..20 {
+            let mut item = make_item("This is ordinary body text in a paragraph.", 1, None);
+            item.font_size = 10.9;
+            item.y = 700.0 - i as f32 * 14.0;
+            lines.push(make_line(vec![item]));
+        }
+        // A few lines at a slightly different size (simulating column B text)
+        for i in 0..10 {
+            let mut item = make_item("Another body text line from the second column.", 1, None);
+            item.font_size = 11.0; // slightly different → non-zero rarity
+            item.y = 700.0 - i as f32 * 14.0;
+            item.x = 320.0; // right column
+            lines.push(make_line(vec![item]));
+        }
+        // One genuine bold heading
+        let mut heading_item = make_item("3 Philosophical Perspectives", 1, None);
+        heading_item.font_size = 10.9;
+        heading_item.is_bold = true;
+        heading_item.y = 200.0;
+        lines.push(make_line(vec![heading_item]));
+
+        let md = to_markdown_from_lines_with_tables_and_images(
+            lines,
+            MarkdownOptions::default(),
+            HashMap::new(),
+            HashMap::new(),
+            &std::collections::HashSet::new(),
+            None,
+        );
+
+        // The bold heading should be detected
+        assert!(
+            md.contains("## 3 Philosophical Perspectives"),
+            "Bold heading should be detected: {md}"
+        );
+
+        // Body text lines should NOT be headings
+        let heading_count = md.lines().filter(|l| l.starts_with("##")).count();
+        assert!(
+            heading_count <= 2,
+            "Expected at most 2 headings but found {heading_count} in:\n{md}"
+        );
+    }
+
     #[test]
     fn test_struct_role_code_multiline_accumulation() {
         let mut line1 = make_item("fn main() {", 1, Some(0));
@@ -1102,4 +1317,79 @@ mod tests {
             "Should not have adjacent close/open fences: {md}"
         );
     }
+
+    #[test]
+    fn test_overused_struct_heading_suppressed() {
+        // Simulate a PDF where H2 is mistagged on body text lines.
+        // 30 lines total: 5 tagged H1 (real headings), 20 tagged H2 (mistagged body),
+        // 5 tagged P.
+        let mut lines = Vec::new();
+        let mut page_roles = HashMap::new();
+        let mut mcid = 0i64;
+
+        for i in 0..30 {
+            let mut item = make_item(&format!("Line {i}"), 1, Some(mcid));
+            item.y = 700.0 - (i as f32 * 15.0);
+            lines.push(make_line(vec![item]));
+
+            let role = if i < 5 {
+                StructRole::H1
+            } else if i < 25 {
+                StructRole::H2
+            } else {
+                StructRole::P
+            };
+            page_roles.insert(mcid, role);
+            mcid += 1;
+        }
+
+        let mut roles = HashMap::new();
+        roles.insert(1u32, page_roles);
+
+        let overused = detect_overused_struct_heading_levels(&lines, Some(&roles));
+        // H2 is on 20/30 = 67% of lines — should be suppressed
+        assert!(
+            overused.contains(&2),
+            "H2 should be detected as overused: {:?}",
+            overused
+        );
+        // H1 is on 5/30 = 17% — should also be suppressed at >15% threshold
+        assert!(
+            overused.contains(&1),
+            "H1 at 17% should also be suppressed: {:?}",
+            overused
+        );
+    }
+
+    #[test]
+    fn test_normal_struct_headings_not_suppressed() {
+        // Normal document: a few headings, mostly body text
+        let mut lines = Vec::new();
+        let mut page_roles = HashMap::new();
+        let mut mcid = 0i64;
+
+        for i in 0..50 {
+            let mut item = make_item(&format!("Line {i}"), 1, Some(mcid));
+            item.y = 700.0 - (i as f32 * 14.0);
+            lines.push(make_line(vec![item]));
+
+            let role = if i % 10 == 0 {
+                StructRole::H1 // 5 headings out of 50 = 10%
+            } else {
+                StructRole::P
+            };
+            page_roles.insert(mcid, role);
+            mcid += 1;
+        }
+
+        let mut roles = HashMap::new();
+        roles.insert(1u32, page_roles);
+
+        let overused = detect_overused_struct_heading_levels(&lines, Some(&roles));
+        assert!(
+            overused.is_empty(),
+            "No heading level should be overused: {:?}",
+            overused
+        );
+    }
 }

src/tounicode.rs

@@ -1650,7 +1650,7 @@ fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap
 ///
 /// Returns true if the median CID is >= 0x41 (letter 'A'), indicating
 /// the PDF generator likely used Unicode codepoints as CIDs.
-fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool {
+pub(crate) fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool {
     let w_arr = match cid_font_dict.get(b"W").ok() {
         Some(Object::Array(arr)) => arr,
         _ => return false,

tests/integration_tests.rs

@@ -4,9 +4,10 @@ use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
 use pdf_inspector::extractor::group_into_lines;
 use pdf_inspector::types::TextLine;
 use pdf_inspector::{
-    detect_pdf_type, extract_text, extract_text_in_regions_mem, extract_text_with_positions,
-    process_pdf_mem, process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions,
-    PdfType, TextItem,
+    detect_pdf_type, extract_pages_markdown_mem, extract_tables_in_regions_mem, extract_text,
+    extract_text_in_regions_mem, extract_text_with_positions, process_pdf_mem,
+    process_pdf_with_options, to_markdown, MarkdownOptions, PdfError, PdfOptions, PdfType,
+    TextItem,
 };
 use std::collections::HashSet;
 
@@ -1344,3 +1345,252 @@ fn test_extract_regions_fast_vs_normal_comparison() {
         }
     }
 }
+
+// =========================================================================
+// extract_tables_in_regions_mem tests
+// =========================================================================
+
+#[test]
+fn test_extract_tables_in_regions_table_pdf() {
+    // tnagriculture has a clear table with district names and spice columns
+    let buf = std::fs::read("tests/fixtures/tnagriculture_06_12.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+
+    assert_eq!(results.len(), 1);
+    assert_eq!(results[0].regions.len(), 1);
+
+    let region = &results[0].regions[0];
+    // Should detect a table with pipe-delimited markdown
+    if !region.needs_ocr {
+        assert!(
+            region.text.contains('|'),
+            "Table output should contain pipe delimiters"
+        );
+        // Should have separator row
+        assert!(
+            region.text.lines().any(|l| l.contains("---")),
+            "Table output should contain separator row"
+        );
+    }
+}
+
+#[test]
+fn test_extract_tables_in_regions_non_table_region() {
+    // Use a small region that likely won't contain enough items for a table
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 50.0, 50.0]])]).unwrap();
+
+    assert_eq!(results.len(), 1);
+    assert_eq!(results[0].regions.len(), 1);
+
+    let region = &results[0].regions[0];
+    // Small region with few items should fall back to needs_ocr
+    assert!(
+        region.needs_ocr,
+        "Non-table region should set needs_ocr = true"
+    );
+    assert!(
+        region.text.is_empty(),
+        "Non-table region should have empty text"
+    );
+}
+
+#[test]
+fn test_extract_tables_in_regions_empty_region() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let results = extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 0.0, 0.0]])]).unwrap();
+
+    assert_eq!(results.len(), 1);
+    let region = &results[0].regions[0];
+    assert!(region.needs_ocr);
+    assert!(region.text.is_empty());
+}
+
+#[test]
+fn test_extract_tables_in_regions_identity_h_needs_ocr() {
+    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+
+    assert_eq!(results.len(), 1);
+    let region = &results[0].regions[0];
+    assert!(region.needs_ocr, "Identity-H font should trigger needs_ocr");
+}
+
+#[test]
+fn test_extract_tables_in_regions_not_a_pdf() {
+    let result =
+        extract_tables_in_regions_mem(b"not a pdf", &[(0, vec![[0.0, 0.0, 100.0, 100.0]])]);
+    assert!(result.is_err());
+}
+
+#[test]
+fn test_extract_tables_in_regions_nonexistent_page() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let results =
+        extract_tables_in_regions_mem(&buf, &[(9999, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
+
+    assert_eq!(results.len(), 1);
+    let region = &results[0].regions[0];
+    assert!(region.needs_ocr);
+    assert!(region.text.is_empty());
+}
+
+// =========================================================================
+// extract_pages_markdown_mem tests
+// =========================================================================
+
+#[test]
+fn test_extract_pages_markdown_basic() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0, 1]).unwrap();
+
+    assert_eq!(result.pages.len(), 2);
+    assert_eq!(result.pages[0].page, 0);
+    assert_eq!(result.pages[1].page, 1);
+    // Text-based PDF should produce non-empty markdown
+    assert!(!result.pages[0].markdown.is_empty());
+    assert!(!result.pages[0].needs_ocr);
+}
+
+#[test]
+fn test_extract_pages_markdown_page_ordering() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    // Request pages in non-sequential order
+    let result = extract_pages_markdown_mem(&buf, &[1, 0]).unwrap();
+
+    assert_eq!(result.pages.len(), 2);
+    // Results should match input order, not document order
+    assert_eq!(result.pages[0].page, 1);
+    assert_eq!(result.pages[1].page, 0);
+}
+
+#[test]
+fn test_extract_pages_markdown_out_of_range() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[9999]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert_eq!(result.pages[0].page, 9999);
+    assert!(result.pages[0].markdown.is_empty());
+    assert!(result.pages[0].needs_ocr);
+    assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed
+}
+
+#[test]
+fn test_extract_pages_markdown_empty_pages_list() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[]).unwrap();
+    assert!(result.pages.is_empty());
+}
+
+#[test]
+fn test_extract_pages_markdown_single_page() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert_eq!(result.pages[0].page, 0);
+    assert!(!result.pages[0].markdown.is_empty());
+    assert!(!result.pages[0].needs_ocr);
+}
+
+#[test]
+fn test_extract_pages_markdown_invalid_buffer() {
+    let result = extract_pages_markdown_mem(b"not a pdf", &[0]);
+    assert!(result.is_err());
+}
+
+#[test]
+fn test_extract_pages_markdown_gid_pages_need_ocr() {
+    // shinagawa_identity_h.pdf has GID-encoded fonts
+    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert_eq!(result.pages.len(), 1);
+    assert!(result.pages[0].needs_ocr);
+    assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
+}
+
+#[test]
+fn test_extract_pages_markdown_classification_with_tables() {
+    // nexo-price-en.pdf is known to have tables
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let page_count = process_pdf_mem(&buf).unwrap().page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    assert!(
+        !result.pages_with_tables.is_empty(),
+        "nexo-price-en.pdf should have pages with tables"
+    );
+    assert!(result.is_complex);
+}
+
+#[test]
+fn test_extract_pages_markdown_simple_pdf_no_complexity() {
+    // bare_name_struct.pdf is a simple document with a heading and code block
+    let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
+    let result = extract_pages_markdown_mem(&buf, &[0]).unwrap();
+
+    assert!(result.pages_with_tables.is_empty());
+    assert!(result.pages_with_columns.is_empty());
+    assert!(!result.is_complex);
+}
+
+#[test]
+fn test_extract_pages_markdown_classification_matches_process_pdf() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+    let full = process_pdf_mem(&buf).unwrap();
+    let page_count = full.page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    assert_eq!(
+        result.pages_with_tables, full.layout.pages_with_tables,
+        "pages_with_tables should match process_pdf"
+    );
+    assert_eq!(
+        result.pages_with_columns, full.layout.pages_with_columns,
+        "pages_with_columns should match process_pdf"
+    );
+}
+
+#[test]
+fn test_extract_pages_markdown_consistency_with_process_pdf() {
+    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
+
+    // Get full process_pdf output
+    let full = process_pdf_mem(&buf).unwrap();
+    let full_md = full.markdown.unwrap_or_default();
+
+    // Get per-page output for all pages
+    let page_count = full.page_count;
+    let page_indices: Vec<u32> = (0..page_count).collect();
+    let result = extract_pages_markdown_mem(&buf, &page_indices).unwrap();
+
+    // Concatenated per-page markdown should contain substantial overlap with
+    // the full output (exact match not expected due to header/footer stripping
+    // and cross-page paragraph merging differences)
+    let concat: String = result
+        .pages
+        .iter()
+        .map(|p| p.markdown.as_str())
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    // Both should be non-empty for a text-based PDF
+    assert!(!full_md.is_empty());
+    assert!(!concat.is_empty());
+
+    // The per-page version should contain at least 50% of the full content's
+    // length (accounting for header/footer stripping differences)
+    assert!(
+        concat.len() * 2 >= full_md.len(),
+        "per-page concat ({} chars) is too short vs full ({} chars)",
+        concat.len(),
+        full_md.len()
+    );
+}

tests/snapshots/p1244-1996.md

@@ -76,7 +76,7 @@ forms simpler, we would be happy to hear from you. You can write to the Tax Form
 
 **Unreported Tips.—If you received tips of $20 or** more for any month while working for one employer but did not report them to your employer, you must figure and pay social security and Medicare taxes on the unreported tips when you file your tax return. If you have unreported tips, you must use Form 1040 and Form 4137, Social Security and Medicare Tax on Unreported Tip Income, to report them. You may not use Form 1040A or 1040EZ. Employees subject to the Railroad Retirement Tax Act cannot use Form 4137 to pay railroad retirement tax on unreported tips. To get railroad retirement credit, you must report tips to your employer. If you do not report tips to your employer as required, you may be charged a penalty of 50% of the social security and Medicare taxes (or railroad retirement tax) due on the unreported tips unless there was reasonable cause for not reporting them. **Additional Information.—Get Pub. 531, Reporting** Tip Income, and Form 4137 for more information on tips. If you are an employee of certain large food or beverage establishments, see Pub. 531 for tip allocation rules. **Recordkeeping.—If you do not keep a daily** record of tips, you must keep other reliable proof of the tip income you received. This proof includes copies of restaurant bills and credit card charges that show amounts customers added as tips. Keep your tip income records for as long as the information on them may be needed in the administration of any Internal Revenue law.
 
-**Instructions (continued)**
+### Instructions (continued)
 
 Use this space to total your tips for the year
 

tests/snapshots/thermo-freon12.md

@@ -1,8 +1,8 @@
-**Technical Information**
+##### Technical Information
 
 ## l T-12 SI
 
-DuPont Fluorochemicals
+##### DuPont Fluorochemicals
 
 #### Thermodynamic Properties
 
@@ -20,11 +20,11 @@ Tables of the thermodynamic **Units** properties of R-12 have been developed and
 
 S.A., Lemmon, E.W., and Peskin, Vf = Fluid (liquid) specific volume
 A.P., NIST Standard Reference in cubic meters per kilogram Database 23, NIST thermodynamic and transport properties of Vg = Vapour (gas) specific volume refrigerants and refrigerant in cubic meters per kilogram mixtures – REFPROP version 6.01, Standard Reference Data Program, df and dg = Fluid and Vapour National Institute of Standards and (respectively) densities in Technology, 1998). kilograms per cubic meter
-H = Enthalpy (kJ/kg)
+##### H = Enthalpy (kJ/kg)
 
-S = Entropy (kJ/kg.K)
+##### S = Entropy (kJ/kg.K)
 
-**Physical Properties**
+##### Physical Properties
 
 |Chemical Formula|CCl2F2|
 |---|---|