pdf-inspector/tests/integration_tests.rs

//! Integration tests for pdf-to-markdown library

use pdf_inspector::detector::{DetectionConfig, ScanStrategy};
use pdf_inspector::extractor::group_into_lines;
use pdf_inspector::types::TextLine;
use pdf_inspector::{
    detect_pdf_type, extract_pages_markdown, extract_pages_markdown_mem,
    extract_tables_in_regions_mem, extract_text, extract_text_in_regions_mem,
    extract_text_with_positions, process_pdf_mem, process_pdf_with_options, to_markdown,
    MarkdownOptions, PdfError, PdfOptions, PdfType, TextItem,
};
use std::collections::HashSet;

// Helper to create test TextItems
fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem {
    use pdf_inspector::types::ItemType;
    TextItem {
        text: text.to_string(),
        x,
        y,
        width: text.len() as f32 * font_size * 0.5,
        height: font_size,
        font: "Helvetica".to_string(),
        font_size,
        page,
        is_bold: false,
        is_italic: false,
        item_type: ItemType::Text,
        mcid: None,
    }
}

fn make_text_item_with_font(
    text: &str,
    x: f32,
    y: f32,
    font_size: f32,
    font: &str,
    page: u32,
) -> TextItem {
    use pdf_inspector::extractor::{is_bold_font, is_italic_font, ItemType};
    TextItem {
        text: text.to_string(),
        x,
        y,
        width: text.len() as f32 * font_size * 0.5,
        height: font_size,
        font: font.to_string(),
        font_size,
        page,
        is_bold: is_bold_font(font),
        is_italic: is_italic_font(font),
        item_type: ItemType::Text,
        mcid: None,
    }
}

// ============================================================================
// Detection Config Tests
// ============================================================================

#[test]
fn test_detection_config_default() {
    let config = DetectionConfig::default();
    assert!(matches!(config.strategy, ScanStrategy::Sample(8)));
    assert_eq!(config.min_text_ops_per_page, 3);
    assert!((config.text_page_ratio_threshold - 0.6).abs() < 0.001);
}

#[test]
fn test_detection_config_custom() {
    let config = DetectionConfig {
        strategy: ScanStrategy::Sample(10),
        min_text_ops_per_page: 5,
        text_page_ratio_threshold: 0.8,
    };
    assert!(matches!(config.strategy, ScanStrategy::Sample(10)));
    assert_eq!(config.min_text_ops_per_page, 5);
    assert!((config.text_page_ratio_threshold - 0.8).abs() < 0.001);
}

// ============================================================================
// PdfType Tests
// ============================================================================

#[test]
fn test_pdf_type_equality() {
    assert_eq!(PdfType::TextBased, PdfType::TextBased);
    assert_eq!(PdfType::Scanned, PdfType::Scanned);
    assert_eq!(PdfType::ImageBased, PdfType::ImageBased);
    assert_eq!(PdfType::Mixed, PdfType::Mixed);
    assert_ne!(PdfType::TextBased, PdfType::Scanned);
}

#[test]
fn test_pdf_type_clone() {
    let original = PdfType::TextBased;
    let cloned = original.clone();
    assert_eq!(original, cloned);
}

#[test]
fn test_pdf_type_debug() {
    let pdf_type = PdfType::TextBased;
    let debug_str = format!("{:?}", pdf_type);
    assert_eq!(debug_str, "TextBased");
}

// ============================================================================
// TextItem Tests
// ============================================================================

#[test]
fn test_text_item_creation() {
    let item = make_text_item("Hello", 100.0, 700.0, 12.0, 1);
    assert_eq!(item.text, "Hello");
    assert_eq!(item.x, 100.0);
    assert_eq!(item.y, 700.0);
    assert_eq!(item.font_size, 12.0);
    assert_eq!(item.page, 1);
}

#[test]
fn test_text_item_clone() {
    let item = make_text_item("Test", 50.0, 600.0, 14.0, 2);
    let cloned = item.clone();
    assert_eq!(item.text, cloned.text);
    assert_eq!(item.x, cloned.x);
    assert_eq!(item.y, cloned.y);
}

// ============================================================================
// TextLine Tests
// ============================================================================

#[test]
fn test_text_line_text_method() {
    let items = vec![
        make_text_item("Hello", 100.0, 700.0, 12.0, 1),
        make_text_item("World", 160.0, 700.0, 12.0, 1),
    ];
    let line = TextLine {
        items,
        y: 700.0,
        page: 1,
        adaptive_threshold: 0.10,
    };
    assert_eq!(line.text(), "Hello World");
}

#[test]
fn test_text_line_single_item() {
    let items = vec![make_text_item("Single", 100.0, 700.0, 12.0, 1)];
    let line = TextLine {
        items,
        y: 700.0,
        page: 1,
        adaptive_threshold: 0.10,
    };
    assert_eq!(line.text(), "Single");
}

#[test]
fn test_text_line_empty() {
    let line = TextLine {
        items: vec![],
        y: 700.0,
        page: 1,
        adaptive_threshold: 0.10,
    };
    assert_eq!(line.text(), "");
}

// ============================================================================
// Group Into Lines Tests
// ============================================================================

#[test]
fn test_group_into_lines_empty() {
    let items: Vec<TextItem> = vec![];
    let lines = group_into_lines(items);
    assert!(lines.is_empty());
}

#[test]
fn test_group_into_lines_same_line() {
    let items = vec![
        make_text_item("A", 100.0, 700.0, 12.0, 1),
        make_text_item("B", 120.0, 700.0, 12.0, 1),
        make_text_item("C", 140.0, 700.0, 12.0, 1),
    ];
    let lines = group_into_lines(items);
    assert_eq!(lines.len(), 1);
    assert_eq!(lines[0].items.len(), 3);
    assert_eq!(lines[0].text(), "A B C");
}

#[test]
fn test_group_into_lines_different_lines() {
    let items = vec![
        make_text_item("Line1", 100.0, 700.0, 12.0, 1),
        make_text_item("Line2", 100.0, 680.0, 12.0, 1),
        make_text_item("Line3", 100.0, 660.0, 12.0, 1),
    ];
    let lines = group_into_lines(items);
    assert_eq!(lines.len(), 3);
    assert_eq!(lines[0].text(), "Line1");
    assert_eq!(lines[1].text(), "Line2");
    assert_eq!(lines[2].text(), "Line3");
}

#[test]
fn test_group_into_lines_y_tolerance() {
    // Items within 3.0 Y tolerance should be grouped
    // Note: items are sorted by Y descending, then X ascending
    let items = vec![
        make_text_item("A", 100.0, 700.0, 12.0, 1),
        make_text_item("B", 150.0, 700.0, 12.0, 1), // Same Y
    ];
    let lines = group_into_lines(items);
    assert_eq!(lines.len(), 1);
    assert_eq!(lines[0].text(), "A B");
}

#[test]
fn test_group_into_lines_multiple_pages() {
    let items = vec![
        make_text_item("Page1Text", 100.0, 700.0, 12.0, 1),
        make_text_item("Page2Text", 100.0, 700.0, 12.0, 2),
    ];
    let lines = group_into_lines(items);
    assert_eq!(lines.len(), 2);
    assert_eq!(lines[0].page, 1);
    assert_eq!(lines[1].page, 2);
}

#[test]
fn test_group_into_lines_sorting_by_x() {
    // Items on same line should be sorted by X position
    let items = vec![
        make_text_item("Third", 200.0, 700.0, 12.0, 1),
        make_text_item("First", 50.0, 700.0, 12.0, 1),
        make_text_item("Second", 100.0, 700.0, 12.0, 1),
    ];
    let lines = group_into_lines(items);
    assert_eq!(lines.len(), 1);
    assert_eq!(lines[0].text(), "First Second Third");
}

// ============================================================================
// MarkdownOptions Tests
// ============================================================================

#[test]
fn test_markdown_options_default() {
    let opts = MarkdownOptions::default();
    assert!(opts.detect_headers);
    assert!(opts.detect_lists);
    assert!(opts.detect_code);
    assert!(opts.base_font_size.is_none());
}

#[test]
fn test_markdown_options_custom() {
    let opts = MarkdownOptions {
        detect_headers: false,
        detect_lists: true,
        detect_code: false,
        base_font_size: Some(14.0),
        remove_page_numbers: false,
        format_urls: false,
        fix_hyphenation: false,
        detect_bold: false,
        detect_italic: false,
        include_images: false,
        include_links: false,
        include_page_numbers: false,
        ..Default::default()
    };
    assert!(!opts.detect_headers);
    assert!(opts.detect_lists);
    assert!(!opts.detect_code);
    assert_eq!(opts.base_font_size, Some(14.0));
    assert!(!opts.remove_page_numbers);
    assert!(!opts.format_urls);
    assert!(!opts.fix_hyphenation);
    assert!(!opts.detect_bold);
    assert!(!opts.detect_italic);
    assert!(!opts.include_images);
    assert!(!opts.include_links);
}

// ============================================================================
// Markdown Conversion Tests
// ============================================================================

#[test]
fn test_to_markdown_basic() {
    let text = "Hello World";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("Hello World"));
}

#[test]
fn test_to_markdown_multiple_lines() {
    let text = "Line one\nLine two\nLine three";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("Line one"));
    assert!(md.contains("Line two"));
    assert!(md.contains("Line three"));
}

#[test]
fn test_to_markdown_bullet_list() {
    let text = "• First\n• Second\n• Third";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("- First"));
    assert!(md.contains("- Second"));
    assert!(md.contains("- Third"));
}

#[test]
fn test_to_markdown_dash_list() {
    let text = "- One\n- Two\n- Three";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("- One"));
    assert!(md.contains("- Two"));
}

#[test]
fn test_to_markdown_numbered_list() {
    let text = "1. First\n2. Second\n3. Third";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("1. First"));
    assert!(md.contains("2. Second"));
}

#[test]
fn test_to_markdown_code_detection() {
    let text = "const x = 5;\nlet y = 10;";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("```"));
}

#[test]
fn test_to_markdown_no_code_detection() {
    let text = "const x = 5;";
    let opts = MarkdownOptions {
        detect_code: false,
        ..Default::default()
    };
    let md = to_markdown(text, opts);
    assert!(!md.contains("```"));
}

#[test]
fn test_to_markdown_no_list_detection() {
    let text = "• Item";
    let opts = MarkdownOptions {
        detect_lists: false,
        ..Default::default()
    };
    let md = to_markdown(text, opts);
    // Should keep original bullet character
    assert!(md.contains("•"));
}

#[test]
fn test_to_markdown_empty_lines() {
    let text = "Para one\n\nPara two";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("Para one"));
    assert!(md.contains("Para two"));
}

#[test]
fn test_to_markdown_whitespace_only_lines() {
    let text = "Content\n   \nMore content";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.contains("Content"));
    assert!(md.contains("More content"));
}

// ============================================================================
// Markdown From Items Tests
// ============================================================================

#[test]
fn test_markdown_from_items_empty() {
    use pdf_inspector::markdown::to_markdown_from_items;
    let items: Vec<TextItem> = vec![];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.is_empty());
}

#[test]
fn test_markdown_from_items_single() {
    use pdf_inspector::markdown::to_markdown_from_items;
    let items = vec![make_text_item("Hello", 100.0, 700.0, 12.0, 1)];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("Hello"));
}

#[test]
fn test_markdown_from_items_header_detection() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Need multiple body items to establish base font size
    let items = vec![
        make_text_item("Title", 100.0, 750.0, 24.0, 1), // Large font = H1
        make_text_item("Body text one", 100.0, 700.0, 12.0, 1),
        make_text_item("Body text two", 100.0, 680.0, 12.0, 1),
        make_text_item("Body text three", 100.0, 660.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("# Title"));
    assert!(md.contains("Body text"));
}

#[test]
fn test_markdown_from_items_h2_detection() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Two heading tiers: 24.0 → H1, 18.0 → H2
    let items = vec![
        make_text_item("Title", 100.0, 800.0, 24.0, 1),
        make_text_item("Subtitle", 100.0, 750.0, 18.0, 1),
        make_text_item("Body text one", 100.0, 700.0, 12.0, 1),
        make_text_item("Body text two", 100.0, 680.0, 12.0, 1),
        make_text_item("Body text three", 100.0, 660.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("## Subtitle"));
}

#[test]
fn test_markdown_from_items_monospace_code() {
    use pdf_inspector::markdown::to_markdown_from_items;
    let items = vec![make_text_item_with_font(
        "let x = 5",
        100.0,
        700.0,
        12.0,
        "Courier",
        1,
    )];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("```"));
    assert!(md.contains("let x = 5"));
}

#[test]
fn test_markdown_from_items_page_breaks() {
    use pdf_inspector::markdown::to_markdown_from_items;
    let items = vec![
        make_text_item("Content on first page", 100.0, 700.0, 12.0, 1),
        make_text_item("Content on second page", 100.0, 700.0, 12.0, 2),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    // Pages should be separated by blank lines (no --- markers)
    assert!(!md.contains("---"));
    assert!(md.contains("Content on first page"));
    assert!(md.contains("Content on second page"));
}

// ============================================================================
// Markdown From Lines Tests
// ============================================================================

#[test]
fn test_markdown_from_lines_empty() {
    use pdf_inspector::markdown::to_markdown_from_lines;
    let lines: Vec<TextLine> = vec![];
    let md = to_markdown_from_lines(lines, MarkdownOptions::default());
    assert!(md.is_empty());
}

#[test]
fn test_markdown_from_lines_basic() {
    use pdf_inspector::markdown::to_markdown_from_lines;
    let lines = vec![
        TextLine {
            items: vec![make_text_item("First", 100.0, 700.0, 12.0, 1)],
            y: 700.0,
            page: 1,
            adaptive_threshold: 0.10,
        },
        TextLine {
            items: vec![make_text_item("Second", 100.0, 680.0, 12.0, 1)],
            y: 680.0,
            page: 1,
            adaptive_threshold: 0.10,
        },
    ];
    let md = to_markdown_from_lines(lines, MarkdownOptions::default());
    assert!(md.contains("First"));
    assert!(md.contains("Second"));
}

// ============================================================================
// Error Handling Tests
// ============================================================================

#[test]
fn test_extract_text_nonexistent_file() {
    let result = extract_text("/nonexistent/file.pdf");
    assert!(result.is_err());
}

#[test]
fn test_detect_pdf_type_nonexistent_file() {
    let result = detect_pdf_type("/nonexistent/file.pdf");
    assert!(result.is_err());
}

#[test]
fn test_extract_text_with_positions_nonexistent_file() {
    let result = extract_text_with_positions("/nonexistent/file.pdf");
    assert!(result.is_err());
}

// ============================================================================
// List Pattern Tests
// ============================================================================

#[test]
fn test_bullet_variations() {
    // Unicode bullets get converted to markdown dash
    let unicode_bullets = ["• Item", "○ Item", "● Item", "◦ Item"];
    for bullet in &unicode_bullets {
        let md = to_markdown(bullet, MarkdownOptions::default());
        assert!(md.contains("- Item"), "Failed for: {}", bullet);
    }

    // Markdown-compatible bullets stay as-is
    let md_bullets = ["- Item", "* Item"];
    for bullet in &md_bullets {
        let md = to_markdown(bullet, MarkdownOptions::default());
        assert!(md.contains(bullet), "Failed for: {}", bullet);
    }
}

#[test]
fn test_numbered_list_variations() {
    let lists = ["1. First", "2) Second", "10. Tenth"];
    for item in &lists {
        let md = to_markdown(item, MarkdownOptions::default());
        assert!(md.trim().len() > 0, "Failed for: {}", item);
    }
}

#[test]
fn test_letter_list_items() {
    let md = to_markdown("a. Letter item", MarkdownOptions::default());
    assert!(md.contains("a. Letter item"));
}

// ============================================================================
// Code Detection Tests
// ============================================================================

#[test]
fn test_code_keywords() {
    let keywords = [
        "import foo",
        "export default",
        "const x = 5;",
        "let y = 10;",
        "function test() {",
        "class MyClass {",
        "def func():",
        "pub fn main() {",
        "async fn process() {",
        "impl Trait {",
    ];
    for code in &keywords {
        let md = to_markdown(code, MarkdownOptions::default());
        assert!(md.contains("```"), "Code not detected for: {}", code);
    }
}

#[test]
fn test_code_syntax_patterns() {
    // Patterns that start with code keywords/syntax
    let patterns = [
        "=> value",      // Starts with =>
        "-> Result",     // Starts with ->
        ":: io::Result", // Starts with ::
    ];
    for code in &patterns {
        let md = to_markdown(code, MarkdownOptions::default());
        assert!(md.contains("```"), "Code not detected for: {}", code);
    }
}

#[test]
fn test_code_special_chars() {
    let code = "if (x > 0) { return y; }";
    let md = to_markdown(code, MarkdownOptions::default());
    assert!(md.contains("```"));
}

#[test]
fn test_non_code_text() {
    let text = "This is regular text about programming.";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(!md.contains("```"));
}

// ============================================================================
// Monospace Font Detection Tests
// ============================================================================

#[test]
fn test_monospace_font_names() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Font names that contain the patterns in is_monospace_font
    let monospace_fonts = [
        "Courier",
        "Consolas",
        "Monaco",
        "Menlo",
        "Fira Code",
        "JetBrains Mono",
        "Inconsolata",
        "DejaVu Sans Mono",
        "Liberation Mono",
        "Fixed",
        "Terminal",
    ];

    for font in &monospace_fonts {
        let items = vec![make_text_item_with_font(
            "code", 100.0, 700.0, 12.0, font, 1,
        )];
        let md = to_markdown_from_items(items, MarkdownOptions::default());
        assert!(
            md.contains("```"),
            "Font not detected as monospace: {}",
            font
        );
    }
}

// ============================================================================
// Header Level Detection Tests
// ============================================================================

#[test]
fn test_header_level_h1() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // 24.0 / 12.0 = 2.0x = H1
    // Need multiple body items to establish base font size
    let items = vec![
        make_text_item("H1 Title", 100.0, 700.0, 24.0, 1),
        make_text_item("body text one", 100.0, 650.0, 12.0, 1),
        make_text_item("body text two", 100.0, 630.0, 12.0, 1),
        make_text_item("body text three", 100.0, 610.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("# H1 Title"));
}

#[test]
fn test_single_heading_tier_becomes_h1() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Single heading tier: 18.0pt on 12.0pt base → H1 (not H2)
    let items = vec![
        make_text_item("Section Title", 100.0, 700.0, 18.0, 1),
        make_text_item("body text one", 100.0, 650.0, 12.0, 1),
        make_text_item("body text two", 100.0, 630.0, 12.0, 1),
        make_text_item("body text three", 100.0, 610.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("# Section Title"));
}

#[test]
fn test_header_level_h2() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Two heading tiers: 24.0 → H1, 18.0 → H2
    let items = vec![
        make_text_item("H1 Title", 100.0, 750.0, 24.0, 1),
        make_text_item("H2 Title", 100.0, 700.0, 18.0, 1),
        make_text_item("body text one", 100.0, 650.0, 12.0, 1),
        make_text_item("body text two", 100.0, 630.0, 12.0, 1),
        make_text_item("body text three", 100.0, 610.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("# H1 Title"));
    assert!(md.contains("## H2 Title"));
}

#[test]
fn test_header_level_h3() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Three heading tiers: 24.0 → H1, 18.0 → H2, 15.0 → H3
    let items = vec![
        make_text_item("H1 Title", 100.0, 800.0, 24.0, 1),
        make_text_item("H2 Title", 100.0, 750.0, 18.0, 1),
        make_text_item("H3 Title", 100.0, 700.0, 15.0, 1),
        make_text_item("body text one", 100.0, 650.0, 12.0, 1),
        make_text_item("body text two", 100.0, 630.0, 12.0, 1),
        make_text_item("body text three", 100.0, 610.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("### H3 Title"));
}

#[test]
fn test_header_level_h4() {
    use pdf_inspector::markdown::to_markdown_from_items;
    // Four heading tiers: 24.0 → H1, 18.0 → H2, 15.0 → H3, 14.5 → H4
    let items = vec![
        make_text_item("H1 Title", 100.0, 850.0, 24.0, 1),
        make_text_item("H2 Title", 100.0, 800.0, 18.0, 1),
        make_text_item("H3 Title", 100.0, 750.0, 15.0, 1),
        make_text_item("H4 Title", 100.0, 700.0, 14.5, 1),
        make_text_item("body text one", 100.0, 650.0, 12.0, 1),
        make_text_item("body text two", 100.0, 630.0, 12.0, 1),
        make_text_item("body text three", 100.0, 610.0, 12.0, 1),
    ];
    let md = to_markdown_from_items(items, MarkdownOptions::default());
    assert!(md.contains("#### H4 Title"));
}

// ============================================================================
// Clean Markdown Tests
// ============================================================================

#[test]
fn test_excessive_newlines_preserved_in_plain_text() {
    // Plain text to_markdown preserves structure from input
    let text = "Para one\n\n\n\n\nPara two";
    let md = to_markdown(text, MarkdownOptions::default());
    // The function processes line by line, empty lines become single newlines
    assert!(md.contains("Para one"));
    assert!(md.contains("Para two"));
}

#[test]
fn test_trailing_newline() {
    let text = "Content";
    let md = to_markdown(text, MarkdownOptions::default());
    assert!(md.ends_with('\n'));
    assert!(!md.ends_with("\n\n"));
}

// ============================================================================
// NotAPdf Detection Tests
// ============================================================================

/// Helper: assert that an error is NotAPdf and its message contains the given substring.
fn assert_not_a_pdf(result: Result<impl std::fmt::Debug, PdfError>, expected_hint: &str) {
    match result {
        Err(PdfError::NotAPdf(msg)) => {
            assert!(
                msg.to_lowercase().contains(&expected_hint.to_lowercase()),
                "Expected hint '{}' in NotAPdf message, got: '{}'",
                expected_hint,
                msg,
            );
        }
        other => panic!(
            "Expected Err(NotAPdf) containing '{}', got: {:?}",
            expected_hint, other,
        ),
    }
}

#[test]
fn test_not_a_pdf_html_input() {
    let html = b"<!DOCTYPE html><html><body>Hello</body></html>";
    let result = pdf_inspector::process_pdf_mem(html);
    assert_not_a_pdf(result, "HTML");
}

#[test]
fn test_not_a_pdf_xml_input() {
    let xml = b"<?xml version=\"1.0\"?><root><item>data</item></root>";
    let result = pdf_inspector::process_pdf_mem(xml);
    assert_not_a_pdf(result, "XML");
}

#[test]
fn test_not_a_pdf_json_input() {
    let json = b"{\"error\": \"download failed\"}";
    let result = pdf_inspector::process_pdf_mem(json);
    assert_not_a_pdf(result, "JSON");
}

#[test]
fn test_not_a_pdf_plain_text_input() {
    let text = b"This is a plain text file that is not a PDF at all.";
    let result = pdf_inspector::process_pdf_mem(text);
    assert_not_a_pdf(result, "plain text");
}

#[test]
fn test_not_a_pdf_empty_buffer() {
    let result = pdf_inspector::process_pdf_mem(b"");
    assert_not_a_pdf(result, "empty");
}

#[test]
fn test_valid_pdf_header_not_rejected() {
    // A truncated but valid PDF header should NOT produce NotAPdf —
    // it should fail with Parse or InvalidStructure instead.
    let truncated_pdf = b"%PDF-1.4\ntruncated content";
    let result = pdf_inspector::process_pdf_mem(truncated_pdf);
    match result {
        Err(PdfError::NotAPdf(_)) => panic!("Valid PDF header should not be rejected as NotAPdf"),
        _ => {} // Parse or InvalidStructure is fine
    }
}

#[test]
fn test_bom_prefixed_pdf_header_not_rejected() {
    // UTF-8 BOM + %PDF- should still be recognized as a PDF
    let mut bom_pdf = vec![0xEF, 0xBB, 0xBF];
    bom_pdf.extend_from_slice(b"%PDF-1.7\ntruncated");
    let result = pdf_inspector::process_pdf_mem(&bom_pdf);
    match result {
        Err(PdfError::NotAPdf(_)) => {
            panic!("BOM-prefixed PDF header should not be rejected as NotAPdf")
        }
        _ => {} // Parse or InvalidStructure is fine
    }
}

#[test]
fn test_not_a_pdf_detect_pdf_type_mem() {
    // Verify detect_pdf_type_mem is also guarded
    let html = b"<html><head><title>Not a PDF</title></head></html>";
    let result = pdf_inspector::detector::detect_pdf_type_mem(html);
    assert_not_a_pdf(result, "HTML");
}

#[test]
fn test_not_a_pdf_extract_text_with_positions_mem() {
    // Verify extract_text_with_positions_mem is also guarded
    let html = b"<!DOCTYPE html><html><body>content</body></html>";
    let result = pdf_inspector::extractor::extract_text_with_positions_mem(html);
    assert_not_a_pdf(result, "HTML");
}

#[test]
fn test_not_a_pdf_extract_text_mem() {
    // Verify extract_text_mem is also guarded
    let xml = b"<?xml version=\"1.0\"?><data/>";
    let result = pdf_inspector::extractor::extract_text_mem(xml);
    assert_not_a_pdf(result, "XML");
}

// ============================================================================
// Snapshot Regression Tests (PDF fixtures)
// ============================================================================

/// Process a PDF fixture and compare output against the golden snapshot.
///
/// This catches regressions where code changes silently alter extraction
/// or markdown output. If a change is intentional, update the snapshot:
///   cargo run --release --bin pdf2md -- tests/fixtures/<name>.pdf > tests/snapshots/<name>.md
fn assert_snapshot(fixture: &str) {
    let fixture_path = format!("tests/fixtures/{}.pdf", fixture);
    let snapshot_path = format!("tests/snapshots/{}.md", fixture);

    let result = pdf_inspector::process_pdf(&fixture_path)
        .unwrap_or_else(|e| panic!("Failed to process {}: {}", fixture_path, e));
    let actual = result.markdown.unwrap_or_default();
    let actual = actual.trim_end();

    let expected = std::fs::read_to_string(&snapshot_path)
        .unwrap_or_else(|e| panic!("Failed to read snapshot {}: {}", snapshot_path, e));
    let expected = expected.trim_end();

    if actual != expected {
        // Show a helpful diff summary
        let actual_lines: Vec<&str> = actual.lines().collect();
        let expected_lines: Vec<&str> = expected.lines().collect();

        let mut diffs = Vec::new();
        let max_lines = actual_lines.len().max(expected_lines.len());
        for i in 0..max_lines {
            let a = actual_lines.get(i).unwrap_or(&"<missing>");
            let e = expected_lines.get(i).unwrap_or(&"<missing>");
            if a != e {
                diffs.push(format!(
                    "  line {}: expected {:?}, got {:?}",
                    i + 1,
                    &e[..e.len().min(80)],
                    &a[..a.len().min(80)]
                ));
                if diffs.len() >= 5 {
                    diffs.push("  ... (more diffs truncated)".to_string());
                    break;
                }
            }
        }

        panic!(
            "Snapshot mismatch for {}:\n{}\n\nTo update: cargo run --release --bin pdf2md -- {} > {}",
            fixture,
            diffs.join("\n"),
            fixture_path,
            snapshot_path,
        );
    }
}

#[test]
fn test_snapshot_nexo_price_en() {
    assert_snapshot("nexo-price-en");
}

#[test]
fn test_snapshot_thermo_freon12() {
    assert_snapshot("thermo-freon12");
}

#[test]
fn test_snapshot_td9264() {
    assert_snapshot("td9264");
}

#[test]
fn test_snapshot_p1244() {
    assert_snapshot("p1244-1996");
}

#[test]
fn test_snapshot_real_estate_pricing() {
    assert_snapshot("real-estate-pricing");
}

#[test]
fn test_snapshot_2013_app2() {
    assert_snapshot("2013-app2");
}

// ============================================================================
// Pages Needing OCR Tests
// ============================================================================

#[test]
fn test_pages_needing_ocr_field_accessible() {
    // Compile-time check: verify the field exists on both structs
    let detection_result = pdf_inspector::detector::PdfTypeResult {
        pdf_type: PdfType::TextBased,
        page_count: 1,
        pages_sampled: 1,
        pages_with_text: 1,
        confidence: 1.0,
        title: None,
        ocr_recommended: false,
        pages_needing_ocr: Vec::new(),
    };
    assert!(detection_result.pages_needing_ocr.is_empty());

    let process_result = pdf_inspector::PdfProcessResult {
        pdf_type: PdfType::TextBased,
        markdown: None,
        page_count: 1,
        processing_time_ms: 0,
        pages_needing_ocr: vec![1, 3],
        title: None,
        confidence: 1.0,
        layout: pdf_inspector::LayoutComplexity::default(),
        has_encoding_issues: false,
    };
    assert_eq!(process_result.pages_needing_ocr, vec![1, 3]);
}

#[test]
fn test_text_pdf_process_result_empty_ocr_pages() {
    // A minimal valid PDF that is text-based should have empty pages_needing_ocr.
    // We use a minimal PDF buffer with a text content stream.
    let pdf_bytes = b"%PDF-1.0
1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj
2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj
3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R/Contents 4 0 R>>endobj
4 0 obj<</Length 44>>
stream
BT /F1 12 Tf 100 700 Td (Hello World) Tj ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000009 00000 n
0000000058 00000 n
0000000115 00000 n
0000000206 00000 n
trailer<</Size 5/Root 1 0 R>>
startxref
300
%%EOF";
    let result = pdf_inspector::process_pdf_mem(pdf_bytes);
    // The minimal PDF may fail to parse fully, but if it succeeds,
    // a text-based PDF should have empty pages_needing_ocr.
    if let Ok(result) = result {
        assert!(
            result.pages_needing_ocr.is_empty(),
            "Text-based PDF should have empty pages_needing_ocr, got: {:?}",
            result.pages_needing_ocr
        );
    }
}

#[test]
fn test_firecrawl_tagged_pdf_struct_tree() {
    use lopdf::Document;
    use pdf_inspector::structure_tree::{StructRole, StructTree};

    let doc = Document::load("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap();
    let tree = StructTree::from_doc(&doc).expect("Should have a structure tree");

    // Verify structure tree contains expected roles
    let page_ids = doc.get_pages();
    let roles = tree.mcid_to_roles(&page_ids);
    assert!(!roles.is_empty(), "Should have MCID roles across pages");

    let flat = tree.flatten();
    let has_code = flat.iter().any(|e| matches!(e.role, StructRole::Code));
    let has_h1 = flat.iter().any(|e| matches!(e.role, StructRole::H1));
    let has_li = flat.iter().any(|e| matches!(e.role, StructRole::LI));
    let has_caption = flat.iter().any(|e| matches!(e.role, StructRole::Caption));
    assert!(has_code, "Should have Code elements");
    assert!(has_h1, "Should have H1 elements");
    assert!(has_li, "Should have LI elements");
    assert!(has_caption, "Should have Caption elements");

    // Full conversion: code fences should be generated from Code struct elements
    let buf = std::fs::read("tests/fixtures/firecrawl_docs_tagged.pdf").unwrap();
    let result = pdf_inspector::process_pdf_mem(&buf).unwrap();
    let md = result.markdown.unwrap();
    let fence_count = md.matches("```").count();
    assert!(
        fence_count > 0,
        "Should produce code fences from tagged Code elements"
    );
    // Fences come in open/close pairs
    assert_eq!(fence_count % 2, 0, "Code fences should be balanced");
}

#[test]
fn test_identity_h_no_tounicode_suppresses_garbage() {
    // shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no
    // ToUnicode CMap.  The raw CID values look like random Latin characters.
    // We should suppress the garbage and flag the page for OCR.
    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
    let result = pdf_inspector::process_pdf_mem(&buf).unwrap();

    // Page 1 should be flagged for OCR
    assert!(
        result.pages_needing_ocr.contains(&1),
        "Page with Identity-H font without ToUnicode should be flagged for OCR"
    );

    // Markdown should be empty (garbage suppressed)
    let md = result.markdown.unwrap_or_default();
    assert!(
        md.trim().is_empty(),
        "Garbage CID text should be suppressed, got {} chars: {:?}",
        md.len(),
        &md[..md.len().min(100)]
    );
}

#[test]
fn test_rotated_table_layout_correction() {
    // tnagriculture_06_12.pdf has landscape content in a portrait page via
    // a 90° CCW text matrix [0, b, -b, 0, tx, ty].  Without rotation
    // correction, the table is read sideways (jumbled numbers).
    let result =
        process_pdf_with_options("tests/fixtures/tnagriculture_06_12.pdf", PdfOptions::new())
            .unwrap();
    let md = result.markdown.unwrap_or_default();

    // Title should appear near the top
    assert!(
        md.contains("DISTRICT WISE PRODUCTION OF SPICES AND CONDIMENTS"),
        "Should extract the table title"
    );

    // District names should be readable (not jumbled with numbers)
    assert!(
        md.contains("Ariyalur"),
        "Should extract district name Ariyalur"
    );
    assert!(
        md.contains("Coimbatore"),
        "Should extract district name Coimbatore"
    );

    // Spice column headers should appear
    assert!(
        md.contains("CARDAMOM"),
        "Should extract spice header CARDAMOM"
    );
    assert!(
        md.contains("RED CHILLIES"),
        "Should extract spice header RED CHILLIES"
    );

    // Table should be formatted as markdown table (has pipe delimiters)
    let has_table_row = md
        .lines()
        .any(|l: &str| l.contains('|') && l.contains("Ariyalur"));
    assert!(
        has_table_row,
        "District data should be in a markdown table row"
    );
}

// =========================================================================
// extract_text_in_regions_mem tests
// =========================================================================

/// Build full-page region args for `page_count` pages.
/// Uses a generously large bbox (1200x1200) to capture any page size.
fn full_page_regions(page_count: u32) -> Vec<(u32, Vec<[f32; 4]>)> {
    (0..page_count)
        .map(|p| (p, vec![[0.0, 0.0, 1200.0, 1200.0]]))
        .collect()
}

/// Normalize text for comparison: lowercase, strip non-alphanumeric, split into words.
fn normalize_words(text: &str) -> HashSet<String> {
    text.split(|c: char| !c.is_alphanumeric())
        .map(|w| w.to_lowercase())
        .filter(|w| w.len() > 3)
        .collect()
}

/// Fraction of normalized words in `a` that also appear in `b`.
fn word_overlap_ratio(a: &str, b: &str) -> f64 {
    let words_a = normalize_words(a);
    if words_a.is_empty() {
        return if normalize_words(b).is_empty() {
            1.0
        } else {
            0.0
        };
    }
    let words_b = normalize_words(b);
    let overlap = words_a.intersection(&words_b).count();
    overlap as f64 / words_a.len() as f64
}

#[test]
fn test_extract_regions_mem_basic_text_pdf() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let result = process_pdf_mem(&buf).unwrap();
    let page_count = result.page_count;

    let regions = extract_text_in_regions_mem(&buf, &full_page_regions(page_count)).unwrap();
    assert_eq!(regions.len(), page_count as usize);

    // Each result should have exactly 1 region (we passed one per page)
    for r in &regions {
        assert_eq!(r.regions.len(), 1);
    }

    // First page should have non-empty text
    let first = &regions[0].regions[0];
    assert!(!first.text.trim().is_empty(), "First page should have text");
    assert_eq!(regions[0].page, 0);
}

#[test]
fn test_extract_regions_mem_identity_h_needs_ocr() {
    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
    let regions =
        extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
    assert_eq!(regions.len(), 1);
    assert!(
        regions[0].regions[0].needs_ocr,
        "Identity-H font without ToUnicode should trigger needs_ocr"
    );
}

#[test]
fn test_extract_regions_mem_multiple_regions_per_page() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let regions = extract_text_in_regions_mem(
        &buf,
        &[(
            0,
            vec![
                [0.0, 0.0, 300.0, 100.0],   // small top-left
                [0.0, 0.0, 1200.0, 1200.0], // full page
            ],
        )],
    )
    .unwrap();

    assert_eq!(regions.len(), 1);
    assert_eq!(regions[0].regions.len(), 2);

    let small_len = regions[0].regions[0].text.len();
    let full_len = regions[0].regions[1].text.len();
    assert!(
        full_len >= small_len,
        "Full-page region ({full_len}) should have at least as much text as small region ({small_len})"
    );
}

#[test]
fn test_extract_regions_mem_nonexistent_page() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let regions =
        extract_text_in_regions_mem(&buf, &[(9999, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
    assert_eq!(regions.len(), 1);
    assert!(
        regions[0].regions[0].needs_ocr,
        "Nonexistent page should trigger needs_ocr"
    );
}

#[test]
fn test_extract_regions_mem_empty_region() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let regions = extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 0.0, 0.0]])]).unwrap();
    assert_eq!(regions.len(), 1);
    assert!(
        regions[0].regions[0].needs_ocr,
        "Zero-area region should trigger needs_ocr"
    );
}

#[test]
fn test_extract_regions_mem_not_a_pdf() {
    let result = extract_text_in_regions_mem(b"not a pdf", &[(0, vec![[0.0, 0.0, 100.0, 100.0]])]);
    assert!(result.is_err(), "Non-PDF input should return an error");
}

#[test]
fn test_extract_regions_mem_rotated_page_not_false_empty() {
    let buf = std::fs::read("tests/fixtures/tnagriculture_06_12.pdf").unwrap();
    let regions =
        extract_text_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();
    assert_eq!(regions.len(), 1);
    assert_eq!(regions[0].regions.len(), 1);
    let region = &regions[0].regions[0];
    assert!(
        !region.text.trim().is_empty(),
        "Rotated page full-region extraction should not be empty"
    );
    assert!(
        !region.needs_ocr,
        "Rotated page with native text should not be flagged for OCR fallback"
    );
    assert!(
        region
            .text
            .contains("DISTRICT WISE PRODUCTION OF SPICES AND CONDIMENTS"),
        "Expected known title from rotated fixture in extracted region text"
    );
}

#[test]
fn test_collect_text_in_region_keeps_partial_overlap_items() {
    let item = make_text_item("EdgeWord", 100.0, 700.0, 12.0, 1);
    // Region intersects only the left edge of the item. Center x=124 falls
    // outside x=[95,120], so center-only containment would drop it.
    let text = pdf_inspector::collect_text_in_region(&[item], 95.0, 80.0, 120.0, 110.0, 800.0);
    assert!(
        text.contains("EdgeWord"),
        "Partially overlapping items should be retained in region extraction"
    );
}

#[test]
fn test_collect_text_in_region_uses_rtl_sorting() {
    let items = vec![
        make_text_item("بكم", 240.0, 700.0, 12.0, 1),
        make_text_item("مرحبا", 300.0, 700.0, 12.0, 1),
    ];
    let text = pdf_inspector::collect_text_in_region(&items, 0.0, 0.0, 600.0, 800.0, 800.0);
    assert_eq!(
        text, "مرحبا بكم",
        "Region path should reuse RTL-aware line sorting"
    );
}

// =========================================================================
// Fast vs normal extraction comparison
// =========================================================================

/// For each text-based fixture PDF, compare `extract_text_in_regions_mem` (fast path)
/// against `process_pdf_mem` (normal path). If the fast path claims needs_ocr=false
/// for a page, verify the extracted text has meaningful overlap with the normal
/// markdown output — catching silent quality regressions.
#[test]
fn test_extract_regions_fast_vs_normal_comparison() {
    let fixtures = [
        "tests/fixtures/nexo-price-en.pdf",
        "tests/fixtures/td9264.pdf",
        "tests/fixtures/p1244-1996.pdf",
        "tests/fixtures/real-estate-pricing.pdf",
        "tests/fixtures/2013-app2.pdf",
        "tests/fixtures/firecrawl_docs_tagged.pdf",
        "tests/fixtures/thermo-freon12.pdf",
    ];

    for fixture in &fixtures {
        let buf = std::fs::read(fixture).unwrap();
        let normal = process_pdf_mem(&buf).unwrap();
        let normal_md = normal.markdown.as_deref().unwrap_or("");
        let page_count = normal.page_count;
        let ocr_pages: HashSet<u32> = normal.pages_needing_ocr.iter().copied().collect();

        let regions = extract_text_in_regions_mem(&buf, &full_page_regions(page_count)).unwrap();

        assert_eq!(
            regions.len(),
            page_count as usize,
            "{fixture}: result count should match page count"
        );

        for pr in &regions {
            let region = &pr.regions[0];
            if !region.needs_ocr && !region.text.trim().is_empty() {
                // Fast path claims this text is trustworthy.
                // Check that its words appear in the normal markdown output.
                let overlap = word_overlap_ratio(&region.text, normal_md);
                assert!(
                    overlap >= 0.3,
                    "{fixture} page {}: fast path says needs_ocr=false but only {:.0}% word \
                     overlap with normal extraction (threshold 30%). \
                     Fast text sample: {:?}",
                    pr.page,
                    overlap * 100.0,
                    &region.text[..region.text.len().min(200)],
                );
            }

            // If fast path flags needs_ocr but normal path didn't, that's overly
            // conservative but not a bug — just worth knowing.
            if region.needs_ocr && !ocr_pages.contains(&(pr.page + 1)) {
                eprintln!(
                    "INFO: {fixture} page {}: fast path says needs_ocr=true but normal path extracted fine (conservative, not a bug)",
                    pr.page,
                );
            }
        }
    }
}

// =========================================================================
// extract_tables_in_regions_mem tests
// =========================================================================

#[test]
fn test_extract_tables_in_regions_table_pdf() {
    // tnagriculture has a clear table with district names and spice columns
    let buf = std::fs::read("tests/fixtures/tnagriculture_06_12.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();

    assert_eq!(results.len(), 1);
    assert_eq!(results[0].regions.len(), 1);

    let region = &results[0].regions[0];
    // Should detect a table with pipe-delimited markdown
    if !region.needs_ocr {
        assert!(
            region.text.contains('|'),
            "Table output should contain pipe delimiters"
        );
        // Should have separator row
        assert!(
            region.text.lines().any(|l| l.contains("---")),
            "Table output should contain separator row"
        );
    }
}

#[test]
fn test_extract_tables_in_regions_non_table_region() {
    // Use a small region that likely won't contain enough items for a table
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 50.0, 50.0]])]).unwrap();

    assert_eq!(results.len(), 1);
    assert_eq!(results[0].regions.len(), 1);

    let region = &results[0].regions[0];
    // Small region with few items should fall back to needs_ocr
    assert!(
        region.needs_ocr,
        "Non-table region should set needs_ocr = true"
    );
    assert!(
        region.text.is_empty(),
        "Non-table region should have empty text"
    );
}

#[test]
fn test_extract_tables_in_regions_empty_region() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let results = extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 0.0, 0.0]])]).unwrap();

    assert_eq!(results.len(), 1);
    let region = &results[0].regions[0];
    assert!(region.needs_ocr);
    assert!(region.text.is_empty());
}

#[test]
fn test_extract_tables_in_regions_identity_h_needs_ocr() {
    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(0, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();

    assert_eq!(results.len(), 1);
    let region = &results[0].regions[0];
    assert!(region.needs_ocr, "Identity-H font should trigger needs_ocr");
}

#[test]
fn test_extract_tables_in_regions_not_a_pdf() {
    let result =
        extract_tables_in_regions_mem(b"not a pdf", &[(0, vec![[0.0, 0.0, 100.0, 100.0]])]);
    assert!(result.is_err());
}

#[test]
fn test_extract_tables_in_regions_nonexistent_page() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(9999, vec![[0.0, 0.0, 1200.0, 1200.0]])]).unwrap();

    assert_eq!(results.len(), 1);
    let region = &results[0].regions[0];
    assert!(region.needs_ocr);
    assert!(region.text.is_empty());
}

#[test]
fn test_bits_pilani_page4_table_detection() {
    // Page 4 (0-indexed 3) has a table with multi-line wrapped headers and
    // numeric data columns. The heuristic detector previously failed because:
    // 1. Header items at different X positions than data created extra column
    //    clusters (6 cols instead of 4)
    // 2. Spanning super-header row ("First Degree | First Degree") produced
    //    duplicate header cells that looks_like_partial_table_ex rejected
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(3, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
    assert_eq!(results.len(), 1);
    let region = &results[0].regions[0];
    assert!(
        !region.needs_ocr,
        "Page 4 table should be detected, got needs_ocr=true"
    );
    assert!(
        region.text.contains("BIO"),
        "Should contain department name BIO"
    );
    assert!(region.text.contains("8.23"), "Should contain numeric data");
}

#[test]
fn test_bits_pilani_page8_table_detection() {
    // Page 8 (0-indexed 7) has a numbered-row table that already worked.
    // Verify it still works after changes.
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
    let results =
        extract_tables_in_regions_mem(&buf, &[(7, vec![[0.0, 0.0, 612.0, 792.0]])]).unwrap();
    assert_eq!(results.len(), 1);
    let region = &results[0].regions[0];
    assert!(!region.needs_ocr, "Page 8 table should still be detected");
}

// =========================================================================
// extract_tables_with_structure_mem tests (TSR-aware path)
// =========================================================================

/// Build an 8-element 4-corner polygon `[x1,y1, x2,y1, x2,y2, x1,y2]` from
/// an axis-aligned rect — matches the format SLANet emits for cell bboxes.
fn poly(x1: f32, y1: f32, x2: f32, y2: f32) -> Vec<f32> {
    vec![x1, y1, x2, y1, x2, y2, x1, y2]
}

#[test]
fn test_extract_tables_with_structure_real_pdf_bits_pilani() {
    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
    // Hand-crafted TSR fixture targeting page 4 (0-indexed=3) of
    // bits_pilani_feedback.pdf, which contains a clean tabular layout.
    //
    // We construct a 2×2 table:
    //   row 0 (header): "Department"   "Core Courses"
    //   row 1 (data):   "BIO"          "8.23"
    //
    // The PDF page is US Letter (792pt tall). We render at 72 dpi so
    // image-px maps 1:1 to PDF-pt — that lets us write cell bboxes in
    // the same units as our hand-measured page-pt coordinates.
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();

    // The PDF page is A4 in points (≈595.44 × 841.68). The table sits in
    // the upper part of the page; we crop a window large enough to enclose
    // both rows we care about.
    //
    // Crop bounds in PDF points (top-left origin):
    //   x: 80..280, y: 170..240
    let crop = [80.0_f32, 170.0, 280.0, 240.0];
    let dpi = 72.0_f32;

    // Cell bboxes in CROP image-pixel space (= crop-relative PDF-pt at
    // 72 dpi). The y ranges are tightened against neighbouring rows
    // ("First Degree" above the header at native y=666.7, "Feedback Score"
    // between the header and data rows at native y=640.9, "CE" below the
    // BIO row at native y=591.1) so each cell only overlaps its target
    // text item.
    let cell_bboxes = vec![
        // Header row: y crop-relative (7, 18) → page-pt y (177, 188)
        poly(10.0, 7.0, 100.0, 18.0), // "Department"   (item at page-pt x=107.1)
        poly(110.0, 7.0, 200.0, 18.0), // "Core Courses" (item at page-pt x=199.0)
        // Data row: y crop-relative (35, 60) → page-pt y (205, 230)
        poly(10.0, 35.0, 100.0, 60.0), // "BIO"  (item at page-pt x=104.1)
        poly(110.0, 35.0, 200.0, 60.0), // "8.23" (item at page-pt x=221.2)
    ];

    // Minimal SLANet-style token stream: a 2-row table with a thead and tbody.
    let tokens: Vec<String> = [
        "<table>",
        "<thead>",
        "<tr>",
        "<th></th>",
        "<th></th>",
        "</tr>",
        "</thead>",
        "<tbody>",
        "<tr>",
        "<td></td>",
        "<td></td>",
        "</tr>",
        "</tbody>",
        "</table>",
    ]
    .into_iter()
    .map(String::from)
    .collect();

    let inputs = vec![TsrTableInput {
        page: 3,
        crop_pdf_pt_bbox: crop,
        render_dpi: dpi,
        structure_tokens: tokens,
        cell_bboxes,
    }];

    let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
    assert_eq!(mds.len(), 1);
    let md = &mds[0];

    // Hand-written gold standard for the rendered markdown.
    let expected = "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n";
    assert_eq!(
        md, expected,
        "structured-table markdown should match the gold standard exactly\nactual: {md}"
    );
}

#[test]
fn test_extract_tables_with_structure_input_order_preserved() {
    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();

    // Two inputs; both target the same page but with different shapes.
    // We just need to confirm we get 2 outputs in the same order.
    let make_input = |toks: Vec<&str>, cells: Vec<Vec<f32>>| TsrTableInput {
        page: 3,
        crop_pdf_pt_bbox: [80.0, 170.0, 280.0, 240.0],
        render_dpi: 72.0,
        structure_tokens: toks.into_iter().map(String::from).collect(),
        cell_bboxes: cells,
    };

    let inputs = vec![
        make_input(
            vec!["<table>", "<tr>", "<td></td>", "</tr>", "</table>"],
            vec![poly(10.0, 35.0, 100.0, 60.0)],
        ),
        make_input(
            vec!["<table>", "<tr>", "<td></td>", "</tr>", "</table>"],
            vec![poly(110.0, 35.0, 200.0, 60.0)],
        ),
    ];

    let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
    assert_eq!(mds.len(), 2);
    assert!(
        mds[0].contains("BIO"),
        "input 0 should pull 'BIO': {}",
        mds[0]
    );
    assert!(
        mds[1].contains("8.23"),
        "input 1 should pull '8.23': {}",
        mds[1]
    );
}

#[test]
fn test_extract_tables_with_structure_out_of_range_page() {
    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();

    let inputs = vec![TsrTableInput {
        page: 9999,
        crop_pdf_pt_bbox: [0.0, 0.0, 100.0, 100.0],
        render_dpi: 72.0,
        structure_tokens: vec![
            "<table>".into(),
            "<tr>".into(),
            "<td></td>".into(),
            "</tr>".into(),
            "</table>".into(),
        ],
        cell_bboxes: vec![poly(0.0, 0.0, 50.0, 50.0)],
    }];

    let mds = extract_tables_with_structure_mem(&buf, &inputs).unwrap();
    assert_eq!(mds.len(), 1);
    assert!(
        mds[0].is_empty(),
        "out-of-range page should yield empty string"
    );
}

#[test]
fn test_extract_tables_with_structure_not_a_pdf() {
    use pdf_inspector::extract_tables_with_structure_mem;
    let result = extract_tables_with_structure_mem(b"not a pdf", &[]);
    assert!(result.is_err());
}

#[test]
fn test_extract_tables_with_structure_empty_inputs() {
    use pdf_inspector::extract_tables_with_structure_mem;
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();
    let mds = extract_tables_with_structure_mem(&buf, &[]).unwrap();
    assert!(mds.is_empty());
}

#[test]
fn test_extract_tables_with_structure_cells_real_pdf_bits_pilani() {
    use pdf_inspector::{extract_tables_with_structure_cells_mem, TsrTableInput};
    // Same fixture as test_extract_tables_with_structure_real_pdf_bits_pilani
    // but exercising the cell-level API. Verifies that callers receive
    // structured per-cell metadata (row/col/spans/is_header/page_pt_bbox)
    // alongside the extracted text.
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();

    let crop = [80.0_f32, 170.0, 280.0, 240.0];
    let dpi = 72.0_f32;
    let cell_bboxes = vec![
        poly(10.0, 7.0, 100.0, 18.0),
        poly(110.0, 7.0, 200.0, 18.0),
        poly(10.0, 35.0, 100.0, 60.0),
        poly(110.0, 35.0, 200.0, 60.0),
    ];
    let tokens: Vec<String> = [
        "<table>",
        "<thead>",
        "<tr>",
        "<th></th>",
        "<th></th>",
        "</tr>",
        "</thead>",
        "<tbody>",
        "<tr>",
        "<td></td>",
        "<td></td>",
        "</tr>",
        "</tbody>",
        "</table>",
    ]
    .into_iter()
    .map(String::from)
    .collect();

    let inputs = vec![TsrTableInput {
        page: 3,
        crop_pdf_pt_bbox: crop,
        render_dpi: dpi,
        structure_tokens: tokens,
        cell_bboxes,
    }];

    let cells_lists = extract_tables_with_structure_cells_mem(&buf, &inputs).unwrap();
    assert_eq!(cells_lists.len(), 1);
    let cells = &cells_lists[0];
    assert_eq!(cells.len(), 4);

    // Header row: both cells flagged as headers (they were in <thead>/<th>).
    assert!(cells[0].is_header);
    assert!(cells[1].is_header);
    assert_eq!((cells[0].row, cells[0].col), (0, 0));
    assert_eq!((cells[1].row, cells[1].col), (0, 1));
    assert_eq!(cells[0].text, "Department");
    assert_eq!(cells[1].text, "Core Courses");

    // Data row: not flagged as header.
    assert!(!cells[2].is_header);
    assert!(!cells[3].is_header);
    assert_eq!((cells[2].row, cells[2].col), (1, 0));
    assert_eq!((cells[3].row, cells[3].col), (1, 1));
    assert_eq!(cells[2].text, "BIO");
    assert_eq!(cells[3].text, "8.23");

    // Every cell carries a non-degenerate page-pt bbox.
    for c in cells {
        let [x1, y1, x2, y2] = c.page_pt_bbox;
        assert!(
            x1 < x2 && y1 < y2,
            "cell bbox should be non-empty: {:?}",
            c.page_pt_bbox
        );
    }
}

#[test]
fn test_extract_tables_with_structure_separator_after_thead() {
    use pdf_inspector::{extract_tables_with_structure_mem, TsrTableInput};
    // Re-run the same 2x2 fixture but assert exact markdown output: with
    // <thead> + <th> headers, the separator should land after the header
    // row (which is also row 0 here, so the gold-standard hasn't changed).
    let buf = std::fs::read("tests/fixtures/bits_pilani_feedback.pdf").unwrap();

    let crop = [80.0_f32, 170.0, 280.0, 240.0];
    let dpi = 72.0_f32;
    let cell_bboxes = vec![
        poly(10.0, 7.0, 100.0, 18.0),
        poly(110.0, 7.0, 200.0, 18.0),
        poly(10.0, 35.0, 100.0, 60.0),
        poly(110.0, 35.0, 200.0, 60.0),
    ];
    let tokens: Vec<String> = [
        "<table>",
        "<thead>",
        "<tr>",
        "<th></th>",
        "<th></th>",
        "</tr>",
        "</thead>",
        "<tbody>",
        "<tr>",
        "<td></td>",
        "<td></td>",
        "</tr>",
        "</tbody>",
        "</table>",
    ]
    .into_iter()
    .map(String::from)
    .collect();

    let mds = extract_tables_with_structure_mem(
        &buf,
        &[TsrTableInput {
            page: 3,
            crop_pdf_pt_bbox: crop,
            render_dpi: dpi,
            structure_tokens: tokens,
            cell_bboxes,
        }],
    )
    .unwrap();
    assert_eq!(mds.len(), 1);
    assert_eq!(mds[0], "|Department|Core Courses|\n|---|---|\n|BIO|8.23|\n");
}

// =========================================================================
// extract_pages_markdown_mem tests
// =========================================================================

#[test]
fn test_extract_pages_markdown_basic() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[0, 1])).unwrap();

    assert_eq!(result.pages.len(), 2);
    assert_eq!(result.pages[0].page, 0);
    assert_eq!(result.pages[1].page, 1);
    // Text-based PDF should produce non-empty markdown
    assert!(!result.pages[0].markdown.is_empty());
    assert!(!result.pages[0].needs_ocr);
}

#[test]
fn test_extract_pages_markdown_page_ordering() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    // Request pages in non-sequential order
    let result = extract_pages_markdown_mem(&buf, Some(&[1, 0])).unwrap();

    assert_eq!(result.pages.len(), 2);
    // Results should match input order, not document order
    assert_eq!(result.pages[0].page, 1);
    assert_eq!(result.pages[1].page, 0);
}

#[test]
fn test_extract_pages_markdown_out_of_range() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[9999])).unwrap();

    assert_eq!(result.pages.len(), 1);
    assert_eq!(result.pages[0].page, 9999);
    assert!(result.pages[0].markdown.is_empty());
    assert!(result.pages[0].needs_ocr);
    assert!(result.pages_needing_ocr.contains(&10000)); // 1-indexed
}

#[test]
fn test_extract_pages_markdown_empty_pages_list() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[])).unwrap();
    assert!(result.pages.is_empty());
}

#[test]
fn test_extract_pages_markdown_single_page() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();

    assert_eq!(result.pages.len(), 1);
    assert_eq!(result.pages[0].page, 0);
    assert!(!result.pages[0].markdown.is_empty());
    assert!(!result.pages[0].needs_ocr);
}

#[test]
fn test_extract_pages_markdown_invalid_buffer() {
    let result = extract_pages_markdown_mem(b"not a pdf", Some(&[0]));
    assert!(result.is_err());
}

#[test]
fn test_extract_pages_markdown_gid_pages_need_ocr() {
    // shinagawa_identity_h.pdf has GID-encoded fonts
    let buf = std::fs::read("tests/fixtures/shinagawa_identity_h.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();

    assert_eq!(result.pages.len(), 1);
    assert!(result.pages[0].needs_ocr);
    assert!(result.pages_needing_ocr.contains(&1)); // 1-indexed
}

#[test]
fn test_extract_pages_markdown_classification_with_tables() {
    // nexo-price-en.pdf is known to have tables
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let page_count = process_pdf_mem(&buf).unwrap().page_count;
    let page_indices: Vec<u32> = (0..page_count).collect();
    let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();

    assert!(
        !result.pages_with_tables.is_empty(),
        "nexo-price-en.pdf should have pages with tables"
    );
    assert!(result.is_complex);
}

#[test]
fn test_extract_pages_markdown_simple_pdf_no_complexity() {
    // bare_name_struct.pdf is a simple document with a heading and code block
    let buf = std::fs::read("tests/fixtures/bare_name_struct.pdf").unwrap();
    let result = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();

    assert!(result.pages_with_tables.is_empty());
    assert!(result.pages_with_columns.is_empty());
    assert!(!result.is_complex);
}

#[test]
fn test_extract_pages_markdown_classification_matches_process_pdf() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let full = process_pdf_mem(&buf).unwrap();
    let page_count = full.page_count;
    let page_indices: Vec<u32> = (0..page_count).collect();
    let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();

    assert_eq!(
        result.pages_with_tables, full.layout.pages_with_tables,
        "pages_with_tables should match process_pdf"
    );
    assert_eq!(
        result.pages_with_columns, full.layout.pages_with_columns,
        "pages_with_columns should match process_pdf"
    );
}

#[test]
fn test_extract_pages_markdown_consistency_with_process_pdf() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();

    // Get full process_pdf output
    let full = process_pdf_mem(&buf).unwrap();
    let full_md = full.markdown.unwrap_or_default();

    // Get per-page output for all pages
    let page_count = full.page_count;
    let page_indices: Vec<u32> = (0..page_count).collect();
    let result = extract_pages_markdown_mem(&buf, Some(&page_indices)).unwrap();

    // Concatenated per-page markdown should contain substantial overlap with
    // the full output (exact match not expected due to header/footer stripping
    // and cross-page paragraph merging differences)
    let concat: String = result
        .pages
        .iter()
        .map(|p| p.markdown.as_str())
        .collect::<Vec<_>>()
        .join("\n");

    // Both should be non-empty for a text-based PDF
    assert!(!full_md.is_empty());
    assert!(!concat.is_empty());

    // The per-page version should contain at least 50% of the full content's
    // length (accounting for header/footer stripping differences)
    assert!(
        concat.len() * 2 >= full_md.len(),
        "per-page concat ({} chars) is too short vs full ({} chars)",
        concat.len(),
        full_md.len()
    );
}

#[test]
fn test_extract_pages_markdown_none_returns_all_pages() {
    let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();
    let page_count = process_pdf_mem(&buf).unwrap().page_count;

    let result = extract_pages_markdown_mem(&buf, None).unwrap();

    assert_eq!(result.pages.len() as u32, page_count);
    for (i, page) in result.pages.iter().enumerate() {
        assert_eq!(page.page, i as u32, "pages should be in document order");
    }
}

#[test]
fn test_extract_pages_markdown_path_api() {
    let path = "tests/fixtures/nexo-price-en.pdf";
    let buf = std::fs::read(path).unwrap();

    let via_path = extract_pages_markdown(path, Some(&[0])).unwrap();
    let via_mem = extract_pages_markdown_mem(&buf, Some(&[0])).unwrap();

    assert_eq!(via_path.pages.len(), via_mem.pages.len());
    assert_eq!(via_path.pages[0].markdown, via_mem.pages[0].markdown);
    assert_eq!(via_path.pages[0].needs_ocr, via_mem.pages[0].needs_ocr);
    assert_eq!(via_path.is_complex, via_mem.is_complex);
}

#[test]
fn test_extract_pages_markdown_path_none_returns_all_pages() {
    let path = "tests/fixtures/nexo-price-en.pdf";
    let page_count = process_pdf_mem(&std::fs::read(path).unwrap())
        .unwrap()
        .page_count;

    let result = extract_pages_markdown(path, None).unwrap();
    assert_eq!(result.pages.len() as u32, page_count);
}