mu-ref/pdf-inspector
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix list continuation, improve table detection, and add more font styling detection

Browse Source
This commit is contained in:
Abimael Martell
2026-02-10 13:30:33 -08:00
parent 6389c85ab9
commit 5afbb67b90
4 changed files with 439 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extractor.rs
+207 -41
View File
@@ -26,6 +26,10 @@ pub struct TextItem {
pub font_size: f32,
/// Page number (1-indexed)
pub page: u32,
/// Whether the font is bold
pub is_bold: bool,
/// Whether the font is italic
pub is_italic: bool,
}
/// A line of text (grouped text items)
@@ -38,6 +42,81 @@ pub struct TextLine {
impl TextLine {
pub fn text(&self) -> String {
self.text_with_formatting(false, false)
}
/// Get text with optional bold/italic markdown formatting
pub fn text_with_formatting(&self, format_bold: bool, format_italic: bool) -> String {
if !format_bold && !format_italic {
return self.text_plain();
}
let mut result = String::new();
let mut current_bold = false;
let mut current_italic = false;
for (i, item) in self.items.iter().enumerate() {
let text = item.text.as_str();
let text_trimmed = text.trim();
// Skip empty items
if text_trimmed.is_empty() {
continue;
}
// Determine spacing
let needs_space = if i == 0 || result.is_empty() {
false
} else {
let prev_item = &self.items[i - 1];
self.needs_space_between(prev_item, item, &result)
};
// Check for style changes
let item_bold = format_bold && item.is_bold;
let item_italic = format_italic && item.is_italic;
// Close previous styles if they change
if current_italic && !item_italic {
result.push('*');
current_italic = false;
}
if current_bold && !item_bold {
result.push_str("**");
current_bold = false;
}
// Add space after closing markers if needed
if needs_space {
result.push(' ');
}
// Open new styles
if item_bold && !current_bold {
result.push_str("**");
current_bold = true;
}
if item_italic && !current_italic {
result.push('*');
current_italic = true;
}
result.push_str(text_trimmed);
}
// Close any remaining open styles
if current_italic {
result.push('*');
}
if current_bold {
result.push_str("**");
}
result
}
/// Get plain text without formatting
fn text_plain(&self) -> String {
let mut result = String::new();
for (i, item) in self.items.iter().enumerate() {
let text = item.text.as_str();
@@ -45,53 +124,49 @@ impl TextLine {
result.push_str(text);
} else {
let prev_item = &self.items[i - 1];
// Don't add space before/after hyphens for hyphenated words
let prev_ends_with_hyphen = result.ends_with('-');
let curr_is_hyphen = text.trim() == "-";
let curr_starts_with_hyphen = text.starts_with('-');
// Detect subscript/superscript: smaller font size and/or Y offset
// Subscripts/superscripts are typically 60-80% of normal font size
// and have a vertical offset of 1-3 points
let font_ratio = item.font_size / prev_item.font_size;
let reverse_font_ratio = prev_item.font_size / item.font_size;
let y_diff = (item.y - prev_item.y).abs();
// Current item is subscript/superscript (smaller than previous)
let is_sub_super = font_ratio < 0.85 && y_diff > 1.0;
// Previous item was subscript/superscript (returning to normal size)
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
// Use position-based spacing detection
// This is more reliable than character-case heuristics for determining
// whether text fragments should be joined (e.g., "CONST" + "ANCIA" → "CONSTANCIA")
let should_join = should_join_items(prev_item, item);
// Check if space already exists to avoid double spaces
let prev_ends_with_space = result.ends_with(' ');
let curr_starts_with_space = text.starts_with(' ');
let space_already_exists = prev_ends_with_space || curr_starts_with_space;
if prev_ends_with_hyphen
|| curr_is_hyphen
|| curr_starts_with_hyphen
|| is_sub_super
|| was_sub_super
|| should_join
|| space_already_exists
{
// No space for hyphenated words, subscript/superscript, closely positioned items,
// or when space already exists
result.push_str(text);
} else {
if self.needs_space_between(prev_item, item, &result) {
result.push(' ');
result.push_str(text);
}
result.push_str(text);
}
}
result
}
/// Determine if a space is needed between two items
fn needs_space_between(&self, prev_item: &TextItem, item: &TextItem, result: &str) -> bool {
let text = item.text.as_str();
// Don't add space before/after hyphens for hyphenated words
let prev_ends_with_hyphen = result.ends_with('-');
let curr_is_hyphen = text.trim() == "-";
let curr_starts_with_hyphen = text.starts_with('-');
// Detect subscript/superscript: smaller font size and/or Y offset
let font_ratio = item.font_size / prev_item.font_size;
let reverse_font_ratio = prev_item.font_size / item.font_size;
let y_diff = (item.y - prev_item.y).abs();
let is_sub_super = font_ratio < 0.85 && y_diff > 1.0;
let was_sub_super = reverse_font_ratio < 0.85 && y_diff > 1.0;
// Use position-based spacing detection
let should_join = should_join_items(prev_item, item);
// Check if space already exists
let prev_ends_with_space = result.ends_with(' ');
let curr_starts_with_space = text.starts_with(' ');
let space_already_exists = prev_ends_with_space || curr_starts_with_space;
// Add space unless one of these conditions applies
!(prev_ends_with_hyphen
|| curr_is_hyphen
|| curr_starts_with_hyphen
|| is_sub_super
|| was_sub_super
|| should_join
|| space_already_exists)
}
}
/// Determine if two adjacent text items should be joined without a space
@@ -388,6 +463,11 @@ fn extract_page_text_items(
// Transform position through CTM
let combined = multiply_matrices(&text_matrix, &ctm);
let (x, y) = (combined[4], combined[5]);
// Detect bold/italic from font name
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
items.push(TextItem {
text,
x,
@@ -397,6 +477,8 @@ fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
});
}
}
@@ -426,6 +508,11 @@ fn extract_page_text_items(
// Transform position through CTM
let combined = multiply_matrices(&text_matrix, &ctm);
let (x, y) = (combined[4], combined[5]);
// Detect bold/italic from font name
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
items.push(TextItem {
text: combined_text,
x,
@@ -435,6 +522,8 @@ fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
});
}
}
@@ -460,6 +549,11 @@ fn extract_page_text_items(
// Transform position through CTM
let combined = multiply_matrices(&text_matrix, &ctm);
let (x, y) = (combined[4], combined[5]);
// Detect bold/italic from font name
let base_font = font_base_names
.get(&current_font)
.map(|s| s.as_str())
.unwrap_or(&current_font);
items.push(TextItem {
text,
x,
@@ -469,6 +563,8 @@ fn extract_page_text_items(
font: current_font.clone(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font),
is_italic: is_italic_font(base_font),
});
}
}
@@ -503,6 +599,42 @@ fn effective_font_size(base_size: f32, text_matrix: &[f32; 6]) -> f32 {
base_size * scale
}
/// Detect if a font name indicates bold style
/// Common patterns: "Bold", "Bd", "Black", "Heavy", "Demi", "Semi" (semi-bold)
pub fn is_bold_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
// Check for common bold indicators
// Note: Need to be careful with "Oblique" not matching "Obl" + false positive for bold
lower.contains("bold")
|| lower.contains("-bd")
|| lower.contains("_bd")
|| lower.contains("black")
|| lower.contains("heavy")
|| lower.contains("demibold")
|| lower.contains("semibold")
|| lower.contains("demi-bold")
|| lower.contains("semi-bold")
|| lower.contains("extrabold")
|| lower.contains("ultrabold")
|| lower.contains("medium") && !lower.contains("mediumitalic") // Some fonts use Medium for semi-bold
}
/// Detect if a font name indicates italic/oblique style
/// Common patterns: "Italic", "It", "Oblique", "Obl", "Slant", "Inclined"
pub fn is_italic_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
// Check for common italic indicators
lower.contains("italic")
|| lower.contains("oblique")
|| lower.contains("-it")
|| lower.contains("_it")
|| lower.contains("slant")
|| lower.contains("inclined")
|| lower.contains("kursiv") // German for italic
}
/// Extract text from a text operand, handling encoding
fn extract_text_from_operand(
obj: &Object,
@@ -881,6 +1013,8 @@ mod tests {
font: "F1".into(),
font_size: 12.0,
page: 1,
is_bold: false,
is_italic: false,
},
TextItem {
text: "World".into(),
@@ -891,6 +1025,8 @@ mod tests {
font: "F1".into(),
font_size: 12.0,
page: 1,
is_bold: false,
is_italic: false,
},
TextItem {
text: "Next line".into(),
@@ -901,6 +1037,8 @@ mod tests {
font: "F1".into(),
font_size: 12.0,
page: 1,
is_bold: false,
is_italic: false,
},
];
@@ -909,4 +1047,32 @@ mod tests {
assert_eq!(lines[0].text(), "Hello World");
assert_eq!(lines[1].text(), "Next line");
}
#[test]
fn test_bold_italic_detection() {
// Test bold detection
assert!(is_bold_font("Arial-Bold"));
assert!(is_bold_font("TimesNewRoman-Bold"));
assert!(is_bold_font("Helvetica-BoldOblique"));
assert!(is_bold_font("ABCDEF+ArialMT-Bold"));
assert!(is_bold_font("NotoSans-Black"));
assert!(is_bold_font("Roboto-SemiBold"));
assert!(!is_bold_font("Arial"));
assert!(!is_bold_font("TimesNewRoman-Italic"));
// Test italic detection
assert!(is_italic_font("Arial-Italic"));
assert!(is_italic_font("TimesNewRoman-Italic"));
assert!(is_italic_font("Helvetica-Oblique"));
assert!(is_italic_font("ABCDEF+ArialMT-Italic"));
assert!(is_italic_font("Helvetica-BoldOblique"));
assert!(!is_italic_font("Arial"));
assert!(!is_italic_font("TimesNewRoman-Bold"));
// Test bold-italic detection
assert!(is_bold_font("Arial-BoldItalic"));
assert!(is_italic_font("Arial-BoldItalic"));
assert!(is_bold_font("Helvetica-BoldOblique"));
assert!(is_italic_font("Helvetica-BoldOblique"));
}
}
src/markdown.rs
+104 -33
View File
@@ -28,6 +28,10 @@ pub struct MarkdownOptions {
pub format_urls: bool,
/// Fix hyphenation (broken words across lines)
pub fix_hyphenation: bool,
/// Detect and format bold text from font names
pub detect_bold: bool,
/// Detect and format italic text from font names
pub detect_italic: bool,
}
impl Default for MarkdownOptions {
@@ -40,6 +44,8 @@ impl Default for MarkdownOptions {
remove_page_numbers: true,
format_urls: true,
fix_hyphenation: true,
detect_bold: true,
detect_italic: true,
}
}
}
@@ -214,6 +220,7 @@ fn to_markdown_from_lines_with_tables(
let mut prev_y = f32::MAX;
let mut in_list = false;
let mut in_paragraph = false;
let mut last_list_x: Option<f32> = None;
let mut inserted_tables: HashSet<(u32, usize)> = HashSet::new();
for line in lines {
@@ -265,27 +272,29 @@ fn to_markdown_from_lines_with_tables(
// Paragraph break (large Y gap)
let y_gap = prev_y - line.y;
let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold
if is_para_break {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
}
if in_list {
in_list = false;
}
if is_para_break && in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
}
// Don't immediately end list on paragraph break
// Let the continuation check below decide if we're still in a list
prev_y = line.y;
let text = line.text();
// Get text with optional bold/italic formatting
let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
let trimmed = text.trim();
// Also get plain text for pattern matching (list detection, captions, etc.)
let plain_text = line.text();
let plain_trimmed = plain_text.trim();
if trimmed.is_empty() {
continue;
}
// Detect figure/table captions and source citations
// These should be on their own line followed by a paragraph break
if is_caption_line(trimmed) {
if is_caption_line(plain_trimmed) {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
@@ -296,7 +305,8 @@ fn to_markdown_from_lines_with_tables(
}
// Detect headers by font size
if options.detect_headers && trimmed.len() > 3 {
// Note: Headers typically shouldn't have bold markers since they're already emphasized
if options.detect_headers && plain_trimmed.len() > 3 {
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
if let Some(header_level) = detect_header_level(line_font_size, base_size) {
if in_paragraph {
@@ -304,14 +314,15 @@ fn to_markdown_from_lines_with_tables(
in_paragraph = false;
}
let prefix = "#".repeat(header_level);
output.push_str(&format!("{} {}\n\n", prefix, trimmed));
// Use plain text for headers to avoid redundant formatting
output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
in_list = false;
continue;
}
}
// Detect list items
if options.detect_lists && is_list_item(trimmed) {
if options.detect_lists && is_list_item(plain_trimmed) {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
@@ -320,9 +331,37 @@ fn to_markdown_from_lines_with_tables(
output.push_str(&formatted);
output.push('\n');
in_list = true;
last_list_x = line.items.first().map(|i| i.x);
continue;
} else if in_list && !trimmed.starts_with(char::is_whitespace) {
in_list = false;
} else if in_list {
// Check if this line is a continuation of the previous list item
// Continuations have similar X position and reasonable Y gap
let line_x = line.items.first().map(|i| i.x);
let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) {
// Continuation criteria:
// 1. X is at or past the list text position
// 2. Y gap is not too large (max ~5 line heights)
// 3. Not a new list item
let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0;
let y_ok = y_gap < base_size * 7.0;
x_ok && y_ok && !is_list_item(plain_trimmed)
} else {
false
};
if is_continuation {
// Append to previous list item with a space
if output.ends_with('\n') {
output.pop();
output.push(' ');
}
output.push_str(trimmed);
output.push('\n');
continue;
} else {
in_list = false;
last_list_x = None;
}
}
// Detect code blocks by font
@@ -333,7 +372,8 @@ fn to_markdown_from_lines_with_tables(
output.push_str("\n\n");
in_paragraph = false;
}
output.push_str(&format!("```\n{}\n```\n", trimmed));
// Use plain text for code blocks
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
continue;
}
}
@@ -390,6 +430,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
let mut prev_y = f32::MAX;
let mut in_list = false;
let mut in_paragraph = false;
let mut last_list_x: Option<f32> = None;
for line in lines {
// Page break
@@ -403,32 +444,36 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
}
current_page = line.page;
prev_y = f32::MAX;
in_list = false;
last_list_x = None;
}
// Paragraph break (large Y gap)
let y_gap = prev_y - line.y;
let is_para_break = y_gap > base_size * 1.8; // Slightly lower threshold
if is_para_break {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
}
if in_list {
in_list = false;
}
if is_para_break && in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
}
// Don't immediately end list on paragraph break
// Let the continuation check below decide if we're still in a list
prev_y = line.y;
let text = line.text();
// Get text with optional bold/italic formatting
let text = line.text_with_formatting(options.detect_bold, options.detect_italic);
let trimmed = text.trim();
// Also get plain text for pattern matching
let plain_text = line.text();
let plain_trimmed = plain_text.trim();
if trimmed.is_empty() {
continue;
}
// Detect figure/table captions and source citations
// These should be on their own line followed by a paragraph break
if is_caption_line(trimmed) {
if is_caption_line(plain_trimmed) {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
@@ -440,7 +485,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
// Detect headers by font size
// Skip very short text (likely drop caps or labels)
if options.detect_headers && trimmed.len() > 3 {
if options.detect_headers && plain_trimmed.len() > 3 {
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
if let Some(header_level) = detect_header_level(line_font_size, base_size) {
if in_paragraph {
@@ -448,14 +493,15 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
in_paragraph = false;
}
let prefix = "#".repeat(header_level);
output.push_str(&format!("{} {}\n\n", prefix, trimmed));
// Use plain text for headers to avoid redundant formatting
output.push_str(&format!("{} {}\n\n", prefix, plain_trimmed));
in_list = false;
continue;
}
}
// Detect list items
if options.detect_lists && is_list_item(trimmed) {
if options.detect_lists && is_list_item(plain_trimmed) {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
@@ -464,11 +510,35 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
output.push_str(&formatted);
output.push('\n');
in_list = true;
last_list_x = line.items.first().map(|i| i.x);
continue;
} else if in_list {
// Check if continuing list or ending
if !trimmed.starts_with(char::is_whitespace) {
// Check if this line is a continuation of the previous list item
let line_x = line.items.first().map(|i| i.x);
let is_continuation = if let (Some(list_x), Some(curr_x)) = (last_list_x, line_x) {
// Continuation criteria:
// 1. X is at or past the list text position
// 2. Y gap is not too large (max ~5 line heights)
// 3. Not a new list item
let x_ok = curr_x >= list_x - 5.0 && curr_x <= list_x + 50.0;
let y_ok = y_gap < base_size * 7.0;
x_ok && y_ok && !is_list_item(plain_trimmed)
} else {
false
};
if is_continuation {
// Append to previous list item with a space
if output.ends_with('\n') {
output.pop();
output.push(' ');
}
output.push_str(trimmed);
output.push('\n');
continue;
} else {
in_list = false;
last_list_x = None;
}
}
@@ -480,7 +550,8 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
output.push_str("\n\n");
in_paragraph = false;
}
output.push_str(&format!("```\n{}\n```\n", trimmed));
// Use plain text for code blocks
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
continue;
}
}
@@ -949,7 +1020,7 @@ fn is_page_number_line(trimmed: &str) -> bool {
}
// Pattern 4: "- X -" centered page number
if trimmed.starts_with('-') && trimmed.ends_with('-') {
if trimmed.len() >= 3 && trimmed.starts_with('-') && trimmed.ends_with('-') {
let inner = trimmed[1..trimmed.len() - 1].trim();
if inner.chars().all(|c| c.is_ascii_digit()) && !inner.is_empty() {
return true;
src/tables.rs
+119 -8
View File
@@ -222,6 +222,12 @@ fn detect_table_in_region(items: &[(usize, &TextItem)]) -> Option<Table> {
return None;
}
// Validation 8: Check for Table of Contents pattern
// TOCs have dots (leader lines) and page numbers, not real table data
if is_table_of_contents(&cells) {
return None;
}
Some(Table {
columns,
rows,
@@ -305,9 +311,9 @@ fn has_consistent_columns(cells: &[Vec<String>]) -> bool {
consistent_rows as f32 / cells.len() as f32 > 0.4
}
/// Check if the content looks like table data (numbers, short values)
/// Check if the content looks like table data (numbers, short values, specs)
fn has_table_like_content(cells: &[Vec<String>]) -> bool {
let mut numeric_cells = 0;
let mut data_like_cells = 0;
let mut total_cells = 0;
for row in cells.iter().skip(1) {
@@ -316,9 +322,9 @@ fn has_table_like_content(cells: &[Vec<String>]) -> bool {
let trimmed = cell.trim();
if !trimmed.is_empty() {
total_cells += 1;
// Check if it looks like a number (including decimals)
if looks_like_number(trimmed) {
numeric_cells += 1;
// Check if it looks like table data
if looks_like_table_data(trimmed) {
data_like_cells += 1;
}
}
}
@@ -328,12 +334,65 @@ fn has_table_like_content(cells: &[Vec<String>]) -> bool {
return false;
}
// At least 20% numeric content suggests a data table
// At least 20% data-like content suggests a data table
// OR the table has many columns (structural table)
let pct_numeric = numeric_cells as f32 / total_cells as f32;
let pct_data = data_like_cells as f32 / total_cells as f32;
let num_cols = cells.first().map(|r| r.len()).unwrap_or(0);
pct_numeric > 0.2 || num_cols >= 5
pct_data > 0.2 || num_cols >= 5
}
/// Check if a cell value looks like table data
/// Includes: numbers, part numbers, specifications with units, codes
fn looks_like_table_data(s: &str) -> bool {
let s = s.trim();
if s.is_empty() {
return false;
}
// Pure numbers
if looks_like_number(s) {
return true;
}
// Part numbers / model codes (alphanumeric, typically short)
// e.g., "NA555", "NE555", "LM358"
if s.len() <= 10
&& s.chars().all(|c| c.is_alphanumeric())
&& s.chars().any(|c| c.is_ascii_digit())
{
return true;
}
// Specifications with units (contains numbers and unit symbols)
// e.g., "–40°C to +105°C", "5V", "200mA", "8-pin"
let has_number = s.chars().any(|c| c.is_ascii_digit());
let has_unit = s.contains('°')
|| s.contains('V')
|| s.contains('A')
|| s.contains("Hz")
|| s.contains("mA")
|| s.contains("µ")
|| s.contains("pin")
|| s.contains("MHz")
|| s.contains("kHz");
if has_number && has_unit {
return true;
}
// Package designations with parentheses
// e.g., "D (SOIC, 8)", "P (PDIP, 8)"
if s.contains('(') && s.contains(')') && s.chars().any(|c| c.is_ascii_digit()) {
return true;
}
// Temperature ranges
// e.g., "TA = –40°C to +105°C"
if (s.contains("°C") || s.contains("°F")) && s.contains("to") {
return true;
}
false
}
/// Check if a string looks like a number
@@ -349,6 +408,56 @@ fn looks_like_number(s: &str) -> bool {
&& s.chars().any(|c| c.is_ascii_digit())
}
/// Check if this looks like a Table of Contents
/// TOCs have characteristic patterns: leader dots, page numbers, section names
fn is_table_of_contents(cells: &[Vec<String>]) -> bool {
if cells.is_empty() {
return false;
}
let mut dot_cells = 0;
let mut page_number_cells = 0;
let mut total_cells = 0;
for row in cells {
for cell in row {
let trimmed = cell.trim();
if trimmed.is_empty() {
continue;
}
total_cells += 1;
// Check for leader dots (sequences of periods)
// TOCs often have "........" or ". . . ." patterns
let dot_count = trimmed.chars().filter(|&c| c == '.').count();
let is_mostly_dots = dot_count > trimmed.len() / 2 && dot_count >= 3;
if is_mostly_dots {
dot_cells += 1;
}
// Check for standalone page numbers (1-4 digits, possibly with spaces)
let digits_only: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect();
if digits_only.len() <= 4
&& !digits_only.is_empty()
&& digits_only.chars().all(|c| c.is_ascii_digit())
{
page_number_cells += 1;
}
}
}
if total_cells == 0 {
return false;
}
// If a significant portion of cells are dots or page numbers, it's likely a TOC
let dot_ratio = dot_cells as f32 / total_cells as f32;
let page_num_ratio = page_number_cells as f32 / total_cells as f32;
// TOC typically has >15% dot cells and >10% page number cells
dot_ratio > 0.15 || (dot_ratio > 0.05 && page_num_ratio > 0.15)
}
/// Check what fraction of items align to detected columns
fn check_column_alignment(items: &[(usize, &TextItem)], columns: &[f32]) -> f32 {
let tolerance = 40.0;
@@ -817,6 +926,8 @@ mod tests {
font: "F1".into(),
font_size,
page: 1,
is_bold: false,
is_italic: false,
}
}
tests/integration_tests.rs
+9
View File
@@ -18,6 +18,8 @@ fn make_text_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> Text
font: "Helvetica".to_string(),
font_size,
page,
is_bold: false,
is_italic: false,
}
}
@@ -29,6 +31,7 @@ fn make_text_item_with_font(
font: &str,
page: u32,
) -> TextItem {
use pdf_inspector::extractor::{is_bold_font, is_italic_font};
TextItem {
text: text.to_string(),
x,
@@ -38,6 +41,8 @@ fn make_text_item_with_font(
font: font.to_string(),
font_size,
page,
is_bold: is_bold_font(font),
is_italic: is_italic_font(font),
}
}
@@ -253,6 +258,8 @@ fn test_markdown_options_custom() {
remove_page_numbers: false,
format_urls: false,
fix_hyphenation: false,
detect_bold: false,
detect_italic: false,
};
assert!(!opts.detect_headers);
assert!(opts.detect_lists);
@@ -261,6 +268,8 @@ fn test_markdown_options_custom() {
assert!(!opts.remove_page_numbers);
assert!(!opts.format_urls);
assert!(!opts.fix_hyphenation);
assert!(!opts.detect_bold);
assert!(!opts.detect_italic);
}
// ============================================================================