Caption detection was incorrectly classifying "Table of Contents" as a caption because it starts with "Table ". Now "Table" and "Figure" prefixes require a digit, parenthesis, or hash after them — matching actual captions like "Table 1", "Figure 3.2" but not titles. Also removes debug logging left from previous iteration. Benchmark improvement: MHS 0.52→0.54, overall 0.757→0.761. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
201 lines
5.1 KiB
Rust
201 lines
5.1 KiB
Rust
//! Line classification: captions, lists, code detection.
|
|
|
|
/// Check if text is a figure/table caption or source citation
|
|
pub(crate) fn is_caption_line(text: &str) -> bool {
|
|
let trimmed = text.trim();
|
|
|
|
// Caption prefixes that always match (always followed by identifiers)
|
|
let always_prefixes = [
|
|
"Figura ",
|
|
"Fig. ",
|
|
"Fig ",
|
|
"Tabela ",
|
|
"Source:",
|
|
"Fonte:",
|
|
"Source ",
|
|
"Fonte ",
|
|
"Note:",
|
|
"Nota:",
|
|
"Chart ",
|
|
"Gráfico ",
|
|
"Graph ",
|
|
"Diagram ",
|
|
"Image ",
|
|
"Imagem ",
|
|
"Photo ",
|
|
"Foto ",
|
|
];
|
|
for prefix in &always_prefixes {
|
|
if trimmed.starts_with(prefix) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// "Figure" and "Table" need a digit/reference after them to distinguish
|
|
// captions ("Table 1", "Figure 3.2") from headings ("Table of Contents")
|
|
for prefix in ["Figure ", "Table "] {
|
|
if let Some(rest) = trimmed.strip_prefix(prefix) {
|
|
if rest
|
|
.trim_start()
|
|
.starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check case-insensitive patterns — require digit or punctuation after
|
|
// prefix to avoid matching "Table of Contents" or "Figure drawing" etc.
|
|
let lower = trimmed.to_lowercase();
|
|
for pfx in ["figure ", "table "] {
|
|
if let Some(rest) = lower.strip_prefix(pfx) {
|
|
if rest
|
|
.trim_start()
|
|
.starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
if lower.starts_with("source:") {
|
|
return true;
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Check if text looks like a list item
|
|
pub(crate) fn is_list_item(text: &str) -> bool {
|
|
let trimmed = text.trim_start();
|
|
|
|
// Bullet patterns
|
|
if trimmed.starts_with("• ")
|
|
|| trimmed.starts_with("- ")
|
|
|| trimmed.starts_with("* ")
|
|
|| trimmed.starts_with("○ ")
|
|
|| trimmed.starts_with("● ")
|
|
|| trimmed.starts_with("◦ ")
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Numbered list patterns: "1.", "1)", "(1)", "a.", "a)"
|
|
let first_chars: String = trimmed.chars().take(5).collect();
|
|
if first_chars.contains(|c: char| c.is_ascii_digit()) {
|
|
// Check for "1.", "1)", "10."
|
|
if let Some(idx) = first_chars.find(['.', ')']) {
|
|
let prefix = &first_chars[..idx];
|
|
if prefix.chars().all(|c| c.is_ascii_digit()) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Letter list: "a.", "a)", "(a)"
|
|
let mut chars = trimmed.chars();
|
|
if let (Some(first), Some(second)) = (chars.next(), chars.next()) {
|
|
if first.is_ascii_alphabetic() && (second == '.' || second == ')') {
|
|
return true;
|
|
}
|
|
if first == '(' && chars.next() == Some(')') {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Format list item to markdown
|
|
pub(crate) fn format_list_item(text: &str) -> String {
|
|
let trimmed = text.trim_start();
|
|
|
|
// Convert various bullet styles to markdown
|
|
// Note: bullet characters like • are multi-byte in UTF-8, use char indices
|
|
for bullet in &['•', '○', '●', '◦'] {
|
|
if let Some(rest) = trimmed.strip_prefix(*bullet) {
|
|
return format!("- {}", rest.trim_start());
|
|
}
|
|
}
|
|
|
|
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
|
|
return trimmed.to_string();
|
|
}
|
|
|
|
// Keep numbered lists as-is (markdown supports them)
|
|
trimmed.to_string()
|
|
}
|
|
|
|
/// Check if text looks like code
|
|
pub(crate) fn is_code_like(text: &str) -> bool {
|
|
let trimmed = text.trim();
|
|
|
|
// Code patterns
|
|
let code_patterns = [
|
|
// Language keywords
|
|
"import ",
|
|
"export ",
|
|
"from ",
|
|
"const ",
|
|
"let ",
|
|
"var ",
|
|
"function ",
|
|
"class ",
|
|
"def ",
|
|
"pub fn ",
|
|
"fn ",
|
|
"async fn ",
|
|
"impl ",
|
|
// Syntax patterns
|
|
"=> ",
|
|
"-> ",
|
|
":: ",
|
|
":= ",
|
|
];
|
|
|
|
for pattern in &code_patterns {
|
|
if trimmed.starts_with(pattern) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Check for code-like syntax
|
|
let special_chars: usize = trimmed
|
|
.chars()
|
|
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
|
|
.count();
|
|
|
|
if special_chars >= 3 && trimmed.len() < 200 {
|
|
return true;
|
|
}
|
|
|
|
// Ends with semicolon or braces
|
|
if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') {
|
|
return true;
|
|
}
|
|
|
|
false
|
|
}
|
|
|
|
/// Check if font name indicates monospace
|
|
pub(crate) fn is_monospace_font(font_name: &str) -> bool {
|
|
let lower = font_name.to_lowercase();
|
|
let patterns = [
|
|
"courier",
|
|
"consolas",
|
|
"monaco",
|
|
"menlo",
|
|
"mono",
|
|
"fixed",
|
|
"terminal",
|
|
"typewriter",
|
|
"source code",
|
|
"fira code",
|
|
"jetbrains",
|
|
"inconsolata",
|
|
"dejavu sans mono",
|
|
"liberation mono",
|
|
];
|
|
|
|
patterns.iter().any(|p| lower.contains(p))
|
|
}
|