Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1470dc162a |
@@ -130,6 +130,29 @@ pub(crate) fn has_dot_leaders(text: &str) -> bool {
|
|||||||
dot_groups >= 2
|
dot_groups >= 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Detect a table-of-contents entry: a line ending in a page number preceded by
|
||||||
|
/// a dot-leader group (e.g. "Measurement Lab worksheet ... 3"). `has_dot_leaders`
|
||||||
|
/// misses single-group leaders ("..."), but a trailing "<dots> <number>" is a
|
||||||
|
/// strong TOC signal on its own. Such lines must never be promoted to headings.
|
||||||
|
pub(crate) fn is_toc_entry_line(text: &str) -> bool {
|
||||||
|
let trimmed = text.trim_end();
|
||||||
|
let digits = trimmed
|
||||||
|
.chars()
|
||||||
|
.rev()
|
||||||
|
.take_while(|c| c.is_ascii_digit())
|
||||||
|
.count();
|
||||||
|
if digits == 0 || digits > 4 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let before_number = trimmed[..trimmed.len() - digits].trim_end();
|
||||||
|
let dots = before_number
|
||||||
|
.chars()
|
||||||
|
.rev()
|
||||||
|
.take_while(|c| *c == '.')
|
||||||
|
.count();
|
||||||
|
dots >= 3
|
||||||
|
}
|
||||||
|
|
||||||
/// Compute the Y-gap threshold for paragraph break detection.
|
/// Compute the Y-gap threshold for paragraph break detection.
|
||||||
///
|
///
|
||||||
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
/// Instead of using a fixed multiple of base_size (which fails for double-spaced
|
||||||
@@ -320,3 +343,28 @@ pub(crate) fn detect_header_level(
|
|||||||
Some(4)
|
Some(4)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn toc_entry_with_single_dot_group() {
|
||||||
|
assert!(is_toc_entry_line("Measurement Lab worksheet ... 3"));
|
||||||
|
assert!(is_toc_entry_line("Results ........ 12"));
|
||||||
|
assert!(is_toc_entry_line("Appendix B...42"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_toc_lines_pass() {
|
||||||
|
assert!(!is_toc_entry_line(
|
||||||
|
"6.2. Expectations for Re-Hiring Employees"
|
||||||
|
));
|
||||||
|
assert!(!is_toc_entry_line("What happened in 2020"));
|
||||||
|
assert!(!is_toc_entry_line("IMPLEMENTATION"));
|
||||||
|
// Ellipsis without a trailing page number
|
||||||
|
assert!(!is_toc_entry_line("and so it goes ..."));
|
||||||
|
// Long numbers are data, not page refs
|
||||||
|
assert!(!is_toc_entry_line("ISBN ... 97814"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+11
-3
@@ -7,7 +7,7 @@ use crate::types::TextLine;
|
|||||||
|
|
||||||
use super::analysis::{
|
use super::analysis::{
|
||||||
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
bold_heading_level, calculate_font_stats, compute_heading_tiers, compute_paragraph_threshold,
|
||||||
detect_header_level, font_size_rarity, has_dot_leaders,
|
detect_header_level, font_size_rarity, has_dot_leaders, is_toc_entry_line,
|
||||||
};
|
};
|
||||||
use super::classify::{
|
use super::classify::{
|
||||||
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
|
||||||
@@ -703,6 +703,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
|||||||
&& plain_trimmed.len() > 3
|
&& plain_trimmed.len() > 3
|
||||||
&& plain_trimmed.split_whitespace().count() <= 15
|
&& plain_trimmed.split_whitespace().count() <= 15
|
||||||
&& !starts_with_bullet_marker(plain_trimmed)
|
&& !starts_with_bullet_marker(plain_trimmed)
|
||||||
|
&& !is_toc_entry_line(plain_trimmed)
|
||||||
{
|
{
|
||||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||||
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
detect_header_level(line_font_size, base_size, &heading_tiers).or_else(|| {
|
||||||
@@ -738,7 +739,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
|
|||||||
// paragraph continuity and minor font-size variation
|
// paragraph continuity and minor font-size variation
|
||||||
// inflates rarity scores.
|
// inflates rarity scores.
|
||||||
let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
|
let has_strong_signal = all_bold || isolated || (rarity >= 0.97 && word_count <= 8);
|
||||||
if score >= 0.5 && standalone && word_count >= 2 && has_strong_signal {
|
// Single-word headings ("IMPLEMENTATION", "CONTENTS") are common;
|
||||||
|
// accept them only with the strongest signal combination.
|
||||||
|
let enough_words =
|
||||||
|
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||||
|
if score >= 0.5 && standalone && enough_words && has_strong_signal {
|
||||||
Some(bold_heading_level(&heading_tiers))
|
Some(bold_heading_level(&heading_tiers))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -1037,6 +1042,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
|||||||
if options.detect_headers
|
if options.detect_headers
|
||||||
&& plain_trimmed.len() > 3
|
&& plain_trimmed.len() > 3
|
||||||
&& plain_trimmed.split_whitespace().count() <= 15
|
&& plain_trimmed.split_whitespace().count() <= 15
|
||||||
|
&& !is_toc_entry_line(plain_trimmed)
|
||||||
{
|
{
|
||||||
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
|
||||||
if let Some(header_level) =
|
if let Some(header_level) =
|
||||||
@@ -1059,7 +1065,9 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
|
|||||||
+ if all_bold { 0.3 } else { 0.0 }
|
+ if all_bold { 0.3 } else { 0.0 }
|
||||||
+ if standalone { 0.2 } else { 0.0 }
|
+ if standalone { 0.2 } else { 0.0 }
|
||||||
+ if isolated { 0.3 } else { 0.0 };
|
+ if isolated { 0.3 } else { 0.0 };
|
||||||
if score >= 0.5 && standalone && word_count >= 2 {
|
let enough_words =
|
||||||
|
word_count >= 2 || (all_bold && isolated && plain_trimmed.len() >= 4);
|
||||||
|
if score >= 0.5 && standalone && enough_words {
|
||||||
return Some(bold_heading_level(&heading_tiers));
|
return Some(bold_heading_level(&heading_tiers));
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
|
|||||||
Reference in New Issue
Block a user