Files
pdf-inspector/src/markdown/classify.rs
T
Abimael MartellandCursor b375d6f102 feat(markdown): underline emission, Unicode scripts, style-preserving merges (#117)
* feat(markdown): underline emission, Unicode scripts, style-preserving merges (ENG-5015 2b)

Three formatting losses in the direct-extraction markdown path:

1. text_with_formatting gains <u> run emission (detect_underline option,
   default on) using the geometric is_underline flag from 1.9.9.
   Underline runs stay free of nested bold/italic markers — consumers
   match tag content literally. Heading lines keep plain text for
   bold/italic but preserve <u>: the tag carries meaning `#` doesn't.
2. merge_subscript_items now maps absorbed digit scripts to Unicode
   sub/superscript forms with direction from the baseline offset
   ("H"+"2" -> "H₂", "word"+raised "2" -> "word²", "m"+"3" -> "m³").
   NFKC/NFKD folds these back to plain digits so text matching
   downstream is unaffected; renderers keep the script semantics.
3. merge_text_items no longer merges across bold/italic boundaries —
   absorbing a styled run into a plain neighbor erased the styling
   before markdown emission ever saw it. On eval docs this recovers
   20-82 italic runs per document that previously emitted as plain.

Snapshots regenerated (diffs are the features: CCl₂F₂, m³, underlined
legal section headings, finer bold runs). pdf-evals regression suite:
202/202 real PDFs pass. napi 1.9.9 -> 1.9.10.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(extractor): break merges at underline boundaries too (review)

OR-merging underline stretched the eventual <u> span over neighboring
plain fragments. Merge runs now break on any style-flag change, the
redundant accumulator is gone, and format_list_item learned to move
bullet markers outside <u> wrappers so fully-underlined bullet lines
still render as markdown lists. td9264 snapshot regenerated — spans are
tighter (trailing periods correctly outside the tag).

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(markdown): strip stray spaces before sentence punctuation (review)

Style-boundary item splits can strand a trailing period in its own
fragment, and multiple assembly paths join fragments with spaces,
yielding "word ." artifacts. Rather than chasing every join site, a
postprocess pass removes a space before `.`/`,`/`;` when the mark ends
its token (whitespace, cell boundary `|`, or end of text follows).
Dot leaders/ellipses and mid-token periods are untouched.

Fixes the td9264 "companies ." artifacts and two pre-existing
"armoring ," artifacts in the 2013-app2 snapshot. pdf-evals: zero
markdown diffs across all 203 corpus PDFs vs committed baselines.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(tables): trim spaces inside parenthetical cell fragments

* fix(tables): reject sparse prose row-stripe tables

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-08 23:40:08 -07:00

274 lines
7.8 KiB
Rust

//! Line classification: captions, lists, code detection.
/// Check if text is a figure/table caption or source citation
pub(crate) fn is_caption_line(text: &str) -> bool {
let trimmed = text.trim();
// Caption prefixes that always match (always followed by identifiers)
let always_prefixes = [
"Figura ",
"Fig. ",
"Fig ",
"Tabela ",
"Source:",
"Fonte:",
"Source ",
"Fonte ",
"Note:",
"Nota:",
"Chart ",
"Gráfico ",
"Graph ",
"Diagram ",
"Image ",
"Imagem ",
"Photo ",
"Foto ",
];
for prefix in &always_prefixes {
if trimmed.starts_with(prefix) {
return true;
}
}
// "Figure" and "Table" need a digit/reference after them to distinguish
// captions ("Table 1", "Figure 3.2") from headings ("Table of Contents")
for prefix in ["Figure ", "Table "] {
if let Some(rest) = trimmed.strip_prefix(prefix) {
if rest
.trim_start()
.starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
{
return true;
}
}
}
// Check case-insensitive patterns — require digit or punctuation after
// prefix to avoid matching "Table of Contents" or "Figure drawing" etc.
let lower = trimmed.to_lowercase();
for pfx in ["figure ", "table "] {
if let Some(rest) = lower.strip_prefix(pfx) {
if rest
.trim_start()
.starts_with(|c: char| c.is_ascii_digit() || c == '(' || c == '#')
{
return true;
}
}
}
if lower.starts_with("source:") {
return true;
}
false
}
/// Check if text starts with an unambiguous bullet marker (●, •, ○, ◦).
///
/// Narrower than [`is_list_item`]: it excludes numbered/lettered patterns
/// like `1.` or `a)`, which legitimately appear as section headings in many
/// documents. Used by the heading classifier to reject bullet lines without
/// also demoting numbered headings.
pub(crate) fn starts_with_bullet_marker(text: &str) -> bool {
let trimmed = text.trim_start();
trimmed.starts_with("• ")
|| trimmed.starts_with("● ")
|| trimmed.starts_with("○ ")
|| trimmed.starts_with("◦ ")
|| trimmed.starts_with("- ")
|| trimmed.starts_with("* ")
}
/// Check if text looks like a list item
pub(crate) fn is_list_item(text: &str) -> bool {
let trimmed = text.trim_start();
// Bullet patterns
if trimmed.starts_with("• ")
|| trimmed.starts_with("- ")
|| trimmed.starts_with("* ")
|| trimmed.starts_with("○ ")
|| trimmed.starts_with("● ")
|| trimmed.starts_with("◦ ")
{
return true;
}
// Numbered list patterns: "1.", "1)", "(1)", "a.", "a)"
let first_chars: String = trimmed.chars().take(5).collect();
if first_chars.contains(|c: char| c.is_ascii_digit()) {
// Check for "1.", "1)", "10."
if let Some(idx) = first_chars.find(['.', ')']) {
let prefix = &first_chars[..idx];
if prefix.chars().all(|c| c.is_ascii_digit()) {
return true;
}
}
}
// Letter list: "a.", "a)", "(a)"
let mut chars = trimmed.chars();
if let (Some(first), Some(second)) = (chars.next(), chars.next()) {
if first.is_ascii_alphabetic() && (second == '.' || second == ')') {
return true;
}
if first == '(' && chars.next() == Some(')') {
return true;
}
}
false
}
/// Format list item to markdown
pub(crate) fn format_list_item(text: &str) -> String {
let trimmed = text.trim_start();
// Convert various bullet styles to markdown
// Note: bullet characters like • are multi-byte in UTF-8, use char indices
for bullet in &['•', '○', '●', '◦'] {
if let Some(rest) = trimmed.strip_prefix(*bullet) {
return format!("- {}", rest.trim_start());
}
// Bullet inside a leading style run (e.g. "**● Label:** rest" or
// "<u>● Label</u>"). The run wraps both the marker and the following
// label because both carry the style in the PDF. The marker must move
// outside the wrapper so markdown still sees a list item.
for wrapper in ["**", "*", "<u>"] {
if let Some(after_open) = trimmed.strip_prefix(wrapper) {
if let Some(rest) = after_open.strip_prefix(*bullet) {
return format!("- {}{}", wrapper, rest.trim_start());
}
}
}
}
if trimmed.starts_with("- ") || trimmed.starts_with("* ") {
return trimmed.to_string();
}
// Keep numbered lists as-is (markdown supports them)
trimmed.to_string()
}
/// Check if text looks like code
pub(crate) fn is_code_like(text: &str) -> bool {
let trimmed = text.trim();
// Code patterns
let code_patterns = [
// Language keywords
"import ",
"export ",
"from ",
"const ",
"let ",
"var ",
"function ",
"class ",
"def ",
"pub fn ",
"fn ",
"async fn ",
"impl ",
// Syntax patterns
"=> ",
"-> ",
":: ",
":= ",
];
for pattern in &code_patterns {
if trimmed.starts_with(pattern) {
return true;
}
}
// Check for code-like syntax
let special_chars: usize = trimmed
.chars()
.filter(|c| matches!(c, '{' | '}' | '(' | ')' | '[' | ']' | ';' | '=' | '<' | '>'))
.count();
if special_chars >= 3 && trimmed.len() < 200 {
return true;
}
// Ends with semicolon or braces
if trimmed.ends_with(';') || trimmed.ends_with('{') || trimmed.ends_with('}') {
return true;
}
false
}
/// Check if font name indicates monospace
pub(crate) fn is_monospace_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
let patterns = [
"courier",
"consolas",
"monaco",
"menlo",
"mono",
"fixed",
"terminal",
"typewriter",
"source code",
"fira code",
"jetbrains",
"inconsolata",
"dejavu sans mono",
"liberation mono",
];
patterns.iter().any(|p| lower.contains(p))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn format_list_item_plain_bullet() {
assert_eq!(format_list_item("● Item"), "- Item");
assert_eq!(format_list_item("• Item"), "- Item");
}
#[test]
fn format_list_item_bullet_inside_underline() {
// Fully-underlined bullet line: the marker must move outside the
// <u> wrapper so markdown still renders a list item.
assert_eq!(format_list_item("<u>● Item text</u>"), "- <u>Item text</u>");
}
#[test]
fn format_list_item_bullet_inside_bold() {
// PDF that uses bold font for both the marker and the label produces
// a single bold run like "**● Label:** rest"; the bullet must still
// be stripped and the bold wrapper preserved on the label.
assert_eq!(
format_list_item("**● Fraud: Willing cooperation;**"),
"- **Fraud: Willing cooperation;**"
);
assert_eq!(
format_list_item("**● Label:** rest of line"),
"- **Label:** rest of line"
);
assert_eq!(format_list_item("*● Italic:* rest"), "- *Italic:* rest");
}
#[test]
fn format_list_item_already_dash() {
assert_eq!(format_list_item("- existing"), "- existing");
}
#[test]
fn is_list_item_with_bullet_space() {
assert!(is_list_item("● Item"));
assert!(is_list_item("• Item"));
assert!(is_list_item("- Item"));
}
}