fix(extractor): clip page content to the visible page box (#160)
* fix(extractor): clip page content to the visible page box Single-page extracts and imposed spreads keep neighboring pages' content in the stream, positioned outside the CropBox. Extracting it appends invisible sections to the page, scrambles NID, and poisons font statistics (heading tiers built from off-page text). Clip items (by center), and — only when off-page text was actually found — rects and lines (by overlap) to CropBox-else-MediaBox, walking page-tree inheritance. Rotated pages are left unclipped: their item coordinates are already transformed out of box space. Degenerate boxes (<1 inch) are ignored. opendataloader-bench: overall 0.8445 -> 0.8537, NID +0.008, MHS +0.013; six docs up (best +0.426), none down. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(extractor): guard page-box clipping with coherence and straddle checks Two real-document counterexamples: curved display text leaves short glyph fragments with artifact coordinates outside the box (judge by character mass, not item count), and some PDFs compute inflated coordinates for visible body text (an off-page item continuing an on-page baseline means our transform model is wrong there — skip). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(extractor): clip off-box link annotations when page text was clipped Review follow-up: annotations from the neighboring page bypassed the filter. Form fields are left as-is — they're document-scoped and rare on imposed spreads. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
co-authored by
Claude Fable 5
Claude Fable 5
parent
2ed152e49b
commit
c4cbf49f44
+131
-9
@@ -176,14 +176,89 @@ fn extract_positioned_text_impl(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let ((mut items, rects, lines), has_gid_fonts, _coords_rotated) = extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
let ((mut items, mut rects, mut lines), has_gid_fonts, coords_rotated) =
|
||||
extract_page_text_items(
|
||||
doc,
|
||||
page_id,
|
||||
*page_num,
|
||||
font_cmaps,
|
||||
include_invisible,
|
||||
&mut style_cache,
|
||||
)?;
|
||||
// Clip to the visible page box: single-page extracts and imposed
|
||||
// spreads keep neighboring pages' content in the stream, positioned
|
||||
// outside the CropBox. Extracting it interleaves invisible text into
|
||||
// the page and poisons font statistics. Rotated pages are left alone
|
||||
// — their item coordinates are already transformed out of box space.
|
||||
let mut clipped_box: Option<(f32, f32, f32, f32)> = None;
|
||||
if !coords_rotated {
|
||||
if let Some((bx0, by0, bx1, by1)) = get_page_box(doc, page_id) {
|
||||
const TOL: f32 = 6.0;
|
||||
let outside = |it: &TextItem| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
!(cx >= bx0 - TOL && cx <= bx1 + TOL && it.y >= by0 - TOL && it.y <= by1 + TOL)
|
||||
};
|
||||
// Only clip when the off-page material reads as coherent text
|
||||
// (neighboring-page paragraphs). Curved/rotated display text
|
||||
// leaves short glyph fragments with artifact coordinates
|
||||
// outside the box, and those must stay.
|
||||
let off: Vec<&TextItem> = items.iter().filter(|it| outside(it)).collect();
|
||||
// Judge by character mass: paragraphs are dominated by long
|
||||
// word runs even when interleaved with short math fragments,
|
||||
// while glyph-confetti is short items through and through.
|
||||
let total_chars: usize = off.iter().map(|it| it.text.trim().chars().count()).sum();
|
||||
let wordy_chars: usize = off
|
||||
.iter()
|
||||
.map(|it| it.text.trim().chars().count())
|
||||
.filter(|&n| n >= 4)
|
||||
.sum();
|
||||
// Genuine neighboring-page content is cleanly separated from
|
||||
// on-page text. When an off-page item continues an on-page
|
||||
// line (same baseline, near-adjacent x), the coordinates are
|
||||
// artifacts of transforms we mis-model — don't clip those.
|
||||
let straddles = off.iter().any(|o| {
|
||||
items.iter().any(|i| {
|
||||
!outside(i)
|
||||
&& (i.y - o.y).abs() <= 2.0
|
||||
&& (o.x - (i.x + i.width)).abs() <= 10.0
|
||||
})
|
||||
});
|
||||
let coherent =
|
||||
off.len() >= 10 && wordy_chars * 2 >= total_chars.max(1) && !straddles;
|
||||
if bx1 - bx0 >= 72.0 && by1 - by0 >= 72.0 && coherent {
|
||||
let before = items.len();
|
||||
items.retain(|it| !outside(it));
|
||||
if items.len() < before {
|
||||
debug!(
|
||||
"page {}: clipped {} items outside page box ({:.0},{:.0})-({:.0},{:.0})",
|
||||
page_num,
|
||||
before - items.len(),
|
||||
bx0,
|
||||
by0,
|
||||
bx1,
|
||||
by1
|
||||
);
|
||||
// Only prune off-page geometry when off-page text
|
||||
// existed — same neighboring-page content.
|
||||
let overlaps = |x: f32, y: f32, w: f32, h: f32| {
|
||||
let (x0, x1) = if w < 0.0 { (x + w, x) } else { (x, x + w) };
|
||||
let (y0, y1) = if h < 0.0 { (y + h, y) } else { (y, y + h) };
|
||||
x0 < bx1 + TOL && x1 > bx0 - TOL && y0 < by1 + TOL && y1 > by0 - TOL
|
||||
};
|
||||
rects.retain(|r| overlaps(r.x, r.y, r.width, r.height));
|
||||
clipped_box = Some((bx0, by0, bx1, by1));
|
||||
lines.retain(|l| {
|
||||
overlaps(
|
||||
l.x1.min(l.x2),
|
||||
l.y1.min(l.y2),
|
||||
(l.x2 - l.x1).abs(),
|
||||
(l.y2 - l.y1).abs(),
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_gid_fonts {
|
||||
gid_encoded_pages.insert(*page_num);
|
||||
}
|
||||
@@ -223,7 +298,14 @@ fn extract_positioned_text_impl(
|
||||
all_lines.extend(lines);
|
||||
|
||||
// Extract hyperlinks from page annotations
|
||||
let links = extract_page_links(doc, page_id, *page_num);
|
||||
let mut links = extract_page_links(doc, page_id, *page_num);
|
||||
// Annotations from the neighboring page are off-box too.
|
||||
if let Some((bx0, by0, bx1, by1)) = clipped_box {
|
||||
links.retain(|it| {
|
||||
let cx = it.x + it.width / 2.0;
|
||||
cx >= bx0 - 6.0 && cx <= bx1 + 6.0 && it.y >= by0 - 6.0 && it.y <= by1 + 6.0
|
||||
});
|
||||
}
|
||||
all_items.extend(links);
|
||||
}
|
||||
|
||||
@@ -967,6 +1049,46 @@ pub(crate) fn get_number(obj: &Object) -> Option<f32> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Visible page box: CropBox if present, else MediaBox, walking page-tree
|
||||
/// inheritance (both attributes are inheritable). Returns normalized
|
||||
/// (x0, y0, x1, y1) in PDF space.
|
||||
fn get_page_box(doc: &Document, page_id: ObjectId) -> Option<(f32, f32, f32, f32)> {
|
||||
fn find_box(doc: &Document, page_id: ObjectId, key: &[u8]) -> Option<Vec<f32>> {
|
||||
let mut id = page_id;
|
||||
for _ in 0..32 {
|
||||
let dict = doc.get_dictionary(id).ok()?;
|
||||
if let Ok(obj) = dict.get(key) {
|
||||
let arr = match obj {
|
||||
Object::Array(a) => Some(a.clone()),
|
||||
Object::Reference(r) => match doc.get_object(*r) {
|
||||
Ok(Object::Array(a)) => Some(a.clone()),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
};
|
||||
if let Some(arr) = arr {
|
||||
let vals: Vec<f32> = arr.iter().filter_map(get_number).collect();
|
||||
if vals.len() >= 4 {
|
||||
return Some(vals);
|
||||
}
|
||||
}
|
||||
}
|
||||
match dict.get(b"Parent") {
|
||||
Ok(Object::Reference(p)) => id = *p,
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
let v = find_box(doc, page_id, b"CropBox").or_else(|| find_box(doc, page_id, b"MediaBox"))?;
|
||||
Some((
|
||||
v[0].min(v[2]),
|
||||
v[1].min(v[3]),
|
||||
v[0].max(v[2]),
|
||||
v[1].max(v[3]),
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
Reference in New Issue
Block a user