fix(text): handle Tc/Tw character and word spacing (#11)

* fix(text): handle Tc/Tw character and word spacing in text width computation

PDFs using Tc (character spacing) and Tw (word spacing) operators for text
justification had words incorrectly split across TextItems. The computed
advance width didn't account for these spacing parameters, causing spurious
spaces mid-word (e.g. "deve lopers" instead of "developers").

- Add Tc/Tw operator handling and graphics state save/restore
- Incorporate char_spacing and word_spacing into compute_string_width_ts
- Add adaptive merge threshold: tighter for lowercase→lowercase junctions,
  wider before joining punctuation
- Add unit tests for Tc/Tw width computation and merge behavior

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* test(fonts): add large Tc width computation test

Verifies that large character spacing values are applied in full
without any artificial cap, matching PDF spec behavior.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(text): guard against Tc/Tw-inflated widths in merge and join paths

Two targeted fixes to prevent character-spacing (Tc) and word-spacing
(Tw) inflation from causing data quality regressions:

1. should_join_items: reject large negative gaps (< -font_size) that
   arise when Tc/Tw inflate item widths past adjacent items. Fixes
   FY_2015 merged numbers (e.g. "239.696.0" → "239.69 6.0").

2. merge_text_items: cap effective width for gap computation when Tw
   inflates space-containing items beyond 0.85× font_size per char.
   Prevents column-level gaps from collapsing into merge range,
   recovering table detection for Baldwin-Edwards and similar PDFs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Abimael Martell
2026-03-23 17:30:57 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent a199768c4e
commit 8c4181434f
8 changed files with 292 additions and 42 deletions
+35 -7
View File
@@ -118,12 +118,14 @@ pub(crate) fn extract_page_text_items(
// Graphics state tracking
let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix
let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible
let mut gstate_stack: Vec<([f32; 6], i32)> = Vec::new();
let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new();
// Text state tracking
let mut current_font = String::new();
let mut current_font_size: f32 = 12.0;
let mut text_leading: f32 = 0.0; // TL parameter (in text-space units)
let mut char_spacing: f32 = 0.0; // Tc parameter (extra spacing per character, unscaled)
let mut word_spacing: f32 = 0.0; // Tw parameter (extra spacing per space char, unscaled)
let mut text_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut line_matrix = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0];
let mut in_text_block = false;
@@ -154,13 +156,15 @@ pub(crate) fn extract_page_text_items(
match op.operator.as_str() {
"q" => {
// Save graphics state
gstate_stack.push((ctm, text_rendering_mode));
gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing));
}
"Q" => {
// Restore graphics state
if let Some((saved_ctm, saved_tr)) = gstate_stack.pop() {
if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() {
ctm = saved_ctm;
text_rendering_mode = saved_tr;
char_spacing = saved_tc;
word_spacing = saved_tw;
}
}
"cm" => {
@@ -213,6 +217,18 @@ pub(crate) fn extract_page_text_items(
text_rendering_mode = mode as i32;
}
}
"Tc" => {
// Set character spacing (extra space added after each character)
if let Some(tc) = op.operands.first().and_then(get_number) {
char_spacing = tc;
}
}
"Tw" => {
// Set word spacing (extra space added for each space character)
if let Some(tw) = op.operands.first().and_then(get_number) {
word_spacing = tw;
}
}
"Td" | "TD" => {
// Move text position: TLM = T(tx,ty) × TLM; Tm = TLM
// tx,ty are in text space — must be scaled by the text line matrix
@@ -253,8 +269,15 @@ pub(crate) fn extract_page_text_items(
if in_text_block && !op.operands.is_empty() {
// Advance text matrix regardless of visibility
let w_ts_opt = font_widths.get(&current_font).and_then(|fi| {
get_operand_bytes(&op.operands[0])
.map(|raw| compute_string_width_ts(raw, fi, current_font_size))
get_operand_bytes(&op.operands[0]).map(|raw| {
compute_string_width_ts(
raw,
fi,
current_font_size,
char_spacing,
word_spacing,
)
})
});
// ActualText: suppress glyph extraction, just advance text matrix
if suppress_glyph_extraction {
@@ -408,8 +431,13 @@ pub(crate) fn extract_page_text_items(
}
if let Some(fi) = font_info {
if let Some(raw_bytes) = get_operand_bytes(element) {
total_width_ts +=
compute_string_width_ts(raw_bytes, fi, current_font_size);
total_width_ts += compute_string_width_ts(
raw_bytes,
fi,
current_font_size,
char_spacing,
word_spacing,
);
}
}
if !is_invisible {
+108 -1
View File
@@ -429,15 +429,23 @@ pub(crate) fn parse_cid_w_array(
/// Compute the width of a string in text space units,
/// given raw bytes and font width info.
/// Returns width in text space units (font_units * units_scale * font_size).
///
/// `char_spacing` (Tc) is added per character and `word_spacing` (Tw) is added
/// per space character (byte 0x20), both in unscaled text-space units.
/// Per the PDF spec: tx = (w0 × Tfs + Tc + Tw_if_space) per glyph.
pub(crate) fn compute_string_width_ts(
bytes: &[u8],
font_info: &FontWidthInfo,
font_size: f32,
char_spacing: f32,
word_spacing: f32,
) -> f32 {
let mut total: f32 = 0.0;
if font_info.is_cid {
let mut num_spaces: usize = 0;
let num_chars = if font_info.is_cid {
// 2-byte (big-endian) character codes
let mut j = 0;
let mut count = 0usize;
while j + 1 < bytes.len() {
let cid = u16::from_be_bytes([bytes[j], bytes[j + 1]]);
let w = font_info
@@ -446,8 +454,14 @@ pub(crate) fn compute_string_width_ts(
.copied()
.unwrap_or(font_info.default_width);
total += w as f32;
// CID 32 = space in most CID fonts
if cid == 32 {
num_spaces += 1;
}
count += 1;
j += 2;
}
count
} else {
// 1-byte character codes
for &b in bytes {
@@ -458,10 +472,17 @@ pub(crate) fn compute_string_width_ts(
.copied()
.unwrap_or(font_info.default_width);
total += w as f32;
if b == 0x20 {
num_spaces += 1;
}
}
bytes.len()
};
// Convert from font units to text space using the font's scale factor
// Then add Tc per character and Tw per space character
total * font_info.units_scale * font_size
+ num_chars as f32 * char_spacing
+ num_spaces as f32 * word_spacing
}
/// Extract raw bytes from a PDF operand (String object)
@@ -1070,6 +1091,92 @@ fn score_text(text: &str) -> i32 {
mod tests {
use super::*;
fn make_font_info(widths: &[(u16, u16)], default_width: u16, is_cid: bool) -> FontWidthInfo {
FontWidthInfo {
widths: widths.iter().copied().collect(),
default_width,
space_width: widths
.iter()
.find(|(k, _)| *k == 32)
.map(|(_, v)| *v)
.unwrap_or(default_width),
is_cid,
units_scale: 0.001,
wmode: 0,
}
}
#[test]
fn compute_string_width_ts_no_tc_tw() {
// Without Tc/Tw (both 0), width = glyph widths only
let fi = make_font_info(&[(72, 500), (101, 400), (108, 300)], 600, false);
let bytes = b"Hello"; // H=500, e=400, l=300, l=300, o=600(default)
let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.0);
// (500+400+300+300+600) * 0.001 * 10 = 21.0
assert!((w - 21.0).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_with_positive_tc() {
// Positive Tc adds char_spacing per character
let fi = make_font_info(&[], 500, false);
let bytes = b"ab"; // 2 chars, each 500 default
let w = compute_string_width_ts(bytes, &fi, 10.0, 0.5, 0.0);
// glyph: (500+500)*0.001*10 = 10.0, Tc: 2*0.5 = 1.0, total = 11.0
assert!((w - 11.0).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_with_negative_tc() {
// Negative Tc (tight tracking) reduces width
let fi = make_font_info(&[], 500, false);
let bytes = b"ab";
let w = compute_string_width_ts(bytes, &fi, 10.0, -0.3, 0.0);
// glyph: 10.0, Tc: 2*(-0.3) = -0.6, total = 9.4
assert!((w - 9.4).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_with_tw() {
// Tw applies only to space characters (byte 0x20)
let fi = make_font_info(&[(32, 250)], 500, false);
let bytes = b"a b"; // 'a'=500, ' '=250, 'b'=500
let w = compute_string_width_ts(bytes, &fi, 10.0, 0.0, 0.8);
// glyph: (500+250+500)*0.001*10 = 12.5, Tw: 1*0.8 = 0.8, total = 13.3
assert!((w - 13.3).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_with_tc_and_tw() {
// Both Tc and Tw
let fi = make_font_info(&[(32, 250)], 500, false);
let bytes = b"a b"; // 3 chars, 1 space
let w = compute_string_width_ts(bytes, &fi, 10.0, 0.1, 0.5);
// glyph: 12.5, Tc: 3*0.1 = 0.3, Tw: 1*0.5 = 0.5, total = 13.3
assert!((w - 13.3).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_cid_font() {
// CID font: 2-byte codes, space is CID 32
let fi = make_font_info(&[(65, 500), (32, 250)], 600, true);
// "A " in CID: [0,65, 0,32]
let bytes = &[0u8, 65, 0, 32];
let w = compute_string_width_ts(bytes, &fi, 12.0, 0.2, 0.3);
// glyph: (500+250)*0.001*12 = 9.0, Tc: 2*0.2 = 0.4, Tw: 1*0.3 = 0.3
assert!((w - 9.7).abs() < 0.01);
}
#[test]
fn compute_string_width_ts_large_tc() {
// Large Tc (character-spreading) is applied in full
let fi = make_font_info(&[], 500, false);
let bytes = b"abc"; // 3 chars
let w = compute_string_width_ts(bytes, &fi, 10.0, 5.0, 0.0);
// glyph: (500*3)*0.001*10 = 15.0, Tc: 3*5.0 = 15.0, total = 30.0
assert!((w - 30.0).abs() < 0.01);
}
#[test]
fn score_text_cjk() {
// Correct Japanese text should score well
+115 -4
View File
@@ -272,6 +272,44 @@ pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] {
/// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each
/// group by X, then merges consecutive items that share a similar font size
/// and are close horizontally.
/// Cap item width for merge-gap computation to guard against Tw inflation.
///
/// When PDF word-spacing (Tw) is large (used for text justification), the
/// advance width of strings containing spaces extends far past the visible
/// glyph extent. This inflated width collapses inter-column gaps, making
/// `merge_text_items` incorrectly merge items from different table columns.
///
/// Only applies to non-CJK items whose text contains spaces (where Tw
/// contributes) and whose average width-per-character is abnormally high.
fn effective_merge_width(item: &TextItem) -> f32 {
use crate::text_utils::is_cjk_char;
if item.width <= 0.0 || item.font_size <= 0.0 {
return item.width;
}
// Tw only inflates strings that contain space characters.
if !item.text.contains(' ') {
return item.width;
}
// CJK characters are naturally ~1.0× font_size wide; skip the cap.
if item.text.chars().any(is_cjk_char) {
return item.width;
}
let char_count = item.text.chars().count();
if char_count == 0 {
return item.width;
}
let avg = item.width / char_count as f32;
// Normal proportional text: ~0.5× font_size per char.
// Monospace: ~0.6×. Threshold at 0.85× catches Tw inflation.
if avg > item.font_size * 0.85 {
let capped = char_count as f32 * item.font_size * 0.6;
capped.min(item.width)
} else {
item.width
}
}
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
if items.is_empty() {
return items;
@@ -315,7 +353,7 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
while i < group.len() {
let first = group[i];
let mut text = first.text.clone();
let mut end_x = first.x + first.width;
let mut end_x = first.x + effective_merge_width(first);
let x_gap_max = first.font_size * 0.5;
let mut j = i + 1;
@@ -332,12 +370,30 @@ pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
if gap < -first.font_size * 0.5 {
break;
}
// Insert space at word boundaries
if gap > first.font_size * 0.08 {
// Insert space at word boundaries.
// Base threshold 0.08; raised to 0.13 for lowercase→lowercase
// junctions to accommodate Tc/Tw character-spacing adjustments
// that shift advance widths relative to Td positioning.
let threshold = {
let prev_last = text.trim_end().chars().last();
let next_first = next.text.trim_start().chars().next();
// Never insert space before joining punctuation
if next_first.is_some_and(|c| matches!(c, '.' | ',' | ';' | ')' | ']' | '}')) {
first.font_size * 0.25
} else if prev_last.is_some_and(|c| c.is_lowercase())
&& next_first.is_some_and(|c| c.is_lowercase())
{
// Lowercase→lowercase: likely mid-word, use wider threshold
first.font_size * 0.13
} else {
first.font_size * 0.08
}
};
if gap > threshold {
text.push(' ');
}
text.push_str(&next.text);
end_x = next.x + next.width;
end_x = next.x + effective_merge_width(next);
j += 1;
}
@@ -465,6 +521,61 @@ mod tests {
use crate::types::{ItemType, TextLine};
use layout::{detect_columns, is_newspaper_layout, ColumnRegion};
fn make_merge_item(text: &str, x: f32, width: f32) -> TextItem {
TextItem {
text: text.into(),
x,
y: 700.0,
width,
height: 12.0,
font: "F1".into(),
font_size: 12.0,
page: 1,
is_bold: false,
is_italic: false,
item_type: ItemType::Text,
mcid: None,
}
}
#[test]
fn merge_items_no_space_before_period() {
// Simulate Tc/Tw-adjusted width: "date" width is smaller than the gap
// to "." due to negative Tc, but period should still join without space.
let items = vec![
make_merge_item("date", 227.25, 89.25), // end = 316.50
make_merge_item(".", 318.00, 3.0), // gap = 1.50 (0.125 × fs)
];
let merged = merge_text_items(items);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].text, "date.");
}
#[test]
fn merge_items_lowercase_join_with_tc() {
// Lowercase→lowercase junction: "deve" + "lopers" with Tc-affected gap
// Gap of 0.12 × font_size should merge without space
let items = vec![
make_merge_item("deve", 100.0, 30.0), // end = 130.0
make_merge_item("lopers", 131.44, 40.0), // gap = 1.44 (0.12 × 12)
];
let merged = merge_text_items(items);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].text, "developers");
}
#[test]
fn merge_items_space_at_word_boundary() {
// Word boundary gap (> 0.13 × font_size) should insert space
let items = vec![
make_merge_item("hello", 100.0, 30.0),
make_merge_item("world", 132.0, 30.0), // gap = 2.0 (0.167 × 12)
];
let merged = merge_text_items(items);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].text, "hello world");
}
#[test]
fn test_group_into_lines() {
let items = vec![
+11 -2
View File
@@ -354,6 +354,8 @@ fn extract_form_xobject_text_inner(
raw_bytes,
font_info,
current_font_size,
0.0,
0.0,
);
text_matrix[4] += w_ts * text_matrix[0];
text_matrix[5] += w_ts * text_matrix[1];
@@ -381,6 +383,8 @@ fn extract_form_xobject_text_inner(
raw_bytes,
font_info,
current_font_size,
0.0,
0.0,
);
text_matrix[4] += w_ts * text_matrix[0];
text_matrix[5] += w_ts * text_matrix[1];
@@ -493,8 +497,13 @@ fn extract_form_xobject_text_inner(
}
if let Some(fi) = font_info {
if let Some(raw_bytes) = get_operand_bytes(element) {
total_width_ts +=
compute_string_width_ts(raw_bytes, fi, current_font_size);
total_width_ts += compute_string_width_ts(
raw_bytes,
fi,
current_font_size,
0.0,
0.0,
);
}
}
if !fill_is_white {
+9 -4
View File
@@ -579,8 +579,10 @@ pub(crate) fn should_join_items(
};
let font_size = prev_item.font_size;
// Never join across column-scale gaps
if gap > font_size * 3.0 {
// Never join across column-scale gaps or large overlaps.
// Large negative gaps arise when Tc/Tw inflate item widths past
// where adjacent items actually start.
if gap > font_size * 3.0 || gap < -font_size {
return false;
}
@@ -612,15 +614,18 @@ pub(crate) fn should_join_items(
// are positioned close together are almost always a single number.
// e.g., "34,20" + "8" → "34,208", "+13." + "0" + "%" → "+13.0%"
// Use a generous threshold since word spaces in numbers are rare.
// The lower bound (-font_size) rejects large overlaps caused by
// Tc/Twinflated item widths that make adjacent items appear to
// occupy the same space.
if let (Some(p), Some(c)) = (prev_last, curr_first) {
let prev_is_numeric = p.is_ascii_digit() || p == ',' || p == '.';
let curr_is_numeric = c.is_ascii_digit() || c == '%' || c == '.';
if prev_is_numeric && curr_is_numeric {
return gap < font_size * 0.3;
return gap > -font_size && gap < font_size * 0.3;
}
// Sign characters (+/-) followed by digits
if (p == '+' || p == '-') && c.is_ascii_digit() {
return gap < font_size * 0.3;
return gap > -font_size && gap < font_size * 0.3;
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ Employers name and address (include establishment name, if different) **1** C
Month or shorter period in which tips were received **4** Net tips (lines 1 + 2 - 3) from, 19, to, 19 Signature Date
**Paperwork Reduction Act Notice.— We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; **Learning** **about the law —each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these
**Paperwork Reduction Act Notice.—We ask for the** information on these forms to carry out the Internal Revenue laws of the United States. You are required to give us the information. We need it to ensure that you are complying with these laws and to allow us to figure and collect the right amount of tax. You are not required to provide the information requested on a form that is subject to the Paperwork Reduction Act unless the form displays a valid OMB control number. Books or records relating to a form or its instructions must be retained as long as their contents may become material in the administration of any Internal Revenue law. Generally, tax returns and return information are confidential, as required by Code section 6103. The time needed to complete Forms 4070 and 4070A will vary depending on individual circumstances. The estimated average times are: Recordkeeping—Form 4070, 7 min.; Form 4070A, 3 hr. and 23 min.; Learning **about the law—each form, 2 min.; Preparing Form 4070,** 13 min.; Form 4070A, 55 min.; and Copying and **providing Form 4070, 10 min.; Form 4070A, 14 min.** If you have comments concerning the accuracy of these time estimates or suggestions for making these
forms simpler, we would be happy to hear from you. You can write to the Tax Forms Committee, Western Area Distribution Center, Rancho Cordova, CA 95743-0001. **Purpose.—Use this form to report tips you receive to** your employer. This includes cash tips, tips you receive from other employees, and credit card tips. You must report tips every month regardless of your total wages and tips for the year. However, you do not have to report tips to your employer for any month you received less than $20 in tips while working for that employer. Report tips by the 10th day of the month following the month that you receive them. If the 10th day is a Saturday, Sunday, or legal holiday, report tips by the next day that is not a Saturday, Sunday, or legal holiday. See Pub. 531, Reporting Tip Income, for more information. You can get additional copies of Pub. 1244, Employees Daily Record of Tips and Report to Employer, which contains both Forms 4070A and 4070, by calling 1-800-TAX-FORM (1-800-829-3676).
+9 -19
View File
@@ -206,25 +206,15 @@ CFR part or section where Current OMB identified or described control No.
1.302-2T………………………………………………………………… 1545-2019
1.302-4T………………………………………………………………… 1545-2019
1.331-1T………………………………………………………………… 1545-2019
1.332-6T………………………………………………………………... 1545-2019
1.338-10T………………………………………………………………. 1545-2019
1.351-3T………………………………………………………………… 1545-2019
1.355-5T………………………………………………………………… 1545-2019
1.368-3T………………………………………………………………… 1545-2019
1.381(b)-1T…………………………………………………………….. 1545-2019
1.382-8T………………………………………………………………… 1545-2019
1.382-11T………………………………………………………………. 1545-2019
1.1081-11T……………………………………………………………… 1545-2019
1.1221-2T……………………………………………………………….. 1545-2019
1.1502-13T……………………………………………………………… 1545-2019
1.1502-31T……………………………………………………………… 1545-2019
1.1502-32T……………………………………………………………… 1545-2019
1.1502-33T……………………………………………………………… 1545-2019
1.1502-35T……………………………………………………………… 1545-2019
1.1502-76T……………………………………………………………… 1545-2019
1.1502-95T……………………………………………………………… 1545-2019
1.1563-1T……………………………………………………………….. 1545-2019
|1.331-1T………………………………………………………………… 1545|-2019|
|---|---|
||1.332-6T………………………………………………………………... 1545-2019 1.338-10T………………………………………………………………. 1545-2019|
|1.351-3T………………………………………………………………… 1545|-2019|
|1.355-5T………………………………………………………………… 1545|-2019|
|1.368-3T………………………………………………………………… 1545|-2019 1.381(b)-1T…………………………………………………………….. 1545-2019|
|1.382-8T………………………………………………………………… 1545|-2019 1.382-11T………………………………………………………………. 1545-2019 1.1081-11T……………………………………………………………… 1545-2019 1.1221-2T……………………………………………………………….. 1545-2019|
|1.1502-13T………………………………………………………………|1545-2019 1.1502-31T……………………………………………………………… 1545-2019 1.1502-32T……………………………………………………………… 1545-2019 1.1502-33T……………………………………………………………… 1545-2019 1.1502-35T……………………………………………………………… 1545-2019 1.1502-76T……………………………………………………………… 1545-2019|
|1.1502-95T……………………………………………………………… 1545|-2019 1.1563-1T……………………………………………………………….. 1545-2019|
1.1563-3T……………………………………………………………….. 1545-2019
1.6012-2T……………………………………………………………….. 1545-2019