From b547e685de93bbde67ae74dd80ca5f00cad89f9d Mon Sep 17 00:00:00 2001 From: Abimael Martell Date: Tue, 17 Feb 2026 12:59:09 -0800 Subject: [PATCH] fix(extractor): Include CTM scaling in font_size to fix two-column merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit effective_font_size() was computed from the text matrix alone, ignoring CTM scaling. In PDFs with a small CTM scale (e.g. 0.24×) and large Tm values (e.g. 58), font_size was inflated (58pt instead of ~14pt), causing the merge threshold to bridge inter-column gaps and merge two-column text into single lines. Now compute the combined matrix (text_matrix × CTM) before calling effective_font_size() at all 6 call sites. Also adds PdfRect extraction from `re` operators for future table-grid detection, and related plumbing changes. Co-Authored-By: Claude Opus 4.6 --- src/extractor.rs | 83 ++++++++++++++---- src/lib.rs | 51 ++++++++---- src/markdown.rs | 77 +++++++++++++++-- src/tables.rs | 213 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 381 insertions(+), 43 deletions(-) diff --git a/src/extractor.rs b/src/extractor.rs index e5f6536..42d4a50 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -489,6 +489,16 @@ pub enum ItemType { Link(String), } +/// A rectangle from a PDF `re` operator (cell boundary, border, etc.) +#[derive(Debug, Clone)] +pub struct PdfRect { + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, + pub page: u32, +} + /// A text item with position information #[derive(Debug, Clone)] pub struct TextItem { @@ -887,6 +897,15 @@ pub fn extract_text_with_positions_pages>( path: P, page_filter: Option<&HashSet>, ) -> Result, PdfError> { + let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?; + Ok(items) +} + +/// Extract text with positions and rectangles from a file. +pub(crate) fn extract_text_with_positions_and_rects>( + path: P, + page_filter: Option<&HashSet>, +) -> Result<(Vec, Vec), PdfError> { // Read the raw PDF bytes for ToUnicode extraction let pdf_bytes = std::fs::read(path.as_ref())?; crate::validate_pdf_bytes(&pdf_bytes)?; @@ -906,6 +925,15 @@ pub fn extract_text_with_positions_mem_pages( buffer: &[u8], page_filter: Option<&HashSet>, ) -> Result, PdfError> { + let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?; + Ok(items) +} + +/// Extract text with positions and rectangles from memory buffer. +pub(crate) fn extract_text_with_positions_mem_and_rects( + buffer: &[u8], + page_filter: Option<&HashSet>, +) -> Result<(Vec, Vec), PdfError> { crate::validate_pdf_bytes(buffer)?; // Extract ToUnicode CMaps from raw PDF bytes let font_cmaps = FontCMaps::from_pdf_bytes(buffer); @@ -914,12 +942,12 @@ pub fn extract_text_with_positions_mem_pages( extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter) } -/// Extract positioned text from loaded document +/// Extract positioned text and rectangles from loaded document fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result, PdfError> { +) -> Result<(Vec, Vec), PdfError> { // If raw byte scanning found no CMaps, populate from the document model. // This handles PDFs with compressed object streams where raw scanning fails. let mut font_cmaps_owned; @@ -933,6 +961,7 @@ fn extract_positioned_text_from_doc( let pages = doc.get_pages(); let mut all_items = Vec::new(); + let mut all_rects = Vec::new(); for (page_num, &page_id) in pages.iter() { if let Some(filter) = page_filter { @@ -940,15 +969,16 @@ fn extract_positioned_text_from_doc( continue; } } - let items = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; + let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?; all_items.extend(items); + all_rects.extend(rects); // Extract hyperlinks from page annotations let links = extract_page_links(doc, page_id, *page_num); all_items.extend(links); } - Ok(all_items) + Ok((all_items, all_rects)) } /// Populate FontCMaps from the lopdf document model for ToUnicode streams @@ -1015,16 +1045,17 @@ fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] { ] } -/// Extract text items from a single page +/// Extract text items and rectangles from a single page fn extract_page_text_items( doc: &Document, page_id: ObjectId, page_num: u32, font_cmaps: &FontCMaps, -) -> Result, PdfError> { +) -> Result<(Vec, Vec), PdfError> { use lopdf::content::Content; let mut items = Vec::new(); + let mut rects: Vec = Vec::new(); // Get fonts for encoding let fonts = doc.get_page_fonts(page_id).unwrap_or_default(); @@ -1252,8 +1283,8 @@ fn extract_page_text_items( &font_encodings, &encoding_cache, ) { - let rendered_size = effective_font_size(current_font_size, &text_matrix); let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); let width = if let Some(w_ts) = w_ts_opt { text_matrix[4] += w_ts * text_matrix[0]; @@ -1393,8 +1424,8 @@ fn extract_page_text_items( } // Emit one TextItem per sub-item if !sub_items.is_empty() { - let rendered_size = - effective_font_size(current_font_size, &text_matrix); + let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let base_font = font_base_names .get(¤t_font) .map(|s| s.as_str()) @@ -1464,9 +1495,8 @@ fn extract_page_text_items( &encoding_cache, ) { if !text.trim().is_empty() { - let rendered_size = - effective_font_size(current_font_size, &text_matrix); let combined = multiply_matrices(&text_matrix, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); let base_font = font_base_names .get(¤t_font) @@ -1545,8 +1575,8 @@ fn extract_page_text_items( if let Some(Some(at)) = marked_content_stack.pop() { // Compute width from text matrix advancement during BDC..EMC if let Some(start_tm) = actual_text_start_tm.take() { - let rendered_size = effective_font_size(current_font_size, &start_tm); let combined = multiply_matrices(&start_tm, &ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); // Width in device space from text matrix delta let delta_ts = text_matrix[4] - start_tm[4]; @@ -1575,12 +1605,33 @@ fn extract_page_text_items( suppress_glyph_extraction = marked_content_stack.iter().any(|a| a.is_some()); } } + "re" => { + // Rectangle operator: collect for table-grid detection + if op.operands.len() >= 4 { + let rx = get_number(&op.operands[0]).unwrap_or(0.0); + let ry = get_number(&op.operands[1]).unwrap_or(0.0); + let rw = get_number(&op.operands[2]).unwrap_or(0.0); + let rh = get_number(&op.operands[3]).unwrap_or(0.0); + // Transform origin to device space + let x_dev = rx * ctm[0] + ry * ctm[2] + ctm[4]; + let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5]; + let w_dev = rw * ctm[0]; + let h_dev = rh * ctm[3]; + rects.push(PdfRect { + x: x_dev, + y: y_dev, + width: w_dev, + height: h_dev, + page: page_num, + }); + } + } _ => {} } } let items = merge_text_items(items); - Ok(items) + Ok((items, rects)) } /// Merge adjacent single-character TextItems into words. @@ -1898,8 +1949,8 @@ fn extract_form_xobject_text( &font_encodings, &encoding_cache, ) { - let rendered_size = effective_font_size(current_font_size, &text_matrix); let combined = multiply_matrices(&text_matrix, parent_ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let (x, y) = (combined[4], combined[5]); let width = if let Some(font_info) = font_widths.get(¤t_font) { if let Some(raw_bytes) = get_operand_bytes(&op.operands[0]) { @@ -2043,8 +2094,8 @@ fn extract_form_xobject_text( sub_items.push((current_text, sub_start_width_ts, total_width_ts)); } if !sub_items.is_empty() { - let rendered_size = - effective_font_size(current_font_size, &text_matrix); + let combined = multiply_matrices(&text_matrix, parent_ctm); + let rendered_size = effective_font_size(current_font_size, &combined); let base_font = font_base_names .get(¤t_font) .map(|s| s.as_str()) diff --git a/src/lib.rs b/src/lib.rs index 93be6e6..f7aec57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,9 +17,11 @@ pub use detector::{ detect_pdf_type_with_config, DetectionConfig, PdfType, PdfTypeResult, ScanStrategy, }; pub use extractor::{ - extract_text, extract_text_with_positions, extract_text_with_positions_pages, TextItem, + extract_text, extract_text_with_positions, extract_text_with_positions_pages, PdfRect, TextItem, +}; +pub use markdown::{ + to_markdown, to_markdown_from_items, to_markdown_from_items_with_rects, MarkdownOptions, }; -pub use markdown::{to_markdown, to_markdown_from_items, MarkdownOptions}; use std::path::Path; @@ -66,8 +68,9 @@ pub fn process_pdf>(path: P) -> Result { // Step 2: Full extraction with position-aware reading order - let items = extract_text_with_positions(&path)?; - let markdown = to_markdown_from_items(items, MarkdownOptions::default()); + let (items, rects) = extractor::extract_text_with_positions_and_rects(&path, None)?; + let markdown = + to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects); PdfProcessResult { pdf_type, @@ -95,8 +98,10 @@ pub fn process_pdf>(path: P) -> Result { // Try to extract what we can with position-aware reading order - let items = extract_text_with_positions(&path).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default())); + let result = extractor::extract_text_with_positions_and_rects(&path, None).ok(); + let markdown = result.map(|(items, rects)| { + to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects) + }); PdfProcessResult { pdf_type, @@ -146,8 +151,9 @@ pub fn process_pdf_with_config_pages>( let result = match pdf_type { PdfType::TextBased => { - let items = extract_text_with_positions_pages(&path, page_filter)?; - let markdown = to_markdown_from_items(items, markdown_options); + let (items, rects) = + extractor::extract_text_with_positions_and_rects(&path, page_filter)?; + let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); PdfProcessResult { pdf_type, @@ -171,8 +177,10 @@ pub fn process_pdf_with_config_pages>( confidence, }, PdfType::Mixed => { - let items = extract_text_with_positions_pages(&path, page_filter).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone())); + let result = extractor::extract_text_with_positions_and_rects(&path, page_filter).ok(); + let markdown = result.map(|(items, rects)| { + to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects) + }); PdfProcessResult { pdf_type, @@ -207,8 +215,10 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { let result = match pdf_type { PdfType::TextBased => { // Step 2: Full extraction with position-aware reading order - let items = extractor::extract_text_with_positions_mem(buffer)?; - let markdown = to_markdown_from_items(items, MarkdownOptions::default()); + let (items, rects) = + extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; + let markdown = + to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects); PdfProcessResult { pdf_type, @@ -232,8 +242,10 @@ pub fn process_pdf_mem(buffer: &[u8]) -> Result { confidence, }, PdfType::Mixed => { - let items = extractor::extract_text_with_positions_mem(buffer).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, MarkdownOptions::default())); + let result = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); + let markdown = result.map(|(items, rects)| { + to_markdown_from_items_with_rects(items, MarkdownOptions::default(), &rects) + }); PdfProcessResult { pdf_type, @@ -270,8 +282,9 @@ pub fn process_pdf_mem_with_config( let result = match pdf_type { PdfType::TextBased => { - let items = extractor::extract_text_with_positions_mem(buffer)?; - let markdown = to_markdown_from_items(items, markdown_options); + let (items, rects) = + extractor::extract_text_with_positions_mem_and_rects(buffer, None)?; + let markdown = to_markdown_from_items_with_rects(items, markdown_options, &rects); PdfProcessResult { pdf_type, @@ -295,8 +308,10 @@ pub fn process_pdf_mem_with_config( confidence, }, PdfType::Mixed => { - let items = extractor::extract_text_with_positions_mem(buffer).ok(); - let markdown = items.map(|i| to_markdown_from_items(i, markdown_options.clone())); + let result = extractor::extract_text_with_positions_mem_and_rects(buffer, None).ok(); + let markdown = result.map(|(items, rects)| { + to_markdown_from_items_with_rects(items, markdown_options.clone(), &rects) + }); PdfProcessResult { pdf_type, diff --git a/src/markdown.rs b/src/markdown.rs index 455d0e2..1af6a7f 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -117,8 +117,17 @@ pub fn to_markdown(text: &str, options: MarkdownOptions) -> String { /// Convert positioned text items to markdown with structure detection pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> String { + to_markdown_from_items_with_rects(items, options, &[]) +} + +/// Convert positioned text items to markdown, using rectangle data for table detection +pub fn to_markdown_from_items_with_rects( + items: Vec, + options: MarkdownOptions, + rects: &[crate::extractor::PdfRect], +) -> String { use crate::extractor::ItemType; - use crate::tables::{detect_tables, table_to_markdown}; + use crate::tables::{detect_tables, detect_tables_from_rects, table_to_markdown}; use std::collections::HashSet; if items.is_empty() { @@ -193,25 +202,77 @@ pub fn to_markdown_from_items(items: Vec, options: MarkdownOptions) -> let group = page_groups.get(&page).unwrap(); let page_items: Vec = group.iter().map(|(_, item)| (*item).clone()).collect(); - let tables = detect_tables(&page_items, base_size, false); + // Track which local indices are claimed by rect-based tables + let mut rect_claimed: HashSet = HashSet::new(); - for table in tables { - // Mark items as belonging to a table using pre-computed global indices + // Try rectangle-based table detection first + let rect_tables = detect_tables_from_rects(&page_items, rects, page); + for table in &rect_tables { for &idx in &table.item_indices { + rect_claimed.insert(idx); if let Some(&(global_idx, _)) = group.get(idx) { table_items.insert(global_idx); } } - - // Get Y position for table insertion (use highest Y in table) let table_y = table.rows.first().copied().unwrap_or(0.0); - let table_md = table_to_markdown(&table); - + let table_md = table_to_markdown(table); page_tables .entry(page) .or_default() .push((table_y, table_md)); } + + // Run heuristic detection on unclaimed items only + if rect_claimed.is_empty() { + // No rect tables — run heuristic on all items + let tables = detect_tables(&page_items, base_size, false); + for table in tables { + for &idx in &table.item_indices { + if let Some(&(global_idx, _)) = group.get(idx) { + table_items.insert(global_idx); + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } else { + // Rect tables found — run heuristic on unclaimed items + let unclaimed_items: Vec = page_items + .iter() + .enumerate() + .filter(|(idx, _)| !rect_claimed.contains(idx)) + .map(|(_, item)| item.clone()) + .collect(); + if unclaimed_items.len() >= 6 { + let tables = detect_tables(&unclaimed_items, base_size, false); + for table in tables { + // Remap indices from unclaimed-space back to page-space + let unclaimed_map: Vec = page_items + .iter() + .enumerate() + .filter(|(idx, _)| !rect_claimed.contains(idx)) + .map(|(idx, _)| idx) + .collect(); + for &idx in &table.item_indices { + if let Some(&page_idx) = unclaimed_map.get(idx) { + if let Some(&(global_idx, _)) = group.get(page_idx) { + table_items.insert(global_idx); + } + } + } + let table_y = table.rows.first().copied().unwrap_or(0.0); + let table_md = table_to_markdown(&table); + page_tables + .entry(page) + .or_default() + .push((table_y, table_md)); + } + } + } } // Filter out table items and process the rest diff --git a/src/tables.rs b/src/tables.rs index 323b746..c81a4a3 100644 --- a/src/tables.rs +++ b/src/tables.rs @@ -2,7 +2,7 @@ //! //! Detects tabular data in PDF text items and converts to markdown tables. -use crate::extractor::TextItem; +use crate::extractor::{PdfRect, TextItem}; /// Detection mode controls thresholds for table validation #[derive(Debug, Clone, Copy, PartialEq)] @@ -26,6 +26,217 @@ pub struct Table { pub item_indices: Vec, } +/// Detect tables from explicit rectangle (`re`) operators in the PDF. +/// +/// Many PDFs draw cell borders using `re` (rectangle) operators. Table pages +/// typically have 100-200+ rects while non-table pages have < 30. This function +/// identifies grids of cell-sized rectangles and assigns text items to cells. +pub fn detect_tables_from_rects(items: &[TextItem], rects: &[PdfRect], page: u32) -> Vec { + // Filter rects on this page; normalize negative widths/heights; skip tiny rects. + let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new(); // (x, y, w, h) normalized + for r in rects { + if r.page != page { + continue; + } + let (mut x, mut y, mut w, mut h) = (r.x, r.y, r.width, r.height); + if w < 0.0 { + x += w; + w = -w; + } + if h < 0.0 { + y += h; + h = -h; + } + // Skip tiny rects (borders, dots, decorations) + if w < 5.0 || h < 5.0 { + continue; + } + page_rects.push((x, y, w, h)); + } + + // Need a reasonable number of cell rects to form a table + if page_rects.len() < 6 { + return vec![]; + } + + // Extract unique X and Y edges from all rects + let mut x_edges: Vec = Vec::new(); + let mut y_edges: Vec = Vec::new(); + for &(x, y, w, h) in &page_rects { + x_edges.push(x); + x_edges.push(x + w); + y_edges.push(y); + y_edges.push(y + h); + } + + let x_edges = snap_edges(&x_edges, 2.0); + let y_edges = snap_edges(&y_edges, 2.0); + + if x_edges.len() < 3 || y_edges.len() < 4 { + // Need at least 2 columns (3 edges) and 3 rows (4 edges) + return vec![]; + } + + // Sort column edges left-to-right, row edges top-to-bottom (highest Y first for PDF) + let mut col_edges = x_edges; + col_edges.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let mut row_edges = y_edges; + row_edges.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)); + + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + if num_cols < 2 || num_rows < 2 { + return vec![]; + } + + // Verify that cell-sized rects actually fill the grid + // Count how many grid cells have a matching rect + let mut filled_cells = 0u32; + for row in 0..num_rows { + let y_top = row_edges[row]; + let y_bot = row_edges[row + 1]; + for col in 0..num_cols { + let x_left = col_edges[col]; + let x_right = col_edges[col + 1]; + // Check if any rect approximately covers this cell + let cell_covered = page_rects.iter().any(|&(rx, ry, rw, rh)| { + let tol = 3.0; + rx <= x_left + tol + && (rx + rw) >= x_right - tol + && ry <= y_top + tol + && (ry + rh) >= y_bot - tol + }); + if cell_covered { + filled_cells += 1; + } + } + } + + let total_cells = (num_cols * num_rows) as f32; + let fill_ratio = filled_cells as f32 / total_cells; + + // Require at least 30% of cells to be backed by rects + if fill_ratio < 0.3 { + return vec![]; + } + + // Build table: assign text items to cells + let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + + // Compute column centers and row centers for the Table struct + let columns: Vec = (0..num_cols) + .map(|c| (col_edges[c] + col_edges[c + 1]) / 2.0) + .collect(); + let rows: Vec = (0..num_rows) + .map(|r| (row_edges[r] + row_edges[r + 1]) / 2.0) + .collect(); + + // Skip if no text was assigned + if item_indices.is_empty() { + return vec![]; + } + + // Skip tables with only 1 row of content (header-only) + let non_empty_rows = cells + .iter() + .filter(|row| row.iter().any(|c| !c.trim().is_empty())) + .count(); + if non_empty_rows < 2 { + return vec![]; + } + + vec![Table { + columns, + rows, + cells, + item_indices, + }] +} + +/// Deduplicate nearby edge values within a tolerance, returning sorted unique edges. +fn snap_edges(values: &[f32], tolerance: f32) -> Vec { + let mut sorted: Vec = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let mut snapped: Vec = Vec::new(); + for &v in &sorted { + if let Some(last) = snapped.last() { + if (v - *last).abs() <= tolerance { + continue; // Skip — too close to previous edge + } + } + snapped.push(v); + } + snapped +} + +/// Assign text items to grid cells defined by column/row edges. +/// +/// Returns `(cells, item_indices)` where `cells[row][col]` is the cell text +/// and `item_indices` lists the original item indices that were consumed. +fn assign_items_to_grid( + items: &[TextItem], + col_edges: &[f32], + row_edges: &[f32], + page: u32, +) -> (Vec>, Vec) { + let num_cols = col_edges.len() - 1; + let num_rows = row_edges.len() - 1; + + // Collect items per cell for proper sorting before joining + let mut cell_items: Vec>> = + vec![vec![Vec::new(); num_cols]; num_rows]; + let mut indices = Vec::new(); + + for (idx, item) in items.iter().enumerate() { + if item.page != page { + continue; + } + // Use item center for assignment + let cx = item.x + item.width / 2.0; + let cy = item.y; + + // Find column: cx must be between col_edges[c] and col_edges[c+1] + let col = (0..num_cols).find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0); + // Find row: cy must be between row_edges[r+1] (bottom) and row_edges[r] (top) + let row = (0..num_rows).find(|&r| cy >= row_edges[r + 1] - 2.0 && cy <= row_edges[r] + 2.0); + + if let (Some(c), Some(r)) = (col, row) { + cell_items[r][c].push((idx, item)); + indices.push(idx); + } + } + + // Build cell strings: sort items within each cell by Y descending then X ascending + let mut cells: Vec> = Vec::with_capacity(num_rows); + for row_items in &mut cell_items { + let mut row_cells = Vec::with_capacity(num_cols); + for col_items in row_items.iter_mut() { + col_items.sort_by(|a, b| { + b.1.y + .partial_cmp(&a.1.y) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + a.1.x + .partial_cmp(&b.1.x) + .unwrap_or(std::cmp::Ordering::Equal) + }) + }); + let text: String = col_items + .iter() + .map(|(_, item)| item.text.trim()) + .filter(|t| !t.is_empty()) + .collect::>() + .join(" "); + row_cells.push(text); + } + cells.push(row_cells); + } + + (cells, indices) +} + /// Check if a whitespace-separated token looks like a financial number. /// Must contain at least one digit; all chars must be `0-9 , . ( ) - + %`. fn is_numeric_token(tok: &str) -> bool {