diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 1cf3bf8..4e8c318 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -751,43 +751,7 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( // 2. Line-based detection on unclaimed items (when rects didn't find tables) if rect_claimed.is_empty() { - // Synthesize PdfLine objects from thin filled rects (border lines - // drawn as narrow rectangles instead of stroked paths). - let mut augmented_lines: Vec = band_lines.to_vec(); - for r in band_rects { - let (mut w, mut h) = (r.width, r.height); - let (mut x, mut y) = (r.x, r.y); - if w < 0.0 { - x += w; - w = -w; - } - if h < 0.0 { - y += h; - h = -h; - } - if h < 2.0 && w >= 10.0 { - // Thin horizontal rect → horizontal line - let mid_y = y + h / 2.0; - augmented_lines.push(crate::types::PdfLine { - x1: x, - y1: mid_y, - x2: x + w, - y2: mid_y, - page, - }); - } else if w < 2.0 && h >= 10.0 { - // Thin vertical rect → vertical line - let mid_x = x + w / 2.0; - augmented_lines.push(crate::types::PdfLine { - x1: mid_x, - y1: y, - x2: mid_x, - y2: y + h, - page, - }); - } - } - let line_tables = detect_tables_from_lines(band_items, &augmented_lines, page); + let line_tables = detect_tables_from_lines(band_items, band_lines, page); for table in &line_tables { for &idx in &table.item_indices { rect_claimed.insert(idx); diff --git a/src/tables/detect_lines.rs b/src/tables/detect_lines.rs index f5c6ad4..c60b450 100644 --- a/src/tables/detect_lines.rs +++ b/src/tables/detect_lines.rs @@ -185,11 +185,6 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 .filter(|row| row.iter().any(|cell| !cell.is_empty())) .count(); if non_empty_rows < 2 { - log::debug!( - "detect_lines p{}: rejected — only {} non-empty rows", - page, - non_empty_rows - ); return Vec::new(); } @@ -204,11 +199,6 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 .count(); let density = filled_cells as f32 / total_cells as f32; if density < 0.15 { - log::debug!( - "detect_lines p{}: rejected — low density {:.2}", - page, - density - ); return Vec::new(); } } @@ -253,17 +243,9 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 .map(|s| (s - mean_spacing).powi(2)) .sum::() / spacings.len() as f32; - let cv = variance.sqrt() / mean_spacing; - // CV < 0.02 means nearly identical spacing — likely chart grid. - // Spreadsheet-exported tables often have uniform rows (CV 0.03-0.05), - // so we use a tighter threshold to avoid false negatives. - if cv < 0.02 { - log::debug!( - "detect_lines p{}: rejected — uniform row spacing (cv={:.4}, mean={:.1})", - page, - cv, - mean_spacing - ); + let cv = variance.sqrt() / mean_spacing; // coefficient of variation + // CV < 0.05 means nearly identical spacing — chart grid + if cv < 0.05 { return Vec::new(); } } @@ -281,158 +263,12 @@ pub fn detect_tables_from_lines(items: &[TextItem], lines: &[PdfLine], page: u32 page, num_rows, num_cols, item_indices.len(), page_item_count, non_empty_rows, cols_with_content ); - // Post-process: split rows that contain items at multiple distinct Y - // positions. This happens in stacked sub-tables where "Note:" footer - // text and the next section's "Category" header land between the same - // pair of horizontal rules. Re-assign items to sub-rows by Y proximity. - let cells = split_multi_y_rows(cells, items, &col_edges, &row_edges_desc, page); - - // Split the grid into separate tables at rows that lack vertical border - // coverage (e.g. "Note:" footer text that sits between horizontal rules - // but outside the actual table grid). A row is "unbounded" when fewer - // than 2 vertical lines span its Y range — it's freestanding text, not - // a table cell. - let mut tables = Vec::new(); - let mut current_rows: Vec> = Vec::new(); - - for (r, row) in cells.into_iter().enumerate() { - // Determine Y range for this row - let row_top = if r < row_edges_desc.len() { - row_edges_desc[r] - } else { - row_edges_desc.last().copied().unwrap_or(0.0) - }; - let row_bot = if r + 1 < row_edges_desc.len() { - row_edges_desc[r + 1] - } else { - row_edges_desc.last().copied().unwrap_or(0.0) - 15.0 - }; - - // Count vertical lines that span this row's Y range - let v_covering = verticals - .iter() - .filter(|(_, y_min, y_max)| *y_min <= row_bot + 2.0 && *y_max >= row_top - 2.0) - .count(); - - if v_covering >= 2 { - current_rows.push(row); - } else { - // Flush accumulated rows as a table - if current_rows.len() >= 2 { - tables.push(Table { - columns: col_edges.clone(), - rows: Vec::new(), - cells: std::mem::take(&mut current_rows), - item_indices: item_indices.clone(), - }); - } else { - current_rows.clear(); - } - // The unbounded row's text becomes a standalone "table" with 1 row - // so it gets emitted as text outside the table. - let text = row - .iter() - .map(|c| c.trim()) - .filter(|c| !c.is_empty()) - .collect::>() - .join(" "); - if !text.is_empty() { - // Emit as a 1-cell table which the markdown converter will - // render as a standalone line (single-row tables are just text). - tables.push(Table { - columns: vec![col_edges[0], *col_edges.last().unwrap_or(&col_edges[0])], - rows: Vec::new(), - cells: vec![vec![text]], - item_indices: item_indices.clone(), - }); - } - } - } - // Flush remaining rows - if current_rows.len() >= 2 { - tables.push(Table { - columns: col_edges, - rows: Vec::new(), - cells: current_rows, - item_indices, - }); - } - - tables -} - -/// Split table rows where items within cells span multiple Y positions. -/// Groups items by Y proximity and emits one output row per Y group. -fn split_multi_y_rows( - cells: Vec>, - items: &[TextItem], - col_edges: &[f32], - row_edges: &[f32], - page: u32, -) -> Vec> { - let num_cols = col_edges.len() - 1; - let num_rows = cells.len(); - if num_rows == 0 { - return cells; - } - - // Re-collect items per cell to get Y positions - let mut cell_items: Vec>> = vec![vec![Vec::new(); num_cols]; num_rows]; - for item in items { - if item.page != page { - continue; - } - let cx = item.x + item.width / 2.0; - let cy = item.y; - let col = (0..num_cols).find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0); - let row = (0..num_rows).find(|&r| cy >= row_edges[r + 1] - 2.0 && cy <= row_edges[r] + 2.0); - if let (Some(c), Some(r)) = (col, row) { - cell_items[r][c].push(item); - } - } - - let y_tol = 4.0; // items within 4pt are on the same sub-row - let mut out: Vec> = Vec::new(); - - for (r, _row_cells) in cells.iter().enumerate() { - // Collect all Y positions across all columns in this row - let mut all_ys: Vec = cell_items[r] - .iter() - .flat_map(|items| items.iter().map(|i| i.y)) - .collect(); - if all_ys.is_empty() { - out.push(vec![String::new(); num_cols]); - continue; - } - all_ys.sort_by(|a, b| b.total_cmp(a)); // descending (top first) - all_ys.dedup_by(|a, b| (*a - *b).abs() < y_tol); - - if all_ys.len() <= 1 { - // Single Y level — keep original row - out.push(cells[r].clone()); - } else { - // Multiple Y levels — split into sub-rows - for &sub_y in &all_ys { - let mut sub_row = Vec::with_capacity(num_cols); - for col_items in &cell_items[r] { - let text: String = col_items - .iter() - .filter(|i| (i.y - sub_y).abs() < y_tol) - .map(|i| i.text.trim()) - .filter(|t| !t.is_empty()) - .collect::>() - .join(" "); - sub_row.push(text); - } - // Skip fully empty sub-rows - if sub_row.iter().any(|s| !s.is_empty()) { - out.push(sub_row); - } - } - } - } - - out + vec![Table { + columns: col_edges, + rows: row_edges_desc[..num_rows].to_vec(), + cells, + item_indices, + }] } #[cfg(test)] diff --git a/src/tables/format.rs b/src/tables/format.rs index e740739..14e5aa3 100644 --- a/src/tables/format.rs +++ b/src/tables/format.rs @@ -15,17 +15,6 @@ pub fn table_to_markdown(table: &Table) -> String { } let num_cols = cleaned_cells[0].len(); - - // Single-cell "tables" are standalone text, not real tables. - // Emit as plain text (e.g. "Note: ..." between sub-tables). - if cleaned_cells.len() == 1 && num_cols == 1 { - let text = cleaned_cells[0][0].trim(); - if !text.is_empty() { - return format!("{}\n", text); - } - return String::new(); - } - let mut output = String::new(); // Compact format: no padding, minimal separators. Optimized for token @@ -118,20 +107,11 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { let looks_like_data_row = non_first_cells.len() >= 2 && avg_cell_len <= 10.0 && numeric_cells > non_first_cells.len() / 2; - // Classic continuation: first cell empty, content in other cells. - // Exclude rows where the non-first content is a long label (section - // header like "Category No. 03 - ...") — these are spanning headers, - // not overflow from the previous row. - let has_long_spanning_cell = non_first_cells.len() == 1 && non_first_cells[0].len() > 15; - // Rows filling most columns with short values are header or data rows, - // not text overflow (e.g. "UR | SC | ST | OBC | EWS" column headers). - let looks_like_header_row = non_first_cells.len() >= 3 && avg_cell_len <= 10.0; + // Classic continuation: first cell empty, content in other cells let is_classic_continuation = first_cell.is_empty() && !non_first_cells.is_empty() && !is_short_subheader && !looks_like_data_row - && !looks_like_header_row - && !has_long_spanning_cell && cleaned.len() > 1; // Wrapped-cell continuation: row has fewer filled cells than the header @@ -157,17 +137,11 @@ fn clean_table_cells(cells: &[Vec]) -> (Vec>, Vec) { } else { header_filled.saturating_sub(1) }; - // Don't merge rows where only the first cell has content and it's - // long text — these are section separators (e.g. "Note: ...") or - // section headers (e.g. "Category No. 03 - ..."), not overflow. - let is_first_cell_only = filled_cells == 1 && !first_cell.is_empty(); - let first_cell_long = first_cell.len() > 15; let is_wrapped_continuation = cleaned.len() > 1 && filled_cells <= max_filled_for_merge && prev_filled > filled_cells && !looks_like_data_row - && !is_short_subheader - && !(is_first_cell_only && first_cell_long); + && !is_short_subheader; let is_continuation = is_classic_continuation || is_wrapped_continuation; diff --git a/tests/snapshots/td9264.md b/tests/snapshots/td9264.md index e4d6ff2..8161541 100644 --- a/tests/snapshots/td9264.md +++ b/tests/snapshots/td9264.md @@ -54,24 +54,7 @@ annual statement (or a pro forma annual statement), including the underwriting a ||(3) Foreign insurance companies. The provisions of paragraphs (c)(1) and| |---|---| -||(c)(2) of this section concerning the returns and statements of insurance companies| -||subject to tax under section 801 or section 831 also apply to foreign insurance| -||companies subject to tax under those sections, except that the copy of the annual| -||statement required to be submitted with the return shall, in the case of a foreign| -||insurance company that is not required to file an annual statement, be a copy of the| -||pro forma annual statement relating to the United States business of such company.| -||(4) Exception for insurance companies filing their Federal income tax returns| -||electronically. If an insurance company described in paragraph (c)(1), (c)(2), or| -||(c)(3) of this section files its Federal income tax return electronically, it should not| -||include on or with such return its annual statement (or pro forma annual statement),| -||or any portion thereof. Such statement must be available at all times for inspection| -||by authorized Internal Revenue Service officers or employees and retained for so| -||long as such statements may be material in the administration of any internal| -||revenue law. See §1.6001-1(e).| -||(5) Definition. For purposes of this section, the term annual statement means| -||the annual statement, the form of which is approved by the National Association of| -||Insurance Commissioners (NAIC), which is filed by an insurance company for the| -||year with the insurance departments of States, Territories, and the District of| +||(c)(2) of this section concerning the returns and statements of insurance companies subject to tax under section 801 or section 831 also apply to foreign insurance companies subject to tax under those sections, except that the copy of the annual statement required to be submitted with the return shall, in the case of a foreign insurance company that is not required to file an annual statement, be a copy of the pro forma annual statement relating to the United States business of such company. (4) Exception for insurance companies filing their Federal income tax returns electronically. If an insurance company described in paragraph (c)(1), (c)(2), or (c)(3) of this section files its Federal income tax return electronically, it should not include on or with such return its annual statement (or pro forma annual statement), or any portion thereof. Such statement must be available at all times for inspection by authorized Internal Revenue Service officers or employees and retained for so long as such statements may be material in the administration of any internal revenue law. See §1.6001-1(e). (5) Definition. For purposes of this section, the term annual statement means the annual statement, the form of which is approved by the National Association of Insurance Commissioners (NAIC), which is filed by an insurance company for the year with the insurance departments of States, Territories, and the District of| Columbia. The term annual statement also includes a pro forma annual statement if the insurance company is not required to file the NAIC annual statement. @@ -99,8 +82,7 @@ requirements of section 302(c)(2) are met. ||6|6T| |The last sentence of|paragraph (a)(2)(ii) of this|paragraph (a) of §1.382-| |§1.382-2T(h)(4)(vi)(B)|section|11T| -|The first sentence of|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| -|§1.382-6(b)(2)(i)||| +|The first sentence of §1.382-6(b)(2)(i)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| |The second sentence of|paragraph (c) of this|paragraphs (c)(1), (c)(3),| |§1.382-8(a)|section|(c)(4) and (c)(5) of this| @@ -162,8 +144,7 @@ section and paragraph |---|---|---| |§1.1502-76(b)(2)(ii)(A)(2)|paragraph (b)(2)(ii)(D) of this section|paragraph (b)(2)(ii)(D) of §1.1502-76T| |§1.1502-92(e)(1)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| -|The first sentence of|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| -|§1.1502-92(e)(2)||| +|The first sentence of §1.1502-92(e)(2)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| |The first sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| |The second sentence of §1.1502-94(d)|§1.382-2T(a)(2)(ii)|§1.382-11T(a)| |The last sentence of|paragraph (f) of this|paragraph (f) of §1.1502-| @@ -189,30 +170,16 @@ section and paragraph |||PART 602--OMB CONTROL NUMBERS UNDER THE PAPERWORK|| |---|---|---|---| -||REDUCTION ACT||| -|||Par. 54. The authority citation for part 602 continues to read as follows:|| -||Authority: 26 U.S.C. 7805.||| -|||Par. 55. In §602.101, paragraph (b) is amended to read as follows:|| -||1. The following entries to the table are removed:||| -||§602.101 OMB Control numbers.||| +||REDUCTION ACT Authority: 26 U.S.C. 7805. 1. The following entries to the table are removed: §602.101 OMB Control numbers.|Par. 54. The authority citation for part 602 continues to read as follows: Par. 55. In §602.101, paragraph (b) is amended to read as follows:|| |* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| -|* * * * *|1.332-6………………………………………………………………….||1545-2019| -|||1.382-11……………………………………………………………….. 1545-2019|| -|||1.351-3…………………………………………………………………. 1545-2019|| -|||1.355-5…………………………………………………………………. 1545-2019|| -|||1.368-3…………………………………………………………………. 1545-2019|| -|||1.1081-11………………………………………………………………. 1545-2019|| -|* * * * *|||| -|||______________________________________________________________|| -|||2. The following entries are added in numerical order to the table:|| -||§602.101 OMB Control numbers.||| +|* * * * *|1.332-6………………………………………………………………….|1.382-11……………………………………………………………….. 1545-2019 1.351-3…………………………………………………………………. 1545-2019 1.355-5…………………………………………………………………. 1545-2019 1.368-3…………………………………………………………………. 1545-2019 1.1081-11………………………………………………………………. 1545-2019|1545-2019| +|* * * * *|§602.101 OMB Control numbers.|______________________________________________________________ 2. The following entries are added in numerical order to the table:|| |* * * * *|(b) * * * CFR part or section where identified or described||Current OMB control No.| |* * * * *|1.302-2T………………………………………………………………… 1545 1.302-4T………………………………………………………………… 1545||-2019 -2019| |1.331-1T………………………………………………………………… 1545|-2019| |---|---| -||1.332-6T………………………………………………………………... 1545-2019| -||1.338-10T………………………………………………………………. 1545-2019| +||1.332-6T………………………………………………………………... 1545-2019 1.338-10T………………………………………………………………. 1545-2019| |1.351-3T………………………………………………………………… 1545|-2019| |1.355-5T………………………………………………………………… 1545|-2019| |1.368-3T………………………………………………………………… 1545|-2019 1.381(b)-1T…………………………………………………………….. 1545-2019| diff --git a/tests/snapshots/thermo-freon12.md b/tests/snapshots/thermo-freon12.md index 96155ff..a575f0b 100644 --- a/tests/snapshots/thermo-freon12.md +++ b/tests/snapshots/thermo-freon12.md @@ -29,8 +29,7 @@ S = Entropy (kJ/kg.K) |Chemical Formula|CCl2F2| |---|---| |Molecular mass|120.91| -|Boiling Point|-29.75°C| -|At one atmosphere|| +|Boiling Point At one atmosphere|-29.75°C| |Critical Temperature|111.97°C| |Critical Pressure|4136 kPa| |Critical Density|565.0 kg/m| @@ -46,9 +45,7 @@ l |Temp|Pressure||Volume|||Density||Enthalpy|||Entropy|Temp| |---|---|---|---|---|---|---|---|---|---|---|---|---| -|°C|[kPa]|[m3|/kg]|||[kg/m3]||[kJ/kg]|||[kJ/K-kg]|°C| -|||Liquid||Vapour|Liquid|Vapour|Liquid|Latent|Vapour|Liquid|Vapour|| -|||v f||v g|d f|d g|H f|H fg|H g|S f|S g|| +|°C|[kPa]|[m3 Liquid v f|/kg]|Vapour v g|Liquid d f|[kg/m3] Vapour d g|Liquid H f|[kJ/kg] Latent H fg|Vapour H g|Liquid S f|[kJ/K-kg] Vapour S g|°C| |-100|1.2|0.0006|10.0000|1679.0|0.100|113.3|192.8|306.1|0.6077|1.7210|-100| |---|---|---|---|---|---|---|---|---|---|---|---|