diff --git a/Cargo.toml b/Cargo.toml index 0c76700..b1b7b12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pdf-inspector" -version = "0.1.0" +version = "0.1.1" edition = "2021" autobins = false authors = ["Firecrawl Team"] diff --git a/napi/Cargo.lock b/napi/Cargo.lock index 8aa8837..3afac95 100644 --- a/napi/Cargo.lock +++ b/napi/Cargo.lock @@ -672,8 +672,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lopdf" -version = "0.40.0" -source = "git+https://github.com/J-F-Liu/lopdf?rev=7a05512d831415b1f2b1ce522391d6beab8a1284#7a05512d831415b1f2b1ce522391d6beab8a1284" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d" dependencies = [ "aes", "bitflags", @@ -829,7 +830,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdf-inspector" -version = "0.1.0" +version = "0.1.1" dependencies = [ "env_logger", "log", @@ -844,7 +845,7 @@ dependencies = [ [[package]] name = "pdf-inspector-napi" -version = "0.2.0" +version = "0.2.1" dependencies = [ "napi", "napi-build", diff --git a/napi/Cargo.toml b/napi/Cargo.toml index 55a8730..a97920a 100644 --- a/napi/Cargo.toml +++ b/napi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pdf-inspector-napi" -version = "0.2.0" +version = "0.2.1" edition = "2021" [lib] diff --git a/napi/package.json b/napi/package.json index bb40bb8..bb4a8a6 100644 --- a/napi/package.json +++ b/napi/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/pdf-inspector", - "version": "1.9.5", + "version": "1.9.6", "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.", "main": "index.js", "types": "index.d.ts", diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 4615e2e..85229e7 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -188,7 +188,17 @@ pub(crate) fn extract_page_text_items( // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix let mut text_rendering_mode: i32 = 0; // 0=fill, 1=stroke, 2=fill+stroke, 3=invisible - let mut gstate_stack: Vec<([f32; 6], i32, f32, f32)> = Vec::new(); + #[derive(Clone)] + struct SavedGraphicsState { + ctm: [f32; 6], + text_rendering_mode: i32, + char_spacing: f32, + word_spacing: f32, + text_leading: f32, + current_font: String, + current_font_size: f32, + } + let mut gstate_stack: Vec = Vec::new(); // Text state tracking let mut current_font = String::new(); @@ -227,15 +237,26 @@ pub(crate) fn extract_page_text_items( match op.operator.as_str() { "q" => { // Save graphics state - gstate_stack.push((ctm, text_rendering_mode, char_spacing, word_spacing)); + gstate_stack.push(SavedGraphicsState { + ctm, + text_rendering_mode, + char_spacing, + word_spacing, + text_leading, + current_font: current_font.clone(), + current_font_size, + }); } "Q" => { // Restore graphics state - if let Some((saved_ctm, saved_tr, saved_tc, saved_tw)) = gstate_stack.pop() { - ctm = saved_ctm; - text_rendering_mode = saved_tr; - char_spacing = saved_tc; - word_spacing = saved_tw; + if let Some(saved) = gstate_stack.pop() { + ctm = saved.ctm; + text_rendering_mode = saved.text_rendering_mode; + char_spacing = saved.char_spacing; + word_spacing = saved.word_spacing; + text_leading = saved.text_leading; + current_font = saved.current_font; + current_font_size = saved.current_font_size; } } "cm" => { @@ -1286,6 +1307,91 @@ mod tests { assert!(lines.is_empty()); } + #[test] + fn test_q_restores_current_font_for_text_decoding() { + use crate::tounicode::FontCMaps; + use lopdf::{dictionary, Object, Stream}; + + fn cmap_stream(dst_hex: &str) -> Stream { + let cmap = format!( + r#"/CIDInit /ProcSet findresource begin +12 dict begin +begincmap +/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def +/CMapName /Test-UCS def +/CMapType 2 def +1 begincodespacerange +<00> +endcodespacerange +1 beginbfchar +<41> <{dst_hex}> +endbfchar +endcmap +CMapName currentdict /CMap defineresource pop +end +end"# + ); + Stream::new(dictionary! {}, cmap.into_bytes()) + } + + let mut doc = lopdf::Document::new(); + let f1_cmap = doc.add_object(Object::Stream(cmap_stream("0058"))); // X + let f2_cmap = doc.add_object(Object::Stream(cmap_stream("0059"))); // Y + let f1 = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "ToUnicode" => Object::Reference(f1_cmap), + }); + let f2 = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "ToUnicode" => Object::Reference(f2_cmap), + }); + + let content = b"BT /F1 12 Tf 10 700 Tm <41> Tj ET +q +BT /F2 12 Tf 20 700 Tm <41> Tj ET +Q +BT 30 700 Tm <41> Tj ET"; + let content_id = doc.add_object(Object::Stream(Stream::new( + dictionary! {}, + content.to_vec(), + ))); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Contents" => Object::Reference(content_id), + "Resources" => dictionary! { + "Font" => dictionary! { + "F1" => Object::Reference(f1), + "F2" => Object::Reference(f2), + }, + }, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + let pages_id = doc.add_object(dictionary! { + "Type" => "Pages", + "Count" => Object::Integer(1), + "Kids" => vec![Object::Reference(page_id)], + }); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => Object::Reference(pages_id), + }); + doc.trailer.set("Root", Object::Reference(catalog_id)); + + let font_cmaps = FontCMaps::from_doc(&doc); + let ((items, _, _), _, _) = + extract_page_text_items(&doc, page_id, 1, &font_cmaps, false).unwrap(); + let text = items + .iter() + .map(|item| item.text.as_str()) + .collect::(); + + assert_eq!(text, "XYX"); + } + #[test] fn test_strip_pdf_comments() { // Basic comment stripping diff --git a/src/markdown/postprocess.rs b/src/markdown/postprocess.rs index 6da6090..6c1d1a0 100644 --- a/src/markdown/postprocess.rs +++ b/src/markdown/postprocess.rs @@ -29,6 +29,7 @@ pub(crate) fn clean_markdown(mut text: String, options: &MarkdownOptions) -> Str // text item, which combine with gap-based space insertion to produce // double spaces ("Vice President" instead of "Vice President"). collapse_consecutive_spaces(&mut text); + remove_spaces_before_closing_brackets(&mut text); // Remove excessive newlines (more than 2 in a row) while text.contains("\n\n\n") { @@ -71,6 +72,20 @@ fn collapse_consecutive_spaces(text: &mut String) { *text = result; } +/// Remove spaces before closing square brackets. +/// Unit markers and markdown links occasionally pick up a gap-inserted space +/// before `]` (e.g. `[kg/m3 ]`), which is cosmetic padding. +fn remove_spaces_before_closing_brackets(text: &mut String) { + let mut result = String::with_capacity(text.len()); + for ch in text.chars() { + if ch == ']' && result.ends_with(' ') { + result.pop(); + } + result.push(ch); + } + *text = result; +} + /// Collapse dot leaders (runs of 4+ dots) into " ... " /// Common in tables of contents: "Introduction...............................1" -> "Introduction ... 1" fn collapse_dot_leaders(text: &str) -> String { @@ -342,6 +357,18 @@ mod tests { assert!(result.contains("Chapter 2 ... 20")); } + // --- remove_spaces_before_closing_brackets --- + + #[test] + fn test_remove_spaces_before_closing_brackets() { + let mut input = "Density [kg/m3 ] and [linked text ](https://example.com)".to_string(); + remove_spaces_before_closing_brackets(&mut input); + assert_eq!( + input, + "Density [kg/m3] and [linked text](https://example.com)" + ); + } + // --- fix_hyphenation --- #[test]