Tokenizer

crates/tokenizer/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "tokenizer"
+version = { workspace = true }
+rust-version = { workspace = true }
+edition = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+
+ahash = { workspace = true }
+compact_str = { workspace = true }
+dary_heap = { workspace = true }
+fancy-regex = { workspace = true }
+rayon = { workspace = true }
+serde = { workspace = true }
+strum = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+indicatif = { workspace = true }