Refactor

2025-12-15 21:59:14 -08:00
parent c59ca2164e
commit 993e813b7e
19 changed files with 1583 additions and 1066 deletions
--- a/crates/tokenizer/src/tokenizer.rs
+++ b/crates/tokenizer/src/tokenizer.rs
@@ -19,8 +19,7 @@ use tracing::{debug, info};

 use crate::{progress_big, split::regex_segment};

-// TODO:
-// - maybe don't use regex
+// Maybe don't use regex for performance?

 #[derive(Debug, Clone, thiserror::Error)]
 pub enum TokenizerTrainError {