Refactor

2025-12-15 20:27:34 -08:00
parent c59ca2164e
commit cd79814c91
19 changed files with 1572 additions and 1066 deletions
--- a/crates/tokenizer/src/tokenizer.rs
+++ b/crates/tokenizer/src/tokenizer.rs
@@ -19,8 +19,7 @@ use tracing::{debug, info};

 use crate::{progress_big, split::regex_segment};

-// TODO:
-// - maybe don't use regex
+// Maybe don't use regex for performance?

 #[derive(Debug, Clone, thiserror::Error)]
 pub enum TokenizerTrainError {