update tiktoken rs

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "happycommit"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,7 +9,7 @@ edition = "2021"
 dotenv = "0.15.0"
 async-openai = "0.10.3"
 tokio = { version="1.27.0", features=["full"] }
-tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.1", features=["async-openai"] }
+tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.2", features=["async-openai"] }
 lazy_static = "1.4.0"
 anyhow = "1.0.70"
 futures = "0.3.28"

justfile

@@ -15,4 +15,20 @@ install:
     @cargo install --path .
 
 # Run both 'format' and 'clippy' tasks
-fix: format clippy
+fix: format clippy
+
+# Install dependencies
+deps:
+    @cargo install cargo-bump
+
+# Bump the patch
+bump-patch: deps
+    @cargo bump patch --git-tag
+
+# Bump the minor
+bump-minor: deps
+    @cargo bump minor --git-tag
+
+# Bump the major
+bump-major: deps
+    @cargo bump major --git-tag

src/main.rs

@@ -463,14 +463,13 @@ async fn stream_multipart_commit_message(
 ) -> Result<Vec<String>, anyhow::Error> {
     let tokenizer = cl100k_base().unwrap();
     let max_tokens = 3500;
-    let tokenized_iter = tokenizer.split_by_token_with_special_tokens(code_changes);
+    let tokenized_iter = tokenizer.split_by_token_iter(code_changes, true);
 
     let overlap = 200;
     let mut split_code_changes = Vec::new();
 
     let initial_prompt_tokens = tokenizer
-        .split_by_token_with_special_tokens(initial_prompt)
-        .collect::<Result<Vec<String>, _>>()?
+        .split_by_token(initial_prompt, true)?
         .len();
 
     let max_code_tokens = max_tokens - initial_prompt_tokens;