diff --git a/Cargo.lock b/Cargo.lock index e04ad72..b219379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,8 +1214,8 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.4.1" -source = "git+https://github.com/zurawiki/tiktoken-rs#6c0e061b4db6df902989104656e137714b62ddb0" +version = "0.4.2" +source = "git+https://github.com/zurawiki/tiktoken-rs#4afc584e1cf4d64a49ee26bb3773078cf34926d5" dependencies = [ "anyhow", "async-openai", diff --git a/Cargo.toml b/Cargo.toml index 09c66d8..e2b819f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "happycommit" -version = "0.2.0" +version = "0.2.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -9,7 +9,7 @@ edition = "2021" dotenv = "0.15.0" async-openai = "0.10.3" tokio = { version="1.27.0", features=["full"] } -tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.1", features=["async-openai"] } +tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.2", features=["async-openai"] } lazy_static = "1.4.0" anyhow = "1.0.70" futures = "0.3.28" diff --git a/justfile b/justfile index f1c4b50..9d7317f 100644 --- a/justfile +++ b/justfile @@ -15,4 +15,20 @@ install: @cargo install --path . # Run both 'format' and 'clippy' tasks -fix: format clippy \ No newline at end of file +fix: format clippy + +# Install dependencies +deps: + @cargo install cargo-bump + +# Bump the patch +bump-patch: deps + @cargo bump patch --git-tag + +# Bump the minor +bump-minor: deps + @cargo bump minor --git-tag + +# Bump the major +bump-major: deps + @cargo bump major --git-tag diff --git a/src/main.rs b/src/main.rs index 4cf975b..cea5f8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -463,14 +463,13 @@ async fn stream_multipart_commit_message( ) -> Result, anyhow::Error> { let tokenizer = cl100k_base().unwrap(); let max_tokens = 3500; - let tokenized_iter = tokenizer.split_by_token_with_special_tokens(code_changes); + let tokenized_iter = tokenizer.split_by_token_iter(code_changes, true); let overlap = 200; let mut split_code_changes = Vec::new(); let initial_prompt_tokens = tokenizer - .split_by_token_with_special_tokens(initial_prompt) - .collect::, _>>()? + .split_by_token(initial_prompt, true)? .len(); let max_code_tokens = max_tokens - initial_prompt_tokens;