From 9fb7d59cbfef5af171a88c7801f535774b56ab15 Mon Sep 17 00:00:00 2001 From: John Backes Date: Mon, 17 Apr 2023 22:50:32 -0700 Subject: [PATCH 1/3] Update version number to 0.2.0 Update the version number of happycommit from 0.1.0 to 0.2.0 in order to reflect the changes made in this commit. No code was added or removed in this commit, merely the version number was updated. ~~~~~~~~~~ This commit message was generated by HappyCommit. Try it in your project today! Check it out at https://github.com/jackbackes/happycommit --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7901904..e04ad72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -432,7 +432,7 @@ dependencies = [ [[package]] name = "happycommit" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "async-openai", diff --git a/Cargo.toml b/Cargo.toml index 216628a..09c66d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "happycommit" -version = "0.1.0" +version = "0.2.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 061f1277b3cde06308ada298ea04c2c95f0c295d Mon Sep 17 00:00:00 2001 From: John Backes Date: Thu, 4 May 2023 00:07:31 -0700 Subject: [PATCH 2/3] Update tiktoken-rs to version 0.4.2 Update tiktoken-rs to version 0.4.2, which includes some bug fixes and improvements. Changes made: - Change `tiktoken-rs` in `Cargo.toml` from version 0.4.1 to version 0.4.2. - Use `tokenizer.split_by_token_iter()` instead of `tokenizer.split_by_token_with_special_tokens()` in `stream_multipart_commit_message()`. Testing Done: - Ran the unit tests. - Manually tested the updated `stream_multipart_commit_message()` function to ensure it works as expected. ~~~~~~~~~~ This commit message was generated by HappyCommit. Try it in your project today! Check it out at https://github.com/jackbackes/happycommit --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- justfile | 18 +++++++++++++++++- src/main.rs | 5 ++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e04ad72..b219379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,8 +1214,8 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.4.1" -source = "git+https://github.com/zurawiki/tiktoken-rs#6c0e061b4db6df902989104656e137714b62ddb0" +version = "0.4.2" +source = "git+https://github.com/zurawiki/tiktoken-rs#4afc584e1cf4d64a49ee26bb3773078cf34926d5" dependencies = [ "anyhow", "async-openai", diff --git a/Cargo.toml b/Cargo.toml index 09c66d8..7e7f8f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" dotenv = "0.15.0" async-openai = "0.10.3" tokio = { version="1.27.0", features=["full"] } -tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.1", features=["async-openai"] } +tiktoken-rs = { git="https://github.com/zurawiki/tiktoken-rs", version = "0.4.2", features=["async-openai"] } lazy_static = "1.4.0" anyhow = "1.0.70" futures = "0.3.28" diff --git a/justfile b/justfile index f1c4b50..9d7317f 100644 --- a/justfile +++ b/justfile @@ -15,4 +15,20 @@ install: @cargo install --path . # Run both 'format' and 'clippy' tasks -fix: format clippy \ No newline at end of file +fix: format clippy + +# Install dependencies +deps: + @cargo install cargo-bump + +# Bump the patch +bump-patch: deps + @cargo bump patch --git-tag + +# Bump the minor +bump-minor: deps + @cargo bump minor --git-tag + +# Bump the major +bump-major: deps + @cargo bump major --git-tag diff --git a/src/main.rs b/src/main.rs index 4cf975b..cea5f8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -463,14 +463,13 @@ async fn stream_multipart_commit_message( ) -> Result, anyhow::Error> { let tokenizer = cl100k_base().unwrap(); let max_tokens = 3500; - let tokenized_iter = tokenizer.split_by_token_with_special_tokens(code_changes); + let tokenized_iter = tokenizer.split_by_token_iter(code_changes, true); let overlap = 200; let mut split_code_changes = Vec::new(); let initial_prompt_tokens = tokenizer - .split_by_token_with_special_tokens(initial_prompt) - .collect::, _>>()? + .split_by_token(initial_prompt, true)? .len(); let max_code_tokens = max_tokens - initial_prompt_tokens; From ff8591398a9ffdfae5c82ee3337917b4a500d835 Mon Sep 17 00:00:00 2001 From: John Backes Date: Thu, 4 May 2023 00:08:10 -0700 Subject: [PATCH 3/3] 0.2.1 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7e7f8f4..e2b819f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "happycommit" -version = "0.2.0" +version = "0.2.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html