From aaaef606d84015ac527b8aca1daf9d5889fca4b3 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 17 Jul 2026 19:13:40 -0400 Subject: [PATCH] feat(decode): Add opt-in lossy mode and decode_bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BPE tokens are byte-level, so a single character can span several tokens. The common "trim text to N tokens" pattern can leave a prefix that is not valid UTF-8, and decode raised Tiktoken::UnicodeError on it — diverging from Python tiktoken, whose decode defaults to errors="replace". Add an opt-in errors: :replace that substitutes invalid sequences with U+FFFD, plus decode_bytes for the raw bytes. The default stays :strict to preserve existing behavior. Also define Tiktoken::UnicodeError eagerly at load, since it previously only existed after the first decode error was raised. Closes #122 --- README.md | 27 +++++++++-- ext/tiktoken_ruby/src/core_bpe_wrapper.rs | 50 ++++++++++++++++++- ext/tiktoken_ruby/src/lib.rs | 7 ++- lib/tiktoken_ruby/encoding.rb | 38 +++++++++++++-- spec/tiktoken_ruby_spec.rb | 59 +++++++++++++++++++++++ 5 files changed, 170 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index eed2d15..e7639ef 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ If bundler is not being used to manage dependencies, install the gem by executin ## Usage -Usage should be very similar to the python library. Here's a simple example - -Encode and decode text +Encode and decode text: ```ruby require 'tiktoken_ruby' @@ -27,7 +25,7 @@ enc = Tiktoken.get_encoding("cl100k_base") enc.decode(enc.encode("hello world")) #=> "hello world" ``` -Encoders can also be retrieved by model name +Encoders can also be retrieved by model name: ```ruby require 'tiktoken_ruby' @@ -69,6 +67,27 @@ enc.encode_with_special_tokens(text) All methods round-trip correctly through `decode`. +### Decoding methods + +- `decode(tokens, errors: :strict)` - Decodes tokens back into a UTF-8 string +- `decode_bytes(tokens)` - Decodes tokens into their raw bytes (an `ASCII-8BIT` string), without UTF-8 validation + +Because BPE tokens are byte-level, a single character (an emoji, or non-Latin scripts) can span multiple tokens. Truncating a token array (like, "trim text to N tokens") can leave a prefix that is not valid UTF-8. The `errors:` option controls how those invalid sequences are handled. + +```ruby +enc = Tiktoken.encoding_for_model("gpt-4o") +tokens = enc.encode("🦄") # the emoji spans multiple tokens + +# :strict (default) - raise Tiktoken::UnicodeError on invalid UTF-8 +enc.decode(tokens.first(2)) #=> raises Tiktoken::UnicodeError + +# :replace - substitute invalid sequences with "�" (matches Python tiktoken's default) +enc.decode(tokens.first(2), errors: :replace) #=> "�" + +# decode_bytes - get the raw bytes and handle them yourself +enc.decode_bytes(tokens.first(2)) #=> "\xF0\x9F\xA6" (ASCII-8BIT) +``` + ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. diff --git a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs index c054ff6..25f4345 100644 --- a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs +++ b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs @@ -3,7 +3,7 @@ use std::ffi::c_void; use tiktoken_rs::Rank; -use crate::uncicode_error; +use crate::unicode_error; #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")] pub struct CoreBPEWrapper { @@ -35,6 +35,12 @@ struct DecodeData { result: Result, } +struct DecodeBytesData { + core_bpe: *const tiktoken_rs::CoreBPE, + ids: Vec, + result: Result, String>, +} + unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void { let data = &mut *(data as *mut EncodeOrdinaryData); let core_bpe = &*data.core_bpe; @@ -64,6 +70,13 @@ unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void { std::ptr::null_mut() } +unsafe extern "C" fn decode_bytes_without_gvl(data: *mut c_void) -> *mut c_void { + let data = &mut *(data as *mut DecodeBytesData); + let core_bpe = &*data.core_bpe; + data.result = core_bpe.decode_bytes(&data.ids).map_err(|e| e.to_string()); + std::ptr::null_mut() +} + impl CoreBPEWrapper { pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self { Self { core_bpe } @@ -150,7 +163,7 @@ impl CoreBPEWrapper { } data.result.map_err(|e| { - let error = match uncicode_error() { + let error = match unicode_error() { Ok(error) => error, Err(e) => return e, }; @@ -158,4 +171,37 @@ impl CoreBPEWrapper { magnus::Error::new(error, e) }) } + + pub fn decode_bytes(&self, ids: Vec) -> Result { + let mut data = DecodeBytesData { + core_bpe: &self.core_bpe as *const _, + ids, + result: Err(String::new()), + }; + + unsafe { + rb_sys::rb_thread_call_without_gvl( + Some(decode_bytes_without_gvl), + &mut data as *mut _ as *mut c_void, + None, + std::ptr::null_mut(), + ); + } + + // Build the Ruby string only after the GVL has been re-acquired. + // `str_from_slice` yields an ASCII-8BIT (binary) string, so the bytes + // are handed back verbatim without any UTF-8 validation. We hold the + // GVL here (this is called from Ruby), so `Ruby::get` cannot fail. + let ruby = magnus::Ruby::get().unwrap(); + data.result + .map(|bytes| ruby.str_from_slice(&bytes)) + .map_err(|e| { + let error = match unicode_error() { + Ok(error) => error, + Err(e) => return e, + }; + + magnus::Error::new(error, e) + }) + } } diff --git a/ext/tiktoken_ruby/src/lib.rs b/ext/tiktoken_ruby/src/lib.rs index 102186b..7c3db3c 100644 --- a/ext/tiktoken_ruby/src/lib.rs +++ b/ext/tiktoken_ruby/src/lib.rs @@ -34,7 +34,7 @@ fn module() -> Result { Ruby::get().unwrap().define_module("Tiktoken") } -fn uncicode_error() -> Result { +fn unicode_error() -> Result { module()?.define_error( "UnicodeError", Ruby::get().unwrap().exception_standard_error(), @@ -45,6 +45,10 @@ fn uncicode_error() -> Result { fn init() -> Result<(), Error> { let module = module()?; + // Define Tiktoken::UnicodeError eagerly so the constant exists as soon as + // the extension is loaded, rather than only after the first decode error. + unicode_error()?; + let factory_module = module.define_module("BpeFactory")?; factory_module.define_singleton_method("r50k_base", function!(r50k_base, 0))?; factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?; @@ -67,5 +71,6 @@ fn init() -> Result<(), Error> { )?; bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?; + bpe_class.define_method("decode_bytes", method!(CoreBPEWrapper::decode_bytes, 1))?; Ok(()) } diff --git a/lib/tiktoken_ruby/encoding.rb b/lib/tiktoken_ruby/encoding.rb index 296d92f..50de5c7 100644 --- a/lib/tiktoken_ruby/encoding.rb +++ b/lib/tiktoken_ruby/encoding.rb @@ -47,11 +47,41 @@ def encode_with_special_tokens(text) @ext_base_bpe.encode_with_special_tokens(text) end - # Decodes the tokens back into text + # Decodes the tokens back into text. + # + # BPE tokens are byte-level, so a single character (an emoji, non-Latin + # scripts) can span multiple tokens. Truncating a token array to a limit can + # therefore leave a prefix that is not valid UTF-8. The +errors+ option + # controls how those invalid byte sequences are handled: + # + # * +:strict+ (default) - raise Tiktoken::UnicodeError on invalid UTF-8. + # * +:replace+ - substitute each invalid sequence with the Unicode replacement + # character (U+FFFD, +�+), matching Python tiktoken's default behavior. + # # @param tokens [Array] The tokens to decode - # @return [String] The decoded text - def decode(tokens) - @ext_base_bpe.decode(tokens) + # @param errors [Symbol] How to handle invalid UTF-8 (:strict or :replace) + # @return [String] The decoded text (UTF-8) + def decode(tokens, errors: :strict) + case errors + when :strict + @ext_base_bpe.decode(tokens) + when :replace + decode_bytes(tokens).force_encoding(Encoding::UTF_8).scrub("\u{FFFD}") + else + raise ArgumentError, "errors must be :strict or :replace, got #{errors.inspect}" + end + end + + # Decodes the tokens back into their raw bytes, without any UTF-8 validation. + # + # The returned string has ASCII-8BIT (binary) encoding and is not guaranteed + # to be valid UTF-8 — useful when you want to handle invalid sequences + # yourself. Matches Python tiktoken's +decode_bytes+. + # + # @param tokens [Array] The tokens to decode + # @return [String] The decoded bytes (ASCII-8BIT) + def decode_bytes(tokens) + @ext_base_bpe.decode_bytes(tokens) end private diff --git a/spec/tiktoken_ruby_spec.rb b/spec/tiktoken_ruby_spec.rb index e0c7b18..225141d 100644 --- a/spec/tiktoken_ruby_spec.rb +++ b/spec/tiktoken_ruby_spec.rb @@ -77,6 +77,65 @@ end end + describe "decoding truncated multi-byte characters" do + let(:encoding) { Tiktoken.encoding_for_model("gpt-4o") } + + # An emoji is 4 bytes spread across several tokens, so truncating the token + # array leaves a prefix that is not valid UTF-8. + let(:tokens) { encoding.encode("🦄") } + let(:truncated) { tokens.first(tokens.length - 1) } + + it "encodes the emoji across multiple tokens" do + expect(tokens.length).to be > 1 + end + + it "raises Tiktoken::UnicodeError on invalid UTF-8 by default" do + expect { encoding.decode(truncated) } + .to raise_error(Tiktoken::UnicodeError) + end + + it "raises Tiktoken::UnicodeError with an explicit errors: :strict" do + expect { encoding.decode(truncated, errors: :strict) } + .to raise_error(Tiktoken::UnicodeError) + end + + it "replaces invalid UTF-8 with the replacement character with errors: :replace" do + result = encoding.decode(truncated, errors: :replace) + expect(result.encoding).to eq(Encoding::UTF_8) + expect(result).to include("\u{FFFD}") + end + + it "does not raise with errors: :replace" do + expect { encoding.decode(truncated, errors: :replace) }.not_to raise_error + end + + it "raises ArgumentError for an unknown errors mode" do + expect { encoding.decode(truncated, errors: :bogus) } + .to raise_error(ArgumentError) + end + + it "still round-trips valid tokens exactly" do + expect(encoding.decode(tokens)).to eq("🦄") + end + + describe "#decode_bytes" do + it "returns the raw bytes as an ASCII-8BIT string" do + bytes = encoding.decode_bytes(truncated) + expect(bytes.encoding).to eq(Encoding::ASCII_8BIT) + end + + it "returns bytes that are a valid prefix of the full encoding" do + prefix = encoding.decode_bytes(truncated) + full = encoding.decode_bytes(tokens) + expect(full.b).to start_with(prefix.b) + end + + it "round-trips valid tokens to UTF-8 bytes" do + expect(encoding.decode_bytes(tokens).force_encoding(Encoding::UTF_8)).to eq("🦄") + end + end + end + describe "special token handling" do let(:encoding) { Tiktoken.get_encoding("cl100k_base") } let(:text_with_special) { "Hello<|endoftext|>World" }