diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs index b322be3a09..fdccf8facf 100644 --- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs @@ -1044,6 +1044,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo ( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini // chat + ( "gpt-5.2-", ModelEncoding.O200kBase ), ( "gpt-5.1-", ModelEncoding.O200kBase ), ( "gpt-5-", ModelEncoding.O200kBase ), ( "gpt-4.1-", ModelEncoding.O200kBase ), // e.g., gpt-4.1-mini @@ -1072,6 +1073,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo { "o4-mini", ModelEncoding.O200kBase }, // chat + { "gpt-5.2", ModelEncoding.O200kBase }, { "gpt-5.1", ModelEncoding.O200kBase }, { "gpt-5", ModelEncoding.O200kBase }, { "gpt-4.1", ModelEncoding.O200kBase }, diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index b77f981aec..c7c1e342d8 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -37,6 +37,7 @@ public class TiktokenTests public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o"); public static Tokenizer GPT5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5"); public static Tokenizer GPT5_1 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.1"); + public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2"); public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4"); public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b"); @@ -287,7 +288,7 @@ public void TestEncode5() [Fact] public void TestEncodeO200kBaseEncoding() { - foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1 }) + foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1, GPT5_2 }) { string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); IReadOnlyList encoded = tokenizer.EncodeToIds(text); @@ -418,6 +419,8 @@ public void TestEncodeR50kBase() [InlineData("gpt-5-chat")] [InlineData("gpt-5.1")] [InlineData("gpt-5.1-mini")] + [InlineData("gpt-5.2")] + [InlineData("gpt-5.2-mini")] [InlineData("chatgpt-4o-")] [InlineData("gpt-4")] [InlineData("gpt-4-")] @@ -537,6 +540,7 @@ public void TestEncodingNamesNegativeCases() [InlineData("gpt-4o")] [InlineData("gpt-5")] [InlineData("gpt-5.1")] + [InlineData("gpt-5.2")] [InlineData("o1")] [InlineData("o3")] [InlineData("o4-mini")]