From 3beba1ac77fbc2304620f8b3853a8359c5bebc7c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 1 May 2025 17:17:48 -0700 Subject: [PATCH 1/2] Support Tiktoken Gpt-4.1 Model --- src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs | 2 ++ test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs index 8fb73a5225..071efa2d8b 100644 --- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs @@ -1032,6 +1032,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo // chat ( "o1-", ModelEncoding.O200kBase ), // e.g. o1-mini ( "o3-", ModelEncoding.O200kBase ), // e.g. o3-mini + ( "gpt-4.1-", ModelEncoding.O200kBase), // e.g., gpt-4.1-mini ( "gpt-4o-", ModelEncoding.O200kBase), // e.g., gpt-4o-2024-05-13 ( "gpt-4-", ModelEncoding.Cl100kBase), // e.g., gpt-4-0314, etc., plus gpt-4-32k ( "gpt-3.5-", ModelEncoding.Cl100kBase), // e.g, gpt-3.5-turbo-0301, -0401, etc. @@ -1045,6 +1046,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo { "gpt-4o", ModelEncoding.O200kBase }, { "o1", ModelEncoding.O200kBase }, { "o3", ModelEncoding.O200kBase }, + { "gpt-4.1", ModelEncoding.O200kBase }, { "gpt-4", ModelEncoding.Cl100kBase }, { "gpt-3.5-turbo", ModelEncoding.Cl100kBase }, { "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase }, diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index 049f2cd82b..5cb48d9f5a 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -401,6 +401,8 @@ public void TestEncodeR50kBase() [InlineData("o3")] [InlineData("o3-")] [InlineData("o3-mini")] + [InlineData("gpt-4.1")] + [InlineData("gpt-4.1-mini")] [InlineData("gpt-4o")] [InlineData("gpt-4o-")] [InlineData("gpt-4")] @@ -502,6 +504,7 @@ public void TestEncodingNamesNegativeCases() } [InlineData("gpt-4")] + [InlineData("gpt-4.1")] [InlineData("gpt-4o")] [InlineData("o1")] [InlineData("o3")] From 0f08006ff380859d9d7372d35af882f932f6db21 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 1 May 2025 18:53:57 -0700 Subject: [PATCH 2/2] Add o4-mini --- src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs | 1 + test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs index 071efa2d8b..69cc3dfb9c 100644 --- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs @@ -1046,6 +1046,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo { "gpt-4o", ModelEncoding.O200kBase }, { "o1", ModelEncoding.O200kBase }, { "o3", ModelEncoding.O200kBase }, + { "o4-mini", ModelEncoding.O200kBase }, { "gpt-4.1", ModelEncoding.O200kBase }, { "gpt-4", ModelEncoding.Cl100kBase }, { "gpt-3.5-turbo", ModelEncoding.Cl100kBase }, diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs index 5cb48d9f5a..1ace27f1fb 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs @@ -401,6 +401,7 @@ public void TestEncodeR50kBase() [InlineData("o3")] [InlineData("o3-")] [InlineData("o3-mini")] + [InlineData("o4-mini")] [InlineData("gpt-4.1")] [InlineData("gpt-4.1-mini")] [InlineData("gpt-4o")] @@ -508,6 +509,7 @@ public void TestEncodingNamesNegativeCases() [InlineData("gpt-4o")] [InlineData("o1")] [InlineData("o3")] + [InlineData("o4-mini")] [InlineData("text-davinci-003")] [InlineData("text-curie-001")] [InlineData("text-davinci-edit-001")]