From 66e776ecc8e7900f1a6e1b9e3b234e26dfc49cda Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 25 Jun 2024 14:04:20 +0100 Subject: [PATCH 1/9] Text To Image Service and Unit Tests --- .../Core/ClientCoreTests.cs | 68 ++++++++- .../Services/OpenAITextToImageServiceTests.cs | 135 ++++++++++++++++++ .../Core/ClientCore.Embeddings.cs | 2 - .../Core/ClientCore.TextToImage.cs | 80 +++++++++++ .../Connectors.OpenAIV2/Core/ClientCore.cs | 30 +++- .../OpenAITextEmbbedingGenerationService.cs | 11 +- .../Services/OpenAITextToImageService.cs | 74 ++++++++++ .../AI/TextToImage/ITextToImageService.cs | 8 +- 8 files changed, 395 insertions(+), 13 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs index a3415663459a..d0683a1e4c72 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Services; using Moq; using OpenAI; using Xunit; @@ -23,7 +24,7 @@ public void ItCanBeInstantiatedAndPropertiesSetAsExpected() { // Act var logger = new Mock>().Object; - var openAIClient = new OpenAIClient(new ApiKeyCredential("key")); + var openAIClient = new OpenAIClient("key"); var clientCoreModelConstructor = new ClientCore("model1", "apiKey"); var clientCoreOpenAIClientConstructor = new ClientCore("model1", openAIClient, logger: logger); @@ -67,6 +68,8 @@ public void ItUsesEndpointAsExpected(string? clientBaseAddress, string? provided // Assert Assert.Equal(endpoint ?? client?.BaseAddress ?? new Uri("https://api.openai.com/v1"), clientCore.Endpoint); + Assert.True(clientCore.Attributes.ContainsKey(AIServiceExtensions.EndpointKey)); + Assert.Equal(endpoint?.ToString() ?? client?.BaseAddress?.ToString() ?? "https://api.openai.com/v1", clientCore.Attributes[AIServiceExtensions.EndpointKey]); client?.Dispose(); } @@ -142,7 +145,7 @@ public async Task ItDoNotAddSemanticKernelHeadersWhenOpenAIClientIsProvidedAsync var clientCore = new ClientCore( modelId: "model", openAIClient: new OpenAIClient( - new ApiKeyCredential("test"), + "test", new OpenAIClientOptions() { Transport = new HttpClientPipelineTransport(client), @@ -185,4 +188,65 @@ public void ItAddAttributesButDoesNothingIfNullOrEmpty(string? value) Assert.Equal(value, clientCore.Attributes["key"]); } } + + [Fact] + public void ItAddModelIdAttributeAsExpected() + { + // Arrange + var expectedModelId = "modelId"; + + // Act + var clientCore = new ClientCore(expectedModelId, "apikey"); + var clientCoreBreakingGlass = new ClientCore(expectedModelId, new OpenAIClient(string.Empty)); + + // Assert + Assert.True(clientCore.Attributes.ContainsKey(AIServiceExtensions.ModelIdKey)); + Assert.True(clientCoreBreakingGlass.Attributes.ContainsKey(AIServiceExtensions.ModelIdKey)); + Assert.Equal(expectedModelId, clientCore.Attributes[AIServiceExtensions.ModelIdKey]); + Assert.Equal(expectedModelId, clientCoreBreakingGlass.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItAddOrNotOrganizationIdAttributeWhenProvided() + { + // Arrange + var expectedOrganizationId = "organizationId"; + + // Act + var clientCore = new ClientCore("modelId", "apikey", expectedOrganizationId); + var clientCoreWithoutOrgId = new ClientCore("modelId", "apikey"); + + // Assert + Assert.True(clientCore.Attributes.ContainsKey(ClientCore.OrganizationKey)); + Assert.Equal(expectedOrganizationId, clientCore.Attributes[ClientCore.OrganizationKey]); + Assert.False(clientCoreWithoutOrgId.Attributes.ContainsKey(ClientCore.OrganizationKey)); + } + + [Fact] + public void ItThrowsIfModelIdIsNotProvided() + { + // Act & Assert + Assert.Throws(() => new ClientCore(" ", "apikey")); + Assert.Throws(() => new ClientCore("", "apikey")); + Assert.Throws(() => new ClientCore(null!)); + } + + [Fact] + public void ItThrowsWhenNotUsingCustomEndpintAndApiKeyIsNotProvided() + { + // Act & Assert + Assert.Throws(() => new ClientCore("modelId", " ")); + Assert.Throws(() => new ClientCore("modelId", "")); + Assert.Throws(() => new ClientCore("modelId", apiKey: null!)); + } + + [Fact] + public void ItDoesNotThrowWhenUsingCustomEndpintAndApiKeyIsNotProvided() + { + // Act & Assert + ClientCore? clientCore = null; + clientCore = new ClientCore("modelId", " ", endpoint: new Uri("http://localhost")); + clientCore = new ClientCore("modelId", "", endpoint: new Uri("http://localhost")); + clientCore = new ClientCore("modelId", apiKey: null!, endpoint: new Uri("http://localhost")); + } } diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs new file mode 100644 index 000000000000..20b969c03513 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Services; +using Moq; +using OpenAI; +using Xunit; + +namespace SemanticKernel.Connectors.UnitTests.OpenAI.TextToImage; + +/// +/// Unit tests for class. +/// +public sealed class OpenAITextToImageServiceTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + private readonly Mock _mockLoggerFactory; + + public OpenAITextToImageServiceTests() + { + this._messageHandlerStub = new HttpMessageHandlerStub(); + this._httpClient = new HttpClient(this._messageHandlerStub, false); + this._mockLoggerFactory = new Mock(); + } + + [Fact] + public void ConstructorWorksCorrectly() + { + // Arrange & Act + var sut = new OpenAITextToImageService("api-key", "organization"); + + // Assert + Assert.NotNull(sut); + Assert.Equal("organization", sut.Attributes[ClientCore.OrganizationKey]); + Assert.False(sut.Attributes.ContainsKey(AIServiceExtensions.ModelIdKey)); + } + + [Fact] + public void OpenAIClientConstructorWorksCorrectly() + { + // Arrange + var sut = new OpenAITextToImageService("model", new OpenAIClient("apikey")); + + // Assert + Assert.NotNull(sut); + Assert.Equal("model", sut.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Theory] + [InlineData(256, 256)] + [InlineData(512, 512)] + [InlineData(1024, 1024)] + public async Task GenerateImageWorksCorrectlyAsync(int width, int height) + { + // Arrange + var sut = new OpenAITextToImageService("dall-e-3", "api-key", httpClient: this._httpClient); + Assert.Equal("dall-e-3", sut.Attributes["ModelId"]); + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "created": 1702575371, + "data": [ + { + "url": "https://image-url" + } + ] + } + """, Encoding.UTF8, "application/json") + }; + + // Act + var result = await sut.GenerateImageAsync("description", width, height); + + // Assert + Assert.Equal("https://image-url", result); + } + + [Theory] + [InlineData(123, 456)] + [InlineData(256, 512)] + public async Task GenerateImageThrowsWhenSizeIsNotSupportedAsync(int width, int height) + { + // Arrange + var sut = new OpenAITextToImageService("model", "apiKey"); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GenerateImageAsync("description", width, height)); + } + + [Theory] + [InlineData(123, 456)] + [InlineData(256, 512)] + [InlineData(6546, 545)] + [InlineData(16, 32)] + public async Task GenerateImageAllowCustomSizeWhenNonDefaultEndpointIsUsedAsync(int width, int height) + { + // Arrange + var sut = new OpenAITextToImageService("model", endpoint: new Uri("http://localhost"), httpClient: this._httpClient); + Assert.Equal("model", sut.Attributes[AIServiceExtensions.ModelIdKey]); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "created": 1702575371, + "data": [ + { + "url": "https://image-url" + } + ] + } + """, Encoding.UTF8, "application/json") + }; + + // Act + var result = await sut.GenerateImageAsync("description", width, height); + + // Assert + Assert.Equal("https://image-url", result); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.Embeddings.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.Embeddings.cs index d11e2799addd..aa15de012084 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.Embeddings.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.Embeddings.cs @@ -13,8 +13,6 @@ This class was created to simplify any Text Embeddings Support from the v1 Clien using System.Threading.Tasks; using OpenAI.Embeddings; -#pragma warning disable CA2208 // Instantiate argument exceptions correctly - namespace Microsoft.SemanticKernel.Connectors.OpenAI; /// diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs new file mode 100644 index 000000000000..56e027024964 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft. All rights reserved. + +/* +Phase 02 + +- This class was created focused in the Image Generation using the SDK client instead of the own client in V1. +- Added Checking for empty or whitespace prompt. +- Removed the format parameter as this is never called in V1 code. Plan to implement it in the future once we change the ITextToImageService abstraction, using PromptExecutionSettings. +- Allow custom size for images when the endpoint is not the default OpenAI v1 endpoint. +*/ + +using System; +using System.ClientModel; +using System.Threading; +using System.Threading.Tasks; +using OpenAI.Images; + +namespace Microsoft.SemanticKernel.Connectors.OpenAI; + +/// +/// Base class for AI clients that provides common functionality for interacting with OpenAI services. +/// +internal partial class ClientCore +{ + /// + /// Generates an image with the provided configuration. + /// + /// Prompt to generate the image + /// Width of the image + /// Height of the image + /// The to monitor for cancellation requests. The default is . + /// Url of the generated image + internal async Task GenerateImageAsync( + string prompt, + int width, + int height, + CancellationToken cancellationToken) + { + Verify.NotNullOrWhiteSpace(prompt); + + var size = new GeneratedImageSize(width, height); + + EnsureSizeIsSupported(size, this.Endpoint!, this.ModelId); + + var imageOptions = new ImageGenerationOptions() + { + Size = size, + ResponseFormat = GeneratedImageFormat.Uri + }; + + ClientResult response = await RunRequestAsync(() => this.Client.GetImageClient(this.ModelId).GenerateImageAsync(prompt, imageOptions, cancellationToken)).ConfigureAwait(false); + var generatedImage = response.Value; + + return generatedImage.ImageUri?.ToString() ?? throw new KernelException("The generated image is not in url format"); + } + + /// + /// Ensures by throwing and exception if the requested image size is not supported by the OpenAI image generation models. + /// + /// Target size to ensure support + /// Current client endpoint + /// Target model identifier + /// Size is not supported + private static void EnsureSizeIsSupported(GeneratedImageSize targetSize, Uri endpoint, string modelId) + { + // Only be restrictive to the image size if the endpoint is the default OpenAI v1 endpoint + if (endpoint != new Uri(OpenAIV1Endpoint)) { return; } + + var supported = (targetSize == GeneratedImageSize.W256xH256 && modelId == "dall-e-2") + || (targetSize == GeneratedImageSize.W512xH512 && modelId == "dall-e-2") + || targetSize == GeneratedImageSize.W1024xH1024 + || (targetSize == GeneratedImageSize.W1024xH1792 && modelId == "dall-e-3") + || (targetSize == GeneratedImageSize.W1792xH1024 && modelId == "dall-e-3"); + + if (!supported) + { + throw new ArgumentOutOfRangeException($"The requested image size {targetSize} is not supported. Supported sizes are: 256x256 (dalle-2 only), 512x512 (dalle-2 only), 1024x1024, 1024x1792 (dalle-3 only), 1792x1024 (dalle-3 only)"); + } + } +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs index 12ca2f3d92fe..af8f78214146 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs @@ -4,6 +4,11 @@ Phase 01 : This class was created adapting and merging ClientCore and OpenAIClientCore classes. System.ClientModel changes were added and adapted to the code as this package is now used as a dependency over OpenAI package. All logic from original ClientCore and OpenAIClientCore were preserved. + +Phase 02 : +- Moved AddAttributes usage to the constructor, avoiding the need verify and adding it in the services. +- Added ModelId attribute to the OpenAIClient constructor. +- Added WhiteSpace instead of empty string for ApiKey to avoid exception from OpenAI Client on custom endpoints added an issue in OpenAI SDK repo. https://github.com/openai/openai-dotnet/issues/90 */ using System; @@ -17,6 +22,7 @@ All logic from original ClientCore and OpenAIClientCore were preserved. using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.Services; using OpenAI; #pragma warning disable CA2208 // Instantiate argument exceptions correctly @@ -28,6 +34,16 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI; /// internal partial class ClientCore { + /// + /// White space constant. + /// + private const string WhiteSpace = " "; + + /// + /// Gets the attribute name used to store the organization in the dictionary. + /// + internal const string OrganizationKey = "Organization"; + /// /// Default OpenAI API endpoint. /// @@ -63,15 +79,15 @@ internal partial class ClientCore /// /// Model name. /// OpenAI API Key. - /// OpenAI compatible API endpoint. /// OpenAI Organization Id (usually optional). + /// OpenAI compatible API endpoint. /// Custom for HTTP requests. /// The to use for logging. If null, no logging will be performed. internal ClientCore( string modelId, string? apiKey = null, - Uri? endpoint = null, string? organizationId = null, + Uri? endpoint = null, HttpClient? httpClient = null, ILogger? logger = null) { @@ -80,6 +96,8 @@ internal ClientCore( this.Logger = logger ?? NullLogger.Instance; this.ModelId = modelId; + this.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); + // Accepts the endpoint if provided, otherwise uses the default OpenAI endpoint. this.Endpoint = endpoint ?? httpClient?.BaseAddress; if (this.Endpoint is null) @@ -88,13 +106,17 @@ internal ClientCore( this.Endpoint = new Uri(OpenAIV1Endpoint); } + this.AddAttribute(AIServiceExtensions.EndpointKey, this.Endpoint.ToString()); + var options = GetOpenAIClientOptions(httpClient, this.Endpoint); if (!string.IsNullOrWhiteSpace(organizationId)) { options.AddPolicy(new AddHeaderRequestPolicy("OpenAI-Organization", organizationId!), PipelinePosition.PerCall); + + this.AddAttribute(ClientCore.OrganizationKey, organizationId); } - this.Client = new OpenAIClient(apiKey ?? string.Empty, options); + this.Client = new OpenAIClient(apiKey ?? WhiteSpace, options); } /// @@ -116,6 +138,8 @@ internal ClientCore( this.Logger = logger ?? NullLogger.Instance; this.ModelId = modelId; this.Client = openAIClient; + + this.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); } /// diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs index 49915031b7fc..a64c4e44d1e6 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs @@ -13,6 +13,10 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI; +/* Phase 02 +Adding the non-default endpiont parameter to the constructor. +*/ + /// /// OpenAI implementation of /// @@ -28,6 +32,7 @@ public sealed class OpenAITextEmbeddingGenerationService : ITextEmbeddingGenerat /// Model name /// OpenAI API Key /// OpenAI Organization Id (usually optional) + /// Non-default endpoint for the OpenAI API /// Custom for HTTP requests. /// The to use for logging. If null, no logging will be performed. /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. @@ -35,6 +40,7 @@ public OpenAITextEmbeddingGenerationService( string modelId, string apiKey, string? organization = null, + Uri? endpoint = null, HttpClient? httpClient = null, ILoggerFactory? loggerFactory = null, int? dimensions = null) @@ -42,12 +48,11 @@ public OpenAITextEmbeddingGenerationService( this._core = new( modelId: modelId, apiKey: apiKey, + endpoint: endpoint, organizationId: organization, httpClient: httpClient, logger: loggerFactory?.CreateLogger(typeof(OpenAITextEmbeddingGenerationService))); - this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); - this._dimensions = dimensions; } @@ -65,8 +70,6 @@ public OpenAITextEmbeddingGenerationService( int? dimensions = null) { this._core = new(modelId, openAIClient, loggerFactory?.CreateLogger(typeof(OpenAITextEmbeddingGenerationService))); - this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); - this._dimensions = dimensions; } diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs new file mode 100644 index 000000000000..b31cbcda33fd --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Threading.Tasks; +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TextToImage; +using OpenAI; + +/* Phase 02 +- Breaking the current constructor parameter order to follow the same order as the other services. +- Added custom endpoint support, and removed ApiKey validation, as it is performed by the ClientCore when the Endpoint is not provided. +- Added custom OpenAIClient support. +- Updated "organization" parameter to "organizationId". +- "modelId" parameter is now required in the constructor. + +- Added OpenAIClient breaking glass constructor. +*/ + +namespace Microsoft.SemanticKernel.Connectors.OpenAI; + +/// +/// OpenAI text to image service. +/// +[Experimental("SKEXP0010")] +public class OpenAITextToImageService : ITextToImageService +{ + private readonly ClientCore _core; + + /// + public IReadOnlyDictionary Attributes => this._core.Attributes; + + /// + /// Initializes a new instance of the class. + /// + /// The model to use for image generation. + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// Non-default endpoint for the OpenAI API. + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public OpenAITextToImageService( + string modelId, + string? apiKey = null, + string? organizationId = null, + Uri? endpoint = null, + HttpClient? httpClient = null, + ILoggerFactory? loggerFactory = null) + { + this._core = new(modelId, apiKey, organizationId, endpoint, httpClient, loggerFactory?.CreateLogger(this.GetType())); + } + + /// + /// Initializes a new instance of the class. + /// + /// Model name + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public OpenAITextToImageService( + string modelId, + OpenAIClient openAIClient, + ILoggerFactory? loggerFactory = null) + { + this._core = new(modelId, openAIClient, loggerFactory?.CreateLogger(typeof(OpenAITextEmbeddingGenerationService))); + } + + /// + public Task GenerateImageAsync(string prompt, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) + => this._core.GenerateImageAsync(prompt, width, height, cancellationToken); +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs index c4c967445a6b..a0c01f95951a 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs @@ -5,6 +5,10 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Services; +/* Phase 02 +- Changing "description" parameter to "prompt" to better match the OpenAI API and avoid confusion. +*/ + namespace Microsoft.SemanticKernel.TextToImage; /// @@ -16,7 +20,7 @@ public interface ITextToImageService : IAIService /// /// Generate an image matching the given description /// - /// Image description + /// Image generation prompt /// Image width in pixels /// Image height in pixels /// The containing services, plugins, and other state for use throughout the operation. @@ -24,7 +28,7 @@ public interface ITextToImageService : IAIService /// Generated image in base64 format or image URL [Experimental("SKEXP0001")] public Task GenerateImageAsync( - string description, + string prompt, int width, int height, Kernel? kernel = null, From abc89ce46fb44fa62e5ae2a720f26b153a1a0110 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 25 Jun 2024 17:56:40 +0100 Subject: [PATCH 2/9] Add TextToImage and Embeddings missing UTs --- .../Connectors.OpenAIV2.UnitTests.csproj | 3 + .../Core/ClientCoreTests.cs | 6 +- ...enAITextEmbeddingGenerationServiceTests.cs | 37 +++++++- .../Services/OpenAITextToImageServiceTests.cs | 93 ++++++++++--------- .../TestData/text-to-image-response.txt | 8 ++ .../Utils/MoqExtensions.cs | 24 +++++ .../Connectors.OpenAIV2/Core/ClientCore.cs | 7 +- .../Services/OpenAITextToImageService.cs | 6 +- 8 files changed, 132 insertions(+), 52 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/TestData/text-to-image-response.txt create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj index 0d89e02beb21..27f85dea987a 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj @@ -44,6 +44,9 @@ Always + + Always + diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs index d0683a1e4c72..8961ed19cf94 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs @@ -197,7 +197,7 @@ public void ItAddModelIdAttributeAsExpected() // Act var clientCore = new ClientCore(expectedModelId, "apikey"); - var clientCoreBreakingGlass = new ClientCore(expectedModelId, new OpenAIClient(string.Empty)); + var clientCoreBreakingGlass = new ClientCore(expectedModelId, new OpenAIClient(" ")); // Assert Assert.True(clientCore.Attributes.ContainsKey(AIServiceExtensions.ModelIdKey)); @@ -228,7 +228,7 @@ public void ItThrowsIfModelIdIsNotProvided() // Act & Assert Assert.Throws(() => new ClientCore(" ", "apikey")); Assert.Throws(() => new ClientCore("", "apikey")); - Assert.Throws(() => new ClientCore(null!)); + Assert.Throws(() => new ClientCore(null!)); } [Fact] @@ -237,7 +237,7 @@ public void ItThrowsWhenNotUsingCustomEndpintAndApiKeyIsNotProvided() // Act & Assert Assert.Throws(() => new ClientCore("modelId", " ")); Assert.Throws(() => new ClientCore("modelId", "")); - Assert.Throws(() => new ClientCore("modelId", apiKey: null!)); + Assert.Throws(() => new ClientCore("modelId", apiKey: null!)); } [Fact] diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs index 25cdc4ec61aa..6c72312434ce 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs @@ -6,9 +6,11 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Services; +using Moq; using OpenAI; using Xunit; @@ -43,8 +45,9 @@ public async Task ItGetEmbeddingsAsyncReturnsEmptyWhenProvidedDataIsEmpty() } [Fact] - public async Task IGetEmbeddingsAsyncReturnsEmptyWhenProvidedDataIsWhitespace() + public async Task GetEmbeddingsAsyncReturnsEmptyWhenProvidedDataIsWhitespace() { + // Arrange using HttpMessageHandlerStub handler = new() { ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) @@ -54,7 +57,6 @@ public async Task IGetEmbeddingsAsyncReturnsEmptyWhenProvidedDataIsWhitespace() }; using HttpClient client = new(handler); - // Arrange var sut = new OpenAITextEmbeddingGenerationService("model", "apikey", httpClient: client); // Act @@ -68,6 +70,7 @@ public async Task IGetEmbeddingsAsyncReturnsEmptyWhenProvidedDataIsWhitespace() [Fact] public async Task ItThrowsIfNumberOfResultsDiffersFromInputsAsync() { + // Arrange using HttpMessageHandlerStub handler = new() { ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) @@ -77,10 +80,38 @@ public async Task ItThrowsIfNumberOfResultsDiffersFromInputsAsync() }; using HttpClient client = new(handler); - // Arrange var sut = new OpenAITextEmbeddingGenerationService("model", "apikey", httpClient: client); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GenerateEmbeddingsAsync(["test"], null, CancellationToken.None)); } + + [Fact] + public async Task GetEmbeddingsDoesLogActionAsync() + { + // Arrange + using HttpMessageHandlerStub handler = new() + { + ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-embeddings-response.txt")) + } + }; + using HttpClient client = new(handler); + + var modelId = "dall-e-2"; + var logger = new Mock>(); + logger.Setup(l => l.IsEnabled(It.IsAny())).Returns(true); + + var mockLoggerFactory = new Mock(); + mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(logger.Object); + + var sut = new OpenAITextEmbeddingGenerationService(modelId, "apiKey", httpClient: client, loggerFactory: mockLoggerFactory.Object); + + // Act + await sut.GenerateEmbeddingsAsync(["description"]); + + // Assert + logger.VerifyLog(LogLevel.Information, $"Action: {nameof(OpenAITextEmbeddingGenerationService.GenerateEmbeddingsAsync)}. OpenAI Model ID: {modelId}.", Times.Once()); + } } diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs index 20b969c03513..c484ee93953a 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -1,15 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.ClientModel; +using System.IO; using System.Net.Http; -using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Services; using Moq; using OpenAI; +using SemanticKernel.Connectors.OpenAI.UnitTests; using Xunit; namespace SemanticKernel.Connectors.UnitTests.OpenAI.TextToImage; @@ -25,7 +25,13 @@ public sealed class OpenAITextToImageServiceTests : IDisposable public OpenAITextToImageServiceTests() { - this._messageHandlerStub = new HttpMessageHandlerStub(); + this._messageHandlerStub = new() + { + ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-response.txt")) + } + }; this._httpClient = new HttpClient(this._messageHandlerStub, false); this._mockLoggerFactory = new Mock(); } @@ -34,12 +40,12 @@ public OpenAITextToImageServiceTests() public void ConstructorWorksCorrectly() { // Arrange & Act - var sut = new OpenAITextToImageService("api-key", "organization"); + var sut = new OpenAITextToImageService("model", "api-key", "organization"); // Assert Assert.NotNull(sut); Assert.Equal("organization", sut.Attributes[ClientCore.OrganizationKey]); - Assert.False(sut.Attributes.ContainsKey(AIServiceExtensions.ModelIdKey)); + Assert.Equal("model", sut.Attributes[AIServiceExtensions.ModelIdKey]); } [Fact] @@ -54,39 +60,34 @@ public void OpenAIClientConstructorWorksCorrectly() } [Theory] - [InlineData(256, 256)] - [InlineData(512, 512)] - [InlineData(1024, 1024)] - public async Task GenerateImageWorksCorrectlyAsync(int width, int height) + [InlineData(256, 256, "dall-e-2")] + [InlineData(512, 512, "dall-e-2")] + [InlineData(1024, 1024, "dall-e-2")] + [InlineData(1024, 1024, "dall-e-3")] + [InlineData(1024, 1792, "dall-e-3")] + [InlineData(1792, 1024, "dall-e-3")] + public async Task GenerateImageWorksCorrectlyAsync(int width, int height, string modelId) { // Arrange - var sut = new OpenAITextToImageService("dall-e-3", "api-key", httpClient: this._httpClient); - Assert.Equal("dall-e-3", sut.Attributes["ModelId"]); - this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(""" - { - "created": 1702575371, - "data": [ - { - "url": "https://image-url" - } - ] - } - """, Encoding.UTF8, "application/json") - }; + var sut = new OpenAITextToImageService(modelId, "api-key", httpClient: this._httpClient); + Assert.Equal(modelId, sut.Attributes["ModelId"]); + // Act var result = await sut.GenerateImageAsync("description", width, height); // Assert - Assert.Equal("https://image-url", result); + Assert.Equal("https://image-url/", result); } [Theory] - [InlineData(123, 456)] - [InlineData(256, 512)] - public async Task GenerateImageThrowsWhenSizeIsNotSupportedAsync(int width, int height) + [InlineData(123, 456, "dall-e-2")] + [InlineData(256, 512, "dall-e-2")] + [InlineData(256, 256, "dall-e-3")] + [InlineData(512, 512, "dall-e-3")] + [InlineData(1024, 1792, "dall-e-2")] + [InlineData(1792, 1024, "dall-e-2")] + public async Task GenerateImageThrowsWhenSizeIsNotSupportedAsync(int width, int height, string modelId) { // Arrange var sut = new OpenAITextToImageService("model", "apiKey"); @@ -106,25 +107,31 @@ public async Task GenerateImageAllowCustomSizeWhenNonDefaultEndpointIsUsedAsync( var sut = new OpenAITextToImageService("model", endpoint: new Uri("http://localhost"), httpClient: this._httpClient); Assert.Equal("model", sut.Attributes[AIServiceExtensions.ModelIdKey]); - this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(""" - { - "created": 1702575371, - "data": [ - { - "url": "https://image-url" - } - ] - } - """, Encoding.UTF8, "application/json") - }; - // Act var result = await sut.GenerateImageAsync("description", width, height); // Assert - Assert.Equal("https://image-url", result); + Assert.Equal("https://image-url/", result); + } + + [Fact] + public async Task GenerateImageDoesLogActionAsync() + { + // Assert + var modelId = "dall-e-2"; + var logger = new Mock>(); + logger.Setup(l => l.IsEnabled(It.IsAny())).Returns(true); + + this._mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(logger.Object); + + // Arrange + var sut = new OpenAITextToImageService(modelId, "apiKey", httpClient: this._httpClient, loggerFactory: this._mockLoggerFactory.Object); + + // Act + await sut.GenerateImageAsync("description", 256, 256); + + // Assert + logger.VerifyLog(LogLevel.Information, $"Action: {nameof(OpenAITextToImageService.GenerateImageAsync)}. OpenAI Model ID: {modelId}.", Times.Once()); } public void Dispose() diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/TestData/text-to-image-response.txt b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/TestData/text-to-image-response.txt new file mode 100644 index 000000000000..7d8f7327a5ec --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/TestData/text-to-image-response.txt @@ -0,0 +1,8 @@ +{ + "created": 1702575371, + "data": [ + { + "url": "https://image-url/" + } + ] +} \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs new file mode 100644 index 000000000000..3f51655a8508 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.Logging; +using Moq; + +#pragma warning disable CS8620 // Argument cannot be used for parameter due to differences in the nullability of reference types. + +namespace SemanticKernel.Connectors.OpenAI.UnitTests; + +internal static class MoqExtensions +{ + public static void VerifyLog(this Mock> logger, LogLevel logLevel, string message, Times times) + { + logger.Verify( + x => x.Log( + It.Is(l => l == logLevel), + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains(message)), + It.IsAny(), + It.IsAny>()), + times); + } +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs index af8f78214146..37a66244d927 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs @@ -105,6 +105,11 @@ internal ClientCore( Verify.NotNullOrWhiteSpace(apiKey); // For Public OpenAI Endpoint a key must be provided. this.Endpoint = new Uri(OpenAIV1Endpoint); } + else if (string.IsNullOrEmpty(apiKey)) + { + // Avoids an exception from OpenAI Client when a custom endpoint is provided without an API key. + apiKey = WhiteSpace; + } this.AddAttribute(AIServiceExtensions.EndpointKey, this.Endpoint.ToString()); @@ -116,7 +121,7 @@ internal ClientCore( this.AddAttribute(ClientCore.OrganizationKey, organizationId); } - this.Client = new OpenAIClient(apiKey ?? WhiteSpace, options); + this.Client = new OpenAIClient(apiKey!, options); } /// diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs index b31cbcda33fd..20a93c8d5604 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using System.Threading; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Services; using Microsoft.SemanticKernel.TextToImage; using OpenAI; @@ -70,5 +69,8 @@ public OpenAITextToImageService( /// public Task GenerateImageAsync(string prompt, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) - => this._core.GenerateImageAsync(prompt, width, height, cancellationToken); + { + this._core.LogActionDetails(); + return this._core.GenerateImageAsync(prompt, width, height, cancellationToken); + } } From 1078efe930379a85909cb78a7bfa27196f1bb60b Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 09:58:37 +0100 Subject: [PATCH 3/9] Added IT and missing UTs --- .../KernelBuilderExtensionsTests.cs | 73 +++++++++ .../ServiceCollectionExtensionsTests.cs | 74 +++++++++ .../Services/OpenAITextToImageServiceTests.cs | 1 - .../Extensions/KernelBuilderExtensions.cs | 152 ++++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 146 +++++++++++++++++ .../OpenAI/OpenAITextToImageTests.cs | 61 +++++++ 6 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/KernelBuilderExtensionsTests.cs create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs create mode 100644 dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/KernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/KernelBuilderExtensionsTests.cs new file mode 100644 index 000000000000..f296000c5245 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/KernelBuilderExtensionsTests.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Embeddings; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TextToImage; +using OpenAI; +using Xunit; + +namespace SemanticKernel.Connectors.OpenAI.UnitTests.Extensions; + +public class KernelBuilderExtensionsTests +{ + [Fact] + public void ItCanAddTextEmbeddingGenerationService() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddOpenAITextEmbeddingGeneration("model", "key") + .Build() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddTextEmbeddingGenerationServiceWithOpenAIClient() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddOpenAITextEmbeddingGeneration("model", new OpenAIClient("key")) + .Build() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddTextToImageService() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddOpenAITextToImage("model", "key") + .Build() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddTextToImageServiceWithOpenAIClient() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddOpenAITextToImage("model", new OpenAIClient("key")) + .Build() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000000..65db68eea180 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Embeddings; +using Microsoft.SemanticKernel.Services; +using Microsoft.SemanticKernel.TextToImage; +using OpenAI; +using Xunit; + +namespace SemanticKernel.Connectors.OpenAI.UnitTests.Extensions; + +public class ServiceCollectionExtensionsTests +{ + [Fact] + public void ItCanAddTextEmbeddingGenerationService() + { + // Arrange + var sut = new ServiceCollection(); + + // Act + var service = sut.AddOpenAITextEmbeddingGeneration("model", "key") + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddTextEmbeddingGenerationServiceWithOpenAIClient() + { + // Arrange + var sut = new ServiceCollection(); + + // Act + var service = sut.AddOpenAITextEmbeddingGeneration("model", new OpenAIClient("key")) + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddImageToTextService() + { + // Arrange + var sut = new ServiceCollection(); + + // Act + var service = sut.AddOpenAITextToImage("model", "key") + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } + + [Fact] + public void ItCanAddImageToTextServiceWithOpenAIClient() + { + // Arrange + var sut = new ServiceCollection(); + + // Act + var service = sut.AddOpenAITextToImage("model", new OpenAIClient("key")) + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); + } +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs index c484ee93953a..25379b3d8e05 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -72,7 +72,6 @@ public async Task GenerateImageWorksCorrectlyAsync(int width, int height, string var sut = new OpenAITextToImageService(modelId, "api-key", httpClient: this._httpClient); Assert.Equal(modelId, sut.Attributes["ModelId"]); - // Act var result = await sut.GenerateImageAsync("description", width, height); diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs new file mode 100644 index 000000000000..803a817d1ea7 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Embeddings; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.TextToImage; +using OpenAI; + +namespace Microsoft.SemanticKernel; + +/// +/// Sponsor extensions class for . +/// +public static class KernelBuilderExtensions +{ + #region Text Embedding + /// + /// Adds the OpenAI text embeddings service to the list. + /// + /// The instance to augment. + /// OpenAI model name, see https://platform.openai.com/docs/models + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// A local identifier for the given AI service + /// Non-default endpoint for the OpenAI API. + /// The HttpClient to use with this service. + /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAITextEmbeddingGeneration( + this IKernelBuilder builder, + string modelId, + string apiKey, + string? orgId = null, + string? serviceId = null, + Uri? endpoint = null, + HttpClient? httpClient = null, + int? dimensions = null) + { + Verify.NotNull(builder); + + builder.Services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextEmbeddingGenerationService( + modelId, + apiKey, + orgId, + endpoint, + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + serviceProvider.GetService(), + dimensions)); + + return builder; + } + + /// + /// Adds the OpenAI text embeddings service to the list. + /// + /// The instance to augment. + /// OpenAI model name, see https://platform.openai.com/docs/models + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAITextEmbeddingGeneration( + this IKernelBuilder builder, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null, + int? dimensions = null) + { + Verify.NotNull(builder); + + builder.Services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextEmbeddingGenerationService( + modelId, + openAIClient ?? serviceProvider.GetRequiredService(), + serviceProvider.GetService(), + dimensions)); + + return builder; + } + #endregion + + #region Text to Image + /// + /// Add the OpenAI Dall-E text to image service to the list + /// + /// The instance to augment. + /// OpenAI model name, see https://platform.openai.com/docs/models + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAITextToImage( + this IKernelBuilder builder, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null) + { + Verify.NotNull(builder); + + builder.Services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextToImageService( + modelId, + openAIClient ?? serviceProvider.GetRequiredService(), + serviceProvider.GetService())); + + return builder; + } + + /// + /// Add the OpenAI Dall-E text to image service to the list + /// + /// The instance to augment. + /// The model to use for image generation. + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// A local identifier for the given AI service + /// Non-default endpoint for the OpenAI API. + /// The HttpClient to use with this service. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAITextToImage( + this IKernelBuilder builder, + string modelId, + string apiKey, + string? orgId = null, + string? serviceId = null, + Uri? endpoint = null, + HttpClient? httpClient = null) + { + Verify.NotNull(builder); + + builder.Services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextToImageService( + modelId, + apiKey, + orgId, + endpoint, + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + serviceProvider.GetService())); + + return builder; + } + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 000000000000..459b2463e93f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Embeddings; +using Microsoft.SemanticKernel.Http; +using Microsoft.SemanticKernel.TextToImage; +using OpenAI; + +namespace Microsoft.SemanticKernel; + +/* Phase 02 +- Add endpoint parameter for both Embedding and TextToImage services extensions. +- Removed unnecessary Validation checks (that are already happening in the service/client constructors) +- Added openAIClient extension for TextToImage service. +- Changed parameters order for TextToImage service extension (modelId comes first). +- Made modelId a required parameter of TextToImage services. + +*/ +/// +/// Sponsor extensions class for . +/// +public static class ServiceCollectionExtensions +{ + #region Text Embedding + /// + /// Adds the OpenAI text embeddings service to the list. + /// + /// The instance to augment. + /// OpenAI model name, see https://platform.openai.com/docs/models + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// A local identifier for the given AI service + /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. + /// Non-default endpoint for the OpenAI API. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAITextEmbeddingGeneration( + this IServiceCollection services, + string modelId, + string apiKey, + string? orgId = null, + string? serviceId = null, + int? dimensions = null, + Uri? endpoint = null) + { + Verify.NotNull(services); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextEmbeddingGenerationService( + modelId, + apiKey, + orgId, + endpoint, + HttpClientProvider.GetHttpClient(serviceProvider), + serviceProvider.GetService(), + dimensions)); + } + + /// + /// Adds the OpenAI text embeddings service to the list. + /// + /// The instance to augment. + /// The OpenAI model id. + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAITextEmbeddingGeneration(this IServiceCollection services, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null, + int? dimensions = null) + { + Verify.NotNull(services); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextEmbeddingGenerationService( + modelId, + openAIClient ?? serviceProvider.GetRequiredService(), + serviceProvider.GetService(), + dimensions)); + } + #endregion + + #region Text to Image + /// + /// Add the OpenAI Dall-E text to image service to the list + /// + /// The instance to augment. + /// The model to use for image generation. + /// OpenAI API key, see https://platform.openai.com/account/api-keys + /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. + /// A local identifier for the given AI service + /// Non-default endpoint for the OpenAI API. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAITextToImage(this IServiceCollection services, + string modelId, + string apiKey, + string? orgId = null, + string? serviceId = null, + Uri? endpoint = null) + { + Verify.NotNull(services); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextToImageService( + modelId, + apiKey, + orgId, + endpoint, + HttpClientProvider.GetHttpClient(serviceProvider), + serviceProvider.GetService())); + } + + /// + /// Adds the OpenAI text embeddings service to the list. + /// + /// The instance to augment. + /// The OpenAI model id. + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// The number of dimensions the resulting output embeddings should have. Only supported in "text-embedding-3" and later models. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAITextToImage(this IServiceCollection services, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null, + int? dimensions = null) + { + Verify.NotNull(services); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new OpenAITextToImageService( + modelId, + openAIClient ?? serviceProvider.GetRequiredService(), + serviceProvider.GetService())); + } + #endregion +} diff --git a/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs new file mode 100644 index 000000000000..d5aea6d47227 --- /dev/null +++ b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.TextToImage; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.OpenAI; +public sealed class OpenAITextToImageTests +{ + private readonly IConfigurationRoot _configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + [Fact(Skip = "This test is for manual verification.")] + public async Task OpenAITextToImageTestAsync() + { + // Arrange + OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAITextToImage").Get(); + Assert.NotNull(openAIConfiguration); + + var kernel = Kernel.CreateBuilder() + .AddOpenAITextToImage(modelId: openAIConfiguration.ModelId, apiKey: openAIConfiguration.ApiKey) + .Build(); + + var service = kernel.GetRequiredService(); + + // Act + var result = await service.GenerateImageAsync("The sun rises in the east and sets in the west.", 512, 512); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact(Skip = "This test is for manual verification.")] + public async Task OpenAITextToImageByModelTestAsync() + { + // Arrange + OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAITextToImage").Get(); + Assert.NotNull(openAIConfiguration); + + var kernel = Kernel.CreateBuilder() + .AddOpenAITextToImage(modelId: openAIConfiguration.ModelId, apiKey: openAIConfiguration.ApiKey) + .Build(); + + var service = kernel.GetRequiredService(); + + // Act + var result = await service.GenerateImageAsync("The sun rises in the east and sets in the west.", 1024, 1024); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result); + } +} From 4a8750fea65e626ba2a51058b193b74a1aaeaeff Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:28:20 +0100 Subject: [PATCH 4/9] Update UTs ITs, Passing, typo fix --- .../Core/ClientCoreTests.cs | 4 +-- .../OpenAITextEmbbedingGenerationService.cs | 3 +- .../OpenAI/OpenAITextEmbeddingTests.cs | 4 +-- .../OpenAI/OpenAITextToImageTests.cs | 33 ++++--------------- 4 files changed, 12 insertions(+), 32 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs index 8961ed19cf94..f162e1d7334c 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Core/ClientCoreTests.cs @@ -232,7 +232,7 @@ public void ItThrowsIfModelIdIsNotProvided() } [Fact] - public void ItThrowsWhenNotUsingCustomEndpintAndApiKeyIsNotProvided() + public void ItThrowsWhenNotUsingCustomEndpointAndApiKeyIsNotProvided() { // Act & Assert Assert.Throws(() => new ClientCore("modelId", " ")); @@ -241,7 +241,7 @@ public void ItThrowsWhenNotUsingCustomEndpintAndApiKeyIsNotProvided() } [Fact] - public void ItDoesNotThrowWhenUsingCustomEndpintAndApiKeyIsNotProvided() + public void ItDoesNotThrowWhenUsingCustomEndpointAndApiKeyIsNotProvided() { // Act & Assert ClientCore? clientCore = null; diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs index a64c4e44d1e6..a4dd48ba75e3 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextEmbbedingGenerationService.cs @@ -8,13 +8,12 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Embeddings; -using Microsoft.SemanticKernel.Services; using OpenAI; namespace Microsoft.SemanticKernel.Connectors.OpenAI; /* Phase 02 -Adding the non-default endpiont parameter to the constructor. +Adding the non-default endpoint parameter to the constructor. */ /// diff --git a/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextEmbeddingTests.cs b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextEmbeddingTests.cs index 6eca1909a546..bccc92bfa0f3 100644 --- a/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextEmbeddingTests.cs +++ b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextEmbeddingTests.cs @@ -19,7 +19,7 @@ public sealed class OpenAITextEmbeddingTests .AddUserSecrets() .Build(); - [Theory]//(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] + [Theory(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] [InlineData("test sentence")] public async Task OpenAITestAsync(string testInputString) { @@ -38,7 +38,7 @@ public async Task OpenAITestAsync(string testInputString) Assert.Equal(3, batchResult.Count); } - [Theory]//(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] + [Theory(Skip = "OpenAI will often throttle requests. This test is for manual verification.")] [InlineData(null, 3072)] [InlineData(1024, 1024)] public async Task OpenAIWithDimensionsAsync(int? dimensions, int expectedVectorLength) diff --git a/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs index d5aea6d47227..812d41677b28 100644 --- a/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs +++ b/dotnet/src/IntegrationTestsV2/Connectors/OpenAI/OpenAITextToImageTests.cs @@ -11,48 +11,29 @@ namespace SemanticKernel.IntegrationTests.Connectors.OpenAI; public sealed class OpenAITextToImageTests { private readonly IConfigurationRoot _configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true) .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() .AddUserSecrets() .Build(); - [Fact(Skip = "This test is for manual verification.")] - public async Task OpenAITextToImageTestAsync() + [Theory(Skip = "This test is for manual verification.")] + [InlineData("dall-e-2", 512, 512)] + [InlineData("dall-e-3", 1024, 1024)] + public async Task OpenAITextToImageByModelTestAsync(string modelId, int width, int height) { // Arrange OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAITextToImage").Get(); Assert.NotNull(openAIConfiguration); var kernel = Kernel.CreateBuilder() - .AddOpenAITextToImage(modelId: openAIConfiguration.ModelId, apiKey: openAIConfiguration.ApiKey) + .AddOpenAITextToImage(modelId, apiKey: openAIConfiguration.ApiKey) .Build(); var service = kernel.GetRequiredService(); // Act - var result = await service.GenerateImageAsync("The sun rises in the east and sets in the west.", 512, 512); - - // Assert - Assert.NotNull(result); - Assert.NotEmpty(result); - } - - [Fact(Skip = "This test is for manual verification.")] - public async Task OpenAITextToImageByModelTestAsync() - { - // Arrange - OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAITextToImage").Get(); - Assert.NotNull(openAIConfiguration); - - var kernel = Kernel.CreateBuilder() - .AddOpenAITextToImage(modelId: openAIConfiguration.ModelId, apiKey: openAIConfiguration.ApiKey) - .Build(); - - var service = kernel.GetRequiredService(); - - // Act - var result = await service.GenerateImageAsync("The sun rises in the east and sets in the west.", 1024, 1024); + var result = await service.GenerateImageAsync("The sun rises in the east and sets in the west.", width, height); // Assert Assert.NotNull(result); From 2bc17f7ab1b55c120b831d5723724ced822946e2 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:31:48 +0100 Subject: [PATCH 5/9] Fix warnings --- .../Services/OpenAITextToImageServiceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs index 25379b3d8e05..0605cee85742 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -89,7 +89,7 @@ public async Task GenerateImageWorksCorrectlyAsync(int width, int height, string public async Task GenerateImageThrowsWhenSizeIsNotSupportedAsync(int width, int height, string modelId) { // Arrange - var sut = new OpenAITextToImageService("model", "apiKey"); + var sut = new OpenAITextToImageService(modelId, "apiKey"); // Act & Assert await Assert.ThrowsAsync(() => sut.GenerateImageAsync("description", width, height)); From 14468749c1184bd1411df6e3bc691b330fb10e59 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:48:39 +0100 Subject: [PATCH 6/9] Address PR Comments --- dotnet/SK-dotnet.sln | 3 ++- .../TextToImage/AzureOpenAITextToImageService.cs | 6 +++--- .../TextToImage/OpenAITextToImageService.cs | 8 +++++--- .../Connectors.OpenAIV2.UnitTests.csproj | 7 +++++-- .../Services/OpenAITextEmbeddingGenerationServiceTests.cs | 4 ++++ .../Services/OpenAITextToImageServiceTests.cs | 3 +-- .../src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs | 4 ++-- ...lderExtensions.cs => OpenAIKernelBuilderExtensions.cs} | 2 +- ...Extensions.cs => OpenAIServiceCollectionExtensions.cs} | 2 +- .../Services/OpenAITextToImageService.cs | 2 +- .../Utils => InternalUtilities/test}/MoqExtensions.cs | 2 -- 11 files changed, 25 insertions(+), 18 deletions(-) rename dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/{KernelBuilderExtensions.cs => OpenAIKernelBuilderExtensions.cs} (99%) rename dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/{ServiceCollectionExtensions.cs => OpenAIServiceCollectionExtensions.cs} (99%) rename dotnet/src/{Connectors/Connectors.OpenAIV2.UnitTests/Utils => InternalUtilities/test}/MoqExtensions.cs (93%) diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index 01ffff52057a..326a35a79ff7 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -92,6 +92,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{5C246969-D ProjectSection(SolutionItems) = preProject src\InternalUtilities\test\AssertExtensions.cs = src\InternalUtilities\test\AssertExtensions.cs src\InternalUtilities\test\HttpMessageHandlerStub.cs = src\InternalUtilities\test\HttpMessageHandlerStub.cs + src\Connectors\Connectors.OpenAIV2.UnitTests\Utils\MoqExtensions.cs = src\Connectors\Connectors.OpenAIV2.UnitTests\Utils\MoqExtensions.cs src\InternalUtilities\test\MultipleHttpMessageHandlerStub.cs = src\InternalUtilities\test\MultipleHttpMessageHandlerStub.cs src\InternalUtilities\test\TestInternalUtilities.props = src\InternalUtilities\test\TestInternalUtilities.props EndProjectSection @@ -324,7 +325,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntegrationTestsV2", "src\I EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureOpenAI", "src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj", "{6744272E-8326-48CE-9A3F-6BE227A5E777}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureOpenAI.UnitTests", "src\Connectors\Connectors.AzureOpenAI.UnitTests\Connectors.AzureOpenAI.UnitTests.csproj", "{DB219924-208B-4CDD-8796-EE424689901E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.AzureOpenAI.UnitTests", "src\Connectors\Connectors.AzureOpenAI.UnitTests\Connectors.AzureOpenAI.UnitTests.csproj", "{DB219924-208B-4CDD-8796-EE424689901E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs index efa3ffcc87c0..cbcaafbd7e19 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs @@ -150,13 +150,13 @@ public AzureOpenAITextToImageService( /// public async Task GenerateImageAsync( - string description, + string prompt, // Changing this temporarily to avoid warnings (this class will be removed in the future) int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) { - Verify.NotNull(description); + Verify.NotNull(prompt); var size = (width, height) switch { @@ -173,7 +173,7 @@ public async Task GenerateImageAsync( new ImageGenerationOptions { DeploymentName = this._deploymentName, - Prompt = description, + Prompt = prompt, Size = size, }, cancellationToken).ConfigureAwait(false); } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs index 335fe8cad5ee..a601ed2aac3c 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs @@ -82,15 +82,17 @@ public OpenAITextToImageService( public IReadOnlyDictionary Attributes => this._core.Attributes; /// - public Task GenerateImageAsync(string description, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) + public Task GenerateImageAsync( + string prompt, // Changing this temporarily to avoid warnings (this class will be removed in the future) + int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) { - Verify.NotNull(description); + Verify.NotNull(prompt); if (width != height || (width != 256 && width != 512 && width != 1024)) { throw new ArgumentOutOfRangeException(nameof(width), width, "OpenAI can generate only square images of size 256x256, 512x512, or 1024x1024."); } - return this.GenerateImageAsync(this._modelId, description, width, height, "url", x => x.Url, cancellationToken); + return this.GenerateImageAsync(this._modelId, prompt, width, height, "url", x => x.Url, cancellationToken); } private async Task GenerateImageAsync( diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj index 27f85dea987a..0a100b3c13a6 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Connectors.OpenAIV2.UnitTests.csproj @@ -28,14 +28,17 @@ - - + + + + + diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs index 6c72312434ce..5fb36efc0349 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextEmbeddingGenerationServiceTests.cs @@ -15,6 +15,10 @@ using Xunit; namespace SemanticKernel.Connectors.OpenAI.UnitTests.Services; + +/// +/// Unit tests for class. +/// public class OpenAITextEmbeddingGenerationServiceTests { [Fact] diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs index 0605cee85742..12fefaa96d33 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -9,10 +9,9 @@ using Microsoft.SemanticKernel.Services; using Moq; using OpenAI; -using SemanticKernel.Connectors.OpenAI.UnitTests; using Xunit; -namespace SemanticKernel.Connectors.UnitTests.OpenAI.TextToImage; +namespace SemanticKernel.Connectors.UnitTests.OpenAI.Services; /// /// Unit tests for class. diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs index 37a66244d927..a6be6d20aa46 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.cs @@ -37,7 +37,7 @@ internal partial class ClientCore /// /// White space constant. /// - private const string WhiteSpace = " "; + private const string SingleSpace = " "; /// /// Gets the attribute name used to store the organization in the dictionary. @@ -108,7 +108,7 @@ internal ClientCore( else if (string.IsNullOrEmpty(apiKey)) { // Avoids an exception from OpenAI Client when a custom endpoint is provided without an API key. - apiKey = WhiteSpace; + apiKey = SingleSpace; } this.AddAttribute(AIServiceExtensions.EndpointKey, this.Endpoint.ToString()); diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIKernelBuilderExtensions.cs similarity index 99% rename from dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs rename to dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIKernelBuilderExtensions.cs index 803a817d1ea7..567d82726e4b 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/KernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIKernelBuilderExtensions.cs @@ -16,7 +16,7 @@ namespace Microsoft.SemanticKernel; /// /// Sponsor extensions class for . /// -public static class KernelBuilderExtensions +public static class OpenAIKernelBuilderExtensions { #region Text Embedding /// diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIServiceCollectionExtensions.cs similarity index 99% rename from dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs rename to dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIServiceCollectionExtensions.cs index 459b2463e93f..77355de7f24e 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/ServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Extensions/OpenAIServiceCollectionExtensions.cs @@ -23,7 +23,7 @@ namespace Microsoft.SemanticKernel; /// /// Sponsor extensions class for . /// -public static class ServiceCollectionExtensions +public static class OpenAIServiceCollectionExtensions { #region Text Embedding /// diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs index 20a93c8d5604..66ad5242bdd2 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs @@ -4,8 +4,8 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Net.Http; -using System.Threading.Tasks; using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.TextToImage; using OpenAI; diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs b/dotnet/src/InternalUtilities/test/MoqExtensions.cs similarity index 93% rename from dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs rename to dotnet/src/InternalUtilities/test/MoqExtensions.cs index 3f51655a8508..8fb435e288f9 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Utils/MoqExtensions.cs +++ b/dotnet/src/InternalUtilities/test/MoqExtensions.cs @@ -6,8 +6,6 @@ #pragma warning disable CS8620 // Argument cannot be used for parameter due to differences in the nullability of reference types. -namespace SemanticKernel.Connectors.OpenAI.UnitTests; - internal static class MoqExtensions { public static void VerifyLog(this Mock> logger, LogLevel logLevel, string message, Times times) From 9e9303381396f901d6cdd73db8aa2d609f4e374a Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:53:27 +0100 Subject: [PATCH 7/9] Remove clientside check on image size for OpenAIImageToText service. --- .../Services/OpenAITextToImageServiceTests.cs | 36 ++----------------- .../Core/ClientCore.TextToImage.cs | 26 -------------- 2 files changed, 2 insertions(+), 60 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs index 12fefaa96d33..919b864327e8 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -65,6 +65,8 @@ public void OpenAIClientConstructorWorksCorrectly() [InlineData(1024, 1024, "dall-e-3")] [InlineData(1024, 1792, "dall-e-3")] [InlineData(1792, 1024, "dall-e-3")] + [InlineData(123, 321, "custom-model-1")] + [InlineData(179, 124, "custom-model-2")] public async Task GenerateImageWorksCorrectlyAsync(int width, int height, string modelId) { // Arrange @@ -78,40 +80,6 @@ public async Task GenerateImageWorksCorrectlyAsync(int width, int height, string Assert.Equal("https://image-url/", result); } - [Theory] - [InlineData(123, 456, "dall-e-2")] - [InlineData(256, 512, "dall-e-2")] - [InlineData(256, 256, "dall-e-3")] - [InlineData(512, 512, "dall-e-3")] - [InlineData(1024, 1792, "dall-e-2")] - [InlineData(1792, 1024, "dall-e-2")] - public async Task GenerateImageThrowsWhenSizeIsNotSupportedAsync(int width, int height, string modelId) - { - // Arrange - var sut = new OpenAITextToImageService(modelId, "apiKey"); - - // Act & Assert - await Assert.ThrowsAsync(() => sut.GenerateImageAsync("description", width, height)); - } - - [Theory] - [InlineData(123, 456)] - [InlineData(256, 512)] - [InlineData(6546, 545)] - [InlineData(16, 32)] - public async Task GenerateImageAllowCustomSizeWhenNonDefaultEndpointIsUsedAsync(int width, int height) - { - // Arrange - var sut = new OpenAITextToImageService("model", endpoint: new Uri("http://localhost"), httpClient: this._httpClient); - Assert.Equal("model", sut.Attributes[AIServiceExtensions.ModelIdKey]); - - // Act - var result = await sut.GenerateImageAsync("description", width, height); - - // Assert - Assert.Equal("https://image-url/", result); - } - [Fact] public async Task GenerateImageDoesLogActionAsync() { diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs index 56e027024964..de58e5712537 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs @@ -40,8 +40,6 @@ internal async Task GenerateImageAsync( var size = new GeneratedImageSize(width, height); - EnsureSizeIsSupported(size, this.Endpoint!, this.ModelId); - var imageOptions = new ImageGenerationOptions() { Size = size, @@ -53,28 +51,4 @@ internal async Task GenerateImageAsync( return generatedImage.ImageUri?.ToString() ?? throw new KernelException("The generated image is not in url format"); } - - /// - /// Ensures by throwing and exception if the requested image size is not supported by the OpenAI image generation models. - /// - /// Target size to ensure support - /// Current client endpoint - /// Target model identifier - /// Size is not supported - private static void EnsureSizeIsSupported(GeneratedImageSize targetSize, Uri endpoint, string modelId) - { - // Only be restrictive to the image size if the endpoint is the default OpenAI v1 endpoint - if (endpoint != new Uri(OpenAIV1Endpoint)) { return; } - - var supported = (targetSize == GeneratedImageSize.W256xH256 && modelId == "dall-e-2") - || (targetSize == GeneratedImageSize.W512xH512 && modelId == "dall-e-2") - || targetSize == GeneratedImageSize.W1024xH1024 - || (targetSize == GeneratedImageSize.W1024xH1792 && modelId == "dall-e-3") - || (targetSize == GeneratedImageSize.W1792xH1024 && modelId == "dall-e-3"); - - if (!supported) - { - throw new ArgumentOutOfRangeException($"The requested image size {targetSize} is not supported. Supported sizes are: 256x256 (dalle-2 only), 512x512 (dalle-2 only), 1024x1024, 1024x1792 (dalle-3 only), 1792x1024 (dalle-3 only)"); - } - } } From cb217128ee51b5f56a1933e476931d3956ce60f6 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:59:44 +0100 Subject: [PATCH 8/9] Fix warnings --- .../Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs index de58e5712537..26d8480fd004 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Core/ClientCore.TextToImage.cs @@ -9,7 +9,6 @@ Phase 02 - Allow custom size for images when the endpoint is not the default OpenAI v1 endpoint. */ -using System; using System.ClientModel; using System.Threading; using System.Threading.Tasks; From cae1aa4aec22003e9a0e67f2f766a48403d3f37e Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:50:53 +0100 Subject: [PATCH 9/9] Revert prompt -> description --- .../TextToImage/AzureOpenAITextToImageService.cs | 6 +++--- .../TextToImage/OpenAITextToImageService.cs | 8 +++----- .../Services/OpenAITextToImageService.cs | 4 ++-- .../AI/TextToImage/ITextToImageService.cs | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs index cbcaafbd7e19..efa3ffcc87c0 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/AzureOpenAITextToImageService.cs @@ -150,13 +150,13 @@ public AzureOpenAITextToImageService( /// public async Task GenerateImageAsync( - string prompt, // Changing this temporarily to avoid warnings (this class will be removed in the future) + string description, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) { - Verify.NotNull(prompt); + Verify.NotNull(description); var size = (width, height) switch { @@ -173,7 +173,7 @@ public async Task GenerateImageAsync( new ImageGenerationOptions { DeploymentName = this._deploymentName, - Prompt = prompt, + Prompt = description, Size = size, }, cancellationToken).ConfigureAwait(false); } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs index a601ed2aac3c..335fe8cad5ee 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs @@ -82,17 +82,15 @@ public OpenAITextToImageService( public IReadOnlyDictionary Attributes => this._core.Attributes; /// - public Task GenerateImageAsync( - string prompt, // Changing this temporarily to avoid warnings (this class will be removed in the future) - int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) + public Task GenerateImageAsync(string description, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) { - Verify.NotNull(prompt); + Verify.NotNull(description); if (width != height || (width != 256 && width != 512 && width != 1024)) { throw new ArgumentOutOfRangeException(nameof(width), width, "OpenAI can generate only square images of size 256x256, 512x512, or 1024x1024."); } - return this.GenerateImageAsync(this._modelId, prompt, width, height, "url", x => x.Url, cancellationToken); + return this.GenerateImageAsync(this._modelId, description, width, height, "url", x => x.Url, cancellationToken); } private async Task GenerateImageAsync( diff --git a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs index 66ad5242bdd2..55eca0e112eb 100644 --- a/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAIV2/Services/OpenAITextToImageService.cs @@ -68,9 +68,9 @@ public OpenAITextToImageService( } /// - public Task GenerateImageAsync(string prompt, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) + public Task GenerateImageAsync(string description, int width, int height, Kernel? kernel = null, CancellationToken cancellationToken = default) { this._core.LogActionDetails(); - return this._core.GenerateImageAsync(prompt, width, height, cancellationToken); + return this._core.GenerateImageAsync(description, width, height, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs index a0c01f95951a..b30f78f3c0ca 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/ITextToImageService.cs @@ -20,7 +20,7 @@ public interface ITextToImageService : IAIService /// /// Generate an image matching the given description /// - /// Image generation prompt + /// Image generation prompt /// Image width in pixels /// Image height in pixels /// The containing services, plugins, and other state for use throughout the operation. @@ -28,7 +28,7 @@ public interface ITextToImageService : IAIService /// Generated image in base64 format or image URL [Experimental("SKEXP0001")] public Task GenerateImageAsync( - string prompt, + string description, int width, int height, Kernel? kernel = null,