From 9761f3d3621938a6567bec1cbc4e30e24c2a975e Mon Sep 17 00:00:00 2001 From: SergeyMenshykh Date: Thu, 4 Jul 2024 23:26:19 +0100 Subject: [PATCH 1/4] feat(Azure.AI.OpenAI v2): migrate azure text to audio service. --- ...eOpenAIServiceCollectionExtensionsTests.cs | 20 ++ ...enAIServiceKernelBuilderExtensionsTests.cs | 20 ++ .../AzureOpenAITextToAudioServiceTests.cs | 214 ++++++++++++++++++ .../Connectors.AzureOpenAI.csproj | 10 - .../Core/ClientCore.TextToAudio.cs | 22 +- .../AzureOpenAIKernelBuilderExtensions.cs | 42 ++++ .../AzureOpenAIServiceCollectionExtensions.cs | 41 ++++ .../Services/AzureOpenAITextToAudioService.cs | 26 ++- ...AzureOpenAITextToAudioExecutionSettings.cs | 130 +++++++++++ .../AzureOpenAITextToAudioTests.cs | 44 ++++ 10 files changed, 550 insertions(+), 19 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs create mode 100644 dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs create mode 100644 dotnet/src/IntegrationTestsV2/Connectors/AzureOpenAI/AzureOpenAITextToAudioTests.cs diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs index ca4899258b21..f08a06e804f5 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs @@ -9,6 +9,7 @@ using Microsoft.SemanticKernel.Connectors.AzureOpenAI; using Microsoft.SemanticKernel.Embeddings; using Microsoft.SemanticKernel.TextGeneration; +using Microsoft.SemanticKernel.TextToAudio; namespace SemanticKernel.Connectors.AzureOpenAI.UnitTests.Extensions; @@ -88,6 +89,25 @@ public void ServiceCollectionAddAzureOpenAITextEmbeddingGenerationAddsValidServi #endregion + #region Text to audio + + [Fact] + public void ServiceCollectionAddAzureOpenAITextToAudioAddsValidService() + { + // Arrange + var sut = new ServiceCollection(); + + // Act + var service = sut.AddAzureOpenAITextToAudio("deployment-name", "https://endpoint", "api-key") + .BuildServiceProvider() + .GetRequiredService(); + + // Assert + Assert.IsType(service); + } + + #endregion + public enum InitializationType { ApiKey, diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceKernelBuilderExtensionsTests.cs index 8c5515516ca5..9ea09b795f8f 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceKernelBuilderExtensionsTests.cs @@ -9,6 +9,7 @@ using Microsoft.SemanticKernel.Connectors.AzureOpenAI; using Microsoft.SemanticKernel.Embeddings; using Microsoft.SemanticKernel.TextGeneration; +using Microsoft.SemanticKernel.TextToAudio; namespace SemanticKernel.Connectors.AzureOpenAI.UnitTests.Extensions; @@ -88,6 +89,25 @@ public void KernelBuilderAddAzureOpenAITextEmbeddingGenerationAddsValidService(I #endregion + #region Text to audio + + [Fact] + public void KernelBuilderAddAzureOpenAITextToAudioAddsValidService() + { + // Arrange + var sut = Kernel.CreateBuilder(); + + // Act + var service = sut.AddAzureOpenAITextToAudio("deployment-name", "https://endpoint", "api-key") + .Build() + .GetRequiredService(); + + // Assert + Assert.IsType(service); + } + + #endregion + public enum InitializationType { ApiKey, diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs new file mode 100644 index 000000000000..88456a1eb848 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Connectors.AzureOpenAI; +using Moq; + +namespace SemanticKernel.Connectors.AzureOpenAI.UnitTests.Services; + +/// +/// Unit tests for class. +/// +public sealed class AzureOpenAITextToAudioServiceTests : IDisposable +{ + private readonly HttpMessageHandlerStub _messageHandlerStub; + private readonly HttpClient _httpClient; + private readonly Mock _mockLoggerFactory; + + public AzureOpenAITextToAudioServiceTests() + { + this._messageHandlerStub = new HttpMessageHandlerStub(); + this._httpClient = new HttpClient(this._messageHandlerStub, false); + this._mockLoggerFactory = new Mock(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ConstructorsAddRequiredMetadata(bool includeLoggerFactory) + { + // Arrange & Act + var service = includeLoggerFactory ? + new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", loggerFactory: this._mockLoggerFactory.Object) : + new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id"); + + // Assert + Assert.Equal("model-id", service.Attributes["ModelId"]); + Assert.Equal("deployment-name", service.Attributes["DeploymentName"]); + } + + [Fact] + public void ItThrowsIfModelIdIsNotProvided() + { + // Act & Assert + Assert.Throws(() => new AzureOpenAITextToAudioService(null!, "https://endpoint", "api-key")); + Assert.Throws(() => new AzureOpenAITextToAudioService("", "https://endpoint", "api-key")); + Assert.Throws(() => new AzureOpenAITextToAudioService(" ", "https://endpoint", "api-key")); + } + + [Fact] + public async Task GetAudioContentWithInvalidSettingsThrowsExceptionAsync() + { + // Arrange + var settingsWithInvalidVoice = new AzureOpenAITextToAudioExecutionSettings(""); + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + await using var stream = new MemoryStream(new byte[] { 0x00, 0x00, 0xFF, 0x7F }); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + // Act & Assert + await Assert.ThrowsAsync(() => service.GetAudioContentsAsync("Some text", settingsWithInvalidVoice)); + } + + [Fact] + public async Task GetAudioContentByDefaultWorksCorrectlyAsync() + { + // Arrange + var expectedByteArray = new byte[] { 0x00, 0x00, 0xFF, 0x7F }; + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + await using var stream = new MemoryStream(expectedByteArray); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + // Act + var result = await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings("Nova")); + + // Assert + var audioData = result[0].Data!.Value; + Assert.False(audioData.IsEmpty); + Assert.True(audioData.Span.SequenceEqual(expectedByteArray)); + } + + [Theory] + [InlineData("echo", "wav")] + [InlineData("fable", "opus")] + [InlineData("onyx", "flac")] + [InlineData("nova", "aac")] + [InlineData("shimmer", "pcm")] + public async Task GetAudioContentVoicesWorksCorrectlyAsync(string voice, string format) + { + // Arrange + byte[] expectedByteArray = [0x00, 0x00, 0xFF, 0x7F]; + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + await using var stream = new MemoryStream(expectedByteArray); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + // Act + var result = await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings(voice) { ResponseFormat = format }); + + // Assert + var requestBody = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent!); + Assert.NotNull(requestBody); + Assert.Equal(voice, requestBody["voice"]?.ToString()); + Assert.Equal(format, requestBody["response_format"]?.ToString()); + + var audioData = result[0].Data!.Value; + Assert.False(audioData.IsEmpty); + Assert.True(audioData.Span.SequenceEqual(expectedByteArray)); + } + + [Fact] + public async Task GetAudioContentThrowsWhenVoiceIsNotSupportedAsync() + { + // Arrange + byte[] expectedByteArray = [0x00, 0x00, 0xFF, 0x7F]; + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + + // Act & Assert + await Assert.ThrowsAsync(async () => await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings("voice"))); + } + + [Fact] + public async Task GetAudioContentThrowsWhenFormatIsNotSupportedAsync() + { + // Arrange + byte[] expectedByteArray = [0x00, 0x00, 0xFF, 0x7F]; + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + + // Act & Assert + await Assert.ThrowsAsync(async () => await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings() { ResponseFormat = "not supported" })); + } + + [Theory] + [InlineData(true, "http://local-endpoint")] + [InlineData(false, "https://endpoint")] + public async Task GetAudioContentUsesValidBaseUrlAsync(bool useHttpClientBaseAddress, string expectedBaseAddress) + { + // Arrange + var expectedByteArray = new byte[] { 0x00, 0x00, 0xFF, 0x7F }; + + if (useHttpClientBaseAddress) + { + this._httpClient.BaseAddress = new Uri("http://local-endpoint"); + } + + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + await using var stream = new MemoryStream(expectedByteArray); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + // Act + var result = await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings("Nova")); + + // Assert + Assert.StartsWith(expectedBaseAddress, this._messageHandlerStub.RequestUri!.AbsoluteUri, StringComparison.InvariantCulture); + } + + [Theory] + [InlineData("model-1", "model-2", "deployment", "model-2")] + [InlineData("model-1", null, "deployment", "model-1")] + [InlineData(null, "model-2", "deployment", "model-2")] + [InlineData(null, null, "deployment", "deployment")] + public async Task GetAudioContentPrioritizesModelIdOverDeploymentNameAsync(string? modelInSettings, string? modelInConstructor, string deploymentName, string expectedModel) + { + // Arrange + var expectedByteArray = new byte[] { 0x00, 0x00, 0xFF, 0x7F }; + + var service = new AzureOpenAITextToAudioService(deploymentName, "https://endpoint", "api-key", modelInConstructor, this._httpClient); + await using var stream = new MemoryStream(expectedByteArray); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + // Act + var result = await service.GetAudioContentsAsync("Some text", new AzureOpenAITextToAudioExecutionSettings("Nova") { ModelId = modelInSettings }); + + // Assert + var requestBody = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent!); + Assert.Equal(expectedModel, requestBody?["model"]?.ToString()); + } + + public void Dispose() + { + this._httpClient.Dispose(); + this._messageHandlerStub.Dispose(); + } +} diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj b/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj index 73bba3cb28f6..35c31788610d 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj @@ -21,20 +21,10 @@ Semantic Kernel connectors for Azure OpenAI. Contains clients for text generation, chat completion, embedding and DALL-E text to image. - - - - - - - - - - diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.TextToAudio.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.TextToAudio.cs index d11b5ce81a26..4351b15607bd 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.TextToAudio.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.TextToAudio.cs @@ -19,24 +19,30 @@ internal partial class ClientCore /// /// Prompt to generate the image /// Text to Audio execution settings for the prompt + /// Azure OpenAI model id /// The to monitor for cancellation requests. The default is . /// Url of the generated image internal async Task> GetAudioContentsAsync( string prompt, PromptExecutionSettings? executionSettings, + string? modelId, CancellationToken cancellationToken) { Verify.NotNullOrWhiteSpace(prompt); - OpenAITextToAudioExecutionSettings? audioExecutionSettings = OpenAITextToAudioExecutionSettings.FromExecutionSettings(executionSettings); - var (responseFormat, mimeType) = GetGeneratedSpeechFormatAndMimeType(audioExecutionSettings?.ResponseFormat); + AzureOpenAITextToAudioExecutionSettings audioExecutionSettings = AzureOpenAITextToAudioExecutionSettings.FromExecutionSettings(executionSettings); + + var (responseFormat, mimeType) = GetGeneratedSpeechFormatAndMimeType(audioExecutionSettings.ResponseFormat); + SpeechGenerationOptions options = new() { ResponseFormat = responseFormat, - Speed = audioExecutionSettings?.Speed, + Speed = audioExecutionSettings.Speed, }; - ClientResult response = await RunRequestAsync(() => this.Client.GetAudioClient(this.ModelId).GenerateSpeechFromTextAsync(prompt, GetGeneratedSpeechVoice(audioExecutionSettings?.Voice), options, cancellationToken)).ConfigureAwait(false); + var deploymentOrModel = this.GetModelId(audioExecutionSettings, modelId); + + ClientResult response = await RunRequestAsync(() => this.Client.GetAudioClient(deploymentOrModel).GenerateSpeechFromTextAsync(prompt, GetGeneratedSpeechVoice(audioExecutionSettings?.Voice), options, cancellationToken)).ConfigureAwait(false); return [new AudioContent(response.Value.ToArray(), mimeType)]; } @@ -64,4 +70,12 @@ private static (GeneratedSpeechFormat Format, string MimeType) GetGeneratedSpeec "PCM" => (GeneratedSpeechFormat.Pcm, "audio/l16"), _ => throw new NotSupportedException($"The format '{format}' is not supported.") }; + + private string GetModelId(AzureOpenAITextToAudioExecutionSettings executionSettings, string? modelId) + { + return + !string.IsNullOrWhiteSpace(modelId) ? modelId! : + !string.IsNullOrWhiteSpace(executionSettings.ModelId) ? executionSettings.ModelId! : + this.DeploymentOrModelName; + } } diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs index 0a391e4693c0..b7ccd39c9eaf 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs @@ -13,6 +13,7 @@ using Microsoft.SemanticKernel.Embeddings; using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.TextGeneration; +using Microsoft.SemanticKernel.TextToAudio; #pragma warning disable IDE0039 // Use local function @@ -248,6 +249,47 @@ public static IKernelBuilder AddAzureOpenAITextEmbeddingGeneration( #endregion + #region Text-to-Audio + + /// + /// Adds the Azure OpenAI text-to-audio service to the list. + /// + /// The instance to augment. + /// Azure OpenAI deployment name + /// Azure OpenAI deployment URL + /// Azure OpenAI API key + /// A local identifier for the given AI service + /// Model identifier + /// The HttpClient to use with this service. + /// The same instance as . + [Experimental("SKEXP0001")] + public static IKernelBuilder AddAzureOpenAITextToAudio( + this IKernelBuilder builder, + string deploymentName, + string endpoint, + string apiKey, + string? serviceId = null, + string? modelId = null, + HttpClient? httpClient = null) + { + Verify.NotNull(builder); + Verify.NotNullOrWhiteSpace(endpoint); + Verify.NotNullOrWhiteSpace(apiKey); + + builder.Services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new AzureOpenAITextToAudioService( + deploymentName, + endpoint, + apiKey, + modelId, + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + serviceProvider.GetService())); + + return builder; + } + + #endregion + private static AzureOpenAIClient CreateAzureOpenAIClient(string endpoint, AzureKeyCredential credentials, HttpClient? httpClient) => new(new Uri(endpoint), credentials, ClientCore.GetAzureOpenAIClientOptions(httpClient)); diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs index 0c719ed8b249..7cca6c561f22 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs @@ -13,6 +13,7 @@ using Microsoft.SemanticKernel.Embeddings; using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.TextGeneration; +using Microsoft.SemanticKernel.TextToAudio; #pragma warning disable IDE0039 // Use local function @@ -234,6 +235,46 @@ public static IServiceCollection AddAzureOpenAITextEmbeddingGeneration( #endregion + #region Text-to-Audio + + /// + /// Adds the Azure OpenAI text-to-audio service to the list. + /// + /// The instance to augment. + /// Azure OpenAI deployment name + /// Azure OpenAI deployment URL + /// Azure OpenAI API key + /// A local identifier for the given AI service + /// Model identifier + /// The HttpClient to use with this service. + /// The same instance as . + [Experimental("SKEXP0001")] + public static IServiceCollection AddAzureOpenAITextToAudio( + this IServiceCollection services, + string deploymentName, + string endpoint, + string apiKey, + string? serviceId = null, + string? modelId = null, + HttpClient? httpClient = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(deploymentName); + Verify.NotNullOrWhiteSpace(endpoint); + Verify.NotNullOrWhiteSpace(apiKey); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + new AzureOpenAITextToAudioService( + deploymentName, + endpoint, + apiKey, + modelId, + HttpClientProvider.GetHttpClient(serviceProvider), + serviceProvider.GetService())); + } + + #endregion + private static AzureOpenAIClient CreateAzureOpenAIClient(string endpoint, AzureKeyCredential credentials, HttpClient? httpClient) => new(new Uri(endpoint), credentials, ClientCore.GetAzureOpenAIClientOptions(httpClient)); diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs index 5dd70f6fd38f..950f94266f67 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Azure.AI.OpenAI; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Services; using Microsoft.SemanticKernel.TextToAudio; @@ -18,9 +20,14 @@ namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; public sealed class AzureOpenAITextToAudioService : ITextToAudioService { /// - /// Azure OpenAI text-to-audio client for HTTP operations. + /// Azure OpenAI text-to-audio client. /// - private readonly AzureOpenAITextToAudioClient _client; + private readonly ClientCore _client; + + /// + /// Azure OpenAI model id. + /// + private readonly string? _modelId; /// public IReadOnlyDictionary Attributes => this._client.Attributes; @@ -47,10 +54,19 @@ public AzureOpenAITextToAudioService( HttpClient? httpClient = null, ILoggerFactory? loggerFactory = null) { - this._client = new(deploymentName, endpoint, apiKey, modelId, httpClient, loggerFactory?.CreateLogger(typeof(AzureOpenAITextToAudioService))); + var url = !string.IsNullOrWhiteSpace(httpClient?.BaseAddress?.AbsoluteUri) ? httpClient!.BaseAddress!.AbsoluteUri : endpoint; + + var options = ClientCore.GetAzureOpenAIClientOptions( + httpClient, + AzureOpenAIClientOptions.ServiceVersion.V2024_05_01_Preview); // https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#text-to-speech + + var azureOpenAIClient = new AzureOpenAIClient(new Uri(url), apiKey, options); + + this._client = new(deploymentName, azureOpenAIClient, loggerFactory?.CreateLogger(typeof(AzureOpenAITextToAudioService))); - this._client.AddAttribute(DeploymentNameKey, deploymentName); this._client.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); + + this._modelId = modelId; } /// @@ -59,5 +75,5 @@ public Task> GetAudioContentsAsync( PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default) - => this._client.GetAudioContentsAsync(text, executionSettings, cancellationToken); + => this._client.GetAudioContentsAsync(text, executionSettings, this._modelId, cancellationToken); } diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs new file mode 100644 index 000000000000..0f9cc3c43b9f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Text; + +namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; + +/// +/// Execution settings for Azure OpenAI text-to-audio request. +/// +[Experimental("SKEXP0001")] +public sealed class AzureOpenAITextToAudioExecutionSettings : PromptExecutionSettings +{ + /// + /// The voice to use when generating the audio. Supported voices are alloy, echo, fable, onyx, nova, and shimmer. + /// + [JsonPropertyName("voice")] + public string Voice + { + get => this._voice; + + set + { + this.ThrowIfFrozen(); + this._voice = value; + } + } + + /// + /// The format to audio in. Supported formats are mp3, opus, aac, and flac. + /// + [JsonPropertyName("response_format")] + public string ResponseFormat + { + get => this._responseFormat; + + set + { + this.ThrowIfFrozen(); + this._responseFormat = value; + } + } + + /// + /// The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default. + /// + [JsonPropertyName("speed")] + public float Speed + { + get => this._speed; + + set + { + this.ThrowIfFrozen(); + this._speed = value; + } + } + + /// + /// Creates an instance of class with default voice - "alloy". + /// + public AzureOpenAITextToAudioExecutionSettings() + : this(DefaultVoice) + { + } + + /// + /// Creates an instance of class. + /// + /// The voice to use when generating the audio. Supported voices are alloy, echo, fable, onyx, nova, and shimmer. + public AzureOpenAITextToAudioExecutionSettings(string voice) + { + this._voice = voice; + } + + /// + public override PromptExecutionSettings Clone() + { + return new AzureOpenAITextToAudioExecutionSettings(this.Voice) + { + ModelId = this.ModelId, + ExtensionData = this.ExtensionData is not null ? new Dictionary(this.ExtensionData) : null, + Speed = this.Speed, + ResponseFormat = this.ResponseFormat + }; + } + + /// + /// Converts to derived type. + /// + /// Instance of . + /// Instance of . + public static AzureOpenAITextToAudioExecutionSettings FromExecutionSettings(PromptExecutionSettings? executionSettings) + { + if (executionSettings is null) + { + return new AzureOpenAITextToAudioExecutionSettings(); + } + + if (executionSettings is AzureOpenAITextToAudioExecutionSettings settings) + { + return settings; + } + + var json = JsonSerializer.Serialize(executionSettings); + + var azureOpenAIExecutionSettings = JsonSerializer.Deserialize(json, JsonOptionsCache.ReadPermissive); + + if (azureOpenAIExecutionSettings is not null) + { + return azureOpenAIExecutionSettings; + } + + throw new ArgumentException($"Invalid execution settings, cannot convert to {nameof(AzureOpenAITextToAudioExecutionSettings)}", nameof(executionSettings)); + } + + #region private ================================================================================ + + private const string DefaultVoice = "alloy"; + + private float _speed = 1.0f; + private string _responseFormat = "mp3"; + private string _voice; + + #endregion +} diff --git a/dotnet/src/IntegrationTestsV2/Connectors/AzureOpenAI/AzureOpenAITextToAudioTests.cs b/dotnet/src/IntegrationTestsV2/Connectors/AzureOpenAI/AzureOpenAITextToAudioTests.cs new file mode 100644 index 000000000000..372364ff21ed --- /dev/null +++ b/dotnet/src/IntegrationTestsV2/Connectors/AzureOpenAI/AzureOpenAITextToAudioTests.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.TextToAudio; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; + +namespace SemanticKernel.IntegrationTestsV2.Connectors.AzureOpenAI; + +public sealed class AzureOpenAITextToAudioTests +{ + private readonly IConfigurationRoot _configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + [Fact] + public async Task AzureOpenAITextToAudioTestAsync() + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAITextToAudio").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + var kernel = Kernel.CreateBuilder() + .AddAzureOpenAITextToAudio( + azureOpenAIConfiguration.DeploymentName, + azureOpenAIConfiguration.Endpoint, + azureOpenAIConfiguration.ApiKey) + .Build(); + + var service = kernel.GetRequiredService(); + + // Act + var result = await service.GetAudioContentAsync("The sun rises in the east and sets in the west."); + + // Assert + var audioData = result.Data!.Value; + Assert.False(audioData.IsEmpty); + } +} From 204fdac64de11fdf8177c5731b78459bfb2245fc Mon Sep 17 00:00:00 2001 From: SergeyMenshykh Date: Fri, 5 Jul 2024 00:00:09 +0100 Subject: [PATCH 2/4] fix: improve xmls documentation comments for azure text to audio extension methods. --- .../Extensions/AzureOpenAIKernelBuilderExtensions.cs | 8 ++++---- .../Extensions/AzureOpenAIServiceCollectionExtensions.cs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs index b7ccd39c9eaf..44747cad4081 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs @@ -255,11 +255,11 @@ public static IKernelBuilder AddAzureOpenAITextEmbeddingGeneration( /// Adds the Azure OpenAI text-to-audio service to the list. /// /// The instance to augment. - /// Azure OpenAI deployment name - /// Azure OpenAI deployment URL - /// Azure OpenAI API key + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// A local identifier for the given AI service - /// Model identifier + /// Model identifier, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// The HttpClient to use with this service. /// The same instance as . [Experimental("SKEXP0001")] diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs index 7cca6c561f22..d067e68530a2 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs @@ -241,11 +241,11 @@ public static IServiceCollection AddAzureOpenAITextEmbeddingGeneration( /// Adds the Azure OpenAI text-to-audio service to the list. /// /// The instance to augment. - /// Azure OpenAI deployment name - /// Azure OpenAI deployment URL - /// Azure OpenAI API key + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// A local identifier for the given AI service - /// Model identifier + /// Model identifier, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// The HttpClient to use with this service. /// The same instance as . [Experimental("SKEXP0001")] From 6953d55f38a2726771231f28c84bd1894f2a39e8 Mon Sep 17 00:00:00 2001 From: SergeyMenshykh Date: Fri, 5 Jul 2024 14:30:20 +0100 Subject: [PATCH 3/4] fix: clean-up --- .../Extensions/AzureOpenAIKernelBuilderExtensions.cs | 2 +- .../Extensions/AzureOpenAIServiceCollectionExtensions.cs | 2 +- .../Services/AzureOpenAITextToAudioService.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs index 44747cad4081..6af8ec3e7e9c 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs @@ -252,7 +252,7 @@ public static IKernelBuilder AddAzureOpenAITextEmbeddingGeneration( #region Text-to-Audio /// - /// Adds the Azure OpenAI text-to-audio service to the list. + /// Adds the to the . /// /// The instance to augment. /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs index d067e68530a2..2bfdfb69e2c9 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs @@ -238,7 +238,7 @@ public static IServiceCollection AddAzureOpenAITextEmbeddingGeneration( #region Text-to-Audio /// - /// Adds the Azure OpenAI text-to-audio service to the list. + /// Adds the to the . /// /// The instance to augment. /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs index 950f94266f67..b688f61263b9 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToAudioService.cs @@ -38,7 +38,7 @@ public sealed class AzureOpenAITextToAudioService : ITextToAudioService public static string DeploymentNameKey => "DeploymentName"; /// - /// Creates an instance of the connector with API key auth. + /// Initializes a new instance of the class. /// /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart From 2904704c1d1671720a16da5cc67b5719610a5a7f Mon Sep 17 00:00:00 2001 From: SergeyMenshykh Date: Fri, 5 Jul 2024 16:57:53 +0100 Subject: [PATCH 4/4] fix: address PR comments --- .../Services/AzureOpenAITextToAudioServiceTests.cs | 4 ++-- .../Extensions/AzureOpenAIServiceCollectionExtensions.cs | 2 +- .../Settings/AzureOpenAITextToAudioExecutionSettings.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs index 88456a1eb848..b1f69110bf21 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToAudioServiceTests.cs @@ -162,10 +162,10 @@ public async Task GetAudioContentUsesValidBaseUrlAsync(bool useHttpClientBaseAdd if (useHttpClientBaseAddress) { - this._httpClient.BaseAddress = new Uri("http://local-endpoint"); + this._httpClient.BaseAddress = new Uri("http://local-endpoint/path"); } - var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint", "api-key", "model-id", this._httpClient); + var service = new AzureOpenAITextToAudioService("deployment-name", "https://endpoint/path", "api-key", "model-id", this._httpClient); await using var stream = new MemoryStream(expectedByteArray); this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.OK) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs index c5d908cd562e..4df5711603ab 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.cs @@ -249,7 +249,7 @@ public static IServiceCollection AddAzureOpenAITextEmbeddingGeneration( /// Model identifier, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// The HttpClient to use with this service. /// The same instance as . - [Experimental("SKEXP0001")] + [Experimental("SKEXP0010")] public static IServiceCollection AddAzureOpenAITextToAudio( this IServiceCollection services, string deploymentName, diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs index 0f9cc3c43b9f..1552d56f26ce 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Settings/AzureOpenAITextToAudioExecutionSettings.cs @@ -12,7 +12,7 @@ namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; /// /// Execution settings for Azure OpenAI text-to-audio request. /// -[Experimental("SKEXP0001")] +[Experimental("SKEXP0010")] public sealed class AzureOpenAITextToAudioExecutionSettings : PromptExecutionSettings { ///