diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj b/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj index 73bba3cb28f6..4ee2a67b24e2 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Connectors.AzureOpenAI.csproj @@ -22,7 +22,9 @@ + + @@ -31,7 +33,9 @@ + + diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.AudioToText.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.AudioToText.cs new file mode 100644 index 000000000000..59c1173bd780 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/ClientCore.AudioToText.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using OpenAI.Audio; + +namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; + +/// +/// Base class for AI clients that provides common functionality for interacting with Azure OpenAI services. +/// +internal partial class ClientCore +{ + /// + /// Generates an image with the provided configuration. + /// + /// Input audio to generate the text + /// Audio-to-text execution settings for the prompt + /// The to monitor for cancellation requests. The default is . + /// Url of the generated image + internal async Task> GetTextFromAudioContentsAsync( + AudioContent input, + PromptExecutionSettings? executionSettings, + CancellationToken cancellationToken) + { + if (!input.CanRead) + { + throw new ArgumentException("The input audio content is not readable.", nameof(input)); + } + + OpenAIAudioToTextExecutionSettings audioExecutionSettings = OpenAIAudioToTextExecutionSettings.FromExecutionSettings(executionSettings)!; + AudioTranscriptionOptions? audioOptions = AudioOptionsFromExecutionSettings(audioExecutionSettings); + + Verify.ValidFilename(audioExecutionSettings?.Filename); + + using var memoryStream = new MemoryStream(input.Data!.Value.ToArray()); + + AudioTranscription responseData = (await RunRequestAsync(() => this.Client.GetAudioClient(this.ModelId).TranscribeAudioAsync(memoryStream, audioExecutionSettings?.Filename, audioOptions)).ConfigureAwait(false)).Value; + + return [new(responseData.Text, this.ModelId, metadata: GetResponseMetadata(responseData))]; + } + + /// + /// Converts to type. + /// + /// Instance of . + /// Instance of . + private static AudioTranscriptionOptions? AudioOptionsFromExecutionSettings(OpenAIAudioToTextExecutionSettings executionSettings) + => new() + { + Granularities = ConvertToAudioTimestampGranularities(executionSettings!.Granularities), + Language = executionSettings.Language, + Prompt = executionSettings.Prompt, + Temperature = executionSettings.Temperature + }; + + private static AudioTimestampGranularities ConvertToAudioTimestampGranularities(IEnumerable? granularities) + { + AudioTimestampGranularities result = AudioTimestampGranularities.Default; + + if (granularities is not null) + { + foreach (var granularity in granularities) + { + var openAIGranularity = granularity switch + { + OpenAIAudioToTextExecutionSettings.TimeStampGranularities.Word => AudioTimestampGranularities.Word, + OpenAIAudioToTextExecutionSettings.TimeStampGranularities.Segment => AudioTimestampGranularities.Segment, + _ => AudioTimestampGranularities.Default + }; + + result |= openAIGranularity; + } + } + + return result; + } + + private static Dictionary GetResponseMetadata(AudioTranscription audioTranscription) + => new(3) + { + [nameof(audioTranscription.Language)] = audioTranscription.Language, + [nameof(audioTranscription.Duration)] = audioTranscription.Duration, + [nameof(audioTranscription.Segments)] = audioTranscription.Segments + }; +} diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAIAudioToTextService.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAIAudioToTextService.cs new file mode 100644 index 000000000000..313402adce99 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAIAudioToTextService.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.OpenAI; +using Azure.Core; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AudioToText; +using Microsoft.SemanticKernel.Services; + +namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; + +/// +/// Azure OpenAI audio-to-text service. +/// +[Experimental("SKEXP0001")] +public sealed class AzureOpenAIAudioToTextService : IAudioToTextService +{ + /// Core implementation shared by Azure OpenAI services. + private readonly AzureOpenAIClientCore _core; + + /// + public IReadOnlyDictionary Attributes => this._core.Attributes; + + /// + /// Creates an instance of the with API key auth. + /// + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public AzureOpenAIAudioToTextService( + string deploymentName, + string endpoint, + string apiKey, + string? modelId = null, + HttpClient? httpClient = null, + ILoggerFactory? loggerFactory = null) + { + this._core = new(deploymentName, endpoint, apiKey, httpClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService))); + this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); + } + + /// + /// Creates an instance of the with AAD auth. + /// + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. + /// Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom for HTTP requests. + /// The to use for logging. If null, no logging will be performed. + public AzureOpenAIAudioToTextService( + string deploymentName, + string endpoint, + TokenCredential credentials, + string? modelId = null, + HttpClient? httpClient = null, + ILoggerFactory? loggerFactory = null) + { + this._core = new(deploymentName, endpoint, credentials, httpClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService))); + this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); + } + + /// + /// Creates an instance of the using the specified . + /// + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Custom . + /// Azure OpenAI model id, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// The to use for logging. If null, no logging will be performed. + public AzureOpenAIAudioToTextService( + string deploymentName, + OpenAIClient openAIClient, + string? modelId = null, + ILoggerFactory? loggerFactory = null) + { + this._core = new(deploymentName, openAIClient, loggerFactory?.CreateLogger(typeof(AzureOpenAIAudioToTextService))); + this._core.AddAttribute(AIServiceExtensions.ModelIdKey, modelId); + } + + /// + public Task> GetTextContentsAsync( + AudioContent content, + PromptExecutionSettings? executionSettings = null, + Kernel? kernel = null, + CancellationToken cancellationToken = default) + => this._core.GetTextContentFromAudioAsync(content, executionSettings, cancellationToken); +}