chatClient use SDK

AIDevGallery/ExternalModelUtils/FoundryLocal/FoundryClient.cs

@@ -13,7 +13,7 @@ namespace AIDevGallery.ExternalModelUtils.FoundryLocal;
 
 internal class FoundryClient
 {
-    private readonly Dictionary<string, (string ServiceUrl, string ModelId)> _preparedModels = new();
+    private readonly Dictionary<string, IModel> _preparedModels = new();
     private readonly SemaphoreSlim _prepareLock = new(1, 1);
     private FoundryLocalManager? _manager;
     private ICatalog? _catalog;
@@ -130,7 +130,7 @@ await model.DownloadAsync(
     }
 
     /// <summary>
-    /// Prepares a model for use by loading it and starting the web service.
+    /// Prepares a model for use by loading it (no web service needed).
     /// Should be called after download or when first accessing a cached model.
     /// Thread-safe: multiple concurrent calls for the same alias will only prepare once.
     /// </summary>
@@ -168,18 +168,8 @@ public async Task PrepareModelAsync(string alias, CancellationToken cancellation
                 await model.LoadAsync(cancellationToken);
             }
 
-            if (_manager.Urls == null || _manager.Urls.Length == 0)
-            {
-                await _manager.StartWebServiceAsync(cancellationToken);
-            }
-
-            var serviceUrl = _manager.Urls?.FirstOrDefault();
-            if (string.IsNullOrEmpty(serviceUrl))
-            {
-                throw new InvalidOperationException("Failed to start Foundry Local web service");
-            }
-
-            _preparedModels[alias] = (serviceUrl, model.Id);
+            // Store the model directly - no web service needed
+            _preparedModels[alias] = model;
         }
         finally
         {
@@ -188,13 +178,13 @@ public async Task PrepareModelAsync(string alias, CancellationToken cancellation
     }
 
     /// <summary>
-    /// Gets the service URL and model ID for a prepared model.
+    /// Gets the prepared model.
     /// Returns null if the model hasn't been prepared yet.
     /// </summary>
-    /// <returns>A tuple containing the service URL and model ID, or null if not prepared.</returns>
-    public (string ServiceUrl, string ModelId)? GetPreparedModel(string alias)
+    /// <returns>The IModel instance, or null if not prepared.</returns>
+    public IModel? GetPreparedModel(string alias)
     {
-        return _preparedModels.TryGetValue(alias, out var info) ? info : null;
+        return _preparedModels.TryGetValue(alias, out var model) ? model : null;
     }
 
     public Task<string?> GetServiceUrl()

AIDevGallery/ExternalModelUtils/FoundryLocal/FoundryLocalChatClientAdapter.cs

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.Extensions.AI;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace AIDevGallery.ExternalModelUtils.FoundryLocal;
+
+/// <summary>
+/// Adapter that wraps FoundryLocal SDK's native OpenAIChatClient to work with Microsoft.Extensions.AI.IChatClient.
+/// Uses the SDK's direct model API (no web service) to avoid SSE compatibility issues.
+/// </summary>
+internal class FoundryLocalChatClientAdapter : IChatClient
+{
+    private readonly Microsoft.AI.Foundry.Local.OpenAIChatClient _chatClient;
+    private readonly string _modelId;
+
+    public FoundryLocalChatClientAdapter(Microsoft.AI.Foundry.Local.OpenAIChatClient chatClient, string modelId)
+    {
+        _modelId = modelId;
+        _chatClient = chatClient;
+
+        // CRITICAL: MaxTokens must be set, otherwise the model won't generate any output
+        if (_chatClient.Settings.MaxTokens == null)
+        {
+            _chatClient.Settings.MaxTokens = 512;
+        }
+
+        if (_chatClient.Settings.Temperature == null)
+        {
+            _chatClient.Settings.Temperature = 0.7f;
+        }
+    }
+
+    public ChatClientMetadata Metadata => new("FoundryLocal", new Uri($"foundrylocal:///{_modelId}"), _modelId);
+
+    public Task<ChatResponse> GetResponseAsync(
+        IEnumerable<Microsoft.Extensions.AI.ChatMessage> chatMessages,
+        ChatOptions? options = null,
+        CancellationToken cancellationToken = default) =>
+        GetStreamingResponseAsync(chatMessages, options, cancellationToken).ToChatResponseAsync(cancellationToken: cancellationToken);
+
+    public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
+        IEnumerable<Microsoft.Extensions.AI.ChatMessage> chatMessages,
+        ChatOptions? options = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        var messageList = chatMessages.ToList();
+        var openAIMessages = ConvertToFoundryMessages(messageList);
+
+        // Use FoundryLocal SDK's native streaming API - direct in-memory communication, no HTTP/SSE
+        var streamingResponse = _chatClient.CompleteChatStreamingAsync(openAIMessages, cancellationToken);
+
+        string responseId = Guid.NewGuid().ToString("N");
+        await foreach (var chunk in streamingResponse)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            if (chunk.Choices != null && chunk.Choices.Count > 0)
+            {
+                var content = chunk.Choices[0].Message?.Content;
+                if (!string.IsNullOrEmpty(content))
+                {
+                    yield return new ChatResponseUpdate(ChatRole.Assistant, content)
+                    {
+                        ResponseId = responseId
+                    };
+                }
+            }
+        }
+    }
+
+    public object? GetService(Type serviceType, object? serviceKey = null)
+    {
+        return serviceType?.IsInstanceOfType(this) == true ? this : null;
+    }
+
+    public void Dispose()
+    {
+        // ChatClient doesn't need disposal
+    }
+
+    private static List<Betalgo.Ranul.OpenAI.ObjectModels.RequestModels.ChatMessage> ConvertToFoundryMessages(IList<Microsoft.Extensions.AI.ChatMessage> messages)
+    {
+        return messages.Select(m => new Betalgo.Ranul.OpenAI.ObjectModels.RequestModels.ChatMessage
+        {
+            Role = m.Role.Value,
+            Content = m.Text ?? string.Empty
+        }).ToList();
+    }
+}

AIDevGallery/ExternalModelUtils/FoundryLocalModelProvider.cs

@@ -6,9 +6,7 @@
 using AIDevGallery.Telemetry.Events;
 using AIDevGallery.Utils;
 using Microsoft.Extensions.AI;
-using OpenAI;
 using System;
-using System.ClientModel;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -56,19 +54,18 @@ internal class FoundryLocalModelProvider : IExternalModelProvider
         }
 
         // Must be prepared beforehand via EnsureModelReadyAsync to avoid deadlock
-        var preparedInfo = _foundryManager.GetPreparedModel(alias);
-        if (preparedInfo == null)
+        var model = _foundryManager.GetPreparedModel(alias);
+        if (model == null)
         {
             throw new InvalidOperationException(
                 $"Model '{alias}' is not ready yet. The model is being loaded in the background. Please wait a moment and try again.");
         }
 
-        var (serviceUrl, modelId) = preparedInfo.Value;
+        // Get the native FoundryLocal chat client - direct SDK usage, no web service needed
+        var chatClient = model.GetChatClientAsync().Result;
 
-        return new OpenAIClient(new ApiKeyCredential("none"), new OpenAIClientOptions
-        {
-            Endpoint = new Uri($"{serviceUrl}/v1")
-        }).GetChatClient(modelId).AsIChatClient();
+        // Wrap it in our adapter to implement IChatClient interface
+        return new FoundryLocal.FoundryLocalChatClientAdapter(chatClient, model.Id);
     }
 
     public string? GetIChatClientString(string url)
@@ -80,14 +77,13 @@ internal class FoundryLocalModelProvider : IExternalModelProvider
             return null;
         }
 
-        var preparedInfo = _foundryManager.GetPreparedModel(alias);
-        if (preparedInfo == null)
+        var model = _foundryManager.GetPreparedModel(alias);
+        if (model == null)
         {
             return null;
         }
 
-        var (serviceUrl, modelId) = preparedInfo.Value;
-        return $"new OpenAIClient(new ApiKeyCredential(\"none\"), new OpenAIClientOptions{{ Endpoint = new Uri(\"{serviceUrl}/v1\") }}).GetChatClient(\"{modelId}\").AsIChatClient()";
+        return $"var model = await catalog.GetModelAsync(\"{alias}\"); await model.LoadAsync(); var chatClient = await model.GetChatClientAsync(); /* Use chatClient.CompleteChatStreamingAsync() */";
     }
 
     public async Task<IEnumerable<ModelDetails>> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default)
@@ -168,20 +164,6 @@ private async Task InitializeAsync(CancellationToken cancelationToken = default)
             if (hasCachedVariant)
             {
                 downloadedModels.Add(firstModel);
-
-                _ = Task.Run(
-                    async () =>
-                    {
-                        try
-                        {
-                            await _foundryManager.PrepareModelAsync(catalogModel.Alias, cancelationToken);
-                        }
-                        catch
-                        {
-                            // Silently fail - user will see "not ready" error when attempting to use the model
-                        }
-                    },
-                    cancelationToken);
             }
         }