Fix OpenTelemetryChatClient failing on unknown content types (#6915)

It should fail gracefully by ignoring any content it can't output telemetry for rather than throwing.

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs

@@ -10,6 +10,7 @@
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -216,7 +217,8 @@ public override async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseA
         }
     }
 
-    internal static string SerializeChatMessages(IEnumerable<ChatMessage> messages, ChatFinishReason? chatFinishReason = null)
+    internal static string SerializeChatMessages(
+        IEnumerable<ChatMessage> messages, ChatFinishReason? chatFinishReason = null, JsonSerializerOptions? customContentSerializerOptions = null)
     {
         List<object> output = [];
 
@@ -293,10 +295,28 @@ internal static string SerializeChatMessages(IEnumerable<ChatMessage> messages,
                         break;
 
                     default:
+                        JsonElement element = _emptyObject;
+                        try
+                        {
+                            JsonTypeInfo? unknownContentTypeInfo =
+                                customContentSerializerOptions?.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? ctsi) is true ? ctsi :
+                                _defaultOptions.TryGetTypeInfo(content.GetType(), out JsonTypeInfo? dtsi) ? dtsi :
+                                null;
+
+                            if (unknownContentTypeInfo is not null)
+                            {
+                                element = JsonSerializer.SerializeToElement(content, unknownContentTypeInfo);
+                            }
+                        }
+                        catch
+                        {
+                            // Ignore the contents of any parts that can't be serialized.
+                        }
+
                         m.Parts.Add(new OtelGenericPart
                         {
                             Type = content.GetType().FullName!,
-                            Content = content,
+                            Content = element,
                         });
                         break;
                 }
@@ -558,7 +578,7 @@ private void AddInputMessagesTags(IEnumerable<ChatMessage> messages, ChatOptions
 
             _ = activity.AddTag(
                 OpenTelemetryConsts.GenAI.Input.Messages,
-                SerializeChatMessages(messages));
+                SerializeChatMessages(messages, customContentSerializerOptions: _jsonSerializerOptions));
         }
     }
 
@@ -568,7 +588,7 @@ private void AddOutputMessagesTags(ChatResponse response, Activity? activity)
         {
             _ = activity.AddTag(
                 OpenTelemetryConsts.GenAI.Output.Messages,
-                SerializeChatMessages(response.Messages, response.FinishReason));
+                SerializeChatMessages(response.Messages, response.FinishReason, customContentSerializerOptions: _jsonSerializerOptions));
         }
     }
 
@@ -609,6 +629,7 @@ private sealed class OtelFunction
     }
 
     private static readonly JsonSerializerOptions _defaultOptions = CreateDefaultOptions();
+    private static readonly JsonElement _emptyObject = JsonSerializer.SerializeToElement(new object(), _defaultOptions.GetTypeInfo(typeof(object)));
 
     private static JsonSerializerOptions CreateDefaultOptions()
     {

test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs

@@ -329,7 +329,77 @@ async static IAsyncEnumerable<ChatResponseUpdate> CallbackAsync(
             Assert.False(tags.ContainsKey("gen_ai.system_instructions"));
             Assert.False(tags.ContainsKey("gen_ai.tool.definitions"));
         }
+    }
+
+    [Fact]
+    public async Task UnknownContentTypes_Ignored()
+    {
+        var sourceName = Guid.NewGuid().ToString();
+        var activities = new List<Activity>();
+        using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
+            .AddSource(sourceName)
+            .AddInMemoryExporter(activities)
+            .Build();
+
+        using var innerClient = new TestChatClient
+        {
+            GetResponseAsyncCallback = async (messages, options, cancellationToken) =>
+            {
+                await Task.Yield();
+                return new ChatResponse(new ChatMessage(ChatRole.Assistant, "The blue whale, I think."));
+            },
+        };
+
+        using var chatClient = innerClient
+            .AsBuilder()
+            .UseOpenTelemetry(null, sourceName, configure: instance =>
+            {
+                instance.EnableSensitiveData = true;
+                instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options;
+            })
+            .Build();
 
-        static string ReplaceWhitespace(string? input) => Regex.Replace(input ?? "", @"\s+", " ").Trim();
+        List<ChatMessage> messages =
+        [
+            new(ChatRole.User,
+            [
+                new TextContent("Hello!"),
+                new NonSerializableAIContent(),
+                new TextContent("How are you?"),
+            ]),
+        ];
+
+        var response = await chatClient.GetResponseAsync(messages);
+        Assert.NotNull(response);
+
+        var activity = Assert.Single(activities);
+        Assert.NotNull(activity);
+
+        var inputMessages = activity.Tags.First(kvp => kvp.Key == "gen_ai.input.messages").Value;
+        Assert.Equal(ReplaceWhitespace("""
+                [
+                  {
+                    "role": "user",
+                    "parts": [
+                      {
+                        "type": "text",
+                        "content": "Hello!"
+                      },
+                      {
+                        "type": "Microsoft.Extensions.AI.OpenTelemetryChatClientTests+NonSerializableAIContent",
+                        "content": {}
+                      },
+                      {
+                          "type": "text",
+                          "content": "How are you?"
+                      }
+                    ]
+                  }
+                ]
+                """), ReplaceWhitespace(inputMessages));
     }
+
+    private sealed class NonSerializableAIContent : AIContent;
+
+    private static string ReplaceWhitespace(string? input) => Regex.Replace(input ?? "", @"\s+", " ").Trim();
 }