diff --git a/dotnet/src/Microsoft.Agents.AI/JsonFixer.cs b/dotnet/src/Microsoft.Agents.AI/JsonFixer.cs new file mode 100644 index 0000000000..46f4da734d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/JsonFixer.cs @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.Agents.AI; + +/// +/// Provides utility methods for fixing common JSON malformations +/// that can arise when consuming JSON output from LLMs. +/// +internal static class JsonFixer +{ + /// + /// Attempts to fix common JSON malformations in the provided text. + /// + /// The raw text potentially containing JSON. + /// The repaired JSON text, or the original if no fix was needed. + /// if a fix was applied; if the text was already valid or no fix was possible. + public static bool TryFix([NotNullWhen(true)] string? text, out string? fixedText) + { + if (string.IsNullOrEmpty(text)) + { + fixedText = null; + return false; + } + + string result = text; + + bool changed = TryStripMarkdownFences(ref result) + | TryFixTrailingCommas(ref result) + | TryFixTruncatedJson(ref result) + | TryUnstringifyNestedJson(ref result); + + fixedText = changed ? result : null; + return changed; + } + + /// + /// Removes markdown code fences (e.g. ```json ... ```) from the text. + /// + public static bool TryStripMarkdownFences(ref string text) + { + const string FenceMarker = "```"; + int start = text.IndexOf(FenceMarker, StringComparison.Ordinal); + if (start < 0) + { + return false; + } + + // Find the end of the fence line (the newline after the opening fence) + int fenceEnd = text.IndexOf('\n', start); + if (fenceEnd < 0) + { + // ``` at start but no newline — treat rest as code + text = text[(start + 3)..].Trim(); + return true; + } + + int contentStart = fenceEnd + 1; + + // Find closing fence — search forward from contentStart + int closeFence = text.IndexOf(FenceMarker, contentStart, StringComparison.Ordinal); + if (closeFence >= contentStart) + { + // Extract content between fences + text = text[contentStart..closeFence].Trim(); + } + else + { + // No closing fence — treat rest as code + text = text[contentStart..].Trim(); + } + + return true; + } + + /// + /// Removes trailing commas before '}', ']', or at the end of the string. + /// Uses a state machine to track string/escape state so comma removal + /// only applies outside string literals. + /// + public static bool TryFixTrailingCommas(ref string text) + { + string original = text; + var sb = new StringBuilder(text.Length); + bool inString = false; + bool escaped = false; + + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + + if (inString) + { + sb.Append(c); + if (escaped) + { + escaped = false; + } + else if (c == '\\') + { + escaped = true; + } + else if (c == '"') + { + inString = false; + } + } + else + { + if (c == '"') + { + inString = true; + sb.Append(c); + } + else if (c == ',') + { + int j = i + 1; + while (j < text.Length && char.IsWhiteSpace(text[j])) + { + j++; + } + if (j >= text.Length || text[j] == '}' || text[j] == ']') + { + i = j - 1; + } + else + { + sb.Append(c); + } + } + else + { + sb.Append(c); + } + } + } + + text = sb.ToString(); + return text != original; + } + + /// + /// Attempts to complete a truncated JSON payload by adding missing closing brackets, + /// braces, and quotes. + /// + public static bool TryFixTruncatedJson(ref string text) + { + string original = text; + var stack = new Stack(); + bool inString = false; + bool escaped = false; + + foreach (char c in text) + { + if (inString) + { + if (escaped) + { + escaped = false; + } + else if (c == '\\') + { + escaped = true; + } + else if (c == '"') + { + inString = false; + } + } + else + { + switch (c) + { + case '{': + case '[': + stack.Push(c); + break; + case '}': + if (stack.Count > 0 && stack.Peek() == '{') + { + stack.Pop(); + } + break; + case ']': + if (stack.Count > 0 && stack.Peek() == '[') + { + stack.Pop(); + } + break; + case '"': + inString = true; + break; + } + } + } + + // Close any unclosed string + if (inString) + { + text += '"'; + } + + // Close any unclosed brackets/braces + while (stack.Count > 0) + { + text += stack.Pop() switch + { + '{' => '}', + '[' => ']', + _ => string.Empty + }; + } + + return text != original; + } + + /// + /// Detects and un-stringifies nested JSON objects that have been embedded + /// as escaped string values (e.g. "arguments": "{\"key\": \"value\"}" + /// becomes "arguments": {"key": "value"}). + /// + public static bool TryUnstringifyNestedJson(ref string text) + { + string original = text; + + // Match pattern: "propertyName": "{...}" + var inlineRegex = new Regex(@"""(\w+)""\s*:\s*""({.*?})"""); + text = inlineRegex.Replace( + text, + m => + { + string propertyName = m.Groups[1].Value; + string potentialJson = m.Groups[2].Value; + + // Unescape using proper JSON string parsing + try + { +#pragma warning disable IL2026, IL3050 // JSON deserialization of a known primitive type + potentialJson = System.Text.Json.JsonSerializer.Deserialize( + "\"" + potentialJson + "\"") ?? potentialJson; +#pragma warning restore IL2026, IL3050 + } + catch + { + // Fall back to original value if JSON parsing fails + } + + // Check if it's valid JSON + try + { + System.Text.Json.JsonDocument.Parse(potentialJson); + // It's valid JSON, so use it directly + return $"\"{propertyName}\": {potentialJson}"; + } + catch + { + // Not valid JSON, keep original + return m.Value; + } + }); + + return text != original; + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/JsonFixerTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/JsonFixerTests.cs new file mode 100644 index 0000000000..715c7e7675 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/JsonFixerTests.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable RCS1118, RCS1192 + +using Xunit; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Tests for . +/// +public class JsonFixerTests +{ + // ---------- Markdown Fence Stripping ---------- + + [Fact] + public void TryStripMarkdownFences_NoFence_ReturnsFalse() + { + string text = @"{""key"": ""value""}"; + string original = text; + bool result = JsonFixer.TryStripMarkdownFences(ref text); + Assert.False(result); + Assert.Equal(original, text); + } + + [Fact] + public void TryStripMarkdownFences_WithFence_RemovesFence() + { + string text = "```json\n{\"key\": \"value\"}\n```"; + string expected = @"{""key"": ""value""}"; + bool result = JsonFixer.TryStripMarkdownFences(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + [Fact] + public void TryStripMarkdownFences_NoClosingFence_StripsOpeningFence() + { + string text = "```json\n{\"key\": \"value\"}"; + string expected = @"{""key"": ""value""}"; + bool result = JsonFixer.TryStripMarkdownFences(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + // ---------- Trailing Comma Fixing ---------- + + [Fact] + public void TryFixTrailingCommas_NoTrailingComma_ReturnsFalse() + { + string text = @"{""a"": 1, ""b"": 2}"; + string original = text; + bool result = JsonFixer.TryFixTrailingCommas(ref text); + Assert.False(result); + Assert.Equal(original, text); + } + + [Fact] + public void TryFixTrailingCommas_BeforeClosingBrace_RemovesComma() + { + string text = @"{""a"": 1,}"; + string expected = @"{""a"": 1}"; + bool result = JsonFixer.TryFixTrailingCommas(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + [Fact] + public void TryFixTrailingCommas_BeforeClosingBracket_RemovesComma() + { + string text = @"[1, 2,]"; + string expected = @"[1, 2]"; + bool result = JsonFixer.TryFixTrailingCommas(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + // ---------- Truncated JSON Fixing ---------- + + [Fact] + public void TryFixTruncatedJson_CompleteJson_ReturnsFalse() + { + string text = @"{""a"": 1, ""b"": [1, 2, 3]}"; + string original = text; + bool result = JsonFixer.TryFixTruncatedJson(ref text); + Assert.False(result); + Assert.Equal(original, text); + } + + [Fact] + public void TryFixTruncatedJson_MissingClosingBrace_AddsIt() + { + string text = @"{""a"": 1"; + string expected = @"{""a"": 1}"; + bool result = JsonFixer.TryFixTruncatedJson(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + [Fact] + public void TryFixTruncatedJson_MissingBracketsAndBraces_AddsThem() + { + string text = @"{""a"": [1, 2"; + string expected = @"{""a"": [1, 2]}"; + bool result = JsonFixer.TryFixTruncatedJson(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + [Fact] + public void TryFixTruncatedJson_UnclosedString_ClosesIt() + { + string text = @"{""a"": ""hello"; + string expected = @"{""a"": ""hello""}"; + bool result = JsonFixer.TryFixTruncatedJson(ref text); + Assert.True(result); + Assert.Equal(expected, text); + } + + // ---------- Nested JSON Unstringifying ---------- + + [Fact] + public void TryUnstringifyNestedJson_ValidNestedJson_GetsInlined() + { + // Arrange + string text = @"{""arguments"": ""{}""}"; + + // Act + bool result = JsonFixer.TryUnstringifyNestedJson(ref text); + + // Assert + Assert.True(result); + Assert.Equal(@"{""arguments"": {}}", text); + } + + [Fact] + public void TryUnstringifyNestedJson_InvalidJsonValue_LeftUntouched() + { + // Arrange + string text = @"{""arguments"": ""not-valid-json""}"; + + // Act + bool result = JsonFixer.TryUnstringifyNestedJson(ref text); + + // Assert + Assert.False(result); + } + + // ---------- Combined TryFix ---------- + + [Fact] + public void TryFix_MarkdownFenceWithCommas_FixesBoth() + { + string text = "```json\n{\"a\": 1,}\n```"; + string expected = @"{""a"": 1}"; + bool result = JsonFixer.TryFix(text, out string? fixedText); + Assert.True(result); + Assert.NotNull(fixedText); + Assert.Equal(expected, fixedText); + } + + [Fact] + public void TryFix_TruncatedWithFence_FixesBoth() + { + string text = "```json\n{\"a\": [1, 2"; + string expected = @"{""a"": [1, 2]}"; + bool result = JsonFixer.TryFix(text, out string? fixedText); + Assert.True(result); + Assert.NotNull(fixedText); + Assert.Equal(expected, fixedText); + } + + [Fact] + public void TryFix_NullText_ReturnsFalse() + { + bool result = JsonFixer.TryFix(null, out string? fixedText); + Assert.False(result); + Assert.Null(fixedText); + } + + [Fact] + public void TryFix_EmptyText_ReturnsFalse() + { + bool result = JsonFixer.TryFix(string.Empty, out string? fixedText); + Assert.False(result); + Assert.Null(fixedText); + } +}