diff --git a/dotnet/server/integration-tests/ScenarioParityTests.cs b/dotnet/server/integration-tests/ScenarioParityTests.cs new file mode 100644 index 00000000..c3b22dc7 --- /dev/null +++ b/dotnet/server/integration-tests/ScenarioParityTests.cs @@ -0,0 +1,303 @@ +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using SmooAI.SmoothOperator.Server.AspNetCore; + +namespace SmooAI.SmoothOperator.Server.IntegrationTests; + +/// +/// Scenario parity runner — the C# port of python/server/tests/test_scenario_parity.py. +/// +/// Runs every scenario in spec/conformance/scenarios/*.json through the C# server and asserts +/// the normalized protocol output matches. This is the shared corpus that holds the five native servers +/// (Rust · C# · Python · TypeScript · Go) to parity: when all five run this corpus green, the servers +/// are at protocol parity. The turn is deterministic because the engine runs on the same scripted mock +/// chat client the scenario declares (mockLlmScript) — no gateway, no flakiness. +/// +public class ScenarioParityTests +{ + private static readonly string ScenariosDir = ResolveScenariosDir(); + + public static IEnumerable Scenarios() + { + foreach (var path in Directory.EnumerateFiles(ScenariosDir, "*.json").OrderBy(p => p, StringComparer.Ordinal)) + { + yield return new object[] { Path.GetFileName(path), path }; + } + } + + [Theory] + [MemberData(nameof(Scenarios))] + public async Task ScenarioParity(string name, string path) + { + _ = name; // surfaced as the test id via MemberData + var scenario = JsonNode.Parse(await File.ReadAllTextAsync(path))!.AsObject(); + var chat = BuildMock(scenario["mockLlmScript"]?.AsArray()); + + await using var app = BuildApp(chat); + await app.StartAsync(); + using var socket = await ConnectAsync(app.GetTestServer()); + + var vars = new Dictionary(); + foreach (var step in scenario["steps"]!.AsArray()) + { + var stepObj = step!.AsObject(); + var send = Subst(stepObj["send"]!, vars); + await SendAsync(socket, send.ToJsonString()); + await MatchExpectedAsync(socket, stepObj["expect"]!.AsArray(), vars); + } + + await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "done", CancellationToken.None); + await app.StopAsync(); + } + + /// Seed the scripted mock chat client from the scenario's mockLlmScript. + private static MockChatClient BuildMock(JsonArray? script) + { + var mock = new MockChatClient(); + if (script is null) + { + return mock; + } + + foreach (var entry in script) + { + var obj = entry!.AsObject(); + var kind = obj["kind"]!.GetValue(); + switch (kind) + { + case "text": + mock.PushText(obj["text"]!.GetValue()); + break; + case "toolCall": + mock.PushToolCall( + obj["id"]?.GetValue() ?? "call-1", + obj["name"]!.GetValue(), + ParseToolArguments(obj["arguments"])); + break; + default: + throw new InvalidOperationException($"unknown mockLlmScript kind: '{kind}'"); + } + } + + return mock; + } + + /// + /// A scenario tool call's arguments is a JSON-object string (per the spec) — parse it into the + /// argument dictionary the mock expects. Tolerate an inline object too. + /// + private static IDictionary ParseToolArguments(JsonNode? arguments) + { + if (arguments is null) + { + return new Dictionary(); + } + + var obj = arguments is JsonValue value && value.TryGetValue(out var json) + ? JsonNode.Parse(json)!.AsObject() + : arguments.AsObject(); + + var result = new Dictionary(); + foreach (var (key, node) in obj) + { + result[key] = node is null ? null : JsonSerializer.Deserialize(node.ToJsonString()); + } + + return result; + } + + /// Match the outbound event stream against an ordered list of matchers (ports the Python runner). + private static async Task MatchExpectedAsync(WebSocket socket, JsonArray matchers, Dictionary vars) + { + JsonObject? pending = null; // one-event lookahead when a `repeat` matcher overruns + foreach (var matcherNode in matchers) + { + var matcher = matcherNode!.AsObject(); + var type = matcher["type"]!.GetValue(); + var repeat = matcher["repeat"]?.GetValue() ?? false; + var accumulateField = matcher["accumulate"]?.GetValue(); + var accumulated = new StringBuilder(); + + while (true) + { + var ev = pending ?? await NextEventAsync(socket); + pending = null; + + if (repeat && ev["type"]!.GetValue() != type) + { + // the repeated run ended; this event belongs to the next matcher + pending = ev; + break; + } + + Assert.Equal(type, ev["type"]!.GetValue()); + + if (matcher["status"] is JsonNode status) + { + Assert.Equal(status.GetValue(), ev["status"]!.GetValue()); + } + + if (matcher["statusGte"] is JsonNode statusGte) + { + Assert.True(ev["status"]!.GetValue() >= statusGte.GetValue(), + $"{type}: status {ev["status"]!.GetValue()} < {statusGte.GetValue()}"); + } + + if (matcher["assert"] is JsonObject asserts) + { + foreach (var (dotPath, expected) in asserts) + { + var actual = Dot(ev, dotPath); + Assert.True(JsonEquals(actual, expected), + $"{type}: {dotPath} = {actual?.ToJsonString() ?? "null"} != {expected?.ToJsonString() ?? "null"}"); + } + } + + if (matcher["capture"] is JsonObject captures) + { + foreach (var (var, dotPathNode) in captures) + { + vars[var] = Dot(ev, dotPathNode!.GetValue()); + } + } + + if (accumulateField is not null) + { + accumulated.Append(Dot(ev, accumulateField)!.GetValue()); + } + + if (!repeat) + { + break; + } + } + + if (matcher["assertAccumulated"] is JsonNode assertAccumulated) + { + Assert.Equal(assertAccumulated.GetValue(), accumulated.ToString()); + } + } + } + + /// Next protocol event, skipping non-semantic keepalive/pong frames (matches the Python runner). + private static async Task NextEventAsync(WebSocket socket) + { + while (true) + { + var ev = await ReceiveAsync(socket); + var type = ev["type"]?.GetValue(); + if (type is not ("keepalive" or "pong")) + { + return ev; + } + } + } + + /// Resolve a dotted path (data.data.response.responseParts) into a nested node. + private static JsonNode? Dot(JsonObject root, string path) + { + JsonNode? cur = root; + foreach (var part in path.Split('.')) + { + cur = cur!.AsObject()[part]; + } + + return cur; + } + + /// Replace {{name}} placeholders in string fields from captured vars (recursively). + private static JsonNode Subst(JsonNode value, Dictionary vars) + { + switch (value) + { + case JsonObject obj: + { + var result = new JsonObject(); + foreach (var (key, child) in obj) + { + result[key] = child is null ? null : Subst(child, vars); + } + + return result; + } + case JsonValue v when v.TryGetValue(out var s) && s.StartsWith("{{", StringComparison.Ordinal) && s.EndsWith("}}", StringComparison.Ordinal): + { + var resolved = vars[s[2..^2]]; + return resolved is null ? JsonValue.Create((string?)null)! : resolved.DeepClone(); + } + default: + return value.DeepClone(); + } + } + + /// Structural JSON equality — compares scalars and arrays/objects by their canonical text. + private static bool JsonEquals(JsonNode? a, JsonNode? b) + { + if (a is null || b is null) + { + return a is null && b is null; + } + + return JsonNode.DeepEquals(a, b); + } + + private static WebApplication BuildApp(IChatClient chat) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Services.AddSingleton(chat); + builder.Services.AddSmoothOperatorServer(); + + var app = builder.Build(); + app.MapSmoothOperatorWebSocket("/ws"); + return app; + } + + private static async Task ConnectAsync(TestServer server) + { + var client = server.CreateWebSocketClient(); + return await client.ConnectAsync(new Uri(server.BaseAddress, "ws"), CancellationToken.None); + } + + private static Task SendAsync(WebSocket socket, string json) => + socket.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, endOfMessage: true, CancellationToken.None); + + private static async Task ReceiveAsync(WebSocket socket) + { + var buffer = new byte[16 * 1024]; + using var stream = new MemoryStream(); + WebSocketReceiveResult result; + do + { + result = await socket.ReceiveAsync(buffer, CancellationToken.None); + stream.Write(buffer, 0, result.Count); + } + while (!result.EndOfMessage); + return JsonNode.Parse(Encoding.UTF8.GetString(stream.ToArray()))!.AsObject(); + } + + /// Walk up from the test assembly to the repo root and resolve the shared scenarios dir. + private static string ResolveScenariosDir() + { + var dir = AppContext.BaseDirectory; + while (dir is not null) + { + var candidate = Path.Combine(dir, "spec", "conformance", "scenarios"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + dir = Path.GetDirectoryName(dir.TrimEnd(Path.DirectorySeparatorChar)); + } + + throw new DirectoryNotFoundException("could not locate spec/conformance/scenarios from " + AppContext.BaseDirectory); + } +} diff --git a/dotnet/server/src/FrameDispatcher.cs b/dotnet/server/src/FrameDispatcher.cs index f76e3c8d..b86cfab9 100644 --- a/dotnet/server/src/FrameDispatcher.cs +++ b/dotnet/server/src/FrameDispatcher.cs @@ -119,7 +119,7 @@ private async Task HandleGetSessionAsync(JsonObject frame, string? requestId, Ac var session = await _store.GetSessionAsync(frame["sessionId"]?.GetValue() ?? string.Empty, cancellationToken).ConfigureAwait(false); if (session is null) { - sink(ProtocolEvents.Error(requestId, "NOT_FOUND", "Session not found")); + sink(ProtocolEvents.Error(requestId, "SESSION_NOT_FOUND", "Session not found")); return; } @@ -139,7 +139,7 @@ private async Task HandleSendMessageAsync(JsonObject frame, string? requestId, A var session = await _store.GetSessionAsync(frame["sessionId"]?.GetValue() ?? string.Empty, cancellationToken).ConfigureAwait(false); if (session is null) { - sink(ProtocolEvents.Error(requestId, "NOT_FOUND", "Session not found")); + sink(ProtocolEvents.Error(requestId, "SESSION_NOT_FOUND", "Session not found")); return; } diff --git a/dotnet/server/src/ProtocolEvents.cs b/dotnet/server/src/ProtocolEvents.cs index 8f0c83ef..05533666 100644 --- a/dotnet/server/src/ProtocolEvents.cs +++ b/dotnet/server/src/ProtocolEvents.cs @@ -91,10 +91,16 @@ public static JsonObject EventualResponse(string requestId, int status, string m public static JsonObject Error(string? requestId, string code, string message) { + // The {code, message} descriptor is duplicated at the envelope top level (`error`) and nested + // under `data.error`, per spec/events/error.schema.json — the top-level copy is "kept for clients + // that pattern-match on the envelope-level `error` field". Mirrors the Python reference server. + var data = new JsonObject { ["error"] = new JsonObject { ["code"] = code, ["message"] = message } }; + if (requestId is not null) data["requestId"] = requestId; var ev = new JsonObject { ["type"] = "error", - ["data"] = new JsonObject { ["error"] = new JsonObject { ["code"] = code, ["message"] = message } }, + ["error"] = new JsonObject { ["code"] = code, ["message"] = message }, + ["data"] = data, ["timestamp"] = NowMs(), }; if (requestId is not null) ev["requestId"] = requestId;