diff --git a/src/Platform/Microsoft.Testing.Platform/IPC/Serializers/RegisterSerializers.cs b/src/Platform/Microsoft.Testing.Platform/IPC/Serializers/RegisterSerializers.cs index 59b204f03d..bf01c6d36a 100644 --- a/src/Platform/Microsoft.Testing.Platform/IPC/Serializers/RegisterSerializers.cs +++ b/src/Platform/Microsoft.Testing.Platform/IPC/Serializers/RegisterSerializers.cs @@ -21,6 +21,7 @@ namespace Microsoft.Testing.Platform.IPC.Serializers; * HandshakeMessageSerializer: 9 * TestInProgressMessagesSerializer: 10 * AzureDevOpsLogMessageSerializer: 11 + * DisplayMessageSerializer: 12 */ [Embedded] @@ -39,5 +40,6 @@ public static void RegisterAllSerializers(this NamedPipeBase namedPipeBase) namedPipeBase.RegisterSerializer(new HandshakeMessageSerializer(), typeof(HandshakeMessage)); namedPipeBase.RegisterSerializer(new TestInProgressMessagesSerializer(), typeof(TestInProgressMessages)); namedPipeBase.RegisterSerializer(new AzureDevOpsLogMessageSerializer(), typeof(AzureDevOpsLogMessage)); + namedPipeBase.RegisterSerializer(new DisplayMessageSerializer(), typeof(DisplayMessage)); } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/DotnetTestPassthroughOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/DotnetTestPassthroughOutputDevice.cs index 3b0900a226..0c34e63fa1 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/DotnetTestPassthroughOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/DotnetTestPassthroughOutputDevice.cs @@ -3,6 +3,7 @@ using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.IPC; using Microsoft.Testing.Platform.IPC.Models; using Microsoft.Testing.Platform.ServerMode; using Microsoft.Testing.Platform.Services; @@ -11,19 +12,30 @@ namespace Microsoft.Testing.Platform.OutputDevice; /// /// The host output device used under the dotnet test pipe protocol. Like -/// it discards regular host output (the SDK's TerminalTestReporter -/// owns user-facing rendering), but it additionally forwards lines marked with -/// to the SDK as so -/// the AzureDevOpsReport extension's logging commands (##[group], ##vso[...]) still reach the pipeline -/// log in multi-assembly runs. +/// it discards regular (informational) host output (the SDK's +/// TerminalTestReporter owns user-facing rendering), but it additionally forwards two classes of host message to +/// the SDK over the pipe so they are not swallowed in multi-assembly runs: +/// +/// Lines marked with are forwarded verbatim as +/// so the AzureDevOpsReport extension's logging commands (##[group], +/// ##vso[...]) still reach the pipeline log (protocol 1.2.0+, only on an Azure DevOps agent where the extension +/// produces them). +/// and are +/// forwarded as so host-side diagnostics produced outside test results (hang/crash +/// dump diagnostics, retry summaries, generic extension/framework warnings and errors) reach the SDK, which +/// routes them to its TerminalTestReporter's WriteWarningMessage / WriteErrorMessage (protocol 1.3.0+). +/// /// /// -/// Forwarding is gated on the SDK negotiating protocol version 1.2.0 or later -/// (); against an older SDK the marked lines -/// are swallowed exactly like the no-op device, so no unknown message id is ever sent. The -/// and the dotnet test execution id are resolved lazily because both -/// become available only after the output device is built (the connection is created and the execution -/// id env var is set during AfterCommonServiceSetupAsync). +/// Forwarding is gated on the SDK negotiating the relevant protocol version +/// ( for Azure DevOps commands, +/// for display messages); against an older +/// SDK the corresponding lines are swallowed exactly like the no-op device, so no unknown message id is ever +/// sent. Informational output (plain / +/// that is not an Azure DevOps marker) is still discarded — only warnings and errors cross the wire. The +/// and the dotnet test execution id are resolved lazily because both become +/// available only after the output device is built (the connection is created and the execution id env var is +/// set during AfterCommonServiceSetupAsync). /// internal sealed class DotnetTestPassthroughOutputDevice : IPlatformOutputDevice { @@ -38,20 +50,35 @@ public DotnetTestPassthroughOutputDevice(IServiceProvider serviceProvider) public string DisplayName => nameof(DotnetTestPassthroughOutputDevice); - public string Description => "Output device that discards host output but forwards Azure DevOps logging commands to the SDK under the dotnet test pipe protocol."; + public string Description => "Output device that discards informational host output but forwards Azure DevOps logging commands and warning/error messages to the SDK under the dotnet test pipe protocol."; // Returning false keeps this device from being registered as a data consumer, matching NopPlatformOutputDevice. public Task IsEnabledAsync() => Task.FromResult(false); public async Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDeviceData data, CancellationToken cancellationToken) { - // Preserve the deliberate pipe-protocol suppression: only Azure DevOps command lines are - // forwarded; everything else is swallowed exactly like NopPlatformOutputDevice. - if (data is not AzureDevOpsCommandOutputDeviceData commandData) + // Preserve the deliberate pipe-protocol suppression: only Azure DevOps command lines and warning/error + // messages are forwarded; everything else (informational text) is swallowed exactly like + // NopPlatformOutputDevice. The switch returns early for non-forwarded data WITHOUT touching the service + // provider, so plain informational output stays as cheap as the no-op device. + switch (data) { - return; + case AzureDevOpsCommandOutputDeviceData commandData: + await ForwardAzureDevOpsAsync(commandData, cancellationToken).ConfigureAwait(false); + break; + + case ErrorMessageOutputDeviceData errorData: + await ForwardDisplayMessageAsync(DisplayMessageLevels.Error, errorData.Message, cancellationToken).ConfigureAwait(false); + break; + + case WarningMessageOutputDeviceData warningData: + await ForwardDisplayMessageAsync(DisplayMessageLevels.Warning, warningData.Message, cancellationToken).ConfigureAwait(false); + break; } + } + private async Task ForwardAzureDevOpsAsync(AzureDevOpsCommandOutputDeviceData commandData, CancellationToken cancellationToken) + { // The dotnet test pipe protocol (DotnetTestConnection) is never active on browser, so the // browser-unsupported members below are unreachable there; suppress CA1416 accordingly. #pragma warning disable CA1416 // Validate platform compatibility @@ -61,11 +88,26 @@ public async Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDevice return; } - string? executionId = _serviceProvider.GetEnvironment().GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID); - await connection.SendMessageAsync(new AzureDevOpsLogMessage(executionId, DotnetTestConnection.InstanceId, commandData.Text)).ConfigureAwait(false); + await connection.SendMessageAsync(new AzureDevOpsLogMessage(GetExecutionId(), DotnetTestConnection.InstanceId, commandData.Text)).ConfigureAwait(false); +#pragma warning restore CA1416 // Validate platform compatibility + } + + private async Task ForwardDisplayMessageAsync(byte level, string text, CancellationToken cancellationToken) + { +#pragma warning disable CA1416 // Validate platform compatibility + if (_serviceProvider.GetService() is not DotnetTestConnection connection + || !connection.IsDisplayMessageForwardingSupported) + { + return; + } + + await connection.SendMessageAsync(new DisplayMessage(GetExecutionId(), DotnetTestConnection.InstanceId, level, text)).ConfigureAwait(false); #pragma warning restore CA1416 // Validate platform compatibility } + private string? GetExecutionId() + => _serviceProvider.GetEnvironment().GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_EXECUTIONID); + public Task DisplayBannerAsync(string? bannerMessage, CancellationToken cancellationToken) => Task.CompletedTask; public Task DisplayBeforeSessionStartAsync(CancellationToken cancellationToken) => Task.CompletedTask; diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index 4ad3c62a45..2fdaedd8d7 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -23,22 +23,22 @@ internal async Task BuildAsync(ServiceProvider serviceProvide // user-facing output, so the host must not produce console output of its own. See #7161 // and dotnet/sdk#51615 for the broader context. // - // Outside Azure DevOps there is nothing to forward, so we keep the pure no-op device. Under - // Azure DevOps the AzureDevOpsReport extension produces logging commands (##[group], - // ##vso[...]) that must still reach the pipeline log; DotnetTestPassthroughOutputDevice - // forwards those marked lines to the SDK over the protocol while discarding everything else. - // The forwarder gates on the agent only (TF_BUILD), NOT the TESTINGPLATFORM_AZDO_OUTPUT opt-out: - // that opt-out is scoped to the platform's automatic ##vso[task.logissue] emission, and honoring - // it here would make multi-assembly forwarding inconsistent with single-assembly runs (where the - // extension's output is gated on TF_BUILD alone). + // DotnetTestPassthroughOutputDevice discards informational output exactly like a no-op device, but + // forwards two classes of host message to the SDK over the protocol so they are not swallowed in + // multi-assembly runs: + // - Warning/error host messages (WarningMessageOutputDeviceData / ErrorMessageOutputDeviceData) as + // DisplayMessage (protocol 1.3.0+). This covers hang/crash dump diagnostics, retry summaries, and + // generic extension/framework warnings and errors — regardless of environment. + // - Azure DevOps logging commands (##[group], ##vso[...]) produced by the AzureDevOpsReport extension as + // AzureDevOpsLogMessage (protocol 1.2.0+). The extension only produces these on an Azure DevOps agent + // (TF_BUILD), so off-agent that branch is naturally inert; the device itself is installed everywhere so + // the warning/error forwarding above is not gated on the agent. The Azure DevOps forwarder gates on the + // agent only (TF_BUILD), NOT the TESTINGPLATFORM_AZDO_OUTPUT opt-out: that opt-out is scoped to the + // platform's automatic ##vso[task.logissue] emission, and honoring it here would make multi-assembly + // forwarding inconsistent with single-assembly runs. if (isPipeProtocol) { - IPlatformOutputDevice pipeProtocolOutputDevice = - AzureDevOpsLogIssueFormatter.IsAzureDevOpsAgent(serviceProvider.GetEnvironment()) - ? new DotnetTestPassthroughOutputDevice(serviceProvider) - : new NopPlatformOutputDevice(); - - return new ProxyOutputDevice(pipeProtocolOutputDevice, serverModeOutputDevice: null); + return new ProxyOutputDevice(new DotnetTestPassthroughOutputDevice(serviceProvider), serverModeOutputDevice: null); } // SetPlatformOutputDevice isn't public yet. Before exposing it, we should decide diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/DotnetTestConnection.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/DotnetTestConnection.cs index e3d06494d4..dcbc7595d2 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/DotnetTestConnection.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/DotnetTestConnection.cs @@ -92,6 +92,11 @@ public async Task HelpInvokedAsync() // SDK (1.0.0/1.1.0) never receives an unknown message id. public bool IsLogForwardingSupported { get; private set; } + // True once the handshake negotiated protocol version 1.3.0 or later, which is when the SDK is able to + // receive generic DisplayMessage forwards (warning/error host diagnostics). The host gates forwarding on this + // so an older SDK (<= 1.2.0) never receives an unknown message id. + public bool IsDisplayMessageForwardingSupported { get; private set; } + public async Task IsCompatibleProtocolAsync(string hostType, IReadOnlyDictionary? additionalHandshakeProperties = null) { RoslynDebug.Assert(_dotnetTestPipeClient is not null); @@ -130,9 +135,9 @@ public async Task IsCompatibleProtocolAsync(string hostType, IReadOnlyDict if (response.Properties?.TryGetValue(HandshakeMessagePropertyNames.SupportedProtocolVersions, out string? protocolVersion) is true) { bool isCompatible = IsVersionCompatible(protocolVersion, supportedProtocolVersions); - IsLogForwardingSupported = isCompatible - && Version.TryParse(protocolVersion, out Version? negotiatedVersion) - && negotiatedVersion >= new Version(1, 2, 0); + bool versionParsed = Version.TryParse(protocolVersion, out Version? negotiatedVersion); + IsLogForwardingSupported = isCompatible && versionParsed && negotiatedVersion >= new Version(1, 2, 0); + IsDisplayMessageForwardingSupported = isCompatible && versionParsed && negotiatedVersion >= new Version(1, 3, 0); return isCompatible; } @@ -174,6 +179,10 @@ public async Task SendMessageAsync(IRequest message) case AzureDevOpsLogMessage azureDevOpsLogMessage: await dotnetTestPipeClient.RequestReplyAsync(azureDevOpsLogMessage, _cancellationTokenSource.CancellationToken).ConfigureAwait(false); break; + + case DisplayMessage displayMessage: + await dotnetTestPipeClient.RequestReplyAsync(displayMessage, _cancellationTokenSource.CancellationToken).ConfigureAwait(false); + break; } } diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Constants.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Constants.cs index 908b9dbaff..700e820a8f 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Constants.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Constants.cs @@ -25,6 +25,17 @@ internal static class SessionEventTypes internal const byte TestSessionEnd = 1; } +[Embedded] +internal static class DisplayMessageLevels +{ + // The severity of a generic host display message forwarded over the pipe. The SDK maps each level + // to its TerminalTestReporter sink: Information -> WriteMessage, Warning -> WriteWarningMessage, + // Error -> WriteErrorMessage. Values must stay stable (they flow over IPC to dotnet test). + internal const byte Information = 0; + internal const byte Warning = 1; + internal const byte Error = 2; +} + [Embedded] internal static class HandshakeMessagePropertyNames { @@ -99,14 +110,23 @@ internal static class ProtocolConstants // TerminalTestReporter so they reach the pipeline log. An older SDK that only negotiates 1.1.0 never // receives the message (the host gates forwarding on the negotiated version), so it stays compatible. // - // NOTE: Under the pipe protocol the host installs a no-op output device for regular output - // regardless of the negotiated protocol version (the SDK's TerminalTestReporter owns user-facing - // output). The sole exception is when running on an Azure DevOps agent with a negotiated version of - // 1.2.0 or later: the host then installs a forwarder that still discards regular output but relays - // Azure DevOps logging commands as AzureDevOpsLogMessage (see OutputDeviceManager.BuildAsync). - // With an old SDK that only supports 1.0.0, both sides will produce no live output (the SDK - // suppresses its TerminalTestReporter to avoid colliding with the host output it expected before - // this change). Users must update to an SDK that negotiates 1.1.0 to see live output via the SDK's + // 1.3.0 adds the generic DisplayMessage: under the pipe protocol the host's forwarding output device still + // discards regular (informational) output, but relays warning/error host messages + // (WarningMessageOutputDeviceData / ErrorMessageOutputDeviceData) to the SDK as DisplayMessage so that + // host-side diagnostics produced outside test results (hang/crash dump diagnostics, retry summaries, generic + // extension/framework warnings and errors) are no longer swallowed in multi-assembly runs. The SDK routes each + // DisplayMessage to its TerminalTestReporter's WriteWarningMessage / WriteErrorMessage. Unlike the AzureDevOps + // path, this is not gated on an Azure DevOps agent. The host gates forwarding on the negotiated version, so an + // older SDK (<= 1.2.0) never receives the message. + // + // NOTE: Under the pipe protocol the host installs a forwarding output device + // (DotnetTestPassthroughOutputDevice) regardless of the negotiated protocol version (the SDK's + // TerminalTestReporter owns user-facing output). It still discards regular (informational) output but, + // depending on the negotiated version, relays: Azure DevOps logging commands as AzureDevOpsLogMessage (1.2.0+, + // only on an Azure DevOps agent) and warning/error host messages as DisplayMessage (1.3.0+, always). See + // OutputDeviceManager.BuildAsync. With an old SDK that only supports 1.0.0, both sides will produce no live + // output (the SDK suppresses its TerminalTestReporter to avoid colliding with the host output it expected + // before this change). Users must update to an SDK that negotiates 1.1.0 to see live output via the SDK's // TerminalTestReporter. - internal const string SupportedVersions = "1.0.0;1.1.0;1.2.0"; + internal const string SupportedVersions = "1.0.0;1.1.0;1.2.0;1.3.0"; } diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Models/DisplayMessage.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Models/DisplayMessage.cs new file mode 100644 index 0000000000..b02ff966cb --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Models/DisplayMessage.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.IPC.Models; + +// A generic host display message forwarded over the pipe so that warning/error diagnostics produced by the +// test host outside of test results (e.g. hang/crash dump diagnostics, retry summaries, generic +// extension/framework warnings and errors) reach the SDK even though the pipe protocol installs a forwarding +// output device that otherwise discards host output. Level is one of DisplayMessageLevels; the SDK maps it to +// its TerminalTestReporter sink (Information -> WriteMessage, Warning -> WriteWarningMessage, +// Error -> WriteErrorMessage). Introduced with protocol version 1.3.0; the host only sends it when the SDK +// negotiates 1.3.0 or later. +internal sealed record DisplayMessage(string? ExecutionId, string? InstanceId, byte Level, string? Text) : IRequest; diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/ObjectFieldIds.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/ObjectFieldIds.cs index b8fd577768..4acf89b1d6 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/ObjectFieldIds.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/ObjectFieldIds.cs @@ -186,3 +186,14 @@ internal static class AzureDevOpsLogMessageFieldsId public const ushort InstanceId = 2; public const ushort LogText = 3; } + +[Embedded] +internal static class DisplayMessageFieldsId +{ + public const int MessagesSerializerId = 12; + + public const ushort ExecutionId = 1; + public const ushort InstanceId = 2; + public const ushort Level = 3; + public const ushort Text = 4; +} diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Serializers/DisplayMessageSerializer.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Serializers/DisplayMessageSerializer.cs new file mode 100644 index 0000000000..26c2ec1abb --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/Serializers/DisplayMessageSerializer.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.IPC.Models; + +namespace Microsoft.Testing.Platform.IPC.Serializers; + +/* + |---FieldCount---| 2 bytes + + |---ExecutionId Id---| (2 bytes) + |---ExecutionId Size---| (4 bytes) + |---ExecutionId Value---| (n bytes) + + |---InstanceId Id---| (2 bytes) + |---InstanceId Size---| (4 bytes) + |---InstanceId Value---| (n bytes) + + |---Level Id---| (2 bytes) + |---Level Size---| (4 bytes) + |---Level Value---| (1 byte) + + |---Text Id---| (2 bytes) + |---Text Size---| (4 bytes) + |---Text Value---| (n bytes) +*/ + +internal sealed class DisplayMessageSerializer : NamedPipeSerializer, INamedPipeSerializer +{ + public override int Id => DisplayMessageFieldsId.MessagesSerializerId; + + protected override DisplayMessage DeserializeCore(Stream stream) + { + string? executionId = null; + string? instanceId = null; + byte level = DisplayMessageLevels.Information; + string? text = null; + + ushort fieldCount = ReadUShort(stream); + + for (int i = 0; i < fieldCount; i++) + { + ushort fieldId = ReadUShort(stream); + int fieldSize = ReadInt(stream); + + switch (fieldId) + { + case DisplayMessageFieldsId.ExecutionId: + executionId = ReadStringValue(stream, fieldSize); + break; + + case DisplayMessageFieldsId.InstanceId: + instanceId = ReadStringValue(stream, fieldSize); + break; + + case DisplayMessageFieldsId.Level: + level = ReadByte(stream); + + // Level is a single byte today, but honor the declared field size so that a future + // protocol revision that widens it (or a frame that reports a different size) does not + // leave extra bytes unread and misalign the remaining fields. + if (fieldSize > 1) + { + SetPosition(stream, stream.Position + (fieldSize - 1)); + } + + break; + + case DisplayMessageFieldsId.Text: + text = ReadStringValue(stream, fieldSize); + break; + + default: + // If we don't recognize the field id, skip the payload corresponding to that field + SetPosition(stream, stream.Position + fieldSize); + break; + } + } + + return new(executionId, instanceId, level, text); + } + + protected override void SerializeCore(DisplayMessage objectToSerialize, Stream stream) + { + RoslynDebug.Assert(stream.CanSeek, "We expect a seekable stream."); + + WriteUShort(stream, GetFieldCount(objectToSerialize)); + + WriteField(stream, DisplayMessageFieldsId.ExecutionId, objectToSerialize.ExecutionId); + WriteField(stream, DisplayMessageFieldsId.InstanceId, objectToSerialize.InstanceId); + WriteField(stream, DisplayMessageFieldsId.Level, objectToSerialize.Level); + WriteField(stream, DisplayMessageFieldsId.Text, objectToSerialize.Text); + } + + // Level is always written (it is a non-nullable byte); the two id strings and the text are optional. + private static ushort GetFieldCount(DisplayMessage message) => + (ushort)((message.ExecutionId is null ? 0 : 1) + + (message.InstanceId is null ? 0 : 1) + + 1 + + (message.Text is null ? 0 : 1)); +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs index 6136cdafa1..e685e28fa2 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs @@ -25,11 +25,11 @@ public class DotnetTestPipeBaselineTests : AcceptanceTestBase +/// Acceptance tests for forwarding generic warning/error host messages over the +/// --server dotnettestcli --dotnet-test-pipe protocol. +/// +/// Under the pipe protocol the host installs a forwarding output device that discards informational output, +/// so host-side diagnostics produced outside test results (hang/crash dump diagnostics, retry summaries, generic +/// extension/framework warnings and errors) would otherwise be swallowed in multi-assembly runs. Protocol 1.3.0 +/// adds DisplayMessage (serializer id 12): when both sides negotiate 1.3.0, the host forwards +/// WarningMessageOutputDeviceData / ErrorMessageOutputDeviceData to the SDK as a +/// DisplayMessage carrying a severity level. These tests assert that contract on the wire via the +/// black-box harness. +/// +/// +[TestClass] +public sealed class DotnetTestPipeDisplayMessageForwardingTests : AcceptanceTestBase +{ + private const string AssetName = "DotnetTestPipeDisplayMessageForwarding"; + + private const string ErrorText = "boom error from the framework"; + private const string WarningText = "careful warning from the framework"; + + // Mirrors ProtocolConstants.SupportedVersions on the host side: 1.3.0 is the version that introduced + // DisplayMessage forwarding. + private const string HostAdvertisedProtocolVersions = "1.0.0;1.1.0;1.2.0;1.3.0"; + + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + public async Task DotnetTestPipe_WhenSdkSupports130_ForwardsWarningAndErrorAsDisplayMessages() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, + supportedProtocolVersions: HostAdvertisedProtocolVersions, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual( + "1.3.0", + result.NegotiatedProtocolVersion, + "When both sides advertise 1.3.0, negotiation should select 1.3.0 to enable DisplayMessage forwarding."); + + (byte? Level, string? Text)[] forwarded = [.. + result.MessagesWithSerializerId(DotnetTestPipeProtocol.SerializerIds.DisplayMessage) + .Select(m => + { + (string? _, string? _, byte? level, string? text) = DotnetTestPipeProtocol.DecodeDisplayMessageBody(m.Body); + return (level, text); + })]; + + Assert.Contains( + m => m.Level == DotnetTestPipeProtocol.DisplayMessageLevels.Error && m.Text == ErrorText, + forwarded, + $"Expected a forwarded error DisplayMessage. Forwarded: [{string.Join(" | ", forwarded.Select(m => $"{m.Level}:{m.Text}"))}]."); + Assert.Contains( + m => m.Level == DotnetTestPipeProtocol.DisplayMessageLevels.Warning && m.Text == WarningText, + forwarded, + $"Expected a forwarded warning DisplayMessage. Forwarded: [{string.Join(" | ", forwarded.Select(m => $"{m.Level}:{m.Text}"))}]."); + + // The forwarded frames carry the same InstanceId as the handshake so the SDK can correlate the + // messages back to the originating assembly in a multi-assembly run. + Assert.IsNotNull(result.ReceivedHandshake); + Assert.IsTrue(result.ReceivedHandshake.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.InstanceId, out string? handshakeInstanceId)); + string[] forwardedInstanceIds = [.. + result.MessagesWithSerializerId(DotnetTestPipeProtocol.SerializerIds.DisplayMessage) + .Select(m => DotnetTestPipeProtocol.DecodeDisplayMessageBody(m.Body).InstanceId) + .Where(id => id is not null) + .Select(id => id!) + .Distinct()]; + string onlyInstanceId = Assert.ContainsSingle(forwardedInstanceIds); + Assert.AreEqual(handshakeInstanceId, onlyInstanceId); + } + + [TestMethod] + public async Task DotnetTestPipe_WhenSdkOnlySupports120_DoesNotForwardDisplayMessages() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + // An older SDK that does not understand DisplayMessage advertises only up to 1.2.0. + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, + supportedProtocolVersions: "1.0.0;1.1.0;1.2.0", + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual( + "1.2.0", + result.NegotiatedProtocolVersion, + "An SDK that supports up to 1.2.0 should negotiate down to 1.2.0."); + + Assert.IsEmpty( + result.MessagesWithSerializerId(DotnetTestPipeProtocol.SerializerIds.DisplayMessage), + "No DisplayMessage should be forwarded when the negotiated protocol is below 1.3.0."); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string Sources = """ +#file DotnetTestPipeDisplayMessageForwarding.csproj + + + $TargetFrameworks$ + Exe + enable + enable + true + preview + + + + + + +#file Program.cs +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework( + _ => new TestFrameworkCapabilities(), + (_, serviceProvider) => new DummyTestFramework(serviceProvider.GetOutputDevice())); + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework, IOutputDeviceDataProducer +{ + private readonly IOutputDevice _outputDevice; + + public DummyTestFramework(IOutputDevice outputDevice) => _outputDevice = outputDevice; + + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Task IsEnabledAsync() => Task.FromResult(true); + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await _outputDevice.DisplayAsync(this, new WarningMessageOutputDeviceData("careful warning from the framework"), CancellationToken.None); + await _outputDevice.DisplayAsync(this, new ErrorMessageOutputDeviceData("boom error from the framework"), CancellationToken.None); + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs index ecca67be51..f62b25f1d2 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs @@ -41,6 +41,7 @@ public static class SerializerIds public const int HandshakeMessage = 9; public const int TestInProgressMessages = 10; public const int AzureDevOpsLogMessage = 11; + public const int DisplayMessage = 12; } public static class HandshakeProperties @@ -79,6 +80,21 @@ public static class AzureDevOpsLogMessageFields public const ushort LogText = 3; } + public static class DisplayMessageFields + { + public const ushort ExecutionId = 1; + public const ushort InstanceId = 2; + public const ushort Level = 3; + public const ushort Text = 4; + } + + public static class DisplayMessageLevels + { + public const byte Information = 0; + public const byte Warning = 1; + public const byte Error = 2; + } + /// /// Computes the OS-level named pipe name from a friendly identifier. Mirrors /// NamedPipeServer.GetPipeName in testfx. @@ -267,6 +283,66 @@ public static (string? ExecutionId, string? InstanceId, string? LogText) DecodeA return (executionId, instanceId, logText); } + /// + /// Decodes the body of a frame. + /// Format: ushort fieldCount; (ushort fieldId, int fieldSize, payload)*fieldCount + /// where the id fields and the text are length-prefixed UTF-8 strings and Level is a single byte. + /// Returns null for absent string fields and null for an absent Level. + /// + public static (string? ExecutionId, string? InstanceId, byte? Level, string? Text) DecodeDisplayMessageBody(byte[] body) + { + string? executionId = null; + string? instanceId = null; + byte? level = null; + string? text = null; + + using MemoryStream stream = new(body, writable: false); + ushort fieldCount = ReadUShort(stream); + for (int i = 0; i < fieldCount; i++) + { + ushort fieldId = ReadUShort(stream); + int fieldSize = ReadInt(stream); + + switch (fieldId) + { + case DisplayMessageFields.ExecutionId: + executionId = ReadFixedSizeString(stream, fieldSize); + break; + case DisplayMessageFields.InstanceId: + instanceId = ReadFixedSizeString(stream, fieldSize); + break; + case DisplayMessageFields.Level: + // Respect the declared field size and handle truncation explicitly: only read a Level byte + // when the field actually carries one (fieldSize >= 1 and a byte is available), then skip any + // extra bytes a future wire revision may add so subsequent fields stay aligned. ReadByte + // returns -1 at end-of-stream, so guard against it rather than casting -1 to a byte. + if (fieldSize >= 1) + { + int read = stream.ReadByte(); + if (read >= 0) + { + level = (byte)read; + } + + if (fieldSize > 1) + { + stream.Seek(fieldSize - 1, SeekOrigin.Current); + } + } + + break; + case DisplayMessageFields.Text: + text = ReadFixedSizeString(stream, fieldSize); + break; + default: + stream.Seek(fieldSize, SeekOrigin.Current); + break; + } + } + + return (executionId, instanceId, level, text); + } + /// /// Decodes the tests carried by a frame, /// including the full discovery details (file path, line number, namespace, type/method name, diff --git a/test/UnitTests/Microsoft.Testing.Platform.DotnetTestProtocolContract.UnitTests/DotnetTestProtocolContractTests.cs b/test/UnitTests/Microsoft.Testing.Platform.DotnetTestProtocolContract.UnitTests/DotnetTestProtocolContractTests.cs index a49aa654bc..5a7021990d 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.DotnetTestProtocolContract.UnitTests/DotnetTestProtocolContractTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.DotnetTestProtocolContract.UnitTests/DotnetTestProtocolContractTests.cs @@ -38,6 +38,7 @@ public void SerializerIds_AreStable() [HandshakeMessageFieldsId.MessagesSerializerId] = nameof(HandshakeMessageFieldsId), [TestInProgressMessagesFieldsId.MessagesSerializerId] = nameof(TestInProgressMessagesFieldsId), [AzureDevOpsLogMessageFieldsId.MessagesSerializerId] = nameof(AzureDevOpsLogMessageFieldsId), + [DisplayMessageFieldsId.MessagesSerializerId] = nameof(DisplayMessageFieldsId), }; Assert.AreEqual(nameof(VoidResponseFieldsId), serializerIds[0]); @@ -53,6 +54,7 @@ public void SerializerIds_AreStable() Assert.AreEqual(nameof(HandshakeMessageFieldsId), serializerIds[9]); Assert.AreEqual(nameof(TestInProgressMessagesFieldsId), serializerIds[10]); Assert.AreEqual(nameof(AzureDevOpsLogMessageFieldsId), serializerIds[11]); + Assert.AreEqual(nameof(DisplayMessageFieldsId), serializerIds[12]); } [TestMethod] @@ -140,6 +142,6 @@ public void ProtocolVersion_IsStable() // Indirect through a collection so the MSTest analyzer does not flag the comparison of a compile-time // constant as "always true" (MSTEST0032). string[] versions = [ProtocolConstants.SupportedVersions]; - Assert.AreEqual("1.0.0;1.1.0;1.2.0", versions[0]); + Assert.AreEqual("1.0.0;1.1.0;1.2.0;1.3.0", versions[0]); } } diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/IPC/ProtocolTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/IPC/ProtocolTests.cs index ab8f3ee485..a23d667e7d 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/IPC/ProtocolTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/IPC/ProtocolTests.cs @@ -57,6 +57,34 @@ public void AzureDevOpsLogMessageSerializeDeserialize_WithNullOptionalFields() Assert.AreEqual("##[endgroup]", actual.LogText); } + [TestMethod] + public void DisplayMessageSerializeDeserialize() + { + object serializer = new DisplayMessageSerializer(); + var message = new DisplayMessage("MyExecId", "MyInstId", DisplayMessageLevels.Error, "Something went wrong"); + + DisplayMessage actual = RoundTrip(serializer, message); + + Assert.AreEqual(message.ExecutionId, actual.ExecutionId); + Assert.AreEqual(message.InstanceId, actual.InstanceId); + Assert.AreEqual(message.Level, actual.Level); + Assert.AreEqual(message.Text, actual.Text); + } + + [TestMethod] + public void DisplayMessageSerializeDeserialize_WithNullOptionalFields() + { + object serializer = new DisplayMessageSerializer(); + var message = new DisplayMessage(null, null, DisplayMessageLevels.Warning, null); + + DisplayMessage actual = RoundTrip(serializer, message); + + Assert.IsNull(actual.ExecutionId); + Assert.IsNull(actual.InstanceId); + Assert.AreEqual(DisplayMessageLevels.Warning, actual.Level); + Assert.IsNull(actual.Text); + } + [TestMethod] public void DiscoveredTestMessagesSerializeDeserialize() { @@ -387,6 +415,7 @@ public void SerializerIds_AreStable() [HandshakeMessageFieldsId.MessagesSerializerId] = nameof(HandshakeMessageFieldsId), [TestInProgressMessagesFieldsId.MessagesSerializerId] = nameof(TestInProgressMessagesFieldsId), [AzureDevOpsLogMessageFieldsId.MessagesSerializerId] = nameof(AzureDevOpsLogMessageFieldsId), + [DisplayMessageFieldsId.MessagesSerializerId] = nameof(DisplayMessageFieldsId), }; Assert.AreEqual(nameof(VoidResponseFieldsId), serializerIds[0]); @@ -402,6 +431,7 @@ public void SerializerIds_AreStable() Assert.AreEqual(nameof(HandshakeMessageFieldsId), serializerIds[9]); Assert.AreEqual(nameof(TestInProgressMessagesFieldsId), serializerIds[10]); Assert.AreEqual(nameof(AzureDevOpsLogMessageFieldsId), serializerIds[11]); + Assert.AreEqual(nameof(DisplayMessageFieldsId), serializerIds[12]); } // The SessionEventTypes byte values flow over IPC to dotnet test in the dotnet/sdk repository. @@ -457,6 +487,6 @@ public void ProtocolVersion_IsStable() // Indirect through a collection so the MSTest analyzer does not flag the comparison of a compile-time // constant as "always true" (MSTEST0032). string[] versions = [ProtocolConstants.SupportedVersions]; - Assert.AreEqual("1.0.0;1.1.0;1.2.0", versions[0]); + Assert.AreEqual("1.0.0;1.1.0;1.2.0;1.3.0", versions[0]); } } diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DotnetTestPassthroughOutputDeviceTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DotnetTestPassthroughOutputDeviceTests.cs index 8990396590..561af2b67d 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DotnetTestPassthroughOutputDeviceTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DotnetTestPassthroughOutputDeviceTests.cs @@ -48,4 +48,45 @@ public async Task DisplayAsync_WithMarkerButNoConnection_IsSwallowedWithoutThrow serviceProvider.Verify(p => p.GetService(typeof(IPushOnlyProtocol)), Times.Once); } + + [TestMethod] + public async Task DisplayAsync_WithWarningButNoConnection_IsSwallowedWithoutThrowing() + { + // A warning message is recognized as forwardable (so the connection is looked up), but with no dotnet + // test connection resolved there is nothing to forward to and the message is dropped. + var serviceProvider = new Mock(); + serviceProvider.Setup(p => p.GetService(typeof(IPushOnlyProtocol))).Returns(null!); + var device = new DotnetTestPassthroughOutputDevice(serviceProvider.Object); + + await device.DisplayAsync(Producer, new WarningMessageOutputDeviceData("a warning"), CancellationToken.None); + + serviceProvider.Verify(p => p.GetService(typeof(IPushOnlyProtocol)), Times.Once); + } + + [TestMethod] + public async Task DisplayAsync_WithErrorButNoConnection_IsSwallowedWithoutThrowing() + { + // An error message is recognized as forwardable (so the connection is looked up), but with no dotnet + // test connection resolved there is nothing to forward to and the message is dropped. + var serviceProvider = new Mock(); + serviceProvider.Setup(p => p.GetService(typeof(IPushOnlyProtocol))).Returns(null!); + var device = new DotnetTestPassthroughOutputDevice(serviceProvider.Object); + + await device.DisplayAsync(Producer, new ErrorMessageOutputDeviceData("an error"), CancellationToken.None); + + serviceProvider.Verify(p => p.GetService(typeof(IPushOnlyProtocol)), Times.Once); + } + + [TestMethod] + public async Task DisplayAsync_WithInformationalText_IsSwallowedWithoutResolvingTheConnection() + { + // Plain informational text (not an Azure DevOps marker, not a warning/error) must be discarded as + // early as NopPlatformOutputDevice would, without touching the service provider. + var serviceProvider = new Mock(MockBehavior.Strict); + var device = new DotnetTestPassthroughOutputDevice(serviceProvider.Object); + + await device.DisplayAsync(Producer, new TextOutputDeviceData("just some info"), CancellationToken.None); + + serviceProvider.Verify(p => p.GetService(It.IsAny()), Times.Never); + } }