diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs index 86d0a10c59..f061158aaa 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs @@ -3,8 +3,14 @@ namespace Microsoft.Testing.Platform.Helpers; -// Copy from https://github.com/dotnet/sdk/tree/eaad2a6f937b2c8d9247c53d71b57204f5d127b2/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs -internal static class LLMEnvironmentDetector +// Adapted from https://github.com/dotnet/sdk/tree/eaad2a6f937b2c8d9247c53d71b57204f5d127b2/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs +// Diverged from the upstream telemetry-only version so detection results can drive +// user-facing platform defaults (ANSI mode, banner, --show-stdout/--show-stderr). +// IMPORTANT: keep the environment-variable list below in sync with +// test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs +// (LLMEnvironmentVariables) so child processes spawned by acceptance tests can be +// deterministically isolated from an ambient agent shell. +internal sealed class LLMEnvironmentDetector { private static readonly EnvironmentDetectionRuleWithResult[] DetectionRules = [ @@ -50,15 +56,21 @@ internal static class LLMEnvironmentDetector new EnvironmentDetectionRuleWithResult("generic_agent", new BooleanEnvironmentRule("AGENT_CLI")), ]; - private static string? LLMEnvironment { get; } = GetLLMEnvironment(); + private readonly IEnvironment _environment; - private static string? GetLLMEnvironment() - { - string?[] results = DetectionRules.Select(r => r.GetResult()).Where(r => r != null).ToArray(); - return results.Length > 0 ? string.Join(", ", results) : null; - } + /// + /// Initializes a new instance of the class. + /// + /// The environment abstraction to use for reading environment variables. + public LLMEnvironmentDetector(IEnvironment environment) + => _environment = environment ?? throw new ArgumentNullException(nameof(environment)); - public static bool IsLLMEnvironment() => !RoslynString.IsNullOrEmpty(LLMEnvironment); + /// + /// Detects if the current environment is hosted by a known LLM/AI agent CLI. + /// + /// true if a known LLM agent environment is detected; otherwise, false. + public bool IsLLMEnvironment() + => DetectionRules.Any(r => r.GetResult(_environment) is not null); /// /// Base class for environment detection rules that can be evaluated against environment variables. @@ -66,10 +78,11 @@ internal static class LLMEnvironmentDetector private abstract class EnvironmentDetectionRule { /// - /// Evaluates the rule against the current environment. + /// Evaluates the rule against the provided environment abstraction. /// + /// The environment abstraction to use for reading environment variables. /// True if the rule matches the current environment; otherwise, false. - public abstract bool IsMatch(); + public abstract bool IsMatch(IEnvironment environment); } /// @@ -82,10 +95,8 @@ private sealed class BooleanEnvironmentRule : EnvironmentDetectionRule public BooleanEnvironmentRule(params string[] variables) => _variables = variables ?? throw new ArgumentNullException(nameof(variables)); - public override bool IsMatch() -#pragma warning disable RS0030 // Do not use banned APIs - fine here. - => _variables.Any(variable => EnvironmentVariableParser.ParseBool(Environment.GetEnvironmentVariable(variable), defaultValue: false)); -#pragma warning restore RS0030 // Do not use banned APIs + public override bool IsMatch(IEnvironment environment) + => _variables.Any(variable => EnvironmentVariableParser.ParseBool(environment.GetEnvironmentVariable(variable), defaultValue: false)); } private static class EnvironmentVariableParser @@ -123,10 +134,8 @@ private sealed class AnyPresentEnvironmentRule : EnvironmentDetectionRule public AnyPresentEnvironmentRule(params string[] variables) => _variables = variables ?? throw new ArgumentNullException(nameof(variables)); - public override bool IsMatch() -#pragma warning disable RS0030 // Do not use banned APIs - fine here. - => _variables.Any(variable => !RoslynString.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable))); -#pragma warning restore RS0030 // Do not use banned APIs + public override bool IsMatch(IEnvironment environment) + => _variables.Any(variable => !RoslynString.IsNullOrEmpty(environment.GetEnvironmentVariable(variable))); } /// @@ -139,8 +148,8 @@ private sealed class AnyMatchEnvironmentRule : EnvironmentDetectionRule public AnyMatchEnvironmentRule(params EnvironmentDetectionRule[] rules) => _rules = rules ?? throw new ArgumentNullException(nameof(rules)); - public override bool IsMatch() - => _rules.Any(rule => rule.IsMatch()); + public override bool IsMatch(IEnvironment environment) + => _rules.Any(rule => rule.IsMatch(environment)); } /// @@ -157,11 +166,9 @@ public EnvironmentVariableValueRule(string variable, string expectedValue) _expectedValue = expectedValue ?? throw new ArgumentNullException(nameof(expectedValue)); } - public override bool IsMatch() + public override bool IsMatch(IEnvironment environment) { -#pragma warning disable RS0030 // Do not use banned APIs - fine here. - string? value = Environment.GetEnvironmentVariable(_variable); -#pragma warning restore RS0030 // Do not use banned APIs + string? value = environment.GetEnvironmentVariable(_variable); return !RoslynString.IsNullOrEmpty(value) && value.Equals(_expectedValue, StringComparison.OrdinalIgnoreCase); } } @@ -184,10 +191,11 @@ public EnvironmentDetectionRuleWithResult(T result, EnvironmentDetectionRule rul } /// - /// Evaluates the rule and returns the result if matched. + /// Evaluates the rule against the provided environment and returns the result if matched. /// + /// The environment abstraction to use for reading environment variables. /// The result value if the rule matches; otherwise, null. - public T? GetResult() - => _rule.IsMatch() ? _result : null; + public T? GetResult(IEnvironment environment) + => _rule.IsMatch(environment) ? _result : null; } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Utilities.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Utilities.cs index ddf1b96504..06d0bfb630 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Utilities.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Utilities.cs @@ -171,7 +171,12 @@ private async Task DisplayBannerIfEnabledAsync( bool isNoBannerSet = loggingState.CommandLineParseResult.IsOptionSet(PlatformCommandLineProvider.NoBannerOptionKey); string? noBannerEnvironmentVar = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_NOBANNER); string? dotnetNoLogoEnvironmentVar = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.DOTNET_NOLOGO); - if (!isNoBannerSet && !(noBannerEnvironmentVar is "1" or "true") && !(dotnetNoLogoEnvironmentVar is "1" or "true")) + + // Skip the banner under detected LLM/AI agent environments to reduce token noise. + // To force the banner back on in an LLM environment, clear the LLM env var (or use a non-LLM shell). + bool isLLMEnvironment = new LLMEnvironmentDetector(_environment).IsLLMEnvironment(); + + if (!isNoBannerSet && !(noBannerEnvironmentVar is "1" or "true") && !(dotnetNoLogoEnvironmentVar is "1" or "true") && !isLLMEnvironment) { IBannerMessageOwnerCapability? bannerMessageOwnerCapability = testFrameworkCapabilities.GetCapability(); string? bannerMessage = bannerMessageOwnerCapability is not null diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 7dd32ab6d9..39499aad51 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -164,6 +164,7 @@ await _policiesService.RegisterOnAbortCallbackAsync( bool effectiveNoAnsi = noAnsi && ansiOverride == AnsiOverride.None; bool inCI = new CIEnvironmentDetector(_environment).IsCIEnvironment(); + bool isLLMEnvironment = new LLMEnvironmentDetector(_environment).IsLLMEnvironment(); AnsiMode ansiMode = ansiOverride switch { @@ -178,7 +179,7 @@ await _policiesService.RegisterOnAbortCallbackAsync( // No --ansi argument was provided, or `--ansi auto` was provided. // Fall back to environment-based detection. // In LLM environments, prefer simple text output so that the LLM can parse it easily. - _ when effectiveNoAnsi || LLMEnvironmentDetector.IsLLMEnvironment() => AnsiMode.NoAnsi, + _ when effectiveNoAnsi || isLLMEnvironment => AnsiMode.NoAnsi, _ when inCI => AnsiMode.SimpleAnsi, _ => AnsiMode.AnsiIfPossible, }; @@ -197,8 +198,8 @@ await _policiesService.RegisterOnAbortCallbackAsync( showPassed = () => true; } - OutputShowMode showStdout = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStdoutOption); - OutputShowMode showStderr = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStderrOption); + OutputShowMode showStdout = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStdoutOption, isLLMEnvironment); + OutputShowMode showStderr = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStderrOption, isLLMEnvironment); Func shouldShowProgress = noProgress || ansiMode is AnsiMode.NoAnsi or AnsiMode.SimpleAnsi // User preference is to not show progress. @@ -229,7 +230,11 @@ await _policiesService.RegisterOnAbortCallbackAsync( }); } - private static OutputShowMode GetShowOutputMode(ICommandLineOptions commandLineOptions, string optionName) + // When the option is absent, default to OutputShowMode.Failed when running under a known + // LLM/AI environment (less token noise for agents) and to OutputShowMode.All otherwise. + // An explicit --show-stdout/--show-stderr value always wins over the LLM-aware default. + // TODO(#8772): Update the --show-stdout/--show-stderr help text to reflect the LLM-aware default. + private static OutputShowMode GetShowOutputMode(ICommandLineOptions commandLineOptions, string optionName, bool isLLMEnvironment) => commandLineOptions.TryGetOptionArgumentList(optionName, out string[]? arguments) && arguments is { Length: > 0 } ? arguments[0] switch { @@ -237,7 +242,7 @@ string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputFailedArg string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputNoneArgument.Equals(s, StringComparison.OrdinalIgnoreCase) => OutputShowMode.None, _ => OutputShowMode.All, } - : OutputShowMode.All; + : isLLMEnvironment ? OutputShowMode.Failed : OutputShowMode.All; private enum AnsiOverride { diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs index 69ec1667da..7f8835950c 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs @@ -63,6 +63,40 @@ public async Task ShowStdout_Default_ShowsStandardOutputForAllTests(string tfm) testHostResult.AssertOutputContains("stdout from passing test"); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_DefaultInLLMEnvironment_ShowsStandardOutputOnlyForFailedTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--output detailed --no-progress --no-ansi", + new Dictionary + { + { "CLAUDECODE", "1" }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stdout from failing test"); + testHostResult.AssertOutputDoesNotContain("stdout from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStdout_All_InLLMEnvironment_StillShowsAllStandardOutput(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stdout all --output detailed --no-progress --no-ansi", + new Dictionary + { + { "CLAUDECODE", "1" }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stdout from failing test"); + testHostResult.AssertOutputContains("stdout from passing test"); + } + [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task ShowStdout_InvalidArgument_ReturnsError(string tfm) @@ -129,6 +163,40 @@ public async Task ShowStderr_Default_ShowsErrorOutputForAllTests(string tfm) testHostResult.AssertOutputContains("stderr from passing test"); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_DefaultInLLMEnvironment_ShowsErrorOutputOnlyForFailedTests(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--output detailed --no-progress --no-ansi", + new Dictionary + { + { "CLAUDECODE", "1" }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stderr from failing test"); + testHostResult.AssertOutputDoesNotContain("stderr from passing test"); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task ShowStderr_All_InLLMEnvironment_StillShowsAllErrorOutput(string tfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--show-stderr all --output detailed --no-progress --no-ansi", + new Dictionary + { + { "CLAUDECODE", "1" }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertOutputContains("stderr from failing test"); + testHostResult.AssertOutputContains("stderr from passing test"); + } + [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task ShowStderr_InvalidArgument_ReturnsError(string tfm) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs index 9e4a6e2207..cd3a5f41bb 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs @@ -54,6 +54,24 @@ public async Task UsingDotnetNoLogo_InTheEnvironmentVars_TheBannerDoesNotAppear( testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern); } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task UsingLLMEnvironmentVar_TheBannerDoesNotAppear(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + null, + new Dictionary + { + // CLAUDECODE matches LLMEnvironmentDetector's claude rule (AnyPresentEnvironmentRule). + { "CLAUDECODE", "1" }, + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.ZeroTests); + testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern); + } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] [TestMethod] public async Task WithoutUsingNoBanner_TheBannerAppears(string tfm) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs index 43023227db..88138bac13 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs @@ -5,6 +5,37 @@ namespace Microsoft.Testing.TestInfrastructure; public static class WellKnownEnvironmentVariables { + /// + /// Environment variables that the Microsoft.Testing.Platform LLM detector inspects. + /// Keep in sync with LLMEnvironmentDetector. + /// + public static readonly IReadOnlyList LLMEnvironmentVariables = + [ + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CURSOR_EDITOR", + "CURSOR_AI", + "GEMINI_CLI", + "GITHUB_COPILOT_CLI_MODE", + "GH_COPILOT_WORKING_DIRECTORY", + "COPILOT_CLI", + "CODEX_CLI", + "CODEX_SANDBOX", + "OR_APP_NAME", + "AMP_HOME", + "QWEN_CODE", + "DROID_CLI", + "OPENCODE_AI", + "ZED_ENVIRONMENT", + "ZED_TERM", + "KIMI_CLI", + "GOOSE_TERMINAL", + "CLINE_TASK_ID", + "ROO_CODE_TASK_ID", + "WINDSURF_SESSION", + "AGENT_CLI", + ]; + public static readonly string[] ToSkipEnvironmentVariables = [ // Skip dotnet root, we redefine it below. @@ -49,6 +80,13 @@ public static class WellKnownEnvironmentVariables "DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY", // Isolate from the skip banner in case of parent, children tests - "TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER" + "TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER", + + // LLM / AI agent CLI environment variables - keep in sync with + // src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs. + // We filter these out so acceptance tests are not affected by the ambient + // shell the developer (or CI) happens to be running them from. Tests that + // need to exercise LLM-aware behavior must set the relevant variable explicitly. + .. LLMEnvironmentVariables, ]; }