Suppress banner and default stdout/stderr to failed in LLM environments

src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs

@@ -3,8 +3,14 @@
 
 namespace Microsoft.Testing.Platform.Helpers;
 
-// Copy from https://github.com/dotnet/sdk/tree/eaad2a6f937b2c8d9247c53d71b57204f5d127b2/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs
-internal static class LLMEnvironmentDetector
+// Adapted from https://github.com/dotnet/sdk/tree/eaad2a6f937b2c8d9247c53d71b57204f5d127b2/src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs
+// Diverged from the upstream telemetry-only version so detection results can drive
+// user-facing platform defaults (ANSI mode, banner, --show-stdout/--show-stderr).
+// IMPORTANT: keep the environment-variable list below in sync with
+// test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs
+// (LLMEnvironmentVariables) so child processes spawned by acceptance tests can be
+// deterministically isolated from an ambient agent shell.
+internal sealed class LLMEnvironmentDetector
 {
     private static readonly EnvironmentDetectionRuleWithResult<string>[] DetectionRules =
     [
@@ -50,26 +56,33 @@ internal static class LLMEnvironmentDetector
         new EnvironmentDetectionRuleWithResult<string>("generic_agent", new BooleanEnvironmentRule("AGENT_CLI")),
     ];
 
-    private static string? LLMEnvironment { get; } = GetLLMEnvironment();
+    private readonly IEnvironment _environment;
 
-    private static string? GetLLMEnvironment()
-    {
-        string?[] results = DetectionRules.Select(r => r.GetResult()).Where(r => r != null).ToArray();
-        return results.Length > 0 ? string.Join(", ", results) : null;
-    }
+    /// <summary>
+    /// Initializes a new instance of the <see cref="LLMEnvironmentDetector"/> class.
+    /// </summary>
+    /// <param name="environment">The environment abstraction to use for reading environment variables.</param>
+    public LLMEnvironmentDetector(IEnvironment environment)
+        => _environment = environment ?? throw new ArgumentNullException(nameof(environment));
 
-    public static bool IsLLMEnvironment() => !RoslynString.IsNullOrEmpty(LLMEnvironment);
+    /// <summary>
+    /// Detects if the current environment is hosted by a known LLM/AI agent CLI.
+    /// </summary>
+    /// <returns><c>true</c> if a known LLM agent environment is detected; otherwise, <c>false</c>.</returns>
+    public bool IsLLMEnvironment()
+        => DetectionRules.Any(r => r.GetResult(_environment) is not null);
 
     /// <summary>
     /// Base class for environment detection rules that can be evaluated against environment variables.
     /// </summary>
     private abstract class EnvironmentDetectionRule
     {
         /// <summary>
-        /// Evaluates the rule against the current environment.
+        /// Evaluates the rule against the provided environment abstraction.
         /// </summary>
+        /// <param name="environment">The environment abstraction to use for reading environment variables.</param>
         /// <returns>True if the rule matches the current environment; otherwise, false.</returns>
-        public abstract bool IsMatch();
+        public abstract bool IsMatch(IEnvironment environment);
     }
 
     /// <summary>
@@ -82,10 +95,8 @@ private sealed class BooleanEnvironmentRule : EnvironmentDetectionRule
         public BooleanEnvironmentRule(params string[] variables)
             => _variables = variables ?? throw new ArgumentNullException(nameof(variables));
 
-        public override bool IsMatch()
-#pragma warning disable RS0030 // Do not use banned APIs - fine here.
-            => _variables.Any(variable => EnvironmentVariableParser.ParseBool(Environment.GetEnvironmentVariable(variable), defaultValue: false));
-#pragma warning restore RS0030 // Do not use banned APIs
+        public override bool IsMatch(IEnvironment environment)
+            => _variables.Any(variable => EnvironmentVariableParser.ParseBool(environment.GetEnvironmentVariable(variable), defaultValue: false));
     }
 
     private static class EnvironmentVariableParser
@@ -123,10 +134,8 @@ private sealed class AnyPresentEnvironmentRule : EnvironmentDetectionRule
         public AnyPresentEnvironmentRule(params string[] variables)
             => _variables = variables ?? throw new ArgumentNullException(nameof(variables));
 
-        public override bool IsMatch()
-#pragma warning disable RS0030 // Do not use banned APIs - fine here.
-            => _variables.Any(variable => !RoslynString.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)));
-#pragma warning restore RS0030 // Do not use banned APIs
+        public override bool IsMatch(IEnvironment environment)
+            => _variables.Any(variable => !RoslynString.IsNullOrEmpty(environment.GetEnvironmentVariable(variable)));
     }
 
     /// <summary>
@@ -139,8 +148,8 @@ private sealed class AnyMatchEnvironmentRule : EnvironmentDetectionRule
         public AnyMatchEnvironmentRule(params EnvironmentDetectionRule[] rules)
             => _rules = rules ?? throw new ArgumentNullException(nameof(rules));
 
-        public override bool IsMatch()
-            => _rules.Any(rule => rule.IsMatch());
+        public override bool IsMatch(IEnvironment environment)
+            => _rules.Any(rule => rule.IsMatch(environment));
     }
 
     /// <summary>
@@ -157,11 +166,9 @@ public EnvironmentVariableValueRule(string variable, string expectedValue)
             _expectedValue = expectedValue ?? throw new ArgumentNullException(nameof(expectedValue));
         }
 
-        public override bool IsMatch()
+        public override bool IsMatch(IEnvironment environment)
         {
-#pragma warning disable RS0030 // Do not use banned APIs - fine here.
-            string? value = Environment.GetEnvironmentVariable(_variable);
-#pragma warning restore RS0030 // Do not use banned APIs
+            string? value = environment.GetEnvironmentVariable(_variable);
             return !RoslynString.IsNullOrEmpty(value) && value.Equals(_expectedValue, StringComparison.OrdinalIgnoreCase);
         }
     }
@@ -184,10 +191,11 @@ public EnvironmentDetectionRuleWithResult(T result, EnvironmentDetectionRule rul
         }
 
         /// <summary>
-        /// Evaluates the rule and returns the result if matched.
+        /// Evaluates the rule against the provided environment and returns the result if matched.
         /// </summary>
+        /// <param name="environment">The environment abstraction to use for reading environment variables.</param>
         /// <returns>The result value if the rule matches; otherwise, null.</returns>
-        public T? GetResult()
-            => _rule.IsMatch() ? _result : null;
+        public T? GetResult(IEnvironment environment)
+            => _rule.IsMatch(environment) ? _result : null;
     }
 }

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Utilities.cs

@@ -171,7 +171,12 @@ private async Task DisplayBannerIfEnabledAsync(
         bool isNoBannerSet = loggingState.CommandLineParseResult.IsOptionSet(PlatformCommandLineProvider.NoBannerOptionKey);
         string? noBannerEnvironmentVar = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_NOBANNER);
         string? dotnetNoLogoEnvironmentVar = _environment.GetEnvironmentVariable(EnvironmentVariableConstants.DOTNET_NOLOGO);
-        if (!isNoBannerSet && !(noBannerEnvironmentVar is "1" or "true") && !(dotnetNoLogoEnvironmentVar is "1" or "true"))
+
+        // Skip the banner under detected LLM/AI agent environments to reduce token noise.
+        // To force the banner back on in an LLM environment, clear the LLM env var (or use a non-LLM shell).
+        bool isLLMEnvironment = new LLMEnvironmentDetector(_environment).IsLLMEnvironment();
+
+        if (!isNoBannerSet && !(noBannerEnvironmentVar is "1" or "true") && !(dotnetNoLogoEnvironmentVar is "1" or "true") && !isLLMEnvironment)
         {
             IBannerMessageOwnerCapability? bannerMessageOwnerCapability = testFrameworkCapabilities.GetCapability<IBannerMessageOwnerCapability>();
             string? bannerMessage = bannerMessageOwnerCapability is not null

src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs

@@ -164,6 +164,7 @@ await _policiesService.RegisterOnAbortCallbackAsync(
         bool effectiveNoAnsi = noAnsi && ansiOverride == AnsiOverride.None;
 
         bool inCI = new CIEnvironmentDetector(_environment).IsCIEnvironment();
+        bool isLLMEnvironment = new LLMEnvironmentDetector(_environment).IsLLMEnvironment();
 
         AnsiMode ansiMode = ansiOverride switch
         {
@@ -178,7 +179,7 @@ await _policiesService.RegisterOnAbortCallbackAsync(
             // No --ansi argument was provided, or `--ansi auto` was provided.
             // Fall back to environment-based detection.
             // In LLM environments, prefer simple text output so that the LLM can parse it easily.
-            _ when effectiveNoAnsi || LLMEnvironmentDetector.IsLLMEnvironment() => AnsiMode.NoAnsi,
+            _ when effectiveNoAnsi || isLLMEnvironment => AnsiMode.NoAnsi,
             _ when inCI => AnsiMode.SimpleAnsi,
             _ => AnsiMode.AnsiIfPossible,
         };
@@ -197,8 +198,8 @@ await _policiesService.RegisterOnAbortCallbackAsync(
             showPassed = () => true;
         }
 
-        OutputShowMode showStdout = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStdoutOption);
-        OutputShowMode showStderr = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStderrOption);
+        OutputShowMode showStdout = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStdoutOption, isLLMEnvironment);
+        OutputShowMode showStderr = GetShowOutputMode(_commandLineOptions, TerminalTestReporterCommandLineOptionsProvider.ShowStderrOption, isLLMEnvironment);
 
         Func<bool?> shouldShowProgress = noProgress || ansiMode is AnsiMode.NoAnsi or AnsiMode.SimpleAnsi
             // User preference is to not show progress.
@@ -229,15 +230,19 @@ await _policiesService.RegisterOnAbortCallbackAsync(
         });
     }
 
-    private static OutputShowMode GetShowOutputMode(ICommandLineOptions commandLineOptions, string optionName)
+    // When the option is absent, default to OutputShowMode.Failed when running under a known
+    // LLM/AI environment (less token noise for agents) and to OutputShowMode.All otherwise.
+    // An explicit --show-stdout/--show-stderr value always wins over the LLM-aware default.
+    // TODO(#8772): Update the --show-stdout/--show-stderr help text to reflect the LLM-aware default.
+    private static OutputShowMode GetShowOutputMode(ICommandLineOptions commandLineOptions, string optionName, bool isLLMEnvironment)
         => commandLineOptions.TryGetOptionArgumentList(optionName, out string[]? arguments) && arguments is { Length: > 0 }
             ? arguments[0] switch
             {
                 string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputFailedArgument.Equals(s, StringComparison.OrdinalIgnoreCase) => OutputShowMode.Failed,
                 string s when TerminalTestReporterCommandLineOptionsProvider.ShowOutputNoneArgument.Equals(s, StringComparison.OrdinalIgnoreCase) => OutputShowMode.None,
                 _ => OutputShowMode.All,
             }
-            : OutputShowMode.All;
+            : isLLMEnvironment ? OutputShowMode.Failed : OutputShowMode.All;
 
     private enum AnsiOverride
     {

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/ShowOutputOptionTests.cs

@@ -63,6 +63,40 @@ public async Task ShowStdout_Default_ShowsStandardOutputForAllTests(string tfm)
         testHostResult.AssertOutputContains("stdout from passing test");
     }
 
+    [TestMethod]
+    [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
+    public async Task ShowStdout_DefaultInLLMEnvironment_ShowsStandardOutputOnlyForFailedTests(string tfm)
+    {
+        var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm);
+        TestHostResult testHostResult = await testHost.ExecuteAsync(
+            "--output detailed --no-progress --no-ansi",
+            new Dictionary<string, string?>
+            {
+                { "CLAUDECODE", "1" },
+            },
+            cancellationToken: TestContext.CancellationToken);
+
+        testHostResult.AssertOutputContains("stdout from failing test");
+        testHostResult.AssertOutputDoesNotContain("stdout from passing test");
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
+    public async Task ShowStdout_All_InLLMEnvironment_StillShowsAllStandardOutput(string tfm)
+    {
+        var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm);
+        TestHostResult testHostResult = await testHost.ExecuteAsync(
+            "--show-stdout all --output detailed --no-progress --no-ansi",
+            new Dictionary<string, string?>
+            {
+                { "CLAUDECODE", "1" },
+            },
+            cancellationToken: TestContext.CancellationToken);
+
+        testHostResult.AssertOutputContains("stdout from failing test");
+        testHostResult.AssertOutputContains("stdout from passing test");
+    }
+
     [TestMethod]
     [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
     public async Task ShowStdout_InvalidArgument_ReturnsError(string tfm)
@@ -129,6 +163,40 @@ public async Task ShowStderr_Default_ShowsErrorOutputForAllTests(string tfm)
         testHostResult.AssertOutputContains("stderr from passing test");
     }
 
+    [TestMethod]
+    [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
+    public async Task ShowStderr_DefaultInLLMEnvironment_ShowsErrorOutputOnlyForFailedTests(string tfm)
+    {
+        var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm);
+        TestHostResult testHostResult = await testHost.ExecuteAsync(
+            "--output detailed --no-progress --no-ansi",
+            new Dictionary<string, string?>
+            {
+                { "CLAUDECODE", "1" },
+            },
+            cancellationToken: TestContext.CancellationToken);
+
+        testHostResult.AssertOutputContains("stderr from failing test");
+        testHostResult.AssertOutputDoesNotContain("stderr from passing test");
+    }
+
+    [TestMethod]
+    [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
+    public async Task ShowStderr_All_InLLMEnvironment_StillShowsAllErrorOutput(string tfm)
+    {
+        var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm);
+        TestHostResult testHostResult = await testHost.ExecuteAsync(
+            "--show-stderr all --output detailed --no-progress --no-ansi",
+            new Dictionary<string, string?>
+            {
+                { "CLAUDECODE", "1" },
+            },
+            cancellationToken: TestContext.CancellationToken);
+
+        testHostResult.AssertOutputContains("stderr from failing test");
+        testHostResult.AssertOutputContains("stderr from passing test");
+    }
+
     [TestMethod]
     [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
     public async Task ShowStderr_InvalidArgument_ReturnsError(string tfm)

test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/NoBannerTests.cs

@@ -54,6 +54,24 @@ public async Task UsingDotnetNoLogo_InTheEnvironmentVars_TheBannerDoesNotAppear(
         testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern);
     }
 
+    [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
+    [TestMethod]
+    public async Task UsingLLMEnvironmentVar_TheBannerDoesNotAppear(string tfm)
+    {
+        var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm);
+        TestHostResult testHostResult = await testHost.ExecuteAsync(
+            null,
+            new Dictionary<string, string?>
+            {
+                // CLAUDECODE matches LLMEnvironmentDetector's claude rule (AnyPresentEnvironmentRule).
+                { "CLAUDECODE", "1" },
+            },
+            cancellationToken: TestContext.CancellationToken);
+
+        testHostResult.AssertExitCodeIs(ExitCode.ZeroTests);
+        testHostResult.AssertOutputDoesNotMatchRegex(_bannerRegexMatchPattern);
+    }
+
     [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))]
     [TestMethod]
     public async Task WithoutUsingNoBanner_TheBannerAppears(string tfm)

test/Utilities/Microsoft.Testing.TestInfrastructure/WellKnownEnvironmentVariables.cs

@@ -5,6 +5,37 @@ namespace Microsoft.Testing.TestInfrastructure;
 
 public static class WellKnownEnvironmentVariables
 {
+    /// <summary>
+    /// Environment variables that the Microsoft.Testing.Platform LLM detector inspects.
+    /// Keep in sync with <c>LLMEnvironmentDetector</c>.
+    /// </summary>
+    public static readonly IReadOnlyList<string> LLMEnvironmentVariables =
+    [
+        "CLAUDECODE",
+        "CLAUDE_CODE_ENTRYPOINT",
+        "CURSOR_EDITOR",
+        "CURSOR_AI",
+        "GEMINI_CLI",
+        "GITHUB_COPILOT_CLI_MODE",
+        "GH_COPILOT_WORKING_DIRECTORY",
+        "COPILOT_CLI",
+        "CODEX_CLI",
+        "CODEX_SANDBOX",
+        "OR_APP_NAME",
+        "AMP_HOME",
+        "QWEN_CODE",
+        "DROID_CLI",
+        "OPENCODE_AI",
+        "ZED_ENVIRONMENT",
+        "ZED_TERM",
+        "KIMI_CLI",
+        "GOOSE_TERMINAL",
+        "CLINE_TASK_ID",
+        "ROO_CODE_TASK_ID",
+        "WINDSURF_SESSION",
+        "AGENT_CLI",
+    ];
+
     public static readonly string[] ToSkipEnvironmentVariables =
     [
         // Skip dotnet root, we redefine it below.
@@ -49,6 +80,13 @@ public static class WellKnownEnvironmentVariables
         "DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY",
 
         // Isolate from the skip banner in case of parent, children tests
-        "TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER"
+        "TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER",
+
+        // LLM / AI agent CLI environment variables - keep in sync with
+        // src/Platform/Microsoft.Testing.Platform/Helpers/LLMEnvironmentDetector.cs.
+        // We filter these out so acceptance tests are not affected by the ambient
+        // shell the developer (or CI) happens to be running them from. Tests that
+        // need to exercise LLM-aware behavior must set the relevant variable explicitly.
+        .. LLMEnvironmentVariables,
     ];
 }