diff --git a/docs/Changelog-Platform.md b/docs/Changelog-Platform.md index e67c15b364..67f7770869 100644 --- a/docs/Changelog-Platform.md +++ b/docs/Changelog-Platform.md @@ -48,6 +48,7 @@ See full log [of v4.2.3...v4.3.0](https://github.com/microsoft/testfx/compare/v4 * Re-print errored assemblies in dotnet test end-of-run recap by @Evangelink in [#9545](https://github.com/microsoft/testfx/pull/9545) * Add server-initiated session cancellation to the dotnet test IPC protocol by @Evangelink in [#9549](https://github.com/microsoft/testfx/pull/9549) * Emit `::warning` annotations for skipped tests in `Microsoft.Testing.Extensions.GitHubActionsReport` by @Evangelink in [#9641](https://github.com/microsoft/testfx/pull/9641) +* Let an explicit `--minimum-expected-tests N` govern the zero-tests verdict, so a run of fewer than N tests reports the minimum-expected violation (exit code 9) instead of "zero tests ran" (exit code 8) even when no tests ran. This lets a `dotnet test --test-modules` orchestrator tell a stricter local-minimum violation apart from an empty module (#7457) by @Copilot in [#9709](https://github.com/microsoft/testfx/pull/9709) ### Fixed diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 5fe1438183..a7ba314c62 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -144,19 +144,31 @@ public int GetProcessExitCode() exitCode = exitCode == ExitCode.Success && _failedTestsCount > 0 ? ExitCode.AtLeastOneTestFailed : exitCode; exitCode = exitCode == ExitCode.Success && _policiesService.IsAbortTriggered ? ExitCode.TestSessionAborted : exitCode; - // Determine whether the run should be treated as having executed zero tests. Skipped tests are excluded - // from `_totalRanTests`. Under the default `allow-skipped` policy (#9385) skipped tests count as run, so only - // a run that discovered nothing at all counts as zero tests; under `strict` an all-skipped run also counts - // as zero tests. - ZeroTestsPolicy zeroTestsPolicy = PlatformCommandLineProvider.GetZeroTestsPolicy(_commandLineOptions); - bool ranZeroTests = zeroTestsPolicy == ZeroTestsPolicy.AllowSkipped - ? _totalRanTests == 0 && _skippedTestsCount == 0 - : _totalRanTests == 0; - exitCode = exitCode == ExitCode.Success && ranZeroTests ? ExitCode.ZeroTests : exitCode; - - if (_commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, out string[]? argumentList)) + // An explicitly-provided `--minimum-expected-tests` governs the count-based verdict and + // supersedes the ZeroTests (8) verdict below: a run of fewer than N tests yields + // ExitCode.MinimumExpectedTestsPolicyViolation (9), even when zero tests ran. This lets callers + // tell an explicit-minimum violation apart from a plain "ran nothing" run (e.g. so a + // `dotnet test --test-modules` orchestrator can distinguish a stricter local minimum from an + // empty module). See issue #7457. + // A malformed value (e.g. present with no argument) is rejected earlier by option validation, but + // we still guard the parse defensively and fall back to the zero-tests verdict if it ever slips through. + if (_commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, out string[]? argumentList) + && argumentList is [string minimumExpectedTestsArgument] + && int.TryParse(minimumExpectedTestsArgument, out int minimumExpectedTests)) { - exitCode = exitCode == ExitCode.Success && _totalRanTests < int.Parse(argumentList[0], CultureInfo.InvariantCulture) ? ExitCode.MinimumExpectedTestsPolicyViolation : exitCode; + exitCode = exitCode == ExitCode.Success && _totalRanTests < minimumExpectedTests ? ExitCode.MinimumExpectedTestsPolicyViolation : exitCode; + } + else + { + // Determine whether the run should be treated as having executed zero tests. Skipped tests are excluded + // from `_totalRanTests`. Under the default `allow-skipped` policy (#9385) skipped tests count as run, so only + // a run that discovered nothing at all counts as zero tests; under `strict` an all-skipped run also counts + // as zero tests. + ZeroTestsPolicy zeroTestsPolicy = PlatformCommandLineProvider.GetZeroTestsPolicy(_commandLineOptions); + bool ranZeroTests = zeroTestsPolicy == ZeroTestsPolicy.AllowSkipped + ? _totalRanTests == 0 && _skippedTestsCount == 0 + : _totalRanTests == 0; + exitCode = exitCode == ExitCode.Success && ranZeroTests ? ExitCode.ZeroTests : exitCode; } // If the user has specified the IgnoreExitCode, then we don't want to return a non-zero exit code if the exit code matches the one specified. diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs index f4755b6142..fe14840d44 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ExecutionTests.cs @@ -172,6 +172,18 @@ public async Task Exec_WhenMinimumExpectedTestsIsNegative_ResultIsNotOk(string t testHostResult.AssertOutputContains("Option '--minimum-expected-tests' has invalid arguments: '--minimum-expected-tests' expects a single non-zero positive integer value"); } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task Exec_WhenMinimumExpectedTestsIsSpecifiedAndNoTestsRun_ResultIsMinimumExpectedTestsPolicyViolation(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + // The uid filter matches no test, so zero tests run. With an explicit minimum, the count-based + // verdict is a minimum-expected violation (exit code 9), not "zero tests ran" (exit code 8). See issue #7457. + TestHostResult testHostResult = await testHost.ExecuteAsync("--filter-uid 2 --minimum-expected-tests 3", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.MinimumExpectedTestsPolicyViolation); + } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] [TestMethod] public async Task Exec_WhenListTestsAndMinimumExpectedTestsAreSpecified_DiscoveryFails(string tfm) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 634fe727a8..4c3637f073 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -212,6 +212,23 @@ TestApplicationResult testApplicationResult Assert.AreEqual((int)ExitCode.MinimumExpectedTestsPolicyViolation, testApplicationResult.GetProcessExitCode()); } + [TestMethod] + public void GetProcessExitCodeAsync_If_MinimumExpectedTests_Set_And_No_Tests_Ran_Returns_MinimumExpectedTestsPolicyViolation() + { + // When an explicit minimum is set, a zero-test run is reported as a minimum-expected violation (9) + // rather than ZeroTests (8), so an orchestrator can tell a stricter-local-minimum violation apart + // from a plain "ran nothing" module. See issue #7457. + TestApplicationResult testApplicationResult + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), + new Mock().Object, + new Mock().Object, + null); + + Assert.AreEqual((int)ExitCode.MinimumExpectedTestsPolicyViolation, testApplicationResult.GetProcessExitCode()); + } + [TestMethod] public async Task GetProcessExitCodeAsync_OnDiscovery_No_Tests_Discovered_Returns_ZeroTests() {