diff --git a/eng/verify-nupkgs.ps1 b/eng/verify-nupkgs.ps1 index be34a6851f..96f5e560f3 100644 --- a/eng/verify-nupkgs.ps1 +++ b/eng/verify-nupkgs.ps1 @@ -21,7 +21,7 @@ function Confirm-NugetPackages { $expectedNumOfFiles = @{ "MSTest.Sdk" = 15 "MSTest.TestFramework" = 105 - "MSTest.TestAdapter" = 49 + "MSTest.TestAdapter" = 53 "MSTest" = 10 "MSTest.Analyzers" = 56 } diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.NonWindows.nuspec b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.NonWindows.nuspec index 56137e1908..b894e0ad5f 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.NonWindows.nuspec +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.NonWindows.nuspec @@ -20,6 +20,7 @@ + @@ -28,6 +29,7 @@ + diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj index 5d44f04283..9efeb89de2 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj @@ -84,6 +84,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.nuspec b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.nuspec index 2d46894d6f..a62f94b4f2 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.nuspec +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.nuspec @@ -32,6 +32,7 @@ + @@ -40,6 +41,7 @@ + @@ -50,6 +52,7 @@ + @@ -60,6 +63,7 @@ + diff --git a/src/Adapter/MSTest.TestAdapter/buildTransitive/Parallelize.targets b/src/Adapter/MSTest.TestAdapter/buildTransitive/Parallelize.targets new file mode 100644 index 0000000000..c15fb82f51 --- /dev/null +++ b/src/Adapter/MSTest.TestAdapter/buildTransitive/Parallelize.targets @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + <_Parameter1>Scope = Microsoft.VisualStudio.TestTools.UnitTesting.ExecutionScope.$(MSTestParallelizeScope) + <_Parameter1_IsLiteral>true + + + + + + + <_Parameter1>Workers = $(MSTestParallelizeWorkers) + <_Parameter1_IsLiteral>true + + + + + + + <_Parameter1>Scope = Microsoft.VisualStudio.TestTools.UnitTesting.ExecutionScope.$(MSTestParallelizeScope) + <_Parameter1_IsLiteral>true + <_Parameter2>Workers = $(MSTestParallelizeWorkers) + <_Parameter2_IsLiteral>true + + + + + + + + + diff --git a/src/Adapter/MSTest.TestAdapter/buildTransitive/common/MSTest.TestAdapter.targets b/src/Adapter/MSTest.TestAdapter/buildTransitive/common/MSTest.TestAdapter.targets index d68f1f6640..4b5401a6a2 100644 --- a/src/Adapter/MSTest.TestAdapter/buildTransitive/common/MSTest.TestAdapter.targets +++ b/src/Adapter/MSTest.TestAdapter/buildTransitive/common/MSTest.TestAdapter.targets @@ -74,4 +74,6 @@ + + diff --git a/src/Adapter/MSTest.TestAdapter/buildTransitive/uwp/MSTest.TestAdapter.targets b/src/Adapter/MSTest.TestAdapter/buildTransitive/uwp/MSTest.TestAdapter.targets index be91415206..a985acd04c 100644 --- a/src/Adapter/MSTest.TestAdapter/buildTransitive/uwp/MSTest.TestAdapter.targets +++ b/src/Adapter/MSTest.TestAdapter/buildTransitive/uwp/MSTest.TestAdapter.targets @@ -51,4 +51,6 @@ + + diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 934d962192..e85040e40e 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -362,4 +362,276 @@ public async Task SettingIsTestApplicationToFalseReducesAddedExtensionsAndMakesP // It's not an executable Assert.DoesNotContain(p => p.Value == "Exe", binLog.FindChildrenRecursive(p => p.Name == "OutputType")); } + + [TestMethod] + public async Task MSTestParallelizeScope_ClassLevel_EmitsParallelizeAttribute() + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "ClassLevel")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + testHostResult.AssertOutputMatchesRegex(@"Test Parallelization enabled .* \(Workers: \d+, Scope: ClassLevel\)"); + } + + [TestMethod] + public async Task MSTestParallelizeScope_MethodLevelWithWorkers_EmitsParallelizeAttribute() + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", """ + MethodLevel + 3 + """)); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + testHostResult.AssertOutputContains("Test Parallelization enabled"); + testHostResult.AssertOutputContains("(Workers: 3, Scope: MethodLevel)"); + } + + [TestMethod] + public async Task MSTestParallelizeWorkers_Only_EmitsParallelizeAttributeWithDefaultScope() + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "2")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + + // ParallelizeAttribute defaults Scope to ClassLevel when only Workers is provided. + testHostResult.AssertOutputContains("Test Parallelization enabled"); + testHostResult.AssertOutputContains("(Workers: 2, Scope: ClassLevel)"); + } + + [TestMethod] + public async Task MSTestParallelizeWorkers_Zero_IsAcceptedAndEmitsParallelizeAttribute() + { + // Workers=0 is a boundary: the regex ^\d+$ accepts it and ParallelizeAttribute + // interprets it as "use the number of available processors". Make sure the build + // succeeds and the runtime picks it up. + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "0")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + testHostResult.AssertOutputContains("Test Parallelization enabled"); + + // 0 workers means "use Environment.ProcessorCount" so the actual reported value is the host CPU count, not 0. + testHostResult.AssertOutputMatchesRegex(@"\(Workers: \d+, Scope: ClassLevel\)"); + } + + [TestMethod] + public async Task MSTestParallelizeScope_None_EmitsDoNotParallelizeAttribute() + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "None")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + + // DoNotParallelize disables parallelization entirely. + testHostResult.AssertOutputDoesNotContain("Test Parallelization enabled"); + } + + [TestMethod] + public async Task MSTestParallelizeScope_GenerateAssemblyInfoFalse_EmitsWarning() + { + // Without GenerateAssemblyInfo, the AssemblyAttribute items are not emitted, so the + // properties would silently do nothing. Verify that the build succeeds but a warning + // is reported to make this case discoverable. + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", """ + false + ClassLevel + """)); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", warnAsError: false, cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + compilationResult.AssertOutputContains("MSTestParallelizeScope and MSTestParallelizeWorkers require GenerateAssemblyInfo to be true to take effect."); + } + + [TestMethod] + public async Task MSTestParallelizeScope_InvalidValue_FailsBuild() + { + // An invalid scope value is emitted into the generated assembly attribute and rejected by the C# compiler (CS0117). + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "NotAValidValue")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( + $"build {testAsset.TargetAssetPath}", + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(1); + compilationResult.AssertOutputContains("CS0117"); + compilationResult.AssertOutputContains("NotAValidValue"); + } + + [TestMethod] + public async Task MSTestParallelizeWorkers_NonInteger_FailsBuild() + { + // A non-integer Workers value is emitted into the generated assembly attribute and rejected by the C# compiler (CS0103). + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", "abc")); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( + $"build {testAsset.TargetAssetPath}", + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(1); + compilationResult.AssertOutputContains("CS0103"); + compilationResult.AssertOutputContains("abc"); + } + + [TestMethod] + public async Task MSTestParallelizeScope_None_With_Workers_FailsBuild() + { + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCode + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", """ + None + 2 + """)); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( + $"build {testAsset.TargetAssetPath}", + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(1); + compilationResult.AssertOutputContains("Property MSTestParallelizeWorkers cannot be set when MSTestParallelizeScope is 'None'."); + } + + [TestMethod] + public async Task MSTestParallelizeScope_VSTestRunner_BuildSucceeds() + { + // The targets emit assembly attributes via WriteCodeFragment, which is runner-agnostic. + // Make sure the targets don't interfere with VSTest projects (build only — observing the + // "Test Parallelization enabled" diagnostic is MTP-specific). + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + AssetName, + SingleTestSourceCodeVSTest + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$ExtraProperties$", """ + true + MethodLevel + 2 + """)); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + } + + // Verifies that the MSTestParallelizeScope/MSTestParallelizeWorkers properties also work for + // projects that don't use MSTest.Sdk (i.e. they reference MSTest.TestAdapter directly via the + // MSTest meta-package), since the targets logic ships in MSTest.TestAdapter. + [TestMethod] + public async Task MSTestParallelizeScope_NonSdkProject_EmitsParallelizeAttribute() + { + const string NonSdkSource = """ +#file MSTestPlain.csproj + + + $TargetFramework$ + Exe + true + x64 + $(NoWarn);NU1507 + MethodLevel + 3 + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestPlainTest +{ + [TestClass] + public class UnitTest1 + { + [TestMethod] + public void TestMethod1() + { + } + } +} +"""; + + const string PlainAssetName = "MSTestPlain"; + + using TestAsset testAsset = await TestAsset.GenerateAssetAsync( + PlainAssetName, + NonSdkSource + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent)); + + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); + compilationResult.AssertExitCodeIs(0); + + var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, PlainAssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + testHostResult.AssertOutputContainsSummary(0, 1, 0); + testHostResult.AssertOutputContains("Test Parallelization enabled"); + testHostResult.AssertOutputContains("(Workers: 3, Scope: MethodLevel)"); + } }