diff --git a/test/Performance/MSTest.Performance.Runner/Program.cs b/test/Performance/MSTest.Performance.Runner/Program.cs index 84c7308305..63b35658b5 100644 --- a/test/Performance/MSTest.Performance.Runner/Program.cs +++ b/test/Performance/MSTest.Performance.Runner/Program.cs @@ -112,6 +112,35 @@ private static int Pipelines(string pipelineNameFilter) .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) .NextStep(() => new CleanupDisposable())); + // Scenario2: data-driven tests (100 classes × 10 methods × 10 DataRow rows = 10 000 executions). + // Matches Scenario1's total execution count for a direct overhead comparison between + // the plain and data-driven execution paths. + pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag => + Pipeline + .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag) + .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug)) + .NextStep(() => new PlainProcess("Scenario2_DataDriven_PlainProcess.zip")) + .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) + .NextStep(() => new CleanupDisposable())); + + // Class-level parallelism variant of the data-driven scenario. + pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_ClassLevel_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag => + Pipeline + .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.ClassLevel), parametersBag) + .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug)) + .NextStep(() => new PlainProcess("Scenario2_DataDriven_ClassLevel_PlainProcess.zip")) + .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) + .NextStep(() => new CleanupDisposable())); + + // dotnet-test path for the data-driven scenario. + pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_DotnetTest_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag => + Pipeline + .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag) + .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug)) + .NextStep(() => new DotnetTestProcess("Scenario2_DataDriven_DotnetTest_PlainProcess.zip", BuildConfiguration.Debug)) + .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) + .NextStep(() => new CleanupDisposable())); + return pipelineRunner.Run(pipelineNameFilter); } } diff --git a/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario2.cs b/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario2.cs new file mode 100644 index 0000000000..913a06ae1d --- /dev/null +++ b/test/Performance/MSTest.Performance.Runner/Scenarios/Scenario2.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.TestInfrastructure; + +namespace MSTest.Performance.Runner.Steps; + +/// +/// Data-driven test scenario: each test method is decorated with multiple +/// [DataRow] attributes so the test runner exercises the data-driven hot path +/// (CloneForDataDrivenIteration, ReflectionTestMethodInfo reuse, ParameterTypes cache, …). +/// +/// Designed to pair with Scenario1 for direct overhead comparison: +/// Scenario1 → 100 classes × 100 methods × 1 row = 10 000 test executions +/// Scenario2 → 100 classes × 10 methods × 10 rows = 10 000 test executions +/// +internal class Scenario2 : IStep +{ + private const string NuGetPackageExtensionName = ".nupkg"; + private const string MSTestTestFrameworkPackageNamePrefix = "MSTest.TestFramework."; + + private readonly int _numberOfClass; + private readonly int _methodsPerClass; + private readonly int _dataRowsPerMethod; + private readonly string _tfm; + private readonly ExecutionScope _executionScope; + private readonly int _workers; + + public Scenario2(int numberOfClass, int methodsPerClass, int dataRowsPerMethod, string tfm, ExecutionScope executionScope, int workers = 0) + { + _numberOfClass = numberOfClass; + _methodsPerClass = methodsPerClass; + _dataRowsPerMethod = dataRowsPerMethod; + _tfm = tfm; + _executionScope = executionScope; + _workers = workers; + } + + public string Description => "create Scenario2 (data-driven)"; + + public async Task ExecuteAsync(NoInputOutput payload, IContext context) + { + Console.WriteLine($"Creating Scenario2 {_numberOfClass} classes, {_methodsPerClass} methods per class, {_dataRowsPerMethod} data rows per method, ExecutionScope {_executionScope} with {_workers} workers"); + + var cpmPropFileDoc = XDocument.Load(Path.Combine(RootFinder.Find(), "Directory.Packages.props")); + string microsoftNETTestSdkVersion = cpmPropFileDoc.Descendants("MicrosoftNETTestSdkVersion").Single().Value; + string msTestVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, MSTestTestFrameworkPackageNamePrefix); + + StringBuilder stringBuilder = new(); + for (int i = 0; i < _numberOfClass; i++) + { + stringBuilder.AppendLine( + CultureInfo.InvariantCulture, + $$""" + + [TestClass] + public class UnitTest{{i}} + { + """); + + for (int k = 1; k < _methodsPerClass + 1; k++) + { + // Emit [DataRow] attributes before [TestMethod] — same count for every method. + for (int d = 0; d < _dataRowsPerMethod; d++) + { + stringBuilder.AppendLine(CultureInfo.InvariantCulture, $" [DataRow({d})]"); + } + + if (k % 2 == 0) + { + stringBuilder.AppendLine( + CultureInfo.InvariantCulture, + $$""" + [TestMethod] + [System.Runtime.CompilerServices.MethodImplAttribute(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + public System.Threading.Tasks.Task TestMethod{{k}}(int data) + { + return System.Threading.Tasks.Task.CompletedTask; + } + + """); + } + else + { + stringBuilder.AppendLine( + CultureInfo.InvariantCulture, + $$""" + [TestMethod] + [System.Runtime.CompilerServices.MethodImplAttribute(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + public void TestMethod{{k}}(int data) + { + } + + """); + } + } + + stringBuilder.AppendLine("}"); + } + + TestAsset generator = await TestAsset.GenerateAssetAsync( + nameof(Scenario2), + CurrentMSTestSourceCode + .PatchCodeWithReplace("$TargetFramework$", $"{_tfm}") + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", microsoftNETTestSdkVersion) + .PatchCodeWithReplace("$MSTestVersion$", msTestVersion) + .PatchCodeWithReplace("$EnableMSTestRunner$", "true") + .PatchCodeWithReplace("$OutputType$", "Exe") + .PatchCodeWithReplace("$Extra$", string.Empty) + .PatchCodeWithReplace("$Tests$", stringBuilder.ToString()) + .PatchCodeWithReplace("$ExecutionScope$", _executionScope.ToString()) + .PatchCodeWithReplace("$Workers$", _workers.ToString(CultureInfo.InvariantCulture)), + addPublicFeeds: true); + + context.AddDisposable(generator); + return new SingleProject([_tfm], generator, nameof(Scenario2)); + } + + private static string ExtractVersionFromPackage(string rootFolder, string packagePrefixName) + { + string[] matches = Directory.GetFiles(rootFolder, packagePrefixName + "*" + NuGetPackageExtensionName, SearchOption.TopDirectoryOnly); + + if (matches.Length > 1) + { + // For some packages the find pattern will match multiple packages, for example: + // Microsoft.Testing.Platform.1.0.0.nupkg + // Microsoft.Testing.Platform.Extensions.1.0.0.nupkg + // Let's take shortest name which should be closest to the package we are looking for. + matches = [matches.OrderBy(x => x.Length).First()]; + } + + if (matches.Length != 1) + { + throw new InvalidOperationException($"Was expecting to find a single NuGet package named '{packagePrefixName}' in '{rootFolder}' but found {matches.Length}."); + } + + string packageFullName = Path.GetFileName(matches[0]); + return packageFullName.Substring(packagePrefixName.Length, packageFullName.Length - packagePrefixName.Length - NuGetPackageExtensionName.Length); + } + + protected const string CurrentMSTestSourceCode = """ +#file Scenario2.csproj + + + + x64 + false + true + $TargetFramework$ + $OutputType$ + $EnableMSTestRunner$ + $Extra$ + + + + + + + + + + + +#file UnitTest1.cs + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = $Workers$, Scope = ExecutionScope.$ExecutionScope$)] + +$Tests$ +"""; +} diff --git a/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs b/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs index 691ffb6a74..fc0044eb8b 100644 --- a/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs +++ b/test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs @@ -8,6 +8,8 @@ namespace MSTest.Performance.Runner.Steps; internal class PlainProcess : IStep { + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true }; + private readonly string _reportFileName; private readonly int _numberOfRun; private readonly string _argument; @@ -52,11 +54,9 @@ public async Task ExecuteAsync(BuildArtifact payload, IContext context) results.Add(result); } -#pragma warning disable CA1869 // Cache and reuse 'JsonSerializerOptions' instances await File.AppendAllTextAsync(Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), JsonSerializer.Serialize( results, - new JsonSerializerOptions { WriteIndented = true })); -#pragma warning restore CA1869 // Cache and reuse 'JsonSerializerOptions' instances + JsonOptions)); string sample = Path.Combine(Path.GetTempPath(), _reportFileName); File.Delete(sample);