perf: add data-driven test Scenario2 to the performance runner

test/Performance/MSTest.Performance.Runner/Program.cs

@@ -112,6 +112,35 @@ private static int Pipelines(string pipelineNameFilter)
             .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results")))
             .NextStep(() => new CleanupDisposable()));
 
+        // Scenario2: data-driven tests (100 classes × 10 methods × 10 DataRow rows = 10 000 executions).
+        // Matches Scenario1's total execution count for a direct overhead comparison between
+        // the plain and data-driven execution paths.
+        pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag =>
+        Pipeline
+            .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag)
+            .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug))
+            .NextStep(() => new PlainProcess("Scenario2_DataDriven_PlainProcess.zip"))
+            .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results")))
+            .NextStep(() => new CleanupDisposable()));
+
+        // Class-level parallelism variant of the data-driven scenario.
+        pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_ClassLevel_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag =>
+        Pipeline
+            .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.ClassLevel), parametersBag)
+            .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug))
+            .NextStep(() => new PlainProcess("Scenario2_DataDriven_ClassLevel_PlainProcess.zip"))
+            .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results")))
+            .NextStep(() => new CleanupDisposable()));
+
+        // dotnet-test path for the data-driven scenario.
+        pipelineRunner.AddPipeline("Default", "Scenario2_DataDriven_DotnetTest_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag =>
+        Pipeline
+            .FirstStep(() => new Scenario2(numberOfClass: 100, methodsPerClass: 10, dataRowsPerMethod: 10, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag)
+            .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug))
+            .NextStep(() => new DotnetTestProcess("Scenario2_DataDriven_DotnetTest_PlainProcess.zip", BuildConfiguration.Debug))
+            .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results")))
+            .NextStep(() => new CleanupDisposable()));
+
         return pipelineRunner.Run(pipelineNameFilter);
     }
 }

test/Performance/MSTest.Performance.Runner/Scenarios/Scenario2.cs

@@ -0,0 +1,172 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.TestInfrastructure;
+
+namespace MSTest.Performance.Runner.Steps;
+
+/// <summary>
+/// Data-driven test scenario: each test method is decorated with multiple
+/// <c>[DataRow]</c> attributes so the test runner exercises the data-driven hot path
+/// (CloneForDataDrivenIteration, ReflectionTestMethodInfo reuse, ParameterTypes cache, …).
+///
+/// Designed to pair with Scenario1 for direct overhead comparison:
+///   Scenario1 → 100 classes × 100 methods × 1  row  = 10 000 test executions
+///   Scenario2 → 100 classes × 10  methods × 10 rows = 10 000 test executions
+/// </summary>
+internal class Scenario2 : IStep<NoInputOutput, SingleProject>
+{
+    private const string NuGetPackageExtensionName = ".nupkg";
+    private const string MSTestTestFrameworkPackageNamePrefix = "MSTest.TestFramework.";
+
+    private readonly int _numberOfClass;
+    private readonly int _methodsPerClass;
+    private readonly int _dataRowsPerMethod;
+    private readonly string _tfm;
+    private readonly ExecutionScope _executionScope;
+    private readonly int _workers;
+
+    public Scenario2(int numberOfClass, int methodsPerClass, int dataRowsPerMethod, string tfm, ExecutionScope executionScope, int workers = 0)
+    {
+        _numberOfClass = numberOfClass;
+        _methodsPerClass = methodsPerClass;
+        _dataRowsPerMethod = dataRowsPerMethod;
+        _tfm = tfm;
+        _executionScope = executionScope;
+        _workers = workers;
+    }
+
+    public string Description => "create Scenario2 (data-driven)";
+
+    public async Task<SingleProject> ExecuteAsync(NoInputOutput payload, IContext context)
+    {
+        Console.WriteLine($"Creating Scenario2 {_numberOfClass} classes, {_methodsPerClass} methods per class, {_dataRowsPerMethod} data rows per method, ExecutionScope {_executionScope} with {_workers} workers");
+
+        var cpmPropFileDoc = XDocument.Load(Path.Combine(RootFinder.Find(), "Directory.Packages.props"));
+        string microsoftNETTestSdkVersion = cpmPropFileDoc.Descendants("MicrosoftNETTestSdkVersion").Single().Value;
+        string msTestVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, MSTestTestFrameworkPackageNamePrefix);
+
+        StringBuilder stringBuilder = new();
+        for (int i = 0; i < _numberOfClass; i++)
+        {
+            stringBuilder.AppendLine(
+                CultureInfo.InvariantCulture,
+                $$"""
+
+                  [TestClass]
+                  public class UnitTest{{i}}
+                  {
+                  """);
+
+            for (int k = 1; k < _methodsPerClass + 1; k++)
+            {
+                // Emit [DataRow] attributes before [TestMethod] — same count for every method.
+                for (int d = 0; d < _dataRowsPerMethod; d++)
+                {
+                    stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"          [DataRow({d})]");
+                }
+
+                if (k % 2 == 0)
+                {
+                    stringBuilder.AppendLine(
+                        CultureInfo.InvariantCulture,
+                        $$"""
+                                  [TestMethod]
+                                  [System.Runtime.CompilerServices.MethodImplAttribute(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
+                                  public System.Threading.Tasks.Task TestMethod{{k}}(int data)
+                                  {
+                                      return System.Threading.Tasks.Task.CompletedTask;
+                                  }
+
+                          """);
+                }
+                else
+                {
+                    stringBuilder.AppendLine(
+                        CultureInfo.InvariantCulture,
+                        $$"""
+                                  [TestMethod]
+                                  [System.Runtime.CompilerServices.MethodImplAttribute(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
+                                  public void TestMethod{{k}}(int data)
+                                  {
+                                  }
+
+                          """);
+                }
+            }
+
+            stringBuilder.AppendLine("}");
+        }
+
+        TestAsset generator = await TestAsset.GenerateAssetAsync(
+            nameof(Scenario2),
+            CurrentMSTestSourceCode
+            .PatchCodeWithReplace("$TargetFramework$", $"<TargetFramework>{_tfm}</TargetFramework>")
+            .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", microsoftNETTestSdkVersion)
+            .PatchCodeWithReplace("$MSTestVersion$", msTestVersion)
+            .PatchCodeWithReplace("$EnableMSTestRunner$", "<EnableMSTestRunner>true</EnableMSTestRunner>")
+            .PatchCodeWithReplace("$OutputType$", "<OutputType>Exe</OutputType>")
+            .PatchCodeWithReplace("$Extra$", string.Empty)
+            .PatchCodeWithReplace("$Tests$", stringBuilder.ToString())
+            .PatchCodeWithReplace("$ExecutionScope$", _executionScope.ToString())
+            .PatchCodeWithReplace("$Workers$", _workers.ToString(CultureInfo.InvariantCulture)),
+            addPublicFeeds: true);
+
+        context.AddDisposable(generator);
+        return new SingleProject([_tfm], generator, nameof(Scenario2));
+    }
+
+    private static string ExtractVersionFromPackage(string rootFolder, string packagePrefixName)
+    {
+        string[] matches = Directory.GetFiles(rootFolder, packagePrefixName + "*" + NuGetPackageExtensionName, SearchOption.TopDirectoryOnly);
+
+        if (matches.Length > 1)
+        {
+            // For some packages the find pattern will match multiple packages, for example:
+            // Microsoft.Testing.Platform.1.0.0.nupkg
+            // Microsoft.Testing.Platform.Extensions.1.0.0.nupkg
+            // Let's take shortest name which should be closest to the package we are looking for.
+            matches = [matches.OrderBy(x => x.Length).First()];
+        }
+
+        if (matches.Length != 1)
+        {
+            throw new InvalidOperationException($"Was expecting to find a single NuGet package named '{packagePrefixName}' in '{rootFolder}' but found {matches.Length}.");
+        }
+
+        string packageFullName = Path.GetFileName(matches[0]);
+        return packageFullName.Substring(packagePrefixName.Length, packageFullName.Length - packagePrefixName.Length - NuGetPackageExtensionName.Length);
+    }
+
+    protected const string CurrentMSTestSourceCode = """
+#file Scenario2.csproj
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PlatformTarget>x64</PlatformTarget>
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+    $TargetFramework$
+    $OutputType$
+    $EnableMSTestRunner$
+    $Extra$
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$MicrosoftNETTestSdkVersion$" />
+    <PackageReference Include="MSTest.TestAdapter" Version="$MSTestVersion$" />
+    <PackageReference Include="MSTest.TestFramework" Version="$MSTestVersion$" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+  </ItemGroup>
+
+</Project>
+
+#file UnitTest1.cs
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+[assembly: Parallelize(Workers = $Workers$, Scope = ExecutionScope.$ExecutionScope$)]
+
+$Tests$
+""";
+}

test/Performance/MSTest.Performance.Runner/Steps/PlainProcess.cs

@@ -8,6 +8,8 @@ namespace MSTest.Performance.Runner.Steps;
 
 internal class PlainProcess : IStep<BuildArtifact, Files>
 {
+    private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true };
+
     private readonly string _reportFileName;
     private readonly int _numberOfRun;
     private readonly string _argument;
@@ -52,11 +54,9 @@ public async Task<Files> ExecuteAsync(BuildArtifact payload, IContext context)
             results.Add(result);
         }
 
-#pragma warning disable CA1869 // Cache and reuse 'JsonSerializerOptions' instances
         await File.AppendAllTextAsync(Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), JsonSerializer.Serialize(
             results,
-            new JsonSerializerOptions { WriteIndented = true }));
-#pragma warning restore CA1869 // Cache and reuse 'JsonSerializerOptions' instances
+            JsonOptions));
 
         string sample = Path.Combine(Path.GetTempPath(), _reportFileName);
         File.Delete(sample);