Align CI test reports (TRX/JUnit/CTRF) across all OSes

azure-pipelines.yml

@@ -37,6 +37,21 @@ variables:
     value: False
   - name: _RunAsPublic
     value: True
+  # OS-agnostic Microsoft.Testing.Platform report/dump arguments shared by the Release unit-test
+  # ("--test-modules") commands in the Windows job (below) and the Linux/MacOS jobs
+  # (eng/pipelines/steps/test-non-windows.yml). Keeping them in a single variable guarantees every OS
+  # emits the same set of reports (TRX + CTRF + JUnit + AzDO) so the published TestResults drops stay
+  # in sync. Only the test-module glob, --results-directory, and --diagnostic-output-directory differ
+  # per OS (path separators + .exe vs .dll), so those stay inline in each script.
+  #   --report-trx / --report-trx-filename "{asm}_{tfm}.trx" — TRX per (assembly x TFM).
+  #   --report-ctrf / --report-ctrf-filename "{asm}_{tfm}.ctrf.json" — CTRF JSON per (assembly x TFM).
+  #   --report-junit / --report-junit-filename "{asm}_{tfm}.xml" — JUnit XML per (assembly x TFM).
+  #   --report-azdo — surfaces failed tests in the AzDO UI.
+  #   --report-azdo-summary — Markdown job summary uploaded to the build's Summary tab.
+  #   --hangdump --hangdump-timeout 15m — dump a module that hangs past 15 minutes.
+  # Placeholders ({asm}/{tfm}) are resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper.
+  - name: _ReleaseUnitTestReportArgs
+    value: '--report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-ctrf --report-ctrf-filename "{asm}_{tfm}.ctrf.json" --report-azdo --report-azdo-summary --report-junit --report-junit-filename "{asm}_{tfm}.xml" --hangdump --hangdump-timeout 15m'
 
 stages:
 
@@ -180,27 +195,12 @@ stages:
           #
           # --test-modules bypasses MSBuild evaluation, so test/Directory.Build.targets cannot inject
           # any of its TestingPlatformCommandLineArguments. We therefore restore the subset of those
-          # options that makes sense in Release:
-          #   --report-trx + --report-trx-filename "{asm}_{tfm}.trx" — TRX per (assembly x TFM),
-          #     resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper.
-          #   --report-ctrf + --report-ctrf-filename "{asm}_{tfm}.ctrf.json" — CTRF report per
-          #     (assembly x TFM). Same placeholder resolver as TRX. Dogfoods the new CTRF extension
-          #     so any regression in the report shape surfaces on every CI run.
-          #   --report-azdo — surfaces failed tests in the AzDO UI.
-          #   --report-azdo-progress — emits ##vso[task.logdetail ...;progress=...] timeline records so
-          #     long-running test sessions show live progress on the AzDO timeline. No-op outside AzDO
-          #     (gated on TF_BUILD; emits a warning if the option is set but TF_BUILD is not 'true').
-          #   --report-azdo-summary — writes a Markdown job summary and uploads it via ##vso[task.uploadsummary]
-          #     so failure context shows up on the build's summary tab. No-op outside AzDO (gated on TF_BUILD;
-          #     emits a warning if the option is set but TF_BUILD is not 'true').
-          #   --report-junit + --report-junit-filename "{asm}_{tfm}.xml" — JUnit XML per (assembly x TFM),
-          #     resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper. Dogfoods the
-          #     Microsoft.Testing.Extensions.JUnitReport extension; the .xml output is published alongside
-          #     the .trx files and can be consumed by AzDO PublishTestResults@2 (testResultsFormat: 'JUnit'),
-          #     Jenkins, GitLab, GitHub Actions test reporters, etc.
-          #   --hangdump --hangdump-timeout 15m — captures a dump if a module hangs past 15 minutes.
-          #   --diagnostic + --diagnostic-verbosity trace — produces per-host diagnostic logs;
-          #     directory is published by the Arcade jobs.yml template (artifacts.publish.logs: true).
+          # options that makes sense in Release. The OS-agnostic report/dump flags (TRX + CTRF + JUnit +
+          # AzDO + hangdump) live in the $(_ReleaseUnitTestReportArgs) variable defined at the top of this
+          # file so the Windows and Linux/MacOS commands stay in sync; see that variable for the per-flag
+          # documentation. Only the test-module glob, --results-directory, and --diagnostic-output-directory
+          # are inlined here because they differ per OS (path separators + .exe vs .dll).
+          # Use *.exe to cover both net4x (native exe) and net8+/net9+ (apphost exe) in a single glob.
           # We intentionally do NOT pass:
           #   --crashdump — gated on .NETCoreApp in Directory.Build.targets; the CrashDump extension's
           #     ValidateTestHostEnvironmentVariablesAsync hard-fails the run on non-NETCoreApp hosts
@@ -209,7 +209,7 @@ stages:
           #   --coverage — coverage upload is Debug-only (see "Merge coverage" / "Publish coverage to
           #     Azure DevOps" steps below), so collecting coverage in Release would just write unused
           #     .coverage files.
-          - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-ctrf --report-ctrf-filename "{asm}_{tfm}.ctrf.json" --report-azdo --report-azdo-progress --report-azdo-summary --report-junit --report-junit-filename "{asm}_{tfm}.xml" --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace
+          - script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.exe" --results-directory $(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig) --no-progress --output detailed $(_ReleaseUnitTestReportArgs) --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig) --diagnostic-verbosity trace
             name: TestRelease
             displayName: Test (unit tests only)
             condition: and(succeeded(), eq(variables._BuildConfig, 'Release'))

eng/pipelines/steps/test-non-windows.yml

@@ -24,24 +24,12 @@ steps:
 #
 # --test-modules bypasses MSBuild evaluation, so test/Directory.Build.targets cannot inject
 # any of its TestingPlatformCommandLineArguments. We therefore restore the subset of those
-# options that makes sense in Release:
-#   --report-trx + --report-trx-filename "{asm}_{tfm}.trx" — TRX per (assembly x TFM),
-#     resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper.
-#   --report-azdo — surfaces failed tests in the AzDO UI.
-#   --report-azdo-progress — emits ##vso[task.logdetail ...;progress=...] timeline records so
-#     long-running test sessions show live progress on the AzDO timeline. No-op outside AzDO
-#     (gated on TF_BUILD; emits a warning if the option is set but TF_BUILD is not 'true').
-#   --report-azdo-summary — writes a Markdown job summary and uploads it via ##vso[task.uploadsummary]
-#     so failure context shows up on the build's summary tab. No-op outside AzDO (gated on TF_BUILD;
-#     emits a warning if the option is set but TF_BUILD is not 'true').
-#   --report-junit + --report-junit-filename "{asm}_{tfm}.xml" — JUnit XML per (assembly x TFM),
-#     resolved by Microsoft.Testing.Platform.Services.ArtifactNamingHelper. Dogfoods the
-#     Microsoft.Testing.Extensions.JUnitReport extension; the .xml output is published alongside
-#     the .trx files and can be consumed by AzDO PublishTestResults@2 (testResultsFormat: 'JUnit'),
-#     Jenkins, GitLab, GitHub Actions test reporters, etc.
-#   --hangdump --hangdump-timeout 15m — captures a dump if a module hangs past 15 minutes.
-#   --diagnostic + --diagnostic-verbosity trace — produces per-host diagnostic logs;
-#     directory is published by the Arcade jobs.yml template (artifacts.publish.logs: true).
+# options that makes sense in Release. The OS-agnostic report/dump flags (TRX + CTRF + JUnit +
+# AzDO + hangdump) come from the $(_ReleaseUnitTestReportArgs) variable defined in azure-pipelines.yml
+# (the pipeline that consumes this template) so the Windows and Linux/MacOS commands stay in sync;
+# see that variable for the per-flag documentation. Only the test-module glob, --results-directory,
+# and --diagnostic-output-directory are inlined here because they differ per OS (path separators +
+# .exe vs .dll).
 # We intentionally do NOT pass:
 #   --crashdump — gated on .NETCoreApp in Directory.Build.targets; the CrashDump extension's
 #     ValidateTestHostEnvironmentVariablesAsync hard-fails the run on non-NETCoreApp hosts
@@ -50,7 +38,7 @@ steps:
 #   --coverage — coverage upload is Debug-only (see "Merge coverage" / "Publish coverage to
 #     Azure DevOps" steps in azure-pipelines.yml), so collecting coverage in Release would just
 #     write unused .coverage files.
-- script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.dll" --results-directory $(BUILD.SOURCESDIRECTORY)/artifacts/TestResults/$(_BuildConfig) --no-progress --output detailed --report-trx --report-trx-filename "{asm}_{tfm}.trx" --report-azdo --report-azdo-progress --report-azdo-summary --report-junit --report-junit-filename "{asm}_{tfm}.xml" --hangdump --hangdump-timeout 15m --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)/artifacts/log/$(_BuildConfig) --diagnostic-verbosity trace
+- script: dotnet test --no-build --test-modules "artifacts/bin/*UnitTests*/$(_BuildConfig)/**/*UnitTests.dll" --results-directory $(BUILD.SOURCESDIRECTORY)/artifacts/TestResults/$(_BuildConfig) --no-progress --output detailed $(_ReleaseUnitTestReportArgs) --diagnostic --diagnostic-output-directory $(BUILD.SOURCESDIRECTORY)/artifacts/log/$(_BuildConfig) --diagnostic-verbosity trace
   name: TestRelease
   displayName: Test (unit tests only)
   condition: and(succeeded(), eq(variables._BuildConfig, 'Release'))

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs

@@ -8,7 +8,6 @@ internal static class AzureDevOpsCommandLineOptions
     public const string AzureDevOpsOptionName = "report-azdo";
     public const string AzureDevOpsDemoteKnownFlaky = "report-azdo-demote-known-flaky";
     public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history";
-    public const string AzureDevOpsProgress = "report-azdo-progress";
     public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file";
     public const string AzureDevOpsReportSeverity = "report-azdo-severity";
     public const string AzureDevOpsStackFrameFilter = "report-azdo-stackframe-filter";

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs

@@ -47,7 +47,6 @@ public AzureDevOpsCommandLineProvider()
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky, DemoteKnownFlakyOptionDescriptionFormatted, ArgumentArity.Zero, false),
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
-                new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsProgress, AzureDevOpsResources.ProgressOptionDescription, ArgumentArity.Zero, false),
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false),
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
                 new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter, StackFrameFilterOptionDescriptionFormatted, ArgumentArity.OneOrMore, false),
@@ -89,10 +88,6 @@ public override Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandL
             {
                 errorMessage = AzureDevOpsResources.AzureDevOpsFlakyHistoryRequiresAzureDevOps;
             }
-            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsProgress))
-            {
-                errorMessage = AzureDevOpsResources.AzureDevOpsProgressRequiresAzureDevOps;
-            }
             else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile))
             {
                 errorMessage = AzureDevOpsResources.AzureDevOpsQuarantineFileRequiresAzureDevOps;

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs

@@ -44,15 +44,6 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
                     serviceProvider.GetTestApplicationModuleInfo(),
                     serviceProvider.GetLoggerFactory()));
 
-        var compositeProgressReporter =
-            new CompositeExtensionFactory<AzureDevOpsProgressReporter>(serviceProvider =>
-                new AzureDevOpsProgressReporter(
-                    serviceProvider.GetCommandLineOptions(),
-                    serviceProvider.GetEnvironment(),
-                    serviceProvider.GetOutputDevice(),
-                    serviceProvider.GetTestApplicationModuleInfo(),
-                    serviceProvider.GetLoggerFactory()));
-
         var compositeTestResultsPublisher =
             new CompositeExtensionFactory<AzureDevOpsTestResultsPublisher>(serviceProvider =>
                new AzureDevOpsTestResultsPublisher(
@@ -81,13 +72,11 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
         });
         builder.TestHost.AddDataConsumer(compositeArtifactUploader);
         builder.TestHost.AddDataConsumer(compositeSummaryReporter);
-        builder.TestHost.AddDataConsumer(compositeProgressReporter);
         builder.TestHost.AddDataConsumer(compositeTestResultsPublisher);
         builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider =>
             historyService ??= CreateHistoryService(serviceProvider));
         builder.TestHost.AddTestSessionLifetimeHandler(compositeArtifactUploader);
         builder.TestHost.AddTestSessionLifetimeHandler(compositeSummaryReporter);
-        builder.TestHost.AddTestSessionLifetimeHandler(compositeProgressReporter);
         builder.TestHost.AddTestSessionLifetimeHandler(compositeTestResultsPublisher);
         builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
     }