[release/3.0] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -135,9 +135,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19421.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19422.24">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
+      <Sha>c7f03b2cf06bdfc64dad4140fd0d486127095cd8</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/performance/perfhelixpublish.proj

@@ -5,8 +5,14 @@
     <CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
     <Python>py -3</Python>
     <CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+    <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
     <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
     <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+    <BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+    <ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
+    <DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
+    <Percent>%25%25</Percent>
+    <XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
@@ -24,14 +30,24 @@
     <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
     <Python>python3</Python>
     <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+    <BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
     <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
     <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+    <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+    <ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
+    <DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
+    <Percent>%25</Percent>
+    <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
     <CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
+    <BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(WorkItemCommand)' != ''">
     <WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
   </PropertyGroup>
@@ -57,20 +73,29 @@
     <Partition Include="$(BuildConfig).Partition4" Index="4" />
   </ItemGroup>
 
+  <PropertyGroup Condition="'$(Compare)' == 'true'">
+    <FailOnTestFailure>false</FailOnTestFailure>
+  </PropertyGroup>
+
   <!-- 
     Partition the Microbenchmarks project, but nothing else
   -->
   <ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
     <HelixWorkItem Include="@(Partition)">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(BaselineArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
       <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemGroup>
+
   <ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
     <HelixWorkItem Include="$(BuildConfig).WorkItem">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --artifacts $(ArtifactsDirectory)"</PreCommands>
       <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+      <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemGroup>

eng/common/performance/performance-setup.ps1

@@ -1,6 +1,7 @@
 Param(
     [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
     [string] $CoreRootDirectory,
+    [string] $BaselineCoreRootDirectory,
     [string] $Architecture="x64",
     [string] $Framework="netcoreapp5.0",
     [string] $CompilationMode="Tiered",
@@ -12,11 +13,13 @@ Param(
     [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
     [string] $Kind="micro",
     [switch] $Internal,
+    [switch] $Compare,
     [string] $Configurations="CompilationMode=$CompilationMode"
 )
 
 $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
 $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
 
 $PayloadDirectory = (Join-Path $SourceDirectory "Payload")
 $PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
@@ -29,7 +32,13 @@ $HelixSourcePrefix = "pr"
 $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
 
 if ($Framework.StartsWith("netcoreapp")) {
-    $Queue = "Windows.10.Amd64.ClientRS4.Open"
+    $Queue = "Windows.10.Amd64.ClientRS5.Open"
+}
+
+if ($Compare) {
+    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
+    $PerfLabArguments = ""
+    $ExtraBenchmarkDotNetArguments = ""
 }
 
 if ($Internal) {
@@ -56,6 +65,10 @@ if ($UseCoreRun) {
     $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
     Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
 }
+if ($UseBaselineCoreRun) {
+    $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
+    Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+}
 
 $DocsDir = (Join-Path $PerformanceDirectory "docs")
 robocopy $DocsDir $WorkItemDirectory
@@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria
 Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
 
 # Helix Arguments
 Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false

eng/common/performance/performance-setup.sh

@@ -2,6 +2,7 @@
 
 source_directory=$BUILD_SOURCESDIRECTORY
 core_root_directory=
+baseline_core_root_directory=
 architecture=x64
 framework=netcoreapp5.0
 compilation_mode=tiered
@@ -10,12 +11,14 @@ branch=$BUILD_SOURCEBRANCH
 commit_sha=$BUILD_SOURCEVERSION
 build_number=$BUILD_BUILDNUMBER
 internal=false
+compare=false
 kind="micro"
 run_categories="coreclr corefx"
 csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
 configurations=
 run_from_perf_repo=false
 use_core_run=true
+use_baseline_core_run=true
 
 while (($# > 0)); do
   lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -28,6 +31,10 @@ while (($# > 0)); do
       core_root_directory=$2
       shift 2
       ;;
+    --baselinecorerootdirectory)
+      baseline_core_root_directory=$2
+      shift 2
+      ;;
     --architecture)
       architecture=$2
       shift 2
@@ -72,6 +79,10 @@ while (($# > 0)); do
       internal=true
       shift 1
       ;;
+    --compare)
+      compare=true
+      shift 1
+      ;;
     --configurations)
       configurations=$2
       shift 2
@@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then
     use_core_run=false
 fi
 
+if [ -z "$baseline_core_root_directory" ]; then
+    use_baseline_core_run=false
+fi
+
 payload_directory=$source_directory/Payload
 performance_directory=$payload_directory/performance
 workitem_directory=$source_directory/workitem
@@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open
 creator=$BUILD_DEFINITIONNAME
 helix_source_prefix="pr"
 
+if [[ "$compare" == true ]]; then
+  extra_benchmark_dotnet_arguments=
+  perflab_arguments=
+
+  # No open queues for arm64
+  if [[ "$architecture" = "arm64" ]]; then
+    echo "Compare not available for arm64"
+    exit 1
+  fi
+
+  queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
+fi
+
 if [[ "$internal" == true ]]; then
     perflab_arguments="--upload-to-perflab-container"
     helix_source_prefix="official"
@@ -156,13 +184,19 @@ if [[ "$use_core_run" = true ]]; then
     mv $core_root_directory $new_core_root
 fi
 
+if [[ "$use_baseline_core_run" = true ]]; then
+  new_baseline_core_root=$payload_directory/Baseline_Core_Root
+  mv $baseline_core_root_directory $new_baseline_core_root
+fi
+
 ci=true
 
 _script_dir=$(pwd)/eng/common
 . "$_script_dir/pipeline-logging-functions.sh"
 
 # Make sure all of our variables are available for future steps
 Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
 Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
 Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
 Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
@@ -178,4 +212,5 @@ Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -
 Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
 Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
 Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
-Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false

eng/common/post-build/sourcelink-validation.ps1

@@ -215,7 +215,7 @@ function ValidateSourceLinkLinks {
     }
 
   $ValidationFailures = 0
-  foreach ($Job in $Jobs) {
+  foreach ($Job in @(Get-Job)) {
     $jobResult = Wait-Job -Id $Job.Id | Receive-Job
     if ($jobResult -ne "0") {
       $ValidationFailures++

global.json

@@ -3,6 +3,6 @@
     "dotnet": "3.0.100-preview6-012264"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19421.4"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19422.24"
   }
 }