diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 80808b5573..8c06dce458 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -168,45 +168,44 @@ stages: # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - template: /eng/common/templates/job/publish-build-assets.yml parameters: + publishUsingPipelines: true dependsOn: PrepareSignedArtifacts pool: name: NetCoreInternal-Pool queue: buildpool.windows.10.amd64.vs2017 - # Stages-based publishing entry point - - template: eng\common\templates\post-build\post-build.yml + - template: /eng/stages/publish.yml parameters: - dependsOn: [PrepareForPublish] - # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871 - enableSymbolValidation: false - # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254 - enableSigningValidation: false - # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604 - enableSourceLinkValidation: false - # Allow symbol publish to emit expected warnings without failing the build. Include single - # quotes inside the string so that it passes through to MSBuild without script interference. - symbolPublishingAdditionalParameters: "'-warnAsError:$false'" - - - template: /eng/stages/custom-publish.yml - parameters: - # Custom publish stage dependency configuration. - # - dependsOn: + # Publish channel configuration. The publish template wraps Arcade publishing and adds some + # extras that aren't implemented in Arcade yet. + # + # - dependsOn: # channel: # name: # bar: # storage: dependsOnPublishStages: - - dependsOn: PublishValidation + - dependsOn: NetCore_Dev30_Publish_Validation channel: name: .NET Core 3 Dev bar: PublicDevRelease_30_Channel_Id storage: release/3.0 - - dependsOn: PubRel_PublishValidation + - dependsOn: NetCore_Release30_Publish_Validation channel: name: .NET Core 3 Release bar: PublicRelease_30_Channel_Id storage: release/3.0-preview9 - - dependsOn: NetCore_Dev5_PublishValidation + - dependsOn: NetCore_Dev31_Publish_Validation + channel: + name: .NET Core 3.1 Dev + bar: PublicDevRelease_31_Channel_Id + storage: release/3.1 + - dependsOn: NetCore_Release31_Publish_Validation + channel: + name: .NET Core 3.1 Release + bar: PublicRelease_31_Channel_Id + storage: release/3.1-preview1 + - dependsOn: NetCore_Dev5_Publish_Validation channel: name: .NET Core 5 Dev bar: NetCore_5_Dev_Channel_Id diff --git a/eng/SignCheckExclusionsFile.txt b/eng/SignCheckExclusionsFile.txt new file mode 100644 index 0000000000..f07228545c --- /dev/null +++ b/eng/SignCheckExclusionsFile.txt @@ -0,0 +1,12 @@ +;; Exclusions for SignCheck. Corresponds to info in Signing.props. +;; Format: https://github.com/dotnet/arcade/blob/397316e195639450b6c76bfeb9823b40bee72d6d/src/SignCheck/Microsoft.SignCheck/Verification/Exclusion.cs#L23-L35 +;; +;; This issue tracks a way to implement exclusions via Signing.props and avoid this extra file: https://github.com/dotnet/arcade/issues/2888 + +;; The apphost and comhost are template files, modified by the SDK to produce the executable for FDE +;; and FDD apps. If they are signed, the file that the SDK produces has an invalid signature and +;; can't be signed again. More info at https://github.com/dotnet/core-setup/pull/7549. +*apphost.exe;;Template, https://github.com/dotnet/core-setup/pull/7549 +*comhost.dll;;Template, https://github.com/dotnet/core-setup/pull/7549 +*apphosttemplateapphostexe.exe;;Template, https://github.com/dotnet/core-setup/pull/7549 +*comhosttemplatecomhostdll.dll;;Template, https://github.com/dotnet/core-setup/pull/7549 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 234a5d3f50..18f144df44 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -117,27 +117,43 @@ https://github.com/dotnet/standard f3b1db40ac53eab80c96c3cfa784f5c38bb02d4a - + https://github.com/dotnet/coreclr - eeec95164a0b39ad3f98dfe04012675f30807cb1 + d67367f691e789cf670f48dc646c66e63b804f8d - + https://github.com/dotnet/winforms - 8b19141d6b0b221167bd7d1731cbc6aa9898da75 + dd6650900f5af7283c6af6ec85de4d8d51f28961 - + https://github.com/dotnet/wpf - 105b673bddd8665580a94878db12ac97af8fa794 + ffc3fbd836fb6ff3c28e50fec67be5b55c8b6542 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf-int - 9dbf98944d032f91760e92882c80859cbf45b847 + 83ab428ae9904012fbf4f109c2a5026793136966 - + https://github.com/dotnet/arcade - fa168dd690798a683ef0d1e65d60ce5d6918d987 + 00d8aa82b488f321204a0e69a81399af9df276a1 + + https://github.com/dotnet/arcade + 00d8aa82b488f321204a0e69a81399af9df276a1 + + + https://github.com/dotnet/arcade + 00d8aa82b488f321204a0e69a81399af9df276a1 + + + https://github.com/dotnet/arcade + 00d8aa82b488f321204a0e69a81399af9df276a1 + + + https://github.com/dotnet/sourcelink + + diff --git a/eng/Versions.props b/eng/Versions.props index 288761518e..0ea8a9620c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,9 +36,11 @@ - 1.0.0-beta.19308.1 - 2.2.0-beta.19410.2 - 1.0.0-beta.19308.1 + 2.2.0-beta.19454.31 + 1.0.0-beta.19454.31 + 1.0.0-beta.19454.31 + + 1.0.0-beta2-19367-01 3.0.0-rc1.19454.8 3.1.0-rc1.19454.8 @@ -71,13 +73,13 @@ 2.1.0-prerelease.19454.4 - 3.0.0-preview9.19409.2 + 3.0.0-rc1.19430.3 - 4.8.0-preview9.19415.10 + 4.8.0-preview9.19417.1 - 4.8.0-preview9.19416.2 + 4.8.0-preview9.19417.4 - 4.8.0-preview9.19416.7 + 5.0.0-alpha1.19420.2 15.7.179 $(MicrosoftBuildPackageVersion) @@ -88,8 +90,6 @@ 4.9.4 1.0.5 1.7.0 - - 1.0.0-beta2-18618-05 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8854d979f3..46d175fdfd 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,9 +1,9 @@ param ( $darcVersion = $null, - $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16", + $verbosity = "m" ) -$verbosity = "m" . $PSScriptRoot\tools.ps1 function InstallDarcCli ($darcVersion) { diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index abdd0bc05a..242429bca6 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -3,6 +3,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" +verbosity=m while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -15,6 +16,10 @@ while [[ $# > 0 ]]; do versionEndpoint=$2 shift ;; + --verbosity) + verbosity=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -34,7 +39,6 @@ while [[ -h "$source" ]]; do [[ $source != /* ]] && source="$scriptroot/$source" done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -verbosity=m . "$scriptroot/tools.sh" diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 2a08d5246e..41416862d9 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -152,6 +152,8 @@ function Get-File { } else { Write-Verbose "Downloading $Uri" + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' while($Attempt -Lt $DownloadRetries) { try { diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 05e5f09891..e5826b5323 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -5,8 +5,14 @@ --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP% py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe + %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts + %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline + %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj + %HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe + %25%25 + %HELIX_WORKITEM_ROOT%\testResults.xml @@ -24,14 +30,24 @@ --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP python3 $(BaseDirectory)/Core_Root/corerun + $(BaseDirectory)/Baseline_Core_Root/corerun $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts + $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline + $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj + $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet + %25 + $HELIX_WORKITEM_ROOT/testResults.xml --corerun $(CoreRun) + + --corerun $(BaselineCoreRun) + + $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments) @@ -57,20 +73,29 @@ + + false + + $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 + $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" + $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 4a6706b638..268986246e 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -1,6 +1,7 @@ Param( [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, [string] $CoreRootDirectory, + [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", [string] $Framework="netcoreapp5.0", [string] $CompilationMode="Tiered", @@ -12,11 +13,13 @@ Param( [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", [switch] $Internal, + [switch] $Compare, [string] $Configurations="CompilationMode=$CompilationMode" ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) +$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) $PayloadDirectory = (Join-Path $SourceDirectory "Payload") $PerformanceDirectory = (Join-Path $PayloadDirectory "performance") @@ -29,7 +32,13 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" if ($Framework.StartsWith("netcoreapp")) { - $Queue = "Windows.10.Amd64.ClientRS4.Open" + $Queue = "Windows.10.Amd64.ClientRS5.Open" +} + +if ($Compare) { + $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open" + $PerfLabArguments = "" + $ExtraBenchmarkDotNetArguments = "" } if ($Internal) { @@ -56,6 +65,10 @@ if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot } +if ($UseBaselineCoreRun) { + $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") + Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot +} $DocsDir = (Join-Path $PerformanceDirectory "docs") robocopy $DocsDir $WorkItemDirectory @@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 76126b1f86..550b3ebf18 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -2,6 +2,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= +baseline_core_root_directory= architecture=x64 framework=netcoreapp5.0 compilation_mode=tiered @@ -10,12 +11,14 @@ branch=$BUILD_SOURCEBRANCH commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false +compare=false kind="micro" run_categories="coreclr corefx" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations= run_from_perf_repo=false use_core_run=true +use_baseline_core_run=true while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -28,6 +31,10 @@ while (($# > 0)); do core_root_directory=$2 shift 2 ;; + --baselinecorerootdirectory) + baseline_core_root_directory=$2 + shift 2 + ;; --architecture) architecture=$2 shift 2 @@ -72,6 +79,10 @@ while (($# > 0)); do internal=true shift 1 ;; + --compare) + compare=true + shift 1 + ;; --configurations) configurations=$2 shift 2 @@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then use_core_run=false fi +if [ -z "$baseline_core_root_directory" ]; then + use_baseline_core_run=false +fi + payload_directory=$source_directory/Payload performance_directory=$payload_directory/performance workitem_directory=$source_directory/workitem @@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open creator=$BUILD_DEFINITIONNAME helix_source_prefix="pr" +if [[ "$compare" == true ]]; then + extra_benchmark_dotnet_arguments= + perflab_arguments= + + # No open queues for arm64 + if [[ "$architecture" = "arm64" ]]; then + echo "Compare not available for arm64" + exit 1 + fi + + queue=Ubuntu.1804.Amd64.Tiger.Perf.Open +fi + if [[ "$internal" == true ]]; then perflab_arguments="--upload-to-perflab-container" helix_source_prefix="official" @@ -156,6 +184,11 @@ if [[ "$use_core_run" = true ]]; then mv $core_root_directory $new_core_root fi +if [[ "$use_baseline_core_run" = true ]]; then + new_baseline_core_root=$payload_directory/Baseline_Core_Root + mv $baseline_core_root_directory $new_baseline_core_root +fi + ci=true _script_dir=$(pwd)/eng/common @@ -163,6 +196,7 @@ _script_dir=$(pwd)/eng/common # Make sure all of our variables are available for future steps Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false +Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false @@ -178,4 +212,5 @@ Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" - Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false \ No newline at end of file +Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false +Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 428e8c9607..bbfdacca13 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -215,7 +215,7 @@ function ValidateSourceLinkLinks { } $ValidationFailures = 0 - foreach ($Job in $Jobs) { + foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job if ($jobResult -ne "0") { $ValidationFailures++ diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index d5ec51b150..096ac321d1 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { return "PDB" @@ -159,25 +159,25 @@ function CheckSymbolsAvailable { } } -function Installdotnetsymbol { - $dotnetsymbolPackageName = "dotnet-symbol" +function InstallDotnetSymbol { + $dotnetSymbolPackageName = "dotnet-symbol" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" $toolList = & "$dotnet" tool list --global - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." } else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global + & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global } } try { - Installdotnetsymbol + InstallDotnetSymbol CheckSymbolsAvailable } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 6d9bdcf72b..1159e999d2 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -45,6 +45,7 @@ if ($GuardianPackageName) { $guardianCliLocation = $GuardianCliLocation } +$workingDirectory = (Split-Path $SourceDirectory -Parent) $ValidPath = Test-Path $guardianCliLocation if ($ValidPath -eq $False) @@ -53,13 +54,13 @@ if ($ValidPath -eq $False) exit 1 } -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory (Split-Path $SourceDirectory -Parent) -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path (Split-Path $SourceDirectory -Parent) ".gdn" if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -71,10 +72,10 @@ if ($TsaOnboard) { } if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($UpdateBaseline) { @@ -86,8 +87,8 @@ if ($TsaPublish) { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." exit $LASTEXITCODE diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 26e01c0673..c737eb0e71 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 $LASTEXITCODE = 0 +# Don't display the console progress UI - it's a huge perf hit +$ProgressPreference = 'SilentlyContinue' + # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3f97ac2f16..256ffbfb93 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + - + diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index b90c4399ba..9bc25314ae 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -32,16 +32,16 @@ foreach ($tool in $ToolsList) { Write-Host $tool # We have to manually configure tools that run on source to look at the source directory only if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -56,4 +56,4 @@ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --bas if ($LASTEXITCODE -ne 0) { Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." exit $LASTEXITCODE -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 91621cf88f..a7f9964195 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,7 +1,10 @@ parameters: overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - continueOnError: false # optional: determines whether to continue the build if the step errors; + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; dependsOn: '' # Optional: dependencies of the job jobs: @@ -26,12 +29,12 @@ jobs: -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -45,12 +48,12 @@ jobs: - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/netcore-dev-30.yml similarity index 91% rename from eng/common/templates/post-build/channels/public-dev-release.yml rename to eng/common/templates/post-build/channels/netcore-dev-30.yml index afa9542148..3ce7ce4d6e 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-30.yml @@ -5,11 +5,11 @@ parameters: publishInstallersAndChecksums: false stages: -- stage: Publish +- stage: NetCore_Dev30_Publish dependsOn: validate variables: - template: ../common-variables.yml - displayName: Developer Channel + displayName: .NET Core 3.0 Dev Publishing jobs: - template: ../setup-maestro-vars.yml @@ -39,8 +39,8 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' @@ -85,7 +85,7 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicDevRelease_30_Channel_Id) /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) @@ -93,23 +93,23 @@ stages: /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:Configuration=Release + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) ${{ parameters.artifactsPublishingAdditionalParameters }} - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -123,16 +123,16 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false -- stage: PublishValidation - displayName: Publish Validation +- stage: NetCore_Dev30_Publish_Validation + displayName: .NET Core 3.0 Dev Publish Validation variables: - - template: ../common-variables.yml + - template: ../common-variables.yml jobs: - template: ../setup-maestro-vars.yml @@ -160,6 +160,6 @@ stages: parameters: ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-31.yml b/eng/common/templates/post-build/channels/netcore-dev-31.yml new file mode 100644 index 0000000000..d40aaacc4f --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-dev-31.yml @@ -0,0 +1,165 @@ +parameters: + enableSymbolValidation: true + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: false + +stages: +- stage: NetCore_Dev31_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.1 Dev Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicDevRelease_31_Channel_Id) + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + +- stage: NetCore_Dev31_Publish_Validation + displayName: .NET Core 3.1 Dev Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - template: ../darc-gather-drop.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }} + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-5.yml b/eng/common/templates/post-build/channels/netcore-dev-5.yml index 812def3154..584185c2a7 100644 --- a/eng/common/templates/post-build/channels/netcore-dev-5.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-5.yml @@ -9,7 +9,7 @@ stages: dependsOn: validate variables: - template: ../common-variables.yml - displayName: .NET Core 5 Dev Channel + displayName: .NET Core 5 Dev Publishing jobs: - template: ../setup-maestro-vars.yml @@ -39,8 +39,8 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' @@ -85,7 +85,7 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(NetCore_5_Dev_Channel_Id) /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) @@ -93,15 +93,15 @@ stages: /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) @@ -109,7 +109,7 @@ stages: /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) ${{ parameters.artifactsPublishingAdditionalParameters }} - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -123,16 +123,16 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false -- stage: NetCore_Dev5_PublishValidation - displayName: Publish Validation +- stage: NetCore_Dev5_Publish_Validation + displayName: .NET Core 5 Dev Publish Validation variables: - - template: ../common-variables.yml + - template: ../common-variables.yml jobs: - template: ../setup-maestro-vars.yml @@ -160,6 +160,6 @@ stages: parameters: ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/netcore-internal-30.yml similarity index 91% rename from eng/common/templates/post-build/channels/internal-servicing.yml rename to eng/common/templates/post-build/channels/netcore-internal-30.yml index 4ca36358d9..8bf88fd49e 100644 --- a/eng/common/templates/post-build/channels/internal-servicing.yml +++ b/eng/common/templates/post-build/channels/netcore-internal-30.yml @@ -4,11 +4,11 @@ parameters: artifactsPublishingAdditionalParameters: '' stages: -- stage: IS_Publish +- stage: NetCore_30_Internal_Servicing_Publishing dependsOn: validate variables: - template: ../common-variables.yml - displayName: Internal Servicing + displayName: .NET Core 3.0 Internal Servicing Publishing jobs: - template: ../setup-maestro-vars.yml @@ -38,8 +38,8 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' @@ -84,7 +84,7 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(InternalServicing_30_Channel_Id) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) @@ -95,15 +95,15 @@ stages: /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' - /p:Configuration=Release + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release ${{ parameters.artifactsPublishingAdditionalParameters }} - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -117,7 +117,7 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false @@ -125,11 +125,11 @@ stages: - template: ../trigger-subscription.yml parameters: ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} - -- stage: IS_PublishValidation - displayName: Publish Validation + +- stage: NetCore_30_Internal_Servicing_Publish_Validation + displayName: .NET Core 3.0 Internal Servicing Publish Validation variables: - - template: ../common-variables.yml + - template: ../common-variables.yml jobs: - template: ../setup-maestro-vars.yml diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/netcore-release-30.yml similarity index 87% rename from eng/common/templates/post-build/channels/public-release.yml rename to eng/common/templates/post-build/channels/netcore-release-30.yml index 7ec1f89c08..dc9541d9ec 100644 --- a/eng/common/templates/post-build/channels/public-release.yml +++ b/eng/common/templates/post-build/channels/netcore-release-30.yml @@ -4,11 +4,11 @@ parameters: artifactsPublishingAdditionalParameters: '' stages: -- stage: PubRel_Publish +- stage: NetCore_Release30_Publish dependsOn: validate variables: - template: ../common-variables.yml - displayName: Public Release + displayName: .NET Core 3.0 Release Publishing jobs: - template: ../setup-maestro-vars.yml @@ -38,8 +38,8 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' @@ -84,27 +84,27 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicRelease_30_Channel_Id) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' - /p:Configuration=Release + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release ${{ parameters.artifactsPublishingAdditionalParameters }} - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -118,7 +118,7 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false @@ -127,10 +127,10 @@ stages: parameters: ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} -- stage: PubRel_PublishValidation - displayName: Publish Validation +- stage: NetCore_Release30_Publish_Validation + displayName: .NET Core 3.0 Release Publish Validation variables: - - template: ../common-variables.yml + - template: ../common-variables.yml jobs: - template: ../setup-maestro-vars.yml @@ -138,7 +138,8 @@ stages: - job: displayName: Symbol Availability dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)) + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)), + contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/channels/netcore-release-31.yml b/eng/common/templates/post-build/channels/netcore-release-31.yml new file mode 100644 index 0000000000..4dcf4ba9c8 --- /dev/null +++ b/eng/common/templates/post-build/channels/netcore-release-31.yml @@ -0,0 +1,160 @@ +parameters: + enableSymbolValidation: true + symbolPublishingAdditionalParameters: '' + artifactsPublishingAdditionalParameters: '' + +stages: +- stage: NetCore_Release31_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: .NET Core 3.1 Release Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicRelease_31_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) + /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) + /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) + /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + - template: ../trigger-subscription.yml + parameters: + ChannelId: ${{ variables.PublicRelease_31_Channel_Id }} + +- stage: NetCore_Release31_Publish_Validation + displayName: .NET Core 3.1 Release Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)), + contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-tools-latest.yml b/eng/common/templates/post-build/channels/netcore-tools-latest.yml index c2d2076730..4d9f197d43 100644 --- a/eng/common/templates/post-build/channels/netcore-tools-latest.yml +++ b/eng/common/templates/post-build/channels/netcore-tools-latest.yml @@ -3,13 +3,15 @@ parameters: symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false + publishToAzureDevOpsFeeds: false + azureDevOpsToolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' stages: - stage: NetCore_Tools_Latest_Publish dependsOn: validate variables: - template: ../common-variables.yml - displayName: .NET Tools - Latest + displayName: .NET Tools - Latest Publishing jobs: - template: ../setup-maestro-vars.yml @@ -39,8 +41,8 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' @@ -85,7 +87,7 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(NetCore_Tools_Latest_Channel_Id) /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) @@ -93,23 +95,28 @@ stages: /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }} + /p:AzureDevOpsStaticShippingFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw) ${{ parameters.artifactsPublishingAdditionalParameters }} - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -123,16 +130,16 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false - stage: NetCore_Tools_Latest_PublishValidation - displayName: Publish Validation + displayName: .NET Tools - Latest Publish Validation variables: - - template: ../common-variables.yml + - template: ../common-variables.yml jobs: - template: ../setup-maestro-vars.yml @@ -160,6 +167,6 @@ stages: parameters: ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 12124d6215..e306fa87bf 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -1,13 +1,15 @@ parameters: artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false + publishToAzureDevOpsFeeds: false + azureDevOpsToolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' stages: - stage: PVR_Publish dependsOn: validate variables: - template: ../common-variables.yml - displayName: Validation Channel + displayName: .NET Tools - Validation Publishing jobs: - template: ../setup-maestro-vars.yml @@ -49,7 +51,7 @@ stages: AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) /p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) @@ -57,21 +59,26 @@ stages: /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' /p:Configuration=Release /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }} + /p:AzureDevOpsStaticShippingFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - task: NuGetCommand@2 @@ -87,14 +94,14 @@ stages: displayName: Publish Blobs to AzDO Feed inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) + arguments: -FeedName $(AzDoFeedName) -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) enabled: false - stage: PVR_PublishValidation - displayName: Publish Validation + displayName: .NET Tools - Validation Publish Validation variables: - template: ../common-variables.yml jobs: @@ -104,6 +111,6 @@ stages: parameters: ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} - - template: ../promote-build.yml - parameters: + - template: ../promote-build.yml + parameters: ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b00d85d8ce..49712b3a60 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -6,6 +6,10 @@ variables: - name: PublicDevRelease_30_Channel_Id value: 3 + # .NET Core 3.1 Dev + - name: PublicDevRelease_31_Channel_Id + value: 260 + # .NET Core 5 Dev - name: NetCore_5_Dev_Channel_Id value: 131 @@ -26,6 +30,10 @@ variables: - name: PublicRelease_30_Channel_Id value: 19 + # .NET Core 3.1 Release + - name: PublicRelease_31_Channel_Id + value: 129 + # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} @@ -49,6 +57,6 @@ variables: # Default locations for Installers and checksums - name: ChecksumsBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - - name: InstallersBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json + - name: InstallersBlobFeedUrl + value: https://dotnetcli.blob.core.windows.net/dotnet/index.json diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f239fae2b..ae376b7747 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,11 +1,13 @@ parameters: - enableSourceLinkValidation: true + enableSourceLinkValidation: false enableSigningValidation: true - enableSymbolValidation: true + enableSymbolValidation: false enableNugetValidation: true publishInstallersAndChecksums: false + enableAzDONuGetFeeds: true SDLValidationParameters: enable: false + continueOnError: false params: '' # These parameters let the user customize the call to sdk-task.ps1 for publishing @@ -92,6 +94,7 @@ stages: - template: /eng/common/templates/job/execute-sdl.yml parameters: additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml parameters: @@ -100,7 +103,14 @@ stages: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} -- template: \eng\common\templates\post-build\channels\public-dev-release.yml +- template: \eng\common\templates\post-build\channels\netcore-dev-30.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + +- template: \eng\common\templates\post-build\channels\netcore-dev-31.yml parameters: enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} @@ -113,18 +123,25 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }} - template: \eng\common\templates\post-build\channels\public-validation-release.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }} + +- template: \eng\common\templates\post-build\channels\netcore-release-30.yml + parameters: + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} -- template: \eng\common\templates\post-build\channels\public-release.yml +- template: \eng\common\templates\post-build\channels\netcore-release-31.yml parameters: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} -- template: \eng\common\templates\post-build\channels\internal-servicing.yml +- template: \eng\common\templates\post-build\channels\netcore-internal-30.yml parameters: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9c12b1b4fd..bb5638930a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,6 +163,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript } @@ -282,6 +283,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath Unzip $packagePath $packageDir } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3af9be6157..94a1edd7d0 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -208,12 +208,19 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" - else - wget -q -O "$install_script" "$install_script_url" + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + else + wget -q -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } diff --git a/eng/jobs/run-publish-project.yml b/eng/jobs/run-publish-project.yml new file mode 100644 index 0000000000..734cd31e9d --- /dev/null +++ b/eng/jobs/run-publish-project.yml @@ -0,0 +1,120 @@ +parameters: + projectName: '' + dependency: null + +jobs: + +- template: /eng/common/templates/post-build/setup-maestro-vars.yml + +- job: CustomPublish + displayName: Custom Publish + dependsOn: setupMaestroVars + # Only run this job if setup-maestro-vars says the current branch publishes to this channel. + # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml + condition: contains( + dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], + format('[{0}]', variables['${{ parameters.dependency.channel.bar }}'])) + pool: + name: Hosted VS2017 + # Double the default timeout. + timeoutInMinutes: 120 + workspace: + clean: all + + variables: + # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. + - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: + # Used for publishing individual leg assets to azure blob storage + - group: DotNet-DotNetCli-Storage + # Used for dotnet/versions update + - group: DotNet-Versions-Publish + + # Blob storage publish (installers and checksums) + - name: _AzureAccountName + value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ] + - name: _ContainerName + value: $[ coalesce(variables.ContainerName, 'dotnet') ] + - name: _AzureAccessToken + value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ] + - name: _ChecksumAzureAccountName + value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ] + - name: _ChecksumContainerName + value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ] + - name: _ChecksumAzureAccessToken + value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ] + - name: _CommonPublishArgs + value: >- + /p:AzureAccountName=$(_AzureAccountName) + /p:ContainerName=$(_ContainerName) + /p:AzureAccessToken=$(_AzureAccessToken) + /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName) + /p:ChecksumContainerName=$(_ChecksumContainerName) + /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) + + # dotnet/versions update + - name: _GitHubUser + value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] + - name: _GitHubEmail + value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] + - name: _GitHubAuthToken + value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] + - name: _VersionsRepoOwner + value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] + - name: _VersionsRepo + value: $[ coalesce(variables.VersionsRepo, 'versions') ] + - name: _DotNetVersionsArgs + value: >- + /p:GitHubUser=$(_GitHubUser) + /p:GitHubEmail=$(_GitHubEmail) + /p:GitHubAuthToken=$(_GitHubAuthToken) + /p:VersionsRepoOwner=$(_VersionsRepoOwner) + /p:VersionsRepo=$(_VersionsRepo) + /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + artifactName: PreparedArtifacts + downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload + + - powershell: | + $prefix = "refs/heads/" + $branch = "$(Build.SourceBranch)" + $branchName = $branch + if ($branchName.StartsWith($prefix)) + { + $branchName = $branchName.Substring($prefix.Length) + } + Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" + Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" + displayName: Find true SourceBranchName + + - script: >- + build.cmd -ci + -projects $(Build.SourcesDirectory)\publish\${{ parameters.projectName }}.proj + /p:Channel=${{ parameters.dependency.channel.storage }} + /p:Configuration=Release + $(_CommonPublishArgs) + $(_DotNetVersionsArgs) + /bl:$(Build.SourcesDirectory)\${{ parameters.projectName }}.binlog + displayName: Publish to custom locations + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: Logs-CustomPublish-${{ parameters.dependency.dependsOn }}-${{ parameters.projectName }} + condition: succeededOrFailed() diff --git a/eng/jobs/steps/upload-job-artifacts.yml b/eng/jobs/steps/upload-job-artifacts.yml index 776b284b21..8b9e42f6ae 100644 --- a/eng/jobs/steps/upload-job-artifacts.yml +++ b/eng/jobs/steps/upload-job-artifacts.yml @@ -8,7 +8,9 @@ steps: displayName: Prepare job-specific Artifacts subdirectory inputs: SourceFolder: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)' - Contents: '**/*' + Contents: | + Shipping/**/* + NonShipping/**/* TargetFolder: '$(Build.StagingDirectory)/Artifacts/${{ parameters.name }}' CleanTargetFolder: true condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/eng/stages/custom-publish.yml b/eng/stages/custom-publish.yml deleted file mode 100644 index 952469077a..0000000000 --- a/eng/stages/custom-publish.yml +++ /dev/null @@ -1,129 +0,0 @@ -parameters: - dependsOnPublishStages: [] - -stages: -# Create one stage per pipeline that needs some extra publish steps. -- ${{ each dependency in parameters.dependsOnPublishStages }}: - - stage: CustomPublish_${{ dependency.dependsOn }} - displayName: 'Custom: ${{ dependency.channel.name }}' - dependsOn: - - ${{ dependency.dependsOn }} - variables: - - template: /eng/common/templates/post-build/common-variables.yml - - jobs: - - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - - - job: CustomPublish - displayName: Custom Publish - dependsOn: setupMaestroVars - # Only run this job if setup-maestro-vars says the current branch publishes to this channel. - # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml - condition: contains( - dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], - format('[{0}]', variables['${{ dependency.channel.bar }}'])) - pool: - name: Hosted VS2017 - # Double the default timeout. - timeoutInMinutes: 120 - workspace: - clean: all - - variables: - # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. - - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: - # Used for publishing individual leg assets to azure blob storage - - group: DotNet-DotNetCli-Storage - # Used for dotnet/versions update - - group: DotNet-Versions-Publish - - # Blob storage publish (installers and checksums) - - name: _AzureAccountName - value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ] - - name: _ContainerName - value: $[ coalesce(variables.ContainerName, 'dotnet') ] - - name: _AzureAccessToken - value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ] - - name: _ChecksumAzureAccountName - value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ] - - name: _ChecksumContainerName - value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ] - - name: _ChecksumAzureAccessToken - value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ] - - name: _CommonPublishArgs - value: >- - /p:AzureAccountName=$(_AzureAccountName) - /p:ContainerName=$(_ContainerName) - /p:AzureAccessToken=$(_AzureAccessToken) - /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName) - /p:ChecksumContainerName=$(_ChecksumContainerName) - /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) - - # dotnet/versions update - - name: _GitHubUser - value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] - - name: _GitHubEmail - value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] - - name: _GitHubAuthToken - value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] - - name: _VersionsRepoOwner - value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] - - name: _VersionsRepo - value: $[ coalesce(variables.VersionsRepo, 'versions') ] - - name: _DotNetVersionsArgs - value: >- - /p:GitHubUser=$(_GitHubUser) - /p:GitHubEmail=$(_GitHubEmail) - /p:GitHubAuthToken=$(_GitHubAuthToken) - /p:VersionsRepoOwner=$(_VersionsRepoOwner) - /p:VersionsRepo=$(_VersionsRepo) - /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) - - steps: - - - task: DownloadBuildArtifacts@0 - displayName: Download Artifacts - inputs: - artifactName: PreparedArtifacts - downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload - - - powershell: | - $prefix = "refs/heads/" - $branch = "$(Build.SourceBranch)" - $branchName = $branch - if ($branchName.StartsWith($prefix)) - { - $branchName = $branchName.Substring($prefix.Length) - } - Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" - Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" - displayName: Find true SourceBranchName - - - script: >- - build.cmd -ci - -projects $(Build.SourcesDirectory)\publish\custom-publish.proj - /p:Channel=${{ dependency.channel.storage }} - /p:Configuration=Release - $(_CommonPublishArgs) - $(_DotNetVersionsArgs) - /bl:$(Build.SourcesDirectory)\custom-publish.binlog - displayName: Publish to custom locations - - - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact BuildLogs - inputs: - PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' - ArtifactName: Logs-CustomPublish-${{ dependency.dependsOn }} - condition: succeededOrFailed() diff --git a/eng/stages/publish.yml b/eng/stages/publish.yml new file mode 100644 index 0000000000..25575ff4e5 --- /dev/null +++ b/eng/stages/publish.yml @@ -0,0 +1,46 @@ +parameters: + dependsOnPublishStages: [] + +stages: + +# Create extra stage per BAR channel that needs extra publish steps. +- ${{ each dependency in parameters.dependsOnPublishStages }}: + - stage: PublishBlob_${{ dependency.dependsOn }} + displayName: '${{ dependency.channel.name }} Blob Publish' + dependsOn: PrepareForPublish + variables: + - template: /eng/common/templates/post-build/common-variables.yml + jobs: + - template: /eng/jobs/run-publish-project.yml + parameters: + projectName: publish-blobs + dependency: ${{ dependency }} + +# Stages-based publishing entry point +- template: /eng/common/templates/post-build/post-build.yml + parameters: + dependsOn: + - ${{ each dependency in parameters.dependsOnPublishStages }}: + - PublishBlob_${{ dependency.dependsOn }} + # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604 + enableSourceLinkValidation: false + # Allow symbol publish to emit expected warnings without failing the build. Include single + # quotes inside the string so that it passes through to MSBuild without script interference. + symbolPublishingAdditionalParameters: "'-warnAsError:$false'" + +# Create extra stage per BAR channel that needs extra publish steps. These run after the Arcade +# stages because they depend on Arcade's NuGet package publish being complete. +- ${{ each dependency in parameters.dependsOnPublishStages }}: + - stage: PublishFinal_${{ dependency.dependsOn }} + displayName: '${{ dependency.channel.name }} Finalize' + dependsOn: + - ${{ dependency.dependsOn }} + variables: + - template: /eng/common/templates/post-build/common-variables.yml + jobs: + - template: /eng/jobs/run-publish-project.yml + parameters: + projectName: publish-final + dependency: ${{ dependency }} diff --git a/global.json b/global.json index 5e0119cef7..e42dbcdd60 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100-preview6-012264" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19420.8" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19454.31" } } diff --git a/publish/Directory.Build.targets b/publish/Directory.Build.targets index 3121cbc3fd..bb39273131 100644 --- a/publish/Directory.Build.targets +++ b/publish/Directory.Build.targets @@ -23,7 +23,16 @@ Include="$(DownloadDirectory)**\*.nupkg" Exclude="@(DownloadedSymbolNupkgFile)" /> - + + + + - - - - - - - - + + + + + + + + + + diff --git a/publish/prepare-artifacts.proj b/publish/prepare-artifacts.proj index 7f7bdee661..10429538b9 100644 --- a/publish/prepare-artifacts.proj +++ b/publish/prepare-artifacts.proj @@ -41,9 +41,19 @@ + + + + + - + + diff --git a/publish/publish-blobs.proj b/publish/publish-blobs.proj new file mode 100644 index 0000000000..ed3b2d67d4 --- /dev/null +++ b/publish/publish-blobs.proj @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/publish/custom-publish.proj b/publish/publish-final.proj similarity index 54% rename from publish/custom-publish.proj rename to publish/publish-final.proj index 5ffaa36c35..ce0089bd6b 100644 --- a/publish/custom-publish.proj +++ b/publish/publish-final.proj @@ -1,11 +1,7 @@ - - - - diff --git a/signing/SignBurnBundleFiles.proj b/signing/SignBurnBundleFiles.proj index 28eb9f24b3..55c7a8c581 100644 --- a/signing/SignBurnBundleFiles.proj +++ b/signing/SignBurnBundleFiles.proj @@ -4,7 +4,7 @@ diff --git a/signing/SignMsiFiles.proj b/signing/SignMsiFiles.proj index 7fd756ddce..55dd18dca6 100644 --- a/signing/SignMsiFiles.proj +++ b/signing/SignMsiFiles.proj @@ -12,7 +12,7 @@ DependsOnTargets="GetSharedFrameworkProjects" BeforeTargets="EnsureProjectsBuilt"> - + diff --git a/src/corehost/build.proj b/src/corehost/build.proj index 73f3ce93cd..78a7f2042b 100644 --- a/src/corehost/build.proj +++ b/src/corehost/build.proj @@ -1,13 +1,15 @@ - - + - + + $(NETCoreAppFramework) + + + - $(ArtifactsShippingPackagesDir) + $(ArtifactsShippingPackagesDir);$(ArtifactsNonShippingPackagesDir) false false diff --git a/src/pkg/packaging-tools/packaging-tools.targets b/src/pkg/packaging-tools/packaging-tools.targets index 82bff56ecd..0a25e48298 100644 --- a/src/pkg/packaging-tools/packaging-tools.targets +++ b/src/pkg/packaging-tools/packaging-tools.targets @@ -80,6 +80,7 @@ $(InstallerName)-$(InstallerBuildPart) $(AssetOutputPath)$(InstallerFileNameWithoutExtension)$(InstallerExtension) $(AssetOutputPath)$(InstallerFileNameWithoutExtension).exe + $(AssetOutputPath)$(InstallerFileNameWithoutExtension)-engine.exe $(AssetOutputPath)$(InstallerFileNameWithoutExtension)$(CompressedFileExtension) diff --git a/src/pkg/packaging-tools/windows/wix.targets b/src/pkg/packaging-tools/windows/wix.targets index 1d082736ce..53b761c542 100644 --- a/src/pkg/packaging-tools/windows/wix.targets +++ b/src/pkg/packaging-tools/windows/wix.targets @@ -346,18 +346,22 @@ + DependsOnTargets=" + GetInstallerGenerationFlags; + GetWixBuildConfiguration"> + DependsOnTargets=" + GetInstallerGenerationFlags; + GetWixBuildConfiguration"> diff --git a/src/pkg/projects/netcoreapp/pkg/legacy/Directory.Build.props b/src/pkg/projects/netcoreapp/pkg/legacy/Directory.Build.props index f4b18f4cf7..2bc2d8dbbd 100644 --- a/src/pkg/projects/netcoreapp/pkg/legacy/Directory.Build.props +++ b/src/pkg/projects/netcoreapp/pkg/legacy/Directory.Build.props @@ -4,6 +4,8 @@ true build/$(NETCoreAppFramework)/ + + false diff --git a/src/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj b/src/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj index f3a57e7551..279c09517b 100644 --- a/src/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj +++ b/src/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj @@ -15,6 +15,8 @@ true false + + false diff --git a/src/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj b/src/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj index cba841ca2e..c24c1a8db2 100644 --- a/src/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj +++ b/src/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj @@ -27,8 +27,6 @@ - - false - - - - - + false + diff --git a/src/pkg/projects/windowsdesktop/sfx/Microsoft.WindowsDesktop.App.SharedFx.sfxproj b/src/pkg/projects/windowsdesktop/sfx/Microsoft.WindowsDesktop.App.SharedFx.sfxproj index 31468cbd49..edd3cdb445 100644 --- a/src/pkg/projects/windowsdesktop/sfx/Microsoft.WindowsDesktop.App.SharedFx.sfxproj +++ b/src/pkg/projects/windowsdesktop/sfx/Microsoft.WindowsDesktop.App.SharedFx.sfxproj @@ -8,8 +8,6 @@ - - diff --git a/src/test/Assets/TestProjects/Directory.Build.props b/src/test/Assets/TestProjects/Directory.Build.props index d370c58a76..db2178e295 100644 --- a/src/test/Assets/TestProjects/Directory.Build.props +++ b/src/test/Assets/TestProjects/Directory.Build.props @@ -3,6 +3,7 @@ + diff --git a/src/test/Microsoft.DotNet.CoreSetup.Packaging.Tests/NuGetArtifactTester.cs b/src/test/Microsoft.DotNet.CoreSetup.Packaging.Tests/NuGetArtifactTester.cs index 421311d8e8..b46b78150f 100644 --- a/src/test/Microsoft.DotNet.CoreSetup.Packaging.Tests/NuGetArtifactTester.cs +++ b/src/test/Microsoft.DotNet.CoreSetup.Packaging.Tests/NuGetArtifactTester.cs @@ -36,8 +36,8 @@ public static NuGetArtifactTester OpenOrNull( id = id ?? project; string nuspecPath = Path.Combine( - dirs.BaseBinFolder, - project, + dirs.BaseArtifactsFolder, + "packages", dirs.Configuration, "specs", $"{id}.nuspec"); diff --git a/src/test/PrepareTestAssets/PrepareTestAssets.proj b/src/test/PrepareTestAssets/PrepareTestAssets.proj index bb5542c850..e3906ddeb0 100644 --- a/src/test/PrepareTestAssets/PrepareTestAssets.proj +++ b/src/test/PrepareTestAssets/PrepareTestAssets.proj @@ -41,6 +41,7 @@ Targets="Restore" Properties=" ArtifactsShippingPackagesDir=$(ArtifactsShippingPackagesDir); + ArtifactsNonShippingPackagesDir=$(ArtifactsNonShippingPackagesDir); TestRestorePackagesPath=$(TestRestorePackagesPath); TestTargetRid=$(TestTargetRid); MNAVersion=$(ProductVersion)" />