[release/3.0] Split blob publish into stages run before Arcade

azure-pipelines.yml

@@ -171,24 +171,12 @@ stages:
           name: NetCoreInternal-Pool
           queue: buildpool.windows.10.amd64.vs2017
 
-  # Stages-based publishing entry point
-  - template: eng\common\templates\post-build\post-build.yml
+  - template: /eng/stages/publish.yml
     parameters:
-      dependsOn: [PrepareForPublish]
-      # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871
-      enableSymbolValidation: false
-      # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254
-      enableSigningValidation: false
-      # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604
-      enableSourceLinkValidation: false
-      # Allow symbol publish to emit expected warnings without failing the build. Include single
-      # quotes inside the string so that it passes through to MSBuild without script interference.
-      symbolPublishingAdditionalParameters: "'-warnAsError:$false'"
-
-  - template: /eng/stages/custom-publish.yml
-    parameters:
-      # Custom publish stage dependency configuration.
-      # - dependsOn: <ID of the Arcade stage that custom publish should run after>
+      # Publish channel configuration. The publish template wraps Arcade publishing and adds some
+      # extras that aren't implemented in Arcade yet.
+      #
+      # - dependsOn: <ID of the Arcade stage that finalization should run after>
       #   channel:
       #     name: <Name of this channel, for logging and readability>
       #     bar: <Name of the Arcade variable that contains the ID of this channel in BAR>

eng/jobs/run-publish-project.yml

@@ -0,0 +1,120 @@
+parameters:
+  projectName: ''
+  dependency: null
+
+jobs:
+
+- template: /eng/common/templates/post-build/setup-maestro-vars.yml
+
+- job: CustomPublish
+  displayName: Custom Publish
+  dependsOn: setupMaestroVars
+  # Only run this job if setup-maestro-vars says the current branch publishes to this channel.
+  # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml
+  condition: contains(
+    dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'],
+    format('[{0}]', variables['${{ parameters.dependency.channel.bar }}']))
+  pool:
+    name: Hosted VS2017
+  # Double the default timeout.
+  timeoutInMinutes: 120
+  workspace:
+    clean: all
+
+  variables:
+  # Only get the secret variable groups if the def has the official name. Reduce dev build def risk.
+  - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}:
+    # Used for publishing individual leg assets to azure blob storage
+    - group: DotNet-DotNetCli-Storage
+    # Used for dotnet/versions update
+    - group: DotNet-Versions-Publish
+
+    # Blob storage publish (installers and checksums)
+  - name: _AzureAccountName
+    value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ]
+  - name: _ContainerName
+    value: $[ coalesce(variables.ContainerName, 'dotnet') ]
+  - name: _AzureAccessToken
+    value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ]
+  - name: _ChecksumAzureAccountName
+    value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ]
+  - name: _ChecksumContainerName
+    value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ]
+  - name: _ChecksumAzureAccessToken
+    value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ]
+  - name: _CommonPublishArgs
+    value: >-
+      /p:AzureAccountName=$(_AzureAccountName)
+      /p:ContainerName=$(_ContainerName)
+      /p:AzureAccessToken=$(_AzureAccessToken)
+      /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName)
+      /p:ChecksumContainerName=$(_ChecksumContainerName)
+      /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken)
+
+  # dotnet/versions update
+  - name: _GitHubUser
+    value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ]
+  - name: _GitHubEmail
+    value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ]
+  - name: _GitHubAuthToken
+    value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ]
+  - name: _VersionsRepoOwner
+    value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ]
+  - name: _VersionsRepo
+    value: $[ coalesce(variables.VersionsRepo, 'versions') ]
+  - name: _DotNetVersionsArgs
+    value: >-
+      /p:GitHubUser=$(_GitHubUser)
+      /p:GitHubEmail=$(_GitHubEmail)
+      /p:GitHubAuthToken=$(_GitHubAuthToken)
+      /p:VersionsRepoOwner=$(_VersionsRepoOwner)
+      /p:VersionsRepo=$(_VersionsRepo)
+      /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName)
+
+  steps:
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Download Artifacts
+    inputs:
+      artifactName: PreparedArtifacts
+      downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload
+
+  - powershell: |
+      $prefix = "refs/heads/"
+      $branch = "$(Build.SourceBranch)"
+      $branchName = $branch
+      if ($branchName.StartsWith($prefix))
+      {
+        $branchName = $branchName.Substring($prefix.Length)
+      }
+      Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName"
+      Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName"
+    displayName: Find true SourceBranchName
+
+  - script: >-
+      build.cmd -ci
+      -projects $(Build.SourcesDirectory)\publish\${{ parameters.projectName }}.proj
+      /p:Channel=${{ parameters.dependency.channel.storage }}
+      /p:Configuration=Release
+      $(_CommonPublishArgs)
+      $(_DotNetVersionsArgs)
+      /bl:$(Build.SourcesDirectory)\${{ parameters.projectName }}.binlog
+    displayName: Publish to custom locations
+
+  - task: CopyFiles@2
+    displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)'
+      Contents: |
+        **/*.log
+        **/*.binlog
+      TargetFolder: '$(Build.StagingDirectory)\BuildLogs'
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Artifact BuildLogs
+    inputs:
+      PathtoPublish: '$(Build.StagingDirectory)\BuildLogs'
+      ArtifactName: Logs-CustomPublish-${{ parameters.dependency.dependsOn }}-${{ parameters.projectName }}
+    condition: succeededOrFailed()

eng/stages/publish.yml

@@ -0,0 +1,48 @@
+parameters:
+  dependsOnPublishStages: []
+
+stages:
+
+# Create extra stage per BAR channel that needs extra publish steps.
+- ${{ each dependency in parameters.dependsOnPublishStages }}:
+  - stage: PublishBlob_${{ dependency.dependsOn }}
+    displayName: '${{ dependency.channel.name }} Blob Publish'
+    dependsOn: PrepareForPublish
+    variables:
+    - template: /eng/common/templates/post-build/common-variables.yml
+    jobs:
+    - template: /eng/jobs/run-publish-project.yml
+      parameters:
+        projectName: publish-blobs
+        dependency: ${{ dependency }}
+
+# Stages-based publishing entry point
+- template: /eng/common/templates/post-build/post-build.yml
+  parameters:
+    dependsOn:
+    - ${{ each dependency in parameters.dependsOnPublishStages }}:
+      - PublishBlob_${{ dependency.dependsOn }}
+    # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871
+    enableSymbolValidation: false
+    # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254
+    enableSigningValidation: false
+    # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604
+    enableSourceLinkValidation: false
+    # Allow symbol publish to emit expected warnings without failing the build. Include single
+    # quotes inside the string so that it passes through to MSBuild without script interference.
+    symbolPublishingAdditionalParameters: "'-warnAsError:$false'"
+
+# Create extra stage per BAR channel that needs extra publish steps. These run after the Arcade
+# stages because they depend on Arcade's NuGet package publish being complete.
+- ${{ each dependency in parameters.dependsOnPublishStages }}:
+  - stage: PublishFinal_${{ dependency.dependsOn }}
+    displayName: '${{ dependency.channel.name }} Finalize'
+    dependsOn:
+    - ${{ dependency.dependsOn }}
+    variables:
+    - template: /eng/common/templates/post-build/common-variables.yml
+    jobs:
+    - template: /eng/jobs/run-publish-project.yml
+      parameters:
+        projectName: publish-final
+        dependency: ${{ dependency }}