From 0364ee2da2dc6da2e73f10752560f0725b3997d4 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 15:22:19 -0800 Subject: [PATCH 01/77] Remove prepare-signed-artifacts and publish from each leg directly. --- Build.proj | 2 +- eng/Publishing.props | 143 +++--------- eng/Signing.props | 17 +- eng/pipelines/common/global-build-job.yml | 16 +- .../common/templates/global-build-step.yml | 2 +- .../jobs/prepare-signed-artifacts.yml | 66 ------ eng/pipelines/official/stages/publish.yml | 57 ----- eng/pipelines/runtime-official.yml | 205 +++++------------- 8 files changed, 103 insertions(+), 405 deletions(-) delete mode 100644 eng/pipelines/official/jobs/prepare-signed-artifacts.yml delete mode 100644 eng/pipelines/official/stages/publish.yml diff --git a/Build.proj b/Build.proj index 2f687610b4a93a..0957ae6846e573 100644 --- a/Build.proj +++ b/Build.proj @@ -1,6 +1,6 @@ - + diff --git a/eng/Publishing.props b/eng/Publishing.props index da3d606ed683c4..abed2e967c1b85 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -30,124 +30,53 @@ IsShipping="false" /> - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - + $(ArtifactsPackagesDir)**\runtime.native.*.nupkg + $(ArtifactsPackagesDir)**\*.Transport.*.nupkg" /> + - - - - - + + - - - - - - - - - - - + TODO: Some packages aren't produced in the windows_x64 jobs. In particular, packages matching the following patterns: + - Microsoft.NET.Workload.Mono.Toolchain.*Manifest-* + - Microsoft.NET.Sdk.WebAssembly.Pack.* + + We need to determine where to ship these from and how to specify it in the official build. + + Additionally, some packages matching the above RID-specific patterns aren't actually RID-specific and need to be deduplicated: + - Microsoft.NET.Runtime.*.Sample.Mono + - Microsoft.NET.Runtime.LibraryBuilder.Sdk + - Microsoft.NET.Runtime.MonoAOTCompiler.Task + - Microsoft.NET.Runtime.MonoTargets.Sdk + - Microsoft.NET.Runtime.WebAssembly.Sdk + - Microsoft.NET.Runtime.WebAssembly.Templates.net10 + - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk + - Microsoft.NET.Runtime.WorkloadTesting.Internal + --> + + + + + diff --git a/eng/Signing.props b/eng/Signing.props index c1ff2d8ba476b0..07761eda85163e 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,8 +1,4 @@ - - false - - @@ -45,8 +41,7 @@ - - + - - - \ No newline at end of file diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6bdd96038acf75..e6fc0db08d3908 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -33,6 +33,7 @@ parameters: postBuildSteps: [] extraVariablesTemplates: [] preBuildSteps: [] + PublishRidAgnosticPackagesFromPlatform: windows_x64 templatePath: 'templates' templateContext: '' disableComponentGovernance: '' @@ -52,6 +53,8 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} enablePublishTestResults: ${{ parameters.enablePublishTestResults }} testResultsFormat: ${{ parameters.testResultsFormat }} + enableMicrobuild: ${{ parameters.isOfficialBuild }} + enableMicrobuildForMacAndLinux: ${{ parameters.isOfficialBuild }} ${{ if ne(parameters.templateContext, '') }}: templateContext: ${{ parameters.templateContext }} @@ -123,7 +126,7 @@ jobs: - name: _officialBuildParameter ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:OfficialBuildId=$(Build.BuildNumber) + value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true ${{ if ne(parameters.isOfficialBuild, true) }}: value: '' @@ -132,6 +135,17 @@ jobs: value: /p:BuildDarwinFrameworks=true ${{ if notin(parameters.osGroup, 'ios', 'tvos', 'maccatalyst')}}: value: '' + + - name: _assetManifestFileNameParameter + ${{ if eq(parameters.isOfficialBuild, true) }}: + value: /p:AssetManifestFileName="${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{parameters.archType}}_${{parameters.nameSuffix }}" + ${{ if ne(parameters.isOfficialBuild, true) }}: + value: '' + - name: _publishRidAgnosticArtifactsParameter + ${{ if and(eq(parameters.isOfficialBuild, true), eq(format('{0}{1}_{2}', parameters.osGroup, parameters.osSubGroup, parameters.archType), parameters.PublishRidAgnosticPackagesFromPlatform)) }}: + value: /p:PublishRidAgnosticPackages=true + ${{ if ne(parameters.isOfficialBuild, true) }}: + value: '' # Set no native sanitizers by default - name: _nativeSanitizersArg diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 2a2262d9a4ead4..75cfd580952add 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -10,7 +10,7 @@ parameters: condition: succeeded() steps: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) $(_assetManifestFileNameParameter) $(_publishRidAgnosticArtifactsParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml deleted file mode 100644 index 1440a7a1dfdc35..00000000000000 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ /dev/null @@ -1,66 +0,0 @@ -parameters: - PublishRidAgnosticPackagesFromPlatform: '' - isOfficialBuild: false - logArtifactName: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' - -jobs: -- template: /eng/common/templates-official/job/job.yml - parameters: - name: 'PrepareSignedArtifacts' - displayName: 'Prepare Signed Artifacts' - - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - - # Double the default timeout. - timeoutInMinutes: 240 - - workspace: - clean: all - - enableMicrobuild: true - - variables: - - name: '_SignType' - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - templateContext: - inputs: - - input: checkout - repository: self - clean: true - fetchDepth: 20 - - input: pipelineArtifact - artifactName: IntermediateArtifacts - targetPath: $(Build.SourcesDirectory)\artifacts\PackageDownload\IntermediateArtifacts - outputs: - - output: pipelineArtifact - displayName: 'Publish BuildLogs' - condition: succeededOrFailed() - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.logArtifactName }} - - steps: - - script: >- - build.cmd -restore -sign -publish -ci -configuration Release - /p:RestoreToolsetOnly=true - /p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }} - /p:DownloadDirectory=$(Build.SourcesDirectory)\artifacts\PackageDownload\ - /p:OfficialBuildId=$(Build.BuildNumber) - /p:SignType=$(_SignType) - /p:DotNetSignType=$(_SignType) - /p:DotNetPublishUsingPipelines=true - /bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog - displayName: Prepare artifacts and upload to build - - - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' - continueOnError: true - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml deleted file mode 100644 index 83b059ffc32a65..00000000000000 --- a/eng/pipelines/official/stages/publish.yml +++ /dev/null @@ -1,57 +0,0 @@ -parameters: - PublishRidAgnosticPackagesFromPlatform: windows_x64 - -stages: - -- stage: PrepareForPublish - displayName: Prepare for Publish - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml - parameters: - PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} - - # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - - template: /eng/common/templates-official/job/publish-build-assets.yml - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - dependsOn: PrepareSignedArtifacts - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' - -# Stages-based publishing entry point -- template: /eng/common/templates-official/post-build/post-build.yml - parameters: - validateDependsOn: - - PrepareForPublish - # The following checks are run after the build in the validation and release pipelines - # And thus are not enabled here. They can be enabled for dev builds for spot testing if desired - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishAssetsImmediately: true - SDLValidationParameters: - enable: false - artifactNames: - - PackageArtifacts - - BlobArtifacts - params: >- - -SourceToolsList @("policheck","credscan") - -TsaInstanceURL "$(TsaInstanceURL)" - -TsaProjectName "$(TsaProjectName)" - -TsaNotificationEmail "$(TsaNotificationEmail)" - -TsaCodebaseAdmin "$(TsaCodebaseAdmin)" - -TsaBugAreaPath "$(TsaBugAreaPath)" - -TsaIterationPath "$(TsaIterationPath)" - -TsaRepositoryName "$(TsaRepositoryName)" - -TsaCodebaseName "$(TsaCodebaseName)" - -TsaPublish $True - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' - # Publish to blob storage. - publishInstallersAndChecksums: true diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 4604248840acb0..ac8792df4e0200 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,14 +85,9 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - # # Build all runtime packs # Mac x64/arm64 @@ -136,14 +131,9 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - # # Build all runtime packs for Linux and Linux musl # Upload CoreCLR runtime for CrossDac packaging @@ -161,7 +151,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -179,11 +169,6 @@ extends: PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts PublishLocation: Container ArtifactName: CoreCLRCrossDacArtifacts - - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) extraVariablesTemplates: - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml @@ -198,7 +183,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -226,12 +211,6 @@ extends: basePath: $(crossDacArtifactsPath) isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 30 - postBuildSteps: - # Save packages using the prepare-signed-artifacts format. - # CrossDac packages are expected to be in the windows_x64 folder. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: windows_x64 dependsOn: - build_linux_x64_release_AllRuntimes - build_linux_arm_release_AllRuntimes @@ -272,22 +251,9 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -298,21 +264,9 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -323,22 +277,10 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks # # Build Mono LLVM runtime packs @@ -354,23 +296,11 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) # # Build libraries (all TFMs) and packages @@ -383,13 +313,9 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -pack -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: Libraries_WithPackages timeoutInMinutes: 95 # # Build SourceBuild packages @@ -417,13 +343,9 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO - postBuildSteps: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: PGO timeoutInMinutes: 95 # @@ -441,39 +363,39 @@ extends: preBuildSteps: - task: DownloadPipelineArtifact@2 inputs: - artifact: 'IntermediateArtifacts' + artifact: 'PackageArtifacts' path: $(Build.SourcesDirectory)/artifacts/workloadPackages patterns: | - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.win-x64*.nupkg - IntermediateArtifacts/windows_x86/Shipping/Microsoft.NETCore.App.Runtime.win-x86*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x64*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg + PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - task: CopyFiles@2 displayName: Flatten packages @@ -484,40 +406,7 @@ extends: targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages flattenFolders: true - buildArgs: -s mono.workloads -c $(_BuildConfig) /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads - - postBuildSteps: - # Prepare packages wrapping msis - - task: CopyFiles@2 - displayName: Prepare package artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)' - Contents: | - Shipping/**/* - NonShipping/**/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts1/workloads' - CleanTargetFolder: true - - # Prepare artifacts to be used for generating VS components - - task: CopyFiles@2 - displayName: Prepare VS Insertion artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/VSSetup/$(_BuildConfig)' - Contents: | - Insertion/**/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts2/workloads-vs' - CleanTargetFolder: true - - templateContext: - outputs: - - output: buildArtifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts1' - ArtifactName: IntermediateArtifacts - displayName: 'Publish workload packages' - - output: buildArtifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts2' - ArtifactName: IntermediateArtifacts - displayName: 'Publish workload VS Insertion artifacts' + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -540,7 +429,11 @@ extends: - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes - - ${{ if eq(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/official/stages/publish.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' \ No newline at end of file From 7e284a45dcd94be860b8718617796d55b674e634 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 15:27:59 -0800 Subject: [PATCH 02/77] Publish in a separate stage to make dependency tracking easy --- eng/pipelines/runtime-official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index ac8792df4e0200..068fe007996f59 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -429,6 +429,8 @@ extends: - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes + - stage: Publish + jobs: - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true From d2c9cf864e1dcf5e5b6e66b0011ef26c6784a93d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 16:48:03 -0800 Subject: [PATCH 03/77] Set _SignType so we can pass -sign --- eng/pipelines/common/global-build-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index e6fc0db08d3908..d9211734804ae3 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -93,6 +93,9 @@ jobs: - name: _archParameter value: -arch ${{ parameters.archType }} + - name: _SignType + value: $[ coalesce(variables.OfficialSignType, 'real') ] + - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubGroup, '_bionic')) }}: - name: _osParameter value: -os linux-bionic @@ -126,7 +129,7 @@ jobs: - name: _officialBuildParameter ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true + value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true /p:SignType=$(_SignType) /p:DotNetSignType=$(_SignType) ${{ if ne(parameters.isOfficialBuild, true) }}: value: '' From 3338d4d50088ae69c6146dcc8bcd0f135a7c5ba5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 13:34:31 -0800 Subject: [PATCH 04/77] Remove old validation target --- eng/Publishing.props | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index abed2e967c1b85..b5a2c8e336b6ab 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -140,18 +140,4 @@ RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Artifact.SubBlobFolder)%(Filename)%(Extension)" /> - - - - - - - - - From 01bca6023d4144f37b0067410fc643594a698593 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 13:35:19 -0800 Subject: [PATCH 05/77] !drop remove -sign so we can figure out publishing first --- eng/pipelines/runtime-official.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 068fe007996f59..d0c3e58a21be62 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,7 +85,7 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs # @@ -131,7 +131,7 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # @@ -151,7 +151,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -183,7 +183,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -251,7 +251,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - template: /eng/pipelines/common/platform-matrix.yml @@ -264,7 +264,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -277,7 +277,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -296,7 +296,7 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT @@ -313,7 +313,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -343,7 +343,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -406,7 +406,7 @@ extends: targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages flattenFolders: true - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 88ee36c4fad2e40545ad3e2e4ccfedf453b3ee07 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 14:04:30 -0800 Subject: [PATCH 06/77] Remove InitialTargets --- eng/Publishing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index b5a2c8e336b6ab..b2d05d00626342 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,4 +1,4 @@ - + true From d2fa2444b6e289c6c6e9eef549c46a4c4d18554f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 15:42:56 -0800 Subject: [PATCH 07/77] Only generate product version files once. Fix asset manifest extension. Don't copy workloads packages as they're already in the right format now. --- eng/Publishing.props | 5 ++--- eng/pipelines/common/global-build-job.yml | 2 +- eng/pipelines/runtime-official.yml | 11 +---------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index b2d05d00626342..ec84d512e06d26 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -98,14 +98,13 @@ but we do produce them in both the VMR and the runtime official build. --> - <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' == 'true' and '$(DotNetBuildOrchestrator)' == 'true'">true - <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' != 'true' and '$(DotNetBuildOrchestrator)' != 'true'">true + true + Condition="'$(ShouldGenerateProductVersionFiles)' == 'true'"> Date: Wed, 22 Jan 2025 16:34:13 -0800 Subject: [PATCH 08/77] Completely restructure how we publish to be more explicit that specific jobs publish specific packages --- Directory.Build.props | 99 +---------------- eng/Publishing.props | 68 ++++++------ eng/RuntimeIdentifier.props | 100 ++++++++++++++++++ .../common/templates/packages-to-publish.yml | 35 ++++++ eng/pipelines/runtime-official.yml | 19 ++-- 5 files changed, 181 insertions(+), 140 deletions(-) create mode 100644 eng/RuntimeIdentifier.props create mode 100644 eng/pipelines/common/templates/packages-to-publish.yml diff --git a/Directory.Build.props b/Directory.Build.props index 79ec992824c389..052f4e532e3a12 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -201,104 +201,7 @@ $(MonoCrossAOTTargetOS)+tvos+ios+maccatalyst - - false - true - - - - - - - <_portableOS>$(TargetOS.ToLowerInvariant()) - <_portableOS Condition="'$(_portableOS)' == 'windows'">win - - - <_portableOS Condition="'$(_portableOS)' == 'anyos'">$(__PortableTargetOS) - - - <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-musl'">linux-musl - <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-bionic'">linux-bionic - - - <_portableOS Condition="'$(HostOS)' == 'win' and '$(TargetsMobile)' != 'true'">win - - - - - <_packageOS>$(_portableOS) - - <_packageOS Condition="'$(CrossBuild)' == 'true' and '$(_portableOS)' != 'linux-musl' and '$(_portableOS)' != 'linux-bionic'">$(_hostOS) - - - $(PackageOS)-$(TargetArchitecture) - $(_packageOS)-$(TargetArchitecture) - - - - - - <_portableHostOS>$(_hostOS) - <_portableHostOS Condition="'$(_portableHostOS)' == 'windows'">win - <_portableHostOS Condition="'$(CrossBuild)' != 'true' and '$(_portableOS)' == 'linux-musl'">linux-musl - - - $(ToolsOS)-$(_hostArch) - $(_portableHostOS)-$(_hostArch) - - - $(ToolsRID) - - - - - <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) - <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) - - <_parseDistroRid>$(__DistroRid) - <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) - <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) - - <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) - <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) - - $(_outputOS)-$(TargetArchitecture) - - - - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'microsoft.netcore.app.ref')) diff --git a/eng/Publishing.props b/eng/Publishing.props index ec84d512e06d26..1f4caa1eaca12f 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,6 +8,8 @@ $(TargetArchitecture) + + @@ -30,50 +32,44 @@ IsShipping="false" /> - + - - - + + + + + + + - - + + + - + + + + + + + + + diff --git a/eng/RuntimeIdentifier.props b/eng/RuntimeIdentifier.props new file mode 100644 index 00000000000000..1090d58b5c6187 --- /dev/null +++ b/eng/RuntimeIdentifier.props @@ -0,0 +1,100 @@ + + + false + true + + + + + + + <_portableOS>$(TargetOS.ToLowerInvariant()) + <_portableOS Condition="'$(_portableOS)' == 'windows'">win + + + <_portableOS Condition="'$(_portableOS)' == 'anyos'">$(__PortableTargetOS) + + + <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-musl'">linux-musl + <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-bionic'">linux-bionic + + + <_portableOS Condition="'$(HostOS)' == 'win' and '$(TargetsMobile)' != 'true'">win + + + + + <_packageOS>$(_portableOS) + + <_packageOS Condition="'$(CrossBuild)' == 'true' and '$(_portableOS)' != 'linux-musl' and '$(_portableOS)' != 'linux-bionic'">$(_hostOS) + + + $(PackageOS)-$(TargetArchitecture) + $(_packageOS)-$(TargetArchitecture) + + + + + + <_portableHostOS>$(_hostOS) + <_portableHostOS Condition="'$(_portableHostOS)' == 'windows'">win + <_portableHostOS Condition="'$(CrossBuild)' != 'true' and '$(_portableOS)' == 'linux-musl'">linux-musl + + + $(ToolsOS)-$(_hostArch) + $(_portableHostOS)-$(_hostArch) + + + $(ToolsRID) + + + + + <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) + + <_parseDistroRid>$(__DistroRid) + <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) + <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) + + <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) + <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) + + $(_outputOS)-$(TargetArchitecture) + + + + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + + \ No newline at end of file diff --git a/eng/pipelines/common/templates/packages-to-publish.yml b/eng/pipelines/common/templates/packages-to-publish.yml new file mode 100644 index 00000000000000..6bcc28265f85e9 --- /dev/null +++ b/eng/pipelines/common/templates/packages-to-publish.yml @@ -0,0 +1,35 @@ +parameters: + # Parameters forwarded from global-build-job.yml + targetRid: '' + # Default artifact sets for specific RIDs + ios_arm64AdditionalPackages: + - Microsoft.NET.Runtime.iOS.Sample.Mono + - Microsoft.NET.Runtime.LibraryBuilder.Sdk + - Microsoft.NET.Runtime.MonoAOTCompiler.Task + - Microsoft.NET.Runtime.MonoTargets.Sdk + android_arm64AdditionalPackages: + - Microsoft.NET.Runtime.Android.Sample.Mono + wasi_wasmAdditionalPackages: + - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk + browser_wasmAdditionalPackages: + - Microsoft.NET.Runtime.WebAssembly.Sdk + - Microsoft.NET.Runtime.WebAssembly.Templates.net10 + - Microsoft.NET.Sdk.WebAssembly.Pack + - Microsoft.NET.Workload.Mono.ToolChain + +# TODO-Publish: Move this into Publishing.props except for the win-x64 case. + +variables: + - name: _PublishAdditionalPackagesArg + ${{ if eq(parameters.targetRid, 'ios-arm64') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.ios_arm64AdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'android-arm64') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.android_arm64AdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'wasi-wasm') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.wasi_wasmAdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'browser-wasm') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.browser_wasmAdditionalPackages) }} + ${{ if eq(parameters.targetRid, 'windows_x64') }}: + value: /p:PublishAllPackages=true + ${{ else }}: + value: '' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 6c2c9da1cea6fe..d9bf067231fe0a 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,8 +85,10 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml # # Build all runtime packs @@ -183,7 +185,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -251,9 +253,12 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -264,9 +269,11 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -313,7 +320,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -397,7 +404,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 8ee9ac1b5a750f4e9e8195376ecd7d9d49d5e343 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 11:51:05 -0800 Subject: [PATCH 09/77] Move "non-RID-specific" special packages logic into Publishing.props --- eng/Publishing.props | 33 +++++++++++++---- .../common/templates/packages-to-publish.yml | 35 ------------------- eng/pipelines/runtime-official.yml | 11 +++--- 3 files changed, 31 insertions(+), 48 deletions(-) delete mode 100644 eng/pipelines/common/templates/packages-to-publish.yml diff --git a/eng/Publishing.props b/eng/Publishing.props index 1f4caa1eaca12f..1579fa98616c43 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -38,8 +38,7 @@ For non-RID-specific packages, we have various rules: - A job can specify PublishAllPackages=true to publish all packages it produces. We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - - A job can specify PublishAdditionalSpecificPackages="Package1;Package2" to publish the specified packages. We use this for specific jobs were we produce the same package in multiple jobs, - but we don't produce the package in any jobs where we set PublishAllPackages. + - For some target RIDs, we also include specific RID-agnostic packages. --> @@ -49,6 +48,32 @@ + + + + + + + + + + + + + + + + diff --git a/eng/pipelines/common/templates/packages-to-publish.yml b/eng/pipelines/common/templates/packages-to-publish.yml deleted file mode 100644 index 6bcc28265f85e9..00000000000000 --- a/eng/pipelines/common/templates/packages-to-publish.yml +++ /dev/null @@ -1,35 +0,0 @@ -parameters: - # Parameters forwarded from global-build-job.yml - targetRid: '' - # Default artifact sets for specific RIDs - ios_arm64AdditionalPackages: - - Microsoft.NET.Runtime.iOS.Sample.Mono - - Microsoft.NET.Runtime.LibraryBuilder.Sdk - - Microsoft.NET.Runtime.MonoAOTCompiler.Task - - Microsoft.NET.Runtime.MonoTargets.Sdk - android_arm64AdditionalPackages: - - Microsoft.NET.Runtime.Android.Sample.Mono - wasi_wasmAdditionalPackages: - - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk - browser_wasmAdditionalPackages: - - Microsoft.NET.Runtime.WebAssembly.Sdk - - Microsoft.NET.Runtime.WebAssembly.Templates.net10 - - Microsoft.NET.Sdk.WebAssembly.Pack - - Microsoft.NET.Workload.Mono.ToolChain - -# TODO-Publish: Move this into Publishing.props except for the win-x64 case. - -variables: - - name: _PublishAdditionalPackagesArg - ${{ if eq(parameters.targetRid, 'ios-arm64') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.ios_arm64AdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'android-arm64') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.android_arm64AdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'wasi-wasm') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.wasi_wasmAdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'browser-wasm') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.browser_wasmAdditionalPackages) }} - ${{ if eq(parameters.targetRid, 'windows_x64') }}: - value: /p:PublishAllPackages=true - ${{ else }}: - value: '' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d9bf067231fe0a..1ee6513c279e99 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -70,6 +70,9 @@ extends: - windows_x64 - windows_x86 - windows_arm64 + variables: + - name: PublishAllPackages + value: $[eq(variables.targetRid, 'win-x64')] jobParameters: templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog @@ -85,10 +88,8 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml # # Build all runtime packs @@ -256,8 +257,6 @@ extends: buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -272,8 +271,6 @@ extends: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: From 44553015230b1fcc2c3e8d33725f89c145d215fe Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 13:24:39 -0800 Subject: [PATCH 10/77] Make sure we build the list of skipped packages after we add the RID-specific packages --- eng/Publishing.props | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 1579fa98616c43..f7a867f7169083 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -43,11 +43,9 @@ - + - - @@ -74,6 +72,7 @@ + false - - $(TargetArchitecture) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 0dfa1809a53486..6cbc0c09eeda37 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -33,7 +33,6 @@ parameters: postBuildSteps: [] extraVariablesTemplates: [] preBuildSteps: [] - PublishRidAgnosticPackagesFromPlatform: windows_x64 templatePath: 'templates' templateContext: '' disableComponentGovernance: '' @@ -92,6 +91,9 @@ jobs: value: -os ${{ parameters.osGroup }} - name: _archParameter value: -arch ${{ parameters.archType }} + + - name: _AssetManifestName + value: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.nameSuffix }} - name: _SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] @@ -138,17 +140,6 @@ jobs: value: /p:BuildDarwinFrameworks=true ${{ if notin(parameters.osGroup, 'ios', 'tvos', 'maccatalyst')}}: value: '' - - - name: _assetManifestFileNameParameter - ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:AssetManifestFileName="${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{parameters.archType}}_${{parameters.nameSuffix }}.xml" - ${{ if ne(parameters.isOfficialBuild, true) }}: - value: '' - - name: _publishRidAgnosticArtifactsParameter - ${{ if and(eq(parameters.isOfficialBuild, true), eq(format('{0}{1}_{2}', parameters.osGroup, parameters.osSubGroup, parameters.archType), parameters.PublishRidAgnosticPackagesFromPlatform)) }}: - value: /p:PublishRidAgnosticPackages=true - ${{ if ne(parameters.isOfficialBuild, true) }}: - value: '' # Set no native sanitizers by default - name: _nativeSanitizersArg diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 75cfd580952add..2a2262d9a4ead4 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -10,7 +10,7 @@ parameters: condition: succeeded() steps: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) $(_assetManifestFileNameParameter) $(_publishRidAgnosticArtifactsParameter) + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1ee6513c279e99..b655fb5da8c373 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -281,7 +281,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -300,8 +300,8 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish - /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AssetManifestFileName=$(_AssetManifestName).xml + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -317,7 +317,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -347,7 +347,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -401,7 +401,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 747bffebbf21190199e9e455a77f8e2d22cfc29d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 16:32:27 -0800 Subject: [PATCH 12/77] Extract handling hostOS and hostArch into another props file --- Directory.Build.props | 35 ++--------------------------------- eng/OSArch.props | 35 +++++++++++++++++++++++++++++++++++ eng/Publishing.props | 1 + 3 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 eng/OSArch.props diff --git a/Directory.Build.props b/Directory.Build.props index 052f4e532e3a12..1c2c021f5d7633 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -17,39 +17,8 @@ false - - <_hostOS>linux - <_hostOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx - <_hostOS Condition="$([MSBuild]::IsOSPlatform('FREEBSD'))">freebsd - <_hostOS Condition="$([MSBuild]::IsOSPlatform('NETBSD'))">netbsd - <_hostOS Condition="$([MSBuild]::IsOSPlatform('ILLUMOS'))">illumos - <_hostOS Condition="$([MSBuild]::IsOSPlatform('SOLARIS'))">solaris - <_hostOS Condition="$([MSBuild]::IsOSPlatform('HAIKU'))">haiku - <_hostOS Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">windows - $(_hostOS) - browser - $(_hostOS) - true - true - - - - - <_hostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) - $(_hostArch) - wasm - wasm - arm - armv6 - armel - arm64 - loongarch64 - s390x - ppc64le - x64 - x64 - $(TargetArchitecture) - + + + + <_hostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) + $(_hostArch) + wasm + wasm + arm + armv6 + armel + arm64 + loongarch64 + s390x + ppc64le + x64 + x64 + $(TargetArchitecture) + + \ No newline at end of file diff --git a/eng/Publishing.props b/eng/Publishing.props index 3d3fd70e8ff2e5..2ca4d5db89a6ff 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -6,6 +6,7 @@ false + - + @@ -44,7 +46,7 @@ - + From 6bbbc4cbf39a79ef4c9a8e89186c53f7c9287436 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 24 Jan 2025 13:43:29 -0800 Subject: [PATCH 14/77] Fix asset manifest name uniqueness to ensue we don't overwrite asset manifests. --- eng/Publishing.props | 7 ++++--- eng/pipelines/runtime-official.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 917bd4fd2d6d09..0347fea9c02864 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,14 +1,15 @@ + + true false + + $(PackageRID).xml - - - diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index b655fb5da8c373..35ceca9ad22d9b 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -186,7 +186,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -254,7 +254,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -268,7 +268,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} From c1068eb6b3a325776e3ea920f6af435efc3e2bc4 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 24 Jan 2025 15:38:15 -0800 Subject: [PATCH 15/77] Set AssetManifestFilePath again --- eng/Publishing.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 0347fea9c02864..251fdcec547269 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,6 +8,8 @@ false $(PackageRID).xml + + $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) - + diff --git a/eng/Signing.props b/eng/Signing.props index 07761eda85163e..2dfee7d17628c8 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -23,10 +23,10 @@ - @@ -48,7 +48,8 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi" + $(ArtifactsPackagesDir)**\*.msi + $(ArtifactsDir)VSSetup\**\*.zip;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> @@ -56,4 +57,4 @@ - \ No newline at end of file + From 552bb1d9eb1eb961cfb1c28d62baa4efcdc627c8 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 27 Jan 2025 16:53:45 -0800 Subject: [PATCH 20/77] Add missing semicolon --- eng/Signing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Signing.props b/eng/Signing.props index 2dfee7d17628c8..9e8512c3355eda 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -48,7 +48,7 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi + $(ArtifactsPackagesDir)**\*.msi; $(ArtifactsDir)VSSetup\**\*.zip;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> From 92470278b04e97cffd34f730dfdf7676eb5204e7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 27 Jan 2025 16:57:29 -0800 Subject: [PATCH 21/77] Include VSSetup artifacts separately --- eng/Publishing.props | 5 ----- eng/Signing.props | 10 ++++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index c58d8cca34d625..9332041a8d6687 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -101,11 +101,6 @@ - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 9e8512c3355eda..78688762d0dfcc 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -48,13 +48,19 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi; - $(ArtifactsDir)VSSetup\**\*.zip;" + $(ArtifactsPackagesDir)**\*.msi;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> %(FullPath).sha512 + + + %(FullPath).sha512 + workloads/ + From 1eb6a857e15fbd3aaf5861e14e6439d49438ffd7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 28 Jan 2025 14:25:17 -0800 Subject: [PATCH 22/77] Re-enable signing (depends on Microbuild signing on Linux/Mac) --- eng/pipelines/runtime-official.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index b3c211bd6792a3..2afda15fa1ebdd 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -88,7 +88,7 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs # @@ -134,7 +134,7 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # @@ -154,7 +154,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -186,7 +186,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -254,7 +254,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -268,7 +268,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -281,7 +281,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -300,7 +300,7 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AssetManifestFileName=$(_AssetManifestName).xml + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AssetManifestFileName=$(_AssetManifestName).xml /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT @@ -317,7 +317,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml + buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -347,7 +347,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -401,7 +401,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 8ce2e0992466436e43d2330ab5c060055757e5b2 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 29 Jan 2025 11:12:37 -0800 Subject: [PATCH 23/77] PR feedback --- eng/Publishing.props | 10 +--------- eng/Signing.props | 7 +++---- eng/pipelines/runtime-official.yml | 12 ++++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 9332041a8d6687..f324969ef3f8d4 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -7,7 +7,7 @@ false - $(PackageRID).xml + $(AGENT_JOBNAME) $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) @@ -92,14 +92,6 @@ - - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 54aa91c7ede5af..253ede751b44e4 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,4 +1,7 @@ + + + @@ -10,6 +13,9 @@ + + + diff --git a/eng/pipelines/common/macos-sign-with-entitlements.yml b/eng/pipelines/common/macos-sign-with-entitlements.yml deleted file mode 100644 index 72a03b90f340d6..00000000000000 --- a/eng/pipelines/common/macos-sign-with-entitlements.yml +++ /dev/null @@ -1,77 +0,0 @@ -parameters: - filesToSign: [] - timeoutInMinutes: '30' - -steps: - - task: UseDotNet@2 - displayName: Install .NET 6 SDK for signing. - inputs: - packageType: 'sdk' - version: '6.0.x' - installationPath: '$(Agent.TempDirectory)/dotnet' - - - ${{ each file in parameters.filesToSign }}: - - task: CopyFiles@2 - displayName: 'Copy entitled file ${{ file.name }}' - inputs: - contents: '${{ file.path }}/${{ file.name }}' - targetFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled' - overWrite: true - - - task: ArchiveFiles@2 - displayName: 'Zip MacOS files for signing' - inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/mac_entitled' - archiveFile: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' - archiveType: zip - includeRootFolder: true - replaceExistingArchive: true - - - task: EsrpCodeSigning@5 - displayName: 'ESRP CodeSigning' - inputs: - ConnectedServiceName: 'DotNet-Engineering-Services_KeyVault' - AppRegistrationClientId: '28ec6507-2167-4eaa-a294-34408cf5dd0e' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'EngKeyVault' - AuthCertName: 'DotNetCore-ESRP-AuthCert' - AuthSignCertName: 'DotNetCore-ESRP-AuthSignCert' - FolderPath: '$(Build.ArtifactStagingDirectory)/' - Pattern: 'mac_entitled_to_sign.zip' - UseMinimatch: true - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401337-Apple", - "operationCode": "MacAppDeveloperSign", - "parameters" : { - "hardening": "Enable" - }, - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: ${{ parameters.timeoutInMinutes }} - MaxConcurrency: '50' - MaxRetryAttempts: '5' - PendingAnalysisWaitTimeoutMinutes: '5' - env: - DOTNET_MULTILEVEL_LOOKUP: 0 - DOTNET_ROOT: '$(Agent.TempDirectory)/dotnet' - DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR: '$(Agent.TempDirectory)/dotnet' - - - task: ExtractFiles@1 - displayName: 'Extract MacOS after signing' - inputs: - archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' - destinationFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' - - - ${{ each file in parameters.filesToSign }}: - - task: CopyFiles@2 - displayName: 'Copy ${{ file.name }} to destination' - inputs: - contents: ${{ file.name }} - sourceFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' - targetFolder: '${{ file.path }}' - overWrite: true diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1385372daefe3d..ac169e34d6dfba 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -109,37 +109,10 @@ extends: - osx_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + buildArgs: -s -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 - postBuildSteps: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: createdump - path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - - name: corerun - path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - - name: dotnet - path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost - - name: apphost - path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost - - - task: CopyFiles@2 - displayName: 'Copy signed createdump to sharedFramework' - inputs: - contents: createdump - sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework - overWrite: true - - # Now that we've entitled and signed createdump, we can build the rest. - - template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true - displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # # Build all runtime packs for Linux and Linux musl From 220a7dc4baafe28249d6dc21b705e7485083a6d7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 30 Jan 2025 11:29:19 -0800 Subject: [PATCH 25/77] In a VMR-style build, produce the product version file in either the win-x64 leg (for UB) or in all legs (for SB) --- eng/DotNetBuild.props | 1 + eng/Publishing.props | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 223eae78ef3ee3..8e3a42b5c7b5b4 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -78,6 +78,7 @@ $(InnerBuildArgs) /p:MonoBundleLLVMOptimizer=$(DotNetBuildMonoBundleLLVMOptimizer) $(InnerBuildArgs) /p:DotNetBuildMonoCrossAOT=$(DotNetBuildMonoCrossAOT) $(InnerBuildArgs) /p:DotNetBuildAllRuntimePacks=$(DotNetBuildAllRuntimePacks) + $(InnerBuildArgs) /p:DotNetBuildPass=$(DotNetBuildPass) $(InnerBuildArgs) $(FlagParameterPrefix)pgoinstrument diff --git a/eng/Publishing.props b/eng/Publishing.props index 95ee767f3d2d91..62d8fc0929b2ed 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -85,11 +85,10 @@ In non-VMR builds, we can skip publishing RID-agnostic packages entirely when we're not the lane that is supposed to publish them. In VMR builds, we need to publish them for upstack jobs to consume. --> - + - @@ -108,8 +107,9 @@ This ensures that we don't produce these files in the "Repo source build" builds, but we do produce them in both the VMR and the runtime official build. --> - - true + + true + true Date: Fri, 31 Jan 2025 15:10:30 -0800 Subject: [PATCH 26/77] Change where we set PublishAllPackages to make things automatically work when this change flows to the VMR. --- eng/DotNetBuild.props | 8 ++++++++ eng/Publishing.props | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 8e3a42b5c7b5b4..746e74d075fc60 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -110,6 +110,14 @@ $(InnerBuildArgs) /p:NetCoreAppToolCurrentVersion=$(NetCoreAppToolCurrentVersion) + + + $(InnerBuildArgs) /p:PublishAllPackages=true diff --git a/eng/Publishing.props b/eng/Publishing.props index 62d8fc0929b2ed..987823290a9bcb 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -38,7 +38,6 @@ - A job can specify PublishAllPackages=true to publish all packages it produces. We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - In Source-Build scenarios, we want to publish all packages from here as source-build legs are fully self-contained. - For some target RIDs, we also include specific RID-agnostic packages. --> @@ -79,7 +78,7 @@ - + false - - $(AGENT_JOBNAME) - - $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) - $(InnerBuildArgs) /p:PublishAllPackages=true + $(InnerBuildArgs) /p:EnableDefaultArtifacts=true diff --git a/eng/Publishing.props b/eng/Publishing.props index 904caf1af838a8..55b6d2088f151d 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -3,86 +3,106 @@ true false - - - - - - - - + + false + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - $(InnerBuildArgs) /p:EnableDefaultArtifacts=true + $(InnerBuildArgs) /p:EnableDefaultArtifacts=false diff --git a/eng/Publishing.props b/eng/Publishing.props index 55b6d2088f151d..487c8abcc4a53a 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -13,9 +13,11 @@ We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - For some target RIDs, we also include specific RID-agnostic packages. + VMR jobs control whether or not a vertical has EnableDefaultArtifacts set to true or false in DotNetBuild.props. + Packages that do not meet the above rules are added with Vertical visibility in the VMR and excluded in non-VMR builds. --> - false + false false - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 253ede751b44e4..6c78a667bdd91b 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -2,6 +2,23 @@ + + + false + + @@ -68,4 +85,86 @@ SubBlobFolder="workloads/" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cbfbe4a426731f9b34e92aa3128094a2ed48b083 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 11:00:36 -0800 Subject: [PATCH 33/77] Only publish the LLVM runtime packs from LLVM legs --- eng/Signing.props | 61 +++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 6c78a667bdd91b..18d123f01d40b1 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -17,6 +17,10 @@ Packages that do not meet the above rules are added with Vertical visibility in the VMR and excluded in non-VMR builds. --> false + true + false + + true @@ -57,14 +61,7 @@ - - - - - - - - + + + + + + + + + + + + + + + + @@ -146,25 +168,6 @@ Kind="Package" Condition="'$(DotNetBuildOrchestrator)' == 'true'" /> - - - - - - - - - From 60b8af5ee39aca28cd472ed3bed15ddb8f089efe Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 11:45:19 -0800 Subject: [PATCH 34/77] Add the package artifacts to the right item group --- eng/Signing.props | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/Signing.props b/eng/Signing.props index 18d123f01d40b1..746078a6cecef5 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -115,6 +115,9 @@ + From 7b209dde3826cc43a823d94fa9ad94651fd9ff39 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 13:14:06 -0800 Subject: [PATCH 35/77] Fix package name format --- eng/Signing.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 746078a6cecef5..5b6717e0eff634 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -18,7 +18,7 @@ --> false true - false + false true @@ -112,9 +112,9 @@ IsShipping="false" /> - + - + From ca33ca51a16c3d1c97faf3fcb7c79403ae1ddbf1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 16:52:22 -0800 Subject: [PATCH 36/77] Fix handling host-targeting assets as NETCoreSdkRuntimeIdentifier is not available. --- eng/Signing.props | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 5b6717e0eff634..6b6b0d73a3c5d6 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -95,19 +95,24 @@ - - - - - + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> + + From fa2e34f587d0092a07ddd3bca0342bc25eb1a069 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 11 Feb 2025 11:39:33 -0800 Subject: [PATCH 37/77] Item updates for default artifacts must happen in Publishing.props --- eng/Publishing.props | 23 +++++++++++++++++++++++ eng/Signing.props | 28 +++------------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 1f986eff2e9c3e..a68e020582b1de 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -5,6 +5,29 @@ false + + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> + + + + diff --git a/eng/Signing.props b/eng/Signing.props index 6b6b0d73a3c5d6..f9bec2482b5d42 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -93,30 +93,6 @@ - For LLVM builds, we only publish LLVM-specific packages. --> - - - - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> - - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> - - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> - - <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> - - - - @@ -125,7 +101,7 @@ Kind="Package" /> - + @@ -176,6 +152,8 @@ Kind="Package" Condition="'$(DotNetBuildOrchestrator)' == 'true'" /> + + From 9e43f5bc9a4f63ae7ef337bd0fbc8ae0dce88ce7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 26 Feb 2025 11:16:21 -0800 Subject: [PATCH 38/77] Use Arcade's RID-specific publish feature in dotnet/runtime --- eng/DotNetBuild.props | 21 +++++++++++------ eng/Signing.props | 37 ++++++++++-------------------- eng/pipelines/runtime-official.yml | 6 ++--- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 4b64cec303e5d9..1631f76ce95771 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -38,6 +38,19 @@ true + + + + false + + true + + false + + @@ -108,13 +121,7 @@ $(InnerBuildArgs) /p:NetCoreAppToolCurrentVersion=$(NetCoreAppToolCurrentVersion) - - $(InnerBuildArgs) /p:EnableDefaultArtifacts=false + $(InnerBuildArgs) /p:EnableDefaultRidSpecificArtifacts=$(EnableDefaultRidSpecificArtifacts) diff --git a/eng/Signing.props b/eng/Signing.props index a5b0869b43875d..86e162bddcc807 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -4,22 +4,26 @@ - false + false true false + true + true @@ -70,7 +74,6 @@ - - - - - - + - - - - - - diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index e581348a11eb80..4990a3334b3a21 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -73,7 +73,7 @@ extends: - name: _SignDiagnosticFilesArgs value: '' - name: _EnableDefaultArtifactsArg - value: $[iif(and(eq(variables.osGroup, 'windows'), eq(variables.archType, 'x64')),'/p:EnableDefaultArtifacts=true','')] + value: $[iif(and(eq(variables.osGroup, 'windows'), eq(variables.archType, 'x64')),'/p:EnableDefaultRidSpecificArtifacts=false','')] jobParameters: templatePath: 'templates-official' preBuildSteps: @@ -230,7 +230,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:EnableDefaultArtifacts=true + buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:EnableDefaultRidSpecificArtifacts=false nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -314,7 +314,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultArtifacts=true + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultRidSpecificArtifacts=false isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 431a50fcb24edfa1f00a74588dbc637a696dd70b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 26 Feb 2025 11:23:45 -0800 Subject: [PATCH 39/77] Fix dependencies --- eng/pipelines/runtime-official.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 4990a3334b3a21..871f4316ad17cd 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -323,8 +323,8 @@ extends: - Build_android_arm64_release_AllRuntimes - Build_android_x86_release_AllRuntimes - Build_android_x64_release_AllRuntimes - - Build_browser_wasm_Linux_release_Mono - - Build_wasi_wasm_linux_release_Mono + - Build_browser_wasm_Linux_release_AllRuntimes + - Build_wasi_wasm_linux_release_AllRuntimes - Build_ios_arm64_release_AllRuntimes - Build_iossimulator_x64_release_AllRuntimes - Build_iossimulator_arm64_release_AllRuntimes From 7268ac499ea9e6cd08187814aba3a3514b0b9f8f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 26 Feb 2025 11:27:56 -0800 Subject: [PATCH 40/77] Update host asset publishing --- eng/Publishing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index a68e020582b1de..816eac1b873cd2 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -10,7 +10,7 @@ We can't use NETCoreSdkRuntimeIdentifier here as the Arcade SDK projects don't import the .NET SDK. Instead, just make sure we include the assets targeting "not the output rid", which will catch the host assets. --> - + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> From 8f34d79d603edb60c31dbe7745ced986edfb8042 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 26 Feb 2025 11:52:40 -0800 Subject: [PATCH 41/77] Set TargetRid --- eng/Signing.props | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/Signing.props b/eng/Signing.props index 86e162bddcc807..15dc3c0c2fb2b9 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -22,6 +22,9 @@ true false + + $(PackageRID) + true true From d5e6b66c618dbfe76267ff545853ea6d500087c8 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 13:29:48 -0800 Subject: [PATCH 42/77] Adjust for Azure Linux package names and use artifact kinds --- eng/Publishing.props | 2 +- eng/Signing.props | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 816eac1b873cd2..7280fcafc95d08 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -85,7 +85,7 @@ BeforeTargets="PublishToAzureDevOpsArtifacts" AfterTargets="GenerateChecksumsFromArtifacts"> - diff --git a/eng/Signing.props b/eng/Signing.props index 15dc3c0c2fb2b9..7c0296cc80850c 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -73,7 +73,8 @@ This package name has the version in it, so we need to use a wildcard to discover the right name. --> - + + From 429758d41dfbeec2730ebc5d12f259c35461db1e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:31:56 -0800 Subject: [PATCH 43/77] Restore a final join job for signing so we can merge this in and stop having to update the patch. --- Build.proj | 2 +- eng/Publishing.props | 59 +++-- eng/pipelines/common/global-build-job.yml | 17 +- eng/pipelines/official/pipeline.yml | 31 +++ .../official/prepare-signed-artifacts.yml | 71 ++++++ eng/pipelines/runtime-official.yml | 210 ++++++++++++------ 6 files changed, 298 insertions(+), 92 deletions(-) create mode 100644 eng/pipelines/official/pipeline.yml create mode 100644 eng/pipelines/official/prepare-signed-artifacts.yml diff --git a/Build.proj b/Build.proj index 0957ae6846e573..2f687610b4a93a 100644 --- a/Build.proj +++ b/Build.proj @@ -1,6 +1,6 @@ - + diff --git a/eng/Publishing.props b/eng/Publishing.props index 7280fcafc95d08..8c1fd9f3a2d5b7 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -5,28 +5,42 @@ false - - - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> + + true + true + $(ArtifactsDir)/staging/ + $(ArtifactsStagingDir)/assets/$(Configuration) + $(ArtifactsStagingDir)/PDB/$(Configuration) + $(ArtifactsStagingDir)/packages/$(Configuration)/Shipping + $(ArtifactsStagingDir)/packages/$(Configuration)/NonShipping + true + + $(ArtifactsTmpDir)/manifests/$(Configuration) + VerticalManifest.xml + - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> + + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> - <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> - <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" - Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> - - + <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> + + + + + + + diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6cbc0c09eeda37..f57cadd22182c8 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -36,6 +36,8 @@ parameters: templatePath: 'templates' templateContext: '' disableComponentGovernance: '' + # In official builds, the expected artifact name + officialBuildArtifactName: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}${{ replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', '') }}_Artifacts jobs: - template: /eng/common/${{ parameters.templatePath }}/job/job.yml @@ -56,7 +58,20 @@ jobs: enableMicrobuildForMacAndLinux: ${{ parameters.isOfficialBuild }} ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} + templateContext: + ${{ each pair in parameters.templateContext }}: + ${{ if notIn(pair.key, 'outputs') }}: + ${{ pair.key }}: ${{ pair.value }} + outputs: + ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ each output in parameters.templateContext.outputs }}: + ${{ output.key }}: ${{ output.value }} + ${{ if eq(parameters.isOfficialBuild, true) }}: + - output: pipelineArtifact + displayName: 'Publish Pipeline Artifacts' + artifactName: ${{ parameters.officialBuildArtifactName }} + ${{ if eq(parameters.isOfficialBuild, true) }}: + artifactName: ${{ parameters.officialBuildArtifactName }} artifacts: publish: diff --git a/eng/pipelines/official/pipeline.yml b/eng/pipelines/official/pipeline.yml new file mode 100644 index 00000000000000..6f644f2f1a2b11 --- /dev/null +++ b/eng/pipelines/official/pipeline.yml @@ -0,0 +1,31 @@ +parameters: +- name: buildStage + type: stage +- name: otherStages + type: stageList + +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + isOfficialBuild: true + stages: + - ${{ insert }}: ${{ parameters.otherStages }} + - ${{ insert }}: ${{ parameters.buildStage }} + + - stage: Publish + jobs: + - template: /eng/pipelines/official/prepare-signed-artifacts.yml + parameters: + buildStage: ${{ parameters.buildStage }} + logArtifactName: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + - template: /eng/common/templates-official/job/publish-build-assets.yml + parameters: + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' diff --git a/eng/pipelines/official/prepare-signed-artifacts.yml b/eng/pipelines/official/prepare-signed-artifacts.yml new file mode 100644 index 00000000000000..d2a2ff10bf6d25 --- /dev/null +++ b/eng/pipelines/official/prepare-signed-artifacts.yml @@ -0,0 +1,71 @@ +parameters: + - name: logArtifactName + type: string + default: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' + - name: buildStage + type: stage + +jobs: +- template: /eng/common/templates-official/job/job.yml + parameters: + name: 'PrepareSignedArtifacts' + displayName: 'Prepare Signed Artifacts' + + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + + # Double the default timeout. + timeoutInMinutes: 240 + + workspace: + clean: all + + enableMicrobuild: true + + variables: + - name: '_SignType' + value: $[ coalesce(variables.OfficialSignType, 'real') ] + + templateContext: + inputs: + - input: checkout + repository: self + clean: true + fetchDepth: 20 + - ${{ each job in parameters.buildStage.jobs }}: + - input: pipelineArtifact + artifactName: ${{ job.templateContext.artifactName }} + targetPath: $(Build.SourcesDirectory)\artifacts\packages\ + itemPattern: | + **/* + !_manifest/**/* + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)\BuildLogs' + artifactName: ${{ parameters.logArtifactName }} + + steps: + - script: >- + build.cmd -restore -sign -publish -ci -configuration Release + /p:RestoreToolsetOnly=true + /p:OfficialBuildId=$(Build.BuildNumber) + /p:SignType=$(_SignType) + /p:DotNetSignType=$(_SignType) + /p:DotNetPublishUsingPipelines=true + /p:DotNetFinalPublish=true + /bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog + displayName: Prepare artifacts and upload to build + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index c5116bfb2c42eb..a01d1c6c895d02 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -30,11 +30,11 @@ variables: teamName: dotnet-core-acquisition extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/official/pipeline.yml parameters: isOfficialBuild: true - stages: - - stage: Build + otherStages: + - stage: Localization jobs: # @@ -47,7 +47,8 @@ extends: MirrorBranch: main LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-RUNTIME' - + - stage: Source Index + jobs: # # Source Index Build # @@ -55,7 +56,9 @@ extends: - template: /eng/common/templates-official/job/source-index-stage1.yml parameters: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true - + buildStage: + - stage: Build + jobs: # # Build CoreCLR runtime packs # Windows x64/arm64 @@ -79,21 +82,21 @@ extends: preBuildSteps: - template: /eng/pipelines/coreclr/templates/install-diagnostic-certs.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true certNames: - 'dotnetesrp-diagnostics-aad-ssl-cert' - 'dotnet-diagnostics-esrp-pki-onecert' vaultName: 'clrdiag-esrp-id' azureSubscription: 'diagnostics-esrp-kvcertuser' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true $(_SignDiagnosticFilesArgs) $(_EnableDefaultArtifactsArg) + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true $(_SignDiagnosticFilesArgs) $(_EnableDefaultArtifactsArg) nameSuffix: AllRuntimes - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true timeoutInMinutes: 120 postBuildSteps: - template: /eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true # # Build all runtime packs for MacOS, Linux, Linux musl, and mobile @@ -130,9 +133,9 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true timeoutInMinutes: 120 # @@ -148,7 +151,7 @@ extends: templatePath: 'templates-official' buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs) nameSuffix: CrossDac - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true timeoutInMinutes: 120 preBuildSteps: - task: DownloadPipelineArtifact@2 @@ -156,13 +159,13 @@ extends: inputs: artifact: 'PackageArtifacts' path: $(Build.SourcesDirectory)/artifacts/RuntimeDownload - patterns: | + itemPattern: | PackageArtifacts/linux_*/Shipping/Microsoft.NETCore.App.Runtime.linux-*.nupkg - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.SourcesDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs - template: /eng/pipelines/coreclr/templates/install-diagnostic-certs.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true certNames: - 'dotnetesrp-diagnostics-aad-ssl-cert' - 'dotnet-diagnostics-esrp-pki-onecert' @@ -171,7 +174,7 @@ extends: postBuildSteps: - template: /eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true dependsOn: - build_linux_x64_release_AllRuntimes - build_linux_arm_release_AllRuntimes @@ -194,9 +197,9 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true /p:WasmEnableThreads=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true /p:WasmEnableThreads=true nameSuffix: Mono_multithread - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true runtimeVariant: multithread # @@ -213,11 +216,11 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true # # Build libraries (all TFMs) and packages @@ -230,10 +233,11 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools.illink+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:EnableDefaultRidSpecificArtifacts=false + buildArgs: -s tools.illink+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:EnableDefaultRidSpecificArtifacts=false nameSuffix: Libraries_WithPackages - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true timeoutInMinutes: 95 + officialBuildArtifactName: Libraries_WithPackages_Artifacts # # Build SourceBuild packages # @@ -260,10 +264,11 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true - isOfficialBuild: ${{ variables.isOfficialBuild }} + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + isOfficialBuild: true nameSuffix: PGO timeoutInMinutes: 95 + officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_PGO_Artifacts # # Build Workloads @@ -277,46 +282,120 @@ extends: jobParameters: templatePath: 'templates-official' nameSuffix: Workloads - preBuildSteps: - - task: DownloadPipelineArtifact@2 + templateContext: inputs: - artifact: 'PackageArtifacts' - path: $(Build.SourcesDirectory)/artifacts/workloadPackages - patterns: | - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg - PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg - PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg - PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg - PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg - PackageArtifacts/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg - PackageArtifacts/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg - PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg - PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg - PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x64*.nupkg - PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg - PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg + - input: pipelineArtifact + artifactName: windows_x64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg + Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg + Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg + Microsoft.NETCore.App.Runtime.win-x64*.nupkg + - input: pipelineArtifact + artifactName: windows_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg + Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg + Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg + Microsoft.NETCore.App.Runtime.win-arm64*.nupkg + - input: pipelineArtifact + artifactName: windows_x86_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.win-x86*.nupkg + - input: pipelineArtifact + artifactName: android_x64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + - input: pipelineArtifact + artifactName: android_x86_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + - input: pipelineArtifact + artifactName: android_arm_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + - input: pipelineArtifact + artifactName: android_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + - input: pipelineArtifact + artifactName: browser_wasm_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg + Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg + Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg + Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg + Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg + Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg + Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg + Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg + Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg + Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg + Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg + - input: pipelineArtifact + artifactName: browser_wasm_multithread_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + - input: pipelineArtifact + artifactName: wasi_wasm_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg + Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg + - input: pipelineArtifact + artifactName: ios_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg + - input: pipelineArtifact + artifactName: iossimulator_x64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + - input: pipelineArtifact + artifactName: iossimulator_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + - input: pipelineArtifact + artifactName: maccatalyst_x64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + - input: pipelineArtifact + artifactName: maccatalyst_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + - input: pipelineArtifact + artifactName: tvos_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg + - input: pipelineArtifact + artifactName: tvossimulator_x64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + - input: pipelineArtifact + artifactName: tvossimulator_arm64_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + itemPattern: | + Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultRidSpecificArtifacts=false + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultRidSpecificArtifacts=false - isOfficialBuild: ${{ variables.isOfficialBuild }} + officialBuildArtifactName: Workloads_Artifacts + isOfficialBuild: true timeoutInMinutes: 120 dependsOn: - Build_android_arm_release_AllRuntimes @@ -336,14 +415,3 @@ extends: - Build_windows_x64_release_AllRuntimes - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes - - - stage: Publish - jobs: - - template: /eng/common/templates-official/job/publish-build-assets.yml - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' From fee4328998598c0893a7143a1c55def6208d3f0d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:32:59 -0800 Subject: [PATCH 44/77] Remove unused parameter --- eng/pipelines/runtime-official.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index a01d1c6c895d02..c2e4368123a0ab 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -32,7 +32,6 @@ variables: extends: template: /eng/pipelines/official/pipeline.yml parameters: - isOfficialBuild: true otherStages: - stage: Localization jobs: From 4ccf449ed3dde8eb6bcc02a359690fddc439460e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:33:30 -0800 Subject: [PATCH 45/77] The build stage shouldn't be a list --- eng/pipelines/runtime-official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index c2e4368123a0ab..c26256ea3479a5 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -56,7 +56,7 @@ extends: parameters: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true buildStage: - - stage: Build + stage: Build jobs: # # Build CoreCLR runtime packs From 9f7682ee8332f063f93621b7b8e2fed913d38416 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:35:48 -0800 Subject: [PATCH 46/77] Add variable for artifact name --- eng/pipelines/common/global-build-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index f57cadd22182c8..635b9b4136d292 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -37,7 +37,7 @@ parameters: templateContext: '' disableComponentGovernance: '' # In official builds, the expected artifact name - officialBuildArtifactName: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}${{ replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', '') }}_Artifacts + officialBuildArtifactName: $(defaultOfficialBuildArtifactName) jobs: - template: /eng/common/${{ parameters.templatePath }}/job/job.yml @@ -160,6 +160,9 @@ jobs: - name: _nativeSanitizersArg value: '' + - name: defaultOfficialBuildArtifactName + value: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}${{ replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', '') }}_Artifacts + - ${{ each variableTemplate in parameters.extraVariablesTemplates }}: - template: ${{ variableTemplate.template }} parameters: From 252e984983445aa26a07eed58b9e5d22b0faa492 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:37:15 -0800 Subject: [PATCH 47/77] Try another variant --- eng/pipelines/common/global-build-job.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 635b9b4136d292..69e86e940abbe5 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -36,8 +36,7 @@ parameters: templatePath: 'templates' templateContext: '' disableComponentGovernance: '' - # In official builds, the expected artifact name - officialBuildArtifactName: $(defaultOfficialBuildArtifactName) + officialBuildArtifactName: '' jobs: - template: /eng/common/${{ parameters.templatePath }}/job/job.yml @@ -69,9 +68,9 @@ jobs: ${{ if eq(parameters.isOfficialBuild, true) }}: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' - artifactName: ${{ parameters.officialBuildArtifactName }} + artifactName: ${{ coalesce(parameters.officialBuildArtifactName, format("{0}{1}_{2}{3}_Artifacts", parameters.osGroup, parameters.osSubgroup, parameters.archType, replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', ''))) }} ${{ if eq(parameters.isOfficialBuild, true) }}: - artifactName: ${{ parameters.officialBuildArtifactName }} + artifactName: ${{ coalesce(parameters.officialBuildArtifactName, format("{0}{1}_{2}{3}_Artifacts", parameters.osGroup, parameters.osSubgroup, parameters.archType, replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', ''))) }} artifacts: publish: @@ -160,9 +159,6 @@ jobs: - name: _nativeSanitizersArg value: '' - - name: defaultOfficialBuildArtifactName - value: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}${{ replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', '') }}_Artifacts - - ${{ each variableTemplate in parameters.extraVariablesTemplates }}: - template: ${{ variableTemplate.template }} parameters: From 831c5ac5b6c45d530449580c8660fef6ca76dc63 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:42:05 -0800 Subject: [PATCH 48/77] Try to simplify --- eng/pipelines/common/global-build-job.yml | 4 ++-- eng/pipelines/runtime-official.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 69e86e940abbe5..09dfd8f268372a 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -68,9 +68,9 @@ jobs: ${{ if eq(parameters.isOfficialBuild, true) }}: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' - artifactName: ${{ coalesce(parameters.officialBuildArtifactName, format("{0}{1}_{2}{3}_Artifacts", parameters.osGroup, parameters.osSubgroup, parameters.archType, replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', ''))) }} + artifactName: ${{ coalesce(parameters.officialBuildArtifactName, '$(osGroup)$(osSubGroup)_$(archType)_Artifacts') }} ${{ if eq(parameters.isOfficialBuild, true) }}: - artifactName: ${{ coalesce(parameters.officialBuildArtifactName, format("{0}{1}_{2}{3}_Artifacts", parameters.osGroup, parameters.osSubgroup, parameters.archType, replace(format('_{0}', parameters.runtimeVariant, 'normal'), '_normal', ''))) }} + artifactName: ${{ coalesce(parameters.officialBuildArtifactName, '$(osGroup)$(osSubGroup)_$(archType)_Artifacts') }} artifacts: publish: diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index c26256ea3479a5..3bdd13bf70b8ba 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -200,6 +200,7 @@ extends: nameSuffix: Mono_multithread isOfficialBuild: true runtimeVariant: multithread + officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_multithread_Artifacts # # Build Mono LLVM runtime packs @@ -219,6 +220,7 @@ extends: /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT + officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_LLVMAOT_Artifacts isOfficialBuild: true # From 081db0d7ab3287f21fe3c79db2f2071de9fe42a0 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:46:00 -0800 Subject: [PATCH 49/77] Another try --- eng/pipelines/common/global-build-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 09dfd8f268372a..5073f99b8665f1 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -36,7 +36,7 @@ parameters: templatePath: 'templates' templateContext: '' disableComponentGovernance: '' - officialBuildArtifactName: '' + officialBuildArtifactName: '$(osGroup)$(osSubGroup)_$(archType)_Artifacts' jobs: - template: /eng/common/${{ parameters.templatePath }}/job/job.yml @@ -68,9 +68,9 @@ jobs: ${{ if eq(parameters.isOfficialBuild, true) }}: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' - artifactName: ${{ coalesce(parameters.officialBuildArtifactName, '$(osGroup)$(osSubGroup)_$(archType)_Artifacts') }} + artifactName: ${{ parameters.officialBuildArtifactName }} ${{ if eq(parameters.isOfficialBuild, true) }}: - artifactName: ${{ coalesce(parameters.officialBuildArtifactName, '$(osGroup)$(osSubGroup)_$(archType)_Artifacts') }} + artifactName: ${{ parameters.officialBuildArtifactName }} artifacts: publish: From 6d219841df8bbc8f2c18302fd6de6bb3e55c7c11 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:47:05 -0800 Subject: [PATCH 50/77] Add the target path --- eng/pipelines/common/global-build-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 5073f99b8665f1..212881ce39c417 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -69,6 +69,7 @@ jobs: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' artifactName: ${{ parameters.officialBuildArtifactName }} + targetPath: $(Build.SourceDirectory)/artifacts/staging ${{ if eq(parameters.isOfficialBuild, true) }}: artifactName: ${{ parameters.officialBuildArtifactName }} From 12ae0fd309a7900837778a51cd7b019639346707 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:55:03 -0800 Subject: [PATCH 51/77] Define the artifact name like how we define the build job names for simplicity --- eng/pipelines/common/global-build-job.yml | 6 ++- .../official/prepare-signed-artifacts.yml | 2 +- eng/pipelines/runtime-official.yml | 42 +++++++++---------- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 212881ce39c417..346e3bdaac65bb 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -36,7 +36,6 @@ parameters: templatePath: 'templates' templateContext: '' disableComponentGovernance: '' - officialBuildArtifactName: '$(osGroup)$(osSubGroup)_$(archType)_Artifacts' jobs: - template: /eng/common/${{ parameters.templatePath }}/job/job.yml @@ -68,7 +67,10 @@ jobs: ${{ if eq(parameters.isOfficialBuild, true) }}: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' - artifactName: ${{ parameters.officialBuildArtifactName }} + ${{ if eq(parameters.hostedOs, '') }}: + artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} + ${{ if ne(parameters.hostedOs, '') }}: + artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.hostedOs, parameters.buildConfig, parameters.nameSuffix) }} targetPath: $(Build.SourceDirectory)/artifacts/staging ${{ if eq(parameters.isOfficialBuild, true) }}: artifactName: ${{ parameters.officialBuildArtifactName }} diff --git a/eng/pipelines/official/prepare-signed-artifacts.yml b/eng/pipelines/official/prepare-signed-artifacts.yml index d2a2ff10bf6d25..b4304590ada514 100644 --- a/eng/pipelines/official/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/prepare-signed-artifacts.yml @@ -35,7 +35,7 @@ jobs: fetchDepth: 20 - ${{ each job in parameters.buildStage.jobs }}: - input: pipelineArtifact - artifactName: ${{ job.templateContext.artifactName }} + artifactName: ${{ job.job }}_Artifacts targetPath: $(Build.SourcesDirectory)\artifacts\packages\ itemPattern: | **/* diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 3bdd13bf70b8ba..b4432b8c98db87 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -200,7 +200,6 @@ extends: nameSuffix: Mono_multithread isOfficialBuild: true runtimeVariant: multithread - officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_multithread_Artifacts # # Build Mono LLVM runtime packs @@ -220,7 +219,6 @@ extends: /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT - officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_LLVMAOT_Artifacts isOfficialBuild: true # @@ -238,7 +236,6 @@ extends: nameSuffix: Libraries_WithPackages isOfficialBuild: true timeoutInMinutes: 95 - officialBuildArtifactName: Libraries_WithPackages_Artifacts # # Build SourceBuild packages # @@ -269,7 +266,6 @@ extends: isOfficialBuild: true nameSuffix: PGO timeoutInMinutes: 95 - officialBuildArtifactName: $(osGroup)$(osSubGroup)_$(archType)_PGO_Artifacts # # Build Workloads @@ -286,7 +282,7 @@ extends: templateContext: inputs: - input: pipelineArtifact - artifactName: windows_x64_Artifacts + artifactName: Build_Build_windows_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg @@ -294,7 +290,7 @@ extends: Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg Microsoft.NETCore.App.Runtime.win-x64*.nupkg - input: pipelineArtifact - artifactName: windows_arm64_Artifacts + artifactName: Build_windows_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg @@ -302,32 +298,32 @@ extends: Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - input: pipelineArtifact - artifactName: windows_x86_Artifacts + artifactName: Build_windows_x86_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.win-x86*.nupkg - input: pipelineArtifact - artifactName: android_x64_Artifacts + artifactName: Build_android_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact - artifactName: android_x86_Artifacts + artifactName: Build_android_x86_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact - artifactName: android_arm_Artifacts + artifactName: Build_android_arm_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact - artifactName: android_arm64_Artifacts + artifactName: Build_android_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact - artifactName: browser_wasm_Artifacts + artifactName: Build_browser_wasm_Linux_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg @@ -342,60 +338,59 @@ extends: Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - input: pipelineArtifact - artifactName: browser_wasm_multithread_Artifacts + artifactName: Build_browser_wasm_Linux_Mono_Multithread_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - input: pipelineArtifact - artifactName: wasi_wasm_Artifacts + artifactName: Build_wasi_wasm_Linux_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - input: pipelineArtifact - artifactName: ios_arm64_Artifacts + artifactName: Build_ios_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - input: pipelineArtifact - artifactName: iossimulator_x64_Artifacts + artifactName: Build_iossimulator_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact - artifactName: iossimulator_arm64_Artifacts + artifactName: Build_iossimulator_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact - artifactName: maccatalyst_x64_Artifacts + artifactName: Build_maccatalyst_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact - artifactName: maccatalyst_arm64_Artifacts + artifactName: Build_maccatalyst_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact - artifactName: tvos_arm64_Artifacts + artifactName: Build_tvos_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - input: pipelineArtifact - artifactName: tvossimulator_x64_Artifacts + artifactName: Build_tvossimulator_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - input: pipelineArtifact - artifactName: tvossimulator_arm64_Artifacts + artifactName: Build_tvossimulator_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultRidSpecificArtifacts=false - officialBuildArtifactName: Workloads_Artifacts isOfficialBuild: true timeoutInMinutes: 120 dependsOn: @@ -416,3 +411,4 @@ extends: - Build_windows_x64_release_AllRuntimes - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes + - Build_browser_wasm_Linux_Mono_multithread From 61df0b7c33116df983c26f667c4a94fc182d3614 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:55:38 -0800 Subject: [PATCH 52/77] Make some items a list --- eng/pipelines/common/global-build-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 346e3bdaac65bb..f194d96cc21a6b 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -61,10 +61,10 @@ jobs: ${{ if notIn(pair.key, 'outputs') }}: ${{ pair.key }}: ${{ pair.value }} outputs: - ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ if ne(parameters.templateContext.outputs, '') }}: - ${{ each output in parameters.templateContext.outputs }}: ${{ output.key }}: ${{ output.value }} - ${{ if eq(parameters.isOfficialBuild, true) }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - output: pipelineArtifact displayName: 'Publish Pipeline Artifacts' ${{ if eq(parameters.hostedOs, '') }}: From b4ade05b6ab0cce5151af2fa2ea259c8bd9ab611 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:56:27 -0800 Subject: [PATCH 53/77] Rewrite how we insert the "other" stages --- eng/pipelines/official/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/official/pipeline.yml b/eng/pipelines/official/pipeline.yml index 6f644f2f1a2b11..d6ac33c00f0ccb 100644 --- a/eng/pipelines/official/pipeline.yml +++ b/eng/pipelines/official/pipeline.yml @@ -9,7 +9,8 @@ extends: parameters: isOfficialBuild: true stages: - - ${{ insert }}: ${{ parameters.otherStages }} + - ${{ each stage in parameters.otherStages }}: + - ${{ insert }}: ${{ stage }} - ${{ insert }}: ${{ parameters.buildStage }} - stage: Publish From 8bf1832d232e6bceec15f7d73a5ae6ffa1b0f0ca Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:57:52 -0800 Subject: [PATCH 54/77] Remove duplicate pool --- eng/pipelines/official/pipeline.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/eng/pipelines/official/pipeline.yml b/eng/pipelines/official/pipeline.yml index d6ac33c00f0ccb..45f0e62e05e7d0 100644 --- a/eng/pipelines/official/pipeline.yml +++ b/eng/pipelines/official/pipeline.yml @@ -19,9 +19,6 @@ extends: parameters: buildStage: ${{ parameters.buildStage }} logArtifactName: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true From 1a2a5b915ac77e12f4fc0308278a97b1c0a9d5b2 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:59:04 -0800 Subject: [PATCH 55/77] Fix stage name --- eng/pipelines/runtime-official.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index b4432b8c98db87..7c237e38da3445 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -46,7 +46,8 @@ extends: MirrorBranch: main LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-RUNTIME' - - stage: Source Index + - stage: Source_Index + displayName: Source Index jobs: # # Source Index Build From acb8be48a42f36ff559784d64c0124a9137aace1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 14:59:48 -0800 Subject: [PATCH 56/77] Fix conditions --- eng/pipelines/runtime-official.yml | 34 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 7c237e38da3445..f63b30df2d6e00 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -33,29 +33,27 @@ extends: template: /eng/pipelines/official/pipeline.yml parameters: otherStages: - - stage: Localization - jobs: - - # - # Localization build - # - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - stage: Localization + jobs: + # + # Localization build + # - template: /eng/common/templates-official/job/onelocbuild.yml parameters: MirrorRepo: runtime MirrorBranch: main LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-RUNTIME' - - stage: Source_Index - displayName: Source Index - jobs: - # - # Source Index Build - # - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates-official/job/source-index-stage1.yml - parameters: - sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true + - stage: Source_Index + displayName: Source Index + jobs: + # + # Source Index Build + # + - template: /eng/common/templates-official/job/source-index-stage1.yml + parameters: + sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true buildStage: stage: Build jobs: @@ -412,4 +410,4 @@ extends: - Build_windows_x64_release_AllRuntimes - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes - - Build_browser_wasm_Linux_Mono_multithread + - build_browser_wasm_linux_release_Mono_multithread From f014371d9a81746635ece50b41b8a16707b4bf5e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 17:42:36 -0800 Subject: [PATCH 57/77] Fix expected pipeline artifact names --- eng/pipelines/runtime-official.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index f63b30df2d6e00..0a89766b656aad 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -151,6 +151,28 @@ extends: nameSuffix: CrossDac isOfficialBuild: true timeoutInMinutes: 120 + templateContext: + inputs: + - input: pipelineArtifact + artifactName: Build_linux_x64_release_AllRuntimes_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + itemPattern: | + Microsoft.NETCore.App.Runtime.linux-x64*.nupkg + - input: pipelineArtifact + artifactName: Build_linux_arm64_release_AllRuntimes_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + itemPattern: | + Microsoft.NETCore.App.Runtime.linux-arm64*.nupkg + - input: pipelineArtifact + artifactName: Build_linux_musl_x64_release_AllRuntimes_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + itemPattern: | + Microsoft.NETCore.App.Runtime.linux-musl-x64*.nupkg + - input: pipelineArtifact + artifactName: Build_linux_musl_arm64_release_AllRuntimes_Artifacts + targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + itemPattern: | + Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg preBuildSteps: - task: DownloadPipelineArtifact@2 displayName: Download runtime packs for CrossDac @@ -281,7 +303,7 @@ extends: templateContext: inputs: - input: pipelineArtifact - artifactName: Build_Build_windows_x64_release_AllRuntimes_Artifacts + artifactName: Build_windows_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg @@ -341,7 +363,7 @@ extends: targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - - input: pipelineArtifact + - input: pipelineArtifact artifactName: Build_wasi_wasm_Linux_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | From ef298be465da310ee7166a30c2e7f935f1bed77e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 3 Mar 2025 17:43:36 -0800 Subject: [PATCH 58/77] Always put a templateContext into global-build-job even if one isn't provided --- eng/pipelines/common/global-build-job.yml | 34 +++++++++++------------ eng/pipelines/runtime-official.yml | 7 ----- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index f194d96cc21a6b..62774a25a5c3e4 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -55,25 +55,23 @@ jobs: enableMicrobuild: ${{ parameters.isOfficialBuild }} enableMicrobuildForMacAndLinux: ${{ parameters.isOfficialBuild }} - ${{ if ne(parameters.templateContext, '') }}: - templateContext: + templateContext: + ${{ if ne(parameters.templateContext, '') }}: ${{ each pair in parameters.templateContext }}: ${{ if notIn(pair.key, 'outputs') }}: ${{ pair.key }}: ${{ pair.value }} - outputs: - - ${{ if ne(parameters.templateContext.outputs, '') }}: - - ${{ each output in parameters.templateContext.outputs }}: - ${{ output.key }}: ${{ output.value }} - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - output: pipelineArtifact - displayName: 'Publish Pipeline Artifacts' - ${{ if eq(parameters.hostedOs, '') }}: - artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} - ${{ if ne(parameters.hostedOs, '') }}: - artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.hostedOs, parameters.buildConfig, parameters.nameSuffix) }} - targetPath: $(Build.SourceDirectory)/artifacts/staging - ${{ if eq(parameters.isOfficialBuild, true) }}: - artifactName: ${{ parameters.officialBuildArtifactName }} + outputs: + - ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ each output in parameters.templateContext.outputs }}: + ${{ output.key }}: ${{ output.value }} + - ${{ if eq(parameters.isOfficialBuild, true) }}: + - output: pipelineArtifact + displayName: 'Publish Pipeline Artifacts' + ${{ if eq(parameters.hostedOs, '') }}: + artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} + ${{ if ne(parameters.hostedOs, '') }}: + artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.hostedOs, parameters.buildConfig, parameters.nameSuffix) }} + targetPath: $(Build.SourceDirectory)/artifacts/staging artifacts: publish: @@ -108,7 +106,7 @@ jobs: value: -os ${{ parameters.osGroup }} - name: _archParameter value: -arch ${{ parameters.archType }} - + - name: _AssetManifestName value: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.nameSuffix }} @@ -142,7 +140,7 @@ jobs: - ${{ if ne(parameters.cxxAbiLibrary, '') }}: - name: CxxAbiLibraryArg value: /p:TargetCxxAbiLibrary=${{ parameters.cxxAbiLibrary }} - + - name: TargetCxxLibraryConfigurationArgs value: $(CxxStandardLibraryArg) $(CxxStandardLibraryStaticArg) $(CxxAbiLibraryArg) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 0a89766b656aad..ad24396a4f1499 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -174,13 +174,6 @@ extends: itemPattern: | Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg preBuildSteps: - - task: DownloadPipelineArtifact@2 - displayName: Download runtime packs for CrossDac - inputs: - artifact: 'PackageArtifacts' - path: $(Build.SourcesDirectory)/artifacts/RuntimeDownload - itemPattern: | - PackageArtifacts/linux_*/Shipping/Microsoft.NETCore.App.Runtime.linux-*.nupkg - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.SourcesDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs - template: /eng/pipelines/coreclr/templates/install-diagnostic-certs.yml From 5aa4ea7a1056c88ea4137a398a0b7d8e5d782c9c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 10:06:56 -0800 Subject: [PATCH 59/77] Fix typo and adjust publishing to handle PDBs --- eng/Publishing.props | 19 ++++++++++++------- eng/pipelines/common/global-build-job.yml | 2 +- .../official/prepare-signed-artifacts.yml | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 8c1fd9f3a2d5b7..94231838af39c8 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,11 +8,12 @@ true true - $(ArtifactsDir)/staging/ - $(ArtifactsStagingDir)/assets/$(Configuration) - $(ArtifactsStagingDir)/PDB/$(Configuration) - $(ArtifactsStagingDir)/packages/$(Configuration)/Shipping - $(ArtifactsStagingDir)/packages/$(Configuration)/NonShipping + $(ArtifactsDir)staging/ + $(ArtifactsStagingDir)SymStore + + $(ArtifactsStagingDir)packages/ + $(SourceBuiltAssetsDir)Shipping + $(SourceBuiltAssetsDir)NonShipping true $(ArtifactsTmpDir)/manifests/$(Configuration) @@ -104,10 +105,14 @@ - - + + diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 62774a25a5c3e4..76a8364f74c8dd 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -71,7 +71,7 @@ jobs: artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} ${{ if ne(parameters.hostedOs, '') }}: artifactName: ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}_Artifacts', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.hostedOs, parameters.buildConfig, parameters.nameSuffix) }} - targetPath: $(Build.SourceDirectory)/artifacts/staging + targetPath: $(Build.SourcesDirectory)/artifacts/staging artifacts: publish: diff --git a/eng/pipelines/official/prepare-signed-artifacts.yml b/eng/pipelines/official/prepare-signed-artifacts.yml index b4304590ada514..09f27540480aed 100644 --- a/eng/pipelines/official/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/prepare-signed-artifacts.yml @@ -36,7 +36,7 @@ jobs: - ${{ each job in parameters.buildStage.jobs }}: - input: pipelineArtifact artifactName: ${{ job.job }}_Artifacts - targetPath: $(Build.SourcesDirectory)\artifacts\packages\ + targetPath: $(Build.SourcesDirectory)\artifacts itemPattern: | **/* !_manifest/**/* From 9527b73eb69a2c2587c4e58fd9d16d25ee81b67e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 10:39:25 -0800 Subject: [PATCH 60/77] Set blob artifact kind --- eng/Publishing.props | 2 +- eng/Signing.props | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 94231838af39c8..4e009fb8dcf272 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -11,7 +11,7 @@ $(ArtifactsDir)staging/ $(ArtifactsStagingDir)SymStore - $(ArtifactsStagingDir)packages/ + $(ArtifactsStagingDir)packages/$(Configuration)/ $(SourceBuiltAssetsDir)Shipping $(SourceBuiltAssetsDir)NonShipping true diff --git a/eng/Signing.props b/eng/Signing.props index 7c0296cc80850c..5c455ffd32e35a 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -87,6 +87,7 @@ $(ArtifactsPackagesDir)**\*.exe; $(ArtifactsPackagesDir)**\*.msi;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" + Kind="Blob" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> %(FullPath).sha512 @@ -94,7 +95,7 @@ From 3a6a04f339dd009d1b6896ae3c6c01700fdf3254 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 10:40:54 -0800 Subject: [PATCH 61/77] Fix item name for AzureLinux RPM signing --- eng/Signing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Signing.props b/eng/Signing.props index 5c455ffd32e35a..44dae1c23428e1 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -75,7 +75,7 @@ - + From 3702f3cd808e72dc749b2ef6f8b456bf191a3c54 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 10:54:57 -0800 Subject: [PATCH 62/77] Put the VSSetup blobs in the VSSetup folder and adjust the relative blob paths to make that work --- eng/Publishing.props | 17 ++++++++++------- eng/Signing.props | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 4e009fb8dcf272..b96cea141237c3 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -11,9 +11,9 @@ $(ArtifactsDir)staging/ $(ArtifactsStagingDir)SymStore - $(ArtifactsStagingDir)packages/$(Configuration)/ - $(SourceBuiltAssetsDir)Shipping - $(SourceBuiltAssetsDir)NonShipping + $(ArtifactsStagingDir) + $(ArtifactsStagingDir)packages/$(Configuration)/Shipping + $(ArtifactsStagingDir)packages/$(Configuration)/NonShipping true $(ArtifactsTmpDir)/manifests/$(Configuration) @@ -105,14 +105,17 @@ + + + RelativeBlobPath="packages/$(Configuration)/Shipping/%(Filename)%(Extension)" /> + RelativeBlobPath="packages/$(Configuration)/NonShipping/%(Filename)%(Extension)" /> + RelativeBlobPath="packages/$(Configuration)/Shipping/%(Filename)%(Extension)" /> + RelativeBlobPath="packages/$(Configuration)/NonShipping/%(RecursiveDir)%(Filename)%(Extension)" /> diff --git a/eng/Signing.props b/eng/Signing.props index 44dae1c23428e1..eaceb6fd17ee80 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -97,6 +97,7 @@ Condition="Exists('$(ArtifactsDir)VSSetup')" Kind="Blob" ChecksumPath="%(FullPath).sha512" + IsVSSetup="true" SubBlobFolder="workloads/" /> From 5fe309f82754b5a5bed48ab5a03c1e4786c05584 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 13:33:03 -0800 Subject: [PATCH 63/77] Fix crossdac and workloads downloads --- eng/pipelines/runtime-official.yml | 94 +++++++++++++++++------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index ad24396a4f1499..d8cdbdba9935d6 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -157,22 +157,26 @@ extends: artifactName: Build_linux_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload itemPattern: | - Microsoft.NETCore.App.Runtime.linux-x64*.nupkg + **/Microsoft.NETCore.App.Runtime.linux-x64*.nupkg + !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload itemPattern: | - Microsoft.NETCore.App.Runtime.linux-arm64*.nupkg + **/Microsoft.NETCore.App.Runtime.linux-arm64*.nupkg + !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload itemPattern: | - Microsoft.NETCore.App.Runtime.linux-musl-x64*.nupkg + **/Microsoft.NETCore.App.Runtime.linux-musl-x64*.nupkg + !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload itemPattern: | - Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg + **/Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg + !**/*.symbols.nupkg preBuildSteps: - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.SourcesDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs @@ -299,109 +303,119 @@ extends: artifactName: Build_windows_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg - Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg - Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg - Microsoft.NETCore.App.Runtime.win-x64*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg + **/Microsoft.NETCore.App.Runtime.win-x64*.nupkg - input: pipelineArtifact artifactName: Build_windows_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg - Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg - Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg - Microsoft.NETCore.App.Runtime.win-arm64*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg + **/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg + **/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - input: pipelineArtifact artifactName: Build_windows_x86_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.win-x86*.nupkg + **/Microsoft.NETCore.App.Runtime.win-x86*.nupkg - input: pipelineArtifact artifactName: Build_android_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_x86_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_arm_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_browser_wasm_Linux_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg - Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg - Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg - Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg - Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg - Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg - Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg - Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg - Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg - Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg - Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg + **/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg + **/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg + **/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg + **/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg + **/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg + **/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg + **/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg + **/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg + **/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg + **/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - input: pipelineArtifact - artifactName: Build_browser_wasm_Linux_Mono_Multithread_Artifacts + artifactName: build_browser_wasm_linux_release_Mono_multithread_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - input: pipelineArtifact artifactName: Build_wasi_wasm_Linux_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg - Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg + **/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - input: pipelineArtifact artifactName: Build_ios_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - input: pipelineArtifact artifactName: Build_iossimulator_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_iossimulator_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_maccatalyst_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact artifactName: Build_maccatalyst_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact artifactName: Build_tvos_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - input: pipelineArtifact artifactName: Build_tvossimulator_x64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_tvossimulator_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages itemPattern: | - Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + **/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + + preBuildSteps: + - task: CopyFiles@2 + displayName: Flatten packages + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages + contents: '*/Shipping/*.nupkg' + cleanTargetFolder: false + targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages + flattenFolders: true buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:EnableDefaultRidSpecificArtifacts=false From 2343ebef70229b764a50383e0e9c05c03b98ca6e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 15:59:49 -0800 Subject: [PATCH 64/77] inputs happen before clone. Don't download into the sources directory --- eng/pipelines/runtime-official.yml | 48 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d8cdbdba9935d6..e51e828b104bfa 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -155,30 +155,30 @@ extends: inputs: - input: pipelineArtifact artifactName: Build_linux_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | **/Microsoft.NETCore.App.Runtime.linux-x64*.nupkg !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | **/Microsoft.NETCore.App.Runtime.linux-arm64*.nupkg !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | **/Microsoft.NETCore.App.Runtime.linux-musl-x64*.nupkg !**/*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/RuntimeDownload + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | **/Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg !**/*.symbols.nupkg preBuildSteps: - - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.SourcesDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) + - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs - template: /eng/pipelines/coreclr/templates/install-diagnostic-certs.yml parameters: @@ -301,7 +301,7 @@ extends: inputs: - input: pipelineArtifact artifactName: Build_windows_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg **/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg @@ -309,7 +309,7 @@ extends: **/Microsoft.NETCore.App.Runtime.win-x64*.nupkg - input: pipelineArtifact artifactName: Build_windows_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg **/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg @@ -317,32 +317,32 @@ extends: **/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - input: pipelineArtifact artifactName: Build_windows_x86_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.win-x86*.nupkg - input: pipelineArtifact artifactName: Build_android_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_x86_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_arm_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_android_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - input: pipelineArtifact artifactName: Build_browser_wasm_Linux_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg **/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg @@ -357,53 +357,53 @@ extends: **/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - input: pipelineArtifact artifactName: build_browser_wasm_linux_release_Mono_multithread_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - input: pipelineArtifact artifactName: Build_wasi_wasm_Linux_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg **/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - input: pipelineArtifact artifactName: Build_ios_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - input: pipelineArtifact artifactName: Build_iossimulator_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_iossimulator_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_maccatalyst_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact artifactName: Build_maccatalyst_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - input: pipelineArtifact artifactName: Build_tvos_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - input: pipelineArtifact artifactName: Build_tvossimulator_x64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - input: pipelineArtifact artifactName: Build_tvossimulator_arm64_release_AllRuntimes_Artifacts - targetPath: $(Build.SourcesDirectory)/artifacts/workloadPackages + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages itemPattern: | **/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg @@ -411,7 +411,7 @@ extends: - task: CopyFiles@2 displayName: Flatten packages inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages + sourceFolder: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages contents: '*/Shipping/*.nupkg' cleanTargetFolder: false targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages From b08df1ecff5f0f8c2a11700e34741b76c711ec63 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 18:42:04 -0800 Subject: [PATCH 65/77] Fix paths --- eng/pipelines/runtime-official.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index e51e828b104bfa..d861259bd19aaa 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -157,26 +157,26 @@ extends: artifactName: Build_linux_x64_release_AllRuntimes_Artifacts targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | - **/Microsoft.NETCore.App.Runtime.linux-x64*.nupkg - !**/*.symbols.nupkg + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-x64.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-x64.*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | - **/Microsoft.NETCore.App.Runtime.linux-arm64*.nupkg - !**/*.symbols.nupkg + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-arm64.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-arm64.*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_x64_release_AllRuntimes_Artifacts targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | - **/Microsoft.NETCore.App.Runtime.linux-musl-x64*.nupkg - !**/*.symbols.nupkg + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-x64.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-x64.*.symbols.nupkg - input: pipelineArtifact artifactName: Build_linux_musl_arm64_release_AllRuntimes_Artifacts targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload itemPattern: | - **/Microsoft.NETCore.App.Runtime.linux-musl-arm64*.nupkg - !**/*.symbols.nupkg + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm64.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm64.*.symbols.nupkg preBuildSteps: - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs @@ -412,8 +412,8 @@ extends: displayName: Flatten packages inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/artifacts/workloadPackages - contents: '*/Shipping/*.nupkg' - cleanTargetFolder: false + contents: 'packages/Release/Shipping/*.nupkg' + cleanTargetFolder: true targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages flattenFolders: true From f37e6a5613cf8c02d8a2a6ccda1738070f6e1840 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 4 Mar 2025 19:48:24 -0800 Subject: [PATCH 66/77] Skip extracting packages that don't match so we can see the whole list. --- eng/extract-for-crossdac.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/extract-for-crossdac.ps1 b/eng/extract-for-crossdac.ps1 index 24ce91d1043fd3..2ae2e61e5d3240 100644 --- a/eng/extract-for-crossdac.ps1 +++ b/eng/extract-for-crossdac.ps1 @@ -20,6 +20,6 @@ foreach ($file in Get-ChildItem $DownloadDirectory -Recurse -Filter '*.nupkg') { Write-Host "Extracting Package: $id $ver to $ExtractDirectory/$($id.ToLowerInvariant())/$ver" [System.IO.Compression.ZipFile]::ExtractToDirectory($file.FullName, "$ExtractDirectory/$($id.ToLowerInvariant())/$ver") } else { - throw "Unexpected file name: $($file.Name)" + Write-Host "Skipping non-runtime pack: $($file.Name)" } } From d3b9cfdc22d2b4bcca2f1da61a96d0931630ea23 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 5 Mar 2025 11:15:28 -0800 Subject: [PATCH 67/77] Add arm32 downloads for crossdac --- eng/pipelines/runtime-official.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d861259bd19aaa..123b9a82329d5c 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -177,6 +177,18 @@ extends: itemPattern: | packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm64.*.nupkg !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm64.*.symbols.nupkg + - input: pipelineArtifact + artifactName: Build_linux_arm_release_AllRuntimes_Artifacts + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload + itemPattern: | + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-arm.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-arm.*.symbols.nupkg + - input: pipelineArtifact + artifactName: Build_linux_musl_arm_release_AllRuntimes_Artifacts + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload + itemPattern: | + packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm.*.nupkg + !packages/Release/Shipping/Microsoft.NETCore.App.Runtime.linux-musl-arm.*.symbols.nupkg preBuildSteps: - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.ArtifactStagingDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot) displayName: Extract runtime packs From 2c9029626b4067243ca67c82ac8f9532fa2a6381 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 5 Mar 2025 12:53:24 -0800 Subject: [PATCH 68/77] Publish crossos dac packages --- eng/pipelines/runtime-official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 123b9a82329d5c..9966dff2d5bf4f 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -147,7 +147,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs) + buildArgs: -s crossdacpack -restore -build -publish -c $(_BuildConfig) /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs) nameSuffix: CrossDac isOfficialBuild: true timeoutInMinutes: 120 From a0e9fe6dd9d0718d14d210655d4a174a4bb54ced Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 5 Mar 2025 14:47:43 -0800 Subject: [PATCH 69/77] Don't use RID-specific artifacts for the cross-dac build --- eng/pipelines/runtime-official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 9966dff2d5bf4f..07204910583948 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -147,7 +147,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -restore -build -publish -c $(_BuildConfig) /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs) + buildArgs: -s crossdacpack -restore -build -publish -c $(_BuildConfig) /p:EnableDefaultRidSpecificArtifacts=false /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs) nameSuffix: CrossDac isOfficialBuild: true timeoutInMinutes: 120 From 53c44dbf9aba84802e6c1a66ce96d7366e3ec1e3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 5 Mar 2025 17:43:56 -0800 Subject: [PATCH 70/77] Fix source-build and deps --- eng/pipelines/official/pipeline.yml | 1 + .../official/prepare-signed-artifacts.yml | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/official/pipeline.yml b/eng/pipelines/official/pipeline.yml index 45f0e62e05e7d0..e343edd0e52b3f 100644 --- a/eng/pipelines/official/pipeline.yml +++ b/eng/pipelines/official/pipeline.yml @@ -27,3 +27,4 @@ extends: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals 1es-windows-2022 symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' + dependsOn: PrepareSignedArtifacts diff --git a/eng/pipelines/official/prepare-signed-artifacts.yml b/eng/pipelines/official/prepare-signed-artifacts.yml index 09f27540480aed..ab28ff29899acb 100644 --- a/eng/pipelines/official/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/prepare-signed-artifacts.yml @@ -34,12 +34,14 @@ jobs: clean: true fetchDepth: 20 - ${{ each job in parameters.buildStage.jobs }}: - - input: pipelineArtifact - artifactName: ${{ job.job }}_Artifacts - targetPath: $(Build.SourcesDirectory)\artifacts - itemPattern: | - **/* - !_manifest/**/* + # Source Build publishes itself, so we don't need to publish it + - ${{ if notIn(job.job, 'Source_Build_Linux_x64')}}: + - input: pipelineArtifact + artifactName: ${{ job.job }}_Artifacts + targetPath: $(Build.SourcesDirectory)\artifacts + itemPattern: | + **/* + !_manifest/**/* outputs: - output: pipelineArtifact displayName: 'Publish BuildLogs' From 0e820f406194208510ea81144b9b218e1acfecff Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Mar 2025 11:16:19 -0800 Subject: [PATCH 71/77] Fix join-job signing --- eng/Publishing.props | 17 +++++++++++++++++ eng/Signing.props | 39 ++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index b96cea141237c3..114ae7675bb5ff 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -20,6 +20,23 @@ VerticalManifest.xml + + + + + %(FullPath).sha512 + + + %(FullPath).sha512 - + + + + + + + + + + Date: Thu, 6 Mar 2025 11:22:49 -0800 Subject: [PATCH 72/77] Add back split macos job to the official build. These files need to be signed coming out of runtime and otherwise we'd miss signing them --- .../common/macos-sign-with-entitlements.yml | 77 +++++++++++++++++++ eng/pipelines/runtime-official.yml | 47 ++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 eng/pipelines/common/macos-sign-with-entitlements.yml diff --git a/eng/pipelines/common/macos-sign-with-entitlements.yml b/eng/pipelines/common/macos-sign-with-entitlements.yml new file mode 100644 index 00000000000000..72a03b90f340d6 --- /dev/null +++ b/eng/pipelines/common/macos-sign-with-entitlements.yml @@ -0,0 +1,77 @@ +parameters: + filesToSign: [] + timeoutInMinutes: '30' + +steps: + - task: UseDotNet@2 + displayName: Install .NET 6 SDK for signing. + inputs: + packageType: 'sdk' + version: '6.0.x' + installationPath: '$(Agent.TempDirectory)/dotnet' + + - ${{ each file in parameters.filesToSign }}: + - task: CopyFiles@2 + displayName: 'Copy entitled file ${{ file.name }}' + inputs: + contents: '${{ file.path }}/${{ file.name }}' + targetFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled' + overWrite: true + + - task: ArchiveFiles@2 + displayName: 'Zip MacOS files for signing' + inputs: + rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/mac_entitled' + archiveFile: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' + archiveType: zip + includeRootFolder: true + replaceExistingArchive: true + + - task: EsrpCodeSigning@5 + displayName: 'ESRP CodeSigning' + inputs: + ConnectedServiceName: 'DotNet-Engineering-Services_KeyVault' + AppRegistrationClientId: '28ec6507-2167-4eaa-a294-34408cf5dd0e' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'EngKeyVault' + AuthCertName: 'DotNetCore-ESRP-AuthCert' + AuthSignCertName: 'DotNetCore-ESRP-AuthSignCert' + FolderPath: '$(Build.ArtifactStagingDirectory)/' + Pattern: 'mac_entitled_to_sign.zip' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401337-Apple", + "operationCode": "MacAppDeveloperSign", + "parameters" : { + "hardening": "Enable" + }, + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: ${{ parameters.timeoutInMinutes }} + MaxConcurrency: '50' + MaxRetryAttempts: '5' + PendingAnalysisWaitTimeoutMinutes: '5' + env: + DOTNET_MULTILEVEL_LOOKUP: 0 + DOTNET_ROOT: '$(Agent.TempDirectory)/dotnet' + DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR: '$(Agent.TempDirectory)/dotnet' + + - task: ExtractFiles@1 + displayName: 'Extract MacOS after signing' + inputs: + archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' + destinationFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' + + - ${{ each file in parameters.filesToSign }}: + - task: CopyFiles@2 + displayName: 'Copy ${{ file.name }} to destination' + inputs: + contents: ${{ file.name }} + sourceFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' + targetFolder: '${{ file.path }}' + overWrite: true diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d861259bd19aaa..f465123da5c5d7 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -97,7 +97,52 @@ extends: isOfficialBuild: true # - # Build all runtime packs for MacOS, Linux, Linux musl, and mobile + # Build all runtime packs + # Mac x64/arm64 + # Sign and entitle createdump and corerun after native build. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - osx_arm64 + - osx_x64 + jobParameters: + templatePath: 'templates-official' + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + nameSuffix: AllRuntimes + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + preBuildSteps: + # Build our native assets first so we can sign them. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/macos-sign-with-entitlements.yml + parameters: + filesToSign: + - name: createdump + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + - name: corerun + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + - name: dotnet + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + - name: apphost + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + + - task: CopyFiles@2 + displayName: 'Copy signed createdump to sharedFramework' + inputs: + contents: createdump + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework + overWrite: true + + # + # Build all runtime packs for Linux, Linux musl, and mobile # - template: /eng/pipelines/common/platform-matrix.yml parameters: From 9557e70f64a3b4c0fa02f0df854574978a743903 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Mar 2025 11:29:38 -0800 Subject: [PATCH 73/77] Remove duplicate jobs --- eng/pipelines/runtime-official.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index ae14a000d6c1a6..76cb7455d0e352 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -155,8 +155,6 @@ extends: - linux_musl_x64 - linux_musl_arm - linux_musl_arm64 - - osx_arm64 - - osx_x64 - android_x64 - android_x86 - android_arm From 745a8a16ddabba4a1fa0aaa4df0b7b4607bd6db9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Mar 2025 13:03:03 -0800 Subject: [PATCH 74/77] Adjust commands for macos split job --- eng/pipelines/runtime-official.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 76cb7455d0e352..c6591a5f09e49f 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -110,7 +110,7 @@ extends: - osx_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -restore -build -publish -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -119,7 +119,7 @@ extends: - template: /eng/pipelines/common/templates/global-build-step.yml parameters: buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog - displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs + displayName: Build native CoreCLR and host components - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/pipelines/common/macos-sign-with-entitlements.yml parameters: From a314fc3aa011e1e78a52ecdb00e8e8564007443d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Mar 2025 16:30:22 -0800 Subject: [PATCH 75/77] Use the right property for the signing items base directory --- eng/Signing.props | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 870c79b22e9030..779f818f425b86 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -96,11 +96,11 @@ - + From 5312597c90ab52201d31f99c47a99b16697ac0ba Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 7 Mar 2025 07:27:55 -0800 Subject: [PATCH 76/77] Update Signing.props Co-authored-by: Viktor Hofer --- eng/Signing.props | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 779f818f425b86..74a72f55fbd749 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -169,7 +169,5 @@ Kind="Package" /> - - From f93980b4cadb72631aff93c838776d8ea0bc6f3e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 7 Mar 2025 10:46:47 -0800 Subject: [PATCH 77/77] Remove duplicate signing inputs --- eng/Signing.props | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/Signing.props b/eng/Signing.props index 74a72f55fbd749..1cda36dc4eb1b9 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -98,7 +98,6 @@