From 2f3a14356c52fc6d9be483cfc080c1f96729d63d Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Mon, 4 May 2020 15:28:15 -0400 Subject: [PATCH 01/26] Add LLVM Mono runtime build --- eng/pipelines/runtime-official.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index bc010a5ce1a182..6783ce17c6dc98 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -92,6 +92,29 @@ stages: jobParameters: isOfficialBuild: ${{ variables.isOfficialBuild }} + # + # Build Mono LLVM release + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/mono/templates/build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: + - OSX_x64 + - Linux_x64 + # - Linux_arm + # - Linux_arm64 + # - Linux_musl_x64 + # - Linux_musl_arm64 + # - Windows_NT_x64 + # - Windows_NT_x86 + # - Windows_NT_arm + # - Windows_NT_arm64 + jobParameters: + llvm: true + isOfficialBuild: ${{ variables.isOfficialBuild }} + # # Build libraries using live CoreLib from CoreCLR # From 8c4966638ed9af667ed85230129b9ad2c0e92daa Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Tue, 5 May 2020 11:58:51 -0400 Subject: [PATCH 02/26] Switch from 'llvm' boolean to 'runtimeVariant' freeform string in yaml This makes it easier to add oddball variant builds, without a big pile of booleans for every possible variant --- eng/pipelines/installer/installer-matrix.yml | 2 + eng/pipelines/mono/templates/build-job.yml | 37 ++++++++----------- .../mono/templates/xplat-pipeline-job.yml | 9 ++--- eng/pipelines/runtime-official.yml | 19 +++++++++- eng/pipelines/runtime.yml | 4 +- 5 files changed, 40 insertions(+), 31 deletions(-) diff --git a/eng/pipelines/installer/installer-matrix.yml b/eng/pipelines/installer/installer-matrix.yml index bb5e6e46f96c43..e9ae06bf267690 100644 --- a/eng/pipelines/installer/installer-matrix.yml +++ b/eng/pipelines/installer/installer-matrix.yml @@ -3,6 +3,7 @@ parameters: platforms: [] jobParameters: [] buildConfig: Release + runtimeVariant: '' jobs: @@ -16,5 +17,6 @@ jobs: platforms: ${{ parameters.platforms }} passPlatforms: true runtimeFlavor: ${{ parameters.runtimeFlavor }} + runtimeVariant: ${{ parameters.runtimeVariant }} jobParameters: ${{ insert }}: ${{ parameters.jobParameters }} diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index 136c83cf522d22..df7972690d80da 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -9,7 +9,7 @@ parameters: variables: {} pool: '' condition: true - llvm: false + runtimeVariant: '' isOfficialBuild: false crossrootfsDir: '' @@ -24,20 +24,13 @@ jobs: helixType: 'build/product/' enableMicrobuild: true pool: ${{ parameters.pool }} - llvm: ${{ parameters.llvm }} + runtimeVariant: ${{ parameters.runtimeVariant }} crossrootfsDir: ${{ parameters.crossroofsDir }} condition: ${{ parameters.condition }} # Compute job name from template parameters - ${{ if ne(parameters.llvm, true) }}: - name: ${{ format('mono_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Mono Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - # if LLVM enabled, set a variable we can consume - ${{ if eq(parameters.llvm, true) }}: - name: ${{ format('mono_llvm_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Mono LLVM Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - + name: ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Mono {0} Product Build {1}{2} {3} {4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -81,6 +74,9 @@ jobs: value: wasm - name: osOverride value: '-os Browser' + - ${{ if startsWith(parameters.runtimeVariant, 'llvm') }}: + - name: llvmParameter + value: /p:MonoEnableLLVM=true - ${{ parameters.variables }} steps: @@ -105,10 +101,10 @@ jobs: # Build - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoEnableLLVM=${{ parameters.llvm }} + - script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) displayName: Build product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoEnableLLVM=${{ parameters.llvm }} + - script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) displayName: Build product - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: @@ -129,15 +125,15 @@ jobs: displayName: 'product build' # Build packages - - ${{ if and(ne(parameters.llvm, true), ne(parameters.osGroup, 'Windows_NT')) }}: - - script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoEnableLLVM=${{ parameters.llvm }} -pack $(OutputRidArg) + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) -pack $(OutputRidArg) displayName: Build nupkg - - ${{ if and(ne(parameters.llvm, true), eq(parameters.osGroup, 'Windows_NT')) }}: - - script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoEnableLLVM=${{ parameters.llvm }} -pack $(OutputRidArg) + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) -pack $(OutputRidArg) displayName: Build nupkg # Publish official build - - ${{ if and(ne(parameters.llvm, true), eq(parameters.publishToBlobFeed, 'true')) }}: + - ${{ if eq(parameters.publishToBlobFeed, 'true') }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - script: $(Build.SourcesDirectory)/eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:Configuration=$(_BuildConfig) /p:TargetArchitecture=$(archType) /p:TargetOS=$(osGroup) /p:OSIdentifier=$(osGroup)$(osSubgroup) /bl:"$(Build.SourcesDirectory)/artifacts/log/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj displayName: Publish packages to blob feed @@ -160,9 +156,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - ${{ if ne(parameters.llvm, true) }}: - artifactName: 'BuildLogs_Mono_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - ${{ if eq(parameters.llvm, true) }}: - artifactName: 'BuildLogs_Mono_LLVM_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + artifactName: 'BuildLogs_Mono_$(runtimeVariant)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 052b0f59fcdaa6..62c25c5508b314 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -9,7 +9,7 @@ parameters: liveLibrariesBuildConfig: '' strategy: '' pool: '' - llvm: false + runtimeVariant: '' liveRuntimeBuildConfig: 'release' # arcade-specific parameters @@ -35,7 +35,7 @@ jobs: container: ${{ parameters.container }} strategy: ${{ parameters.strategy }} pool: ${{ parameters.pool }} - llvm: ${{ parameters.llvm }} + runtimeVariant: ${{ parameters.runtimeVariant }} # arcade-specific parameters condition: and(succeeded(), ${{ parameters.condition }}) @@ -55,10 +55,7 @@ jobs: value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).${{ parameters.liveRuntimeBuildConfig }}' - name: buildProductArtifactName - ${{ if ne(parameters.llvm, true) }}: - value: 'MonoProduct_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - ${{ if eq(parameters.llvm, true) }}: - value: 'MonoProduct_LLVM_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'MonoProduct_$(runtimeVariant)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: binTestsPath value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 6783ce17c6dc98..cb17a3fc84820b 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -112,7 +112,7 @@ stages: # - Windows_NT_arm # - Windows_NT_arm64 jobParameters: - llvm: true + runtimeVariant: LLVMJIT isOfficialBuild: ${{ variables.isOfficialBuild }} # @@ -224,6 +224,23 @@ stages: - Android_x86 - Browser_wasm + # + # Installer Build for platforms using Mono + # + - template: /eng/pipelines/installer/installer-matrix.yml + parameters: + jobParameters: + liveRuntimeBuildConfig: release + liveLibrariesBuildConfig: Release + isOfficialBuild: ${{ variables.isOfficialBuild }} + useOfficialAllConfigurations: false + buildFullPlatformManifest: false + runtimeVariant: LLVMJIT + runtimeFlavor: mono + platforms: + - OSX_x64 + - Linux_x64 + - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml parameters: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 40f22e195782a1..e555490af69904 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -341,7 +341,7 @@ jobs: # - Windows_NT_arm # - Windows_NT_arm64 jobParameters: - llvm: true + runtimeVariant: LLVMJIT condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), @@ -369,7 +369,7 @@ jobs: # - Windows_NT_arm # - Windows_NT_arm64 jobParameters: - llvm: true + runtimeVariant: LLVMJIT condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), From 496ffbb06742ec40d3882799c19ab315a00c12b8 Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Tue, 5 May 2020 12:13:29 -0400 Subject: [PATCH 03/26] Add an LLVM suffix to installer nupkgs --- eng/pipelines/installer/jobs/base-job.yml | 1 + src/installer/pkg/projects/Directory.Build.targets | 3 +++ 2 files changed, 4 insertions(+) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 8a62e0ba2952bb..28e64e6bea8902 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -13,6 +13,7 @@ parameters: variables: [] name: '' displayName: '' + runtimeVariant: '' pool: '' packageDistroList: diff --git a/src/installer/pkg/projects/Directory.Build.targets b/src/installer/pkg/projects/Directory.Build.targets index 3617dde42ecad7..25349b4029cff5 100644 --- a/src/installer/pkg/projects/Directory.Build.targets +++ b/src/installer/pkg/projects/Directory.Build.targets @@ -29,6 +29,9 @@ .Mono + + .Mono.LLVM + $(Version) + true From 4187fd86b8eb7146c59aa70ea24bff05421eed58 Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Thu, 14 May 2020 10:33:12 -0400 Subject: [PATCH 24/26] Capitalise Interpreter, so it's prettier on AzDO --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 521ca3fbbd86c4..3c99b048a237df 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -815,7 +815,7 @@ jobs: testGroup: innerloop liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} liveRuntimeBuildConfig: release - runtimeVariant: 'interpreter' + runtimeVariant: 'Interpreter' condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), From 902f2a2beedb4eb61892cfe004ebf002782d4bec Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Mon, 18 May 2020 08:38:54 -0400 Subject: [PATCH 25/26] Remove some Interpreter runtimeVariant stuff, now it's a Scenario instead --- .../common/templates/runtimes/run-test-job.yml | 8 +------- eng/pipelines/coreclr/templates/perf-job.yml | 8 ++------ eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 5 +---- eng/pipelines/mono/templates/xplat-pipeline-job.yml | 10 ++-------- 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index d8b7d7156cddec..760dad39c85d8c 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -61,13 +61,7 @@ jobs: - ${{ if ne(parameters.testGroup, 'innerloop') }}: - '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - ${{ if ne(parameters.stagedBuild, true) }}: - - ${{ if eq(parameters.runtimeVariant, 'interpreter') }}: - ### runtimeVariant affects build artifacts, but we want to use "normal" - ### build artifacts for interpreter - there's no special interpreter - ###artifact to download - - ${{ format('{0}__product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.runtimeVariant, 'interpreter') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 0089e0f3b64668..003e58f276146d 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -80,12 +80,8 @@ jobs: parameters: unpackFolder: $(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) cleanUnpackFolder: false - ${{ if eq(parameters.runtimeVariant, 'interpreter') }}: - artifactFileName: 'MonoProduct__$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' - artifactName: 'MonoProduct__$(osGroup)_$(archType)_$(buildConfig)' - ${{ if ne(parameters.runtimeVariant, 'interpreter') }}: - artifactFileName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' - artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)' + artifactFileName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' + artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)' displayName: 'Mono runtime' # Create Core_Root diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 999f1966c7b671..daa189d88d6da3 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -71,10 +71,7 @@ jobs: # Build product defines what we are trying to build, either coreclr or mono - name: buildProductArtifactName - ${{ if eq(parameters.runtimeVariant, 'interpreter') }}: - value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - ${{ if ne(parameters.runtimeVariant, 'interpreter') }}: - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 0a8c1edabf0726..0701073c669c61 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -48,19 +48,13 @@ jobs: variables: - name: coreClrProductArtifactName - ${{ if eq(parameters.runtimeVariant, 'interpreter') }}: - value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' - ${{ if ne(parameters.runtimeVariant, 'interpreter') }}: - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' - name: coreClrProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(liveRuntimeBuildConfigUpper)' - name: buildProductArtifactName - ${{ if eq(parameters.runtimeVariant, 'interpreter') }}: - value: 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - ${{ if ne(parameters.runtimeVariant, 'interpreter') }}: - value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: binTestsPath value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' From 821d386f5e15228578c6f69eff372b42a0f06b99 Mon Sep 17 00:00:00 2001 From: Joseph Shields Date: Mon, 18 May 2020 14:23:51 -0400 Subject: [PATCH 26/26] Fix tpyo --- src/coreclr/tests/helixpublishwitharcade.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj index 773d20d78b59b4..c1d2de4f8b7131 100644 --- a/src/coreclr/tests/helixpublishwitharcade.proj +++ b/src/coreclr/tests/helixpublishwitharcade.proj @@ -33,7 +33,7 @@ RunInUnloadableContext=$(_RunInUnloadableContext); TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes); TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes); - RuntimeVariant=$(_RuntimeVariant) + RuntimeVariant=$(_RuntimeVariant); BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion)