Generate full platform manifest in official build

eng/liveBuilds.targets

@@ -6,28 +6,73 @@
     <LibrariesOSGroup Condition="'$(LibrariesOSGroup)' == ''">$(OSGroup)</LibrariesOSGroup>
     <LibrariesConfiguration Condition="'$(LibrariesConfiguration)' == ''">$(Configuration)</LibrariesConfiguration>
 
+    <LibrariesOSGroupConfigurationArchitecture Condition="'$(LibrariesOSGroupConfigurationArchitecture)' == ''">$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)</LibrariesOSGroupConfigurationArchitecture>
+  </PropertyGroup>
+
+  <!-- Accept override paths for live artifacts. -->
+  <PropertyGroup>
     <CoreCLRArtifactsPath Condition="'$(CoreCLROverridePath)' != ''">$([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)'))</CoreCLRArtifactsPath>
-    <CoreCLRArtifactsPath Condition="'$(CoreCLRArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)'))</CoreCLRArtifactsPath>
+    <LibrariesArtifactsPath Condition="'$(LibrariesOverridePath)' != ''">$([MSBuild]::NormalizeDirectory('$(LibrariesOverridePath)'))</LibrariesArtifactsPath>
+  </PropertyGroup>
 
-    <CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework'))</CoreCLRSharedFrameworkDir>
+  <!--
+    If this is running and the output RID is not the same as the targeted RID, resolve live assets
+    for the targeted RID, if available. This is used to gather asset metadata for the platform
+    manifest. In CI (multi-machine lab) builds, CoreCLR and Libraries artifacts are all downloaded
+    onto the current machine from all platforms for the Installer portion of the build.
 
+    Higher priority than override paths: in official builds, the overrides are passed to the build,
+    but we need to point to the AllArtifacts locations when building the platform manifest.
+  -->
+  <PropertyGroup Condition="
+    '$(RuntimeIdentifier)' != '$(OutputRid)' and
+    '$(AllArtifactsDownloadPath)' != ''">
+    <!-- Convert the OS component in the RID into names that match the job IDs. -->
+    <ArtifactPlatform>$(RuntimeIdentifier)</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('win-', 'Windows_NT-'))</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('unix-', 'Unix-'))</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('linux-', 'Linux-'))</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('osx-', 'OSX-'))</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('freebsd-', 'FreeBSD-'))</ArtifactPlatform>
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('netbsd-', 'NetBSD-'))</ArtifactPlatform>
+    <!-- Artifact name uses '_' rather than '-'. -->
+    <ArtifactPlatform>$(ArtifactPlatform.Replace('-', '_'))</ArtifactPlatform>
+
+    <CoreCLRArtifactsPath>$([MSBuild]::NormalizeDirectory('$(AllArtifactsDownloadPath)', 'CoreCLRProduct_$(ArtifactPlatform)_$(CoreCLRConfiguration)'))</CoreCLRArtifactsPath>
+    <LibrariesArtifactsPath>$([MSBuild]::NormalizeDirectory('$(AllArtifactsDownloadPath)', 'libraries_bin_$(ArtifactPlatform)_$(LibrariesConfiguration)'))</LibrariesArtifactsPath>
+
+    <!--
+      Use '*', calculating these in a static context based on RID is hard. Note that putting '*'
+      through a dir/path normalization function makes it not work, so avoid that.
+    -->
+    <LibrariesOSGroupConfigurationArchitecture>*</LibrariesOSGroupConfigurationArchitecture>
+  </PropertyGroup>
+
+  <!-- Set up default live asset paths if no overrides provided. -->
+  <PropertyGroup>
+    <CoreCLRArtifactsPath Condition="'$(CoreCLRArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)'))</CoreCLRArtifactsPath>
     <LibrariesArtifactsPath Condition="'$(LibrariesArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts'))</LibrariesArtifactsPath>
+    <LibrariesAllConfigurationsArtifactsPath Condition="'$(LibrariesAllConfigurationsArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts'))</LibrariesAllConfigurationsArtifactsPath>
+  </PropertyGroup>
+
+  <!-- Set up artifact subpaths. -->
+  <PropertyGroup>
+    <CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework'))</CoreCLRSharedFrameworkDir>
 
     <LibrariesPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'packages', '$(LibrariesConfiguration)'))</LibrariesPackagesDir>
     <LibrariesShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping'))</LibrariesShippingPackagesDir>
     <LibrariesNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping'))</LibrariesNonShippingPackagesDir>
 
-    <LibrariesAllConfigurationsArtifactsPath Condition="'$(LibrariesAllConfigurationsArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts'))</LibrariesAllConfigurationsArtifactsPath>
-
     <LibrariesAllConfigPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigurationsArtifactsPath)', 'packages', '$(LibrariesConfiguration)'))</LibrariesAllConfigPackagesDir>
     <LibrariesAllConfigShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigPackagesDir)', 'Shipping'))</LibrariesAllConfigShippingPackagesDir>
     <LibrariesAllConfigNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesAllConfigPackagesDir)', 'NonShipping'))</LibrariesAllConfigNonShippingPackagesDir>
 
     <LibrariesSharedFrameworkRefArtifactsPath Condition="'$(LibrariesSharedFrameworkRefArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)'))</LibrariesSharedFrameworkRefArtifactsPath>
     <LibrariesAllRefArtifactsPath Condition="'$(LibrariesAllRefArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'ref', '$(NetCoreAppCurrent)'))</LibrariesAllRefArtifactsPath>
-    <LibrariesSharedFrameworkBinArtifactsPath Condition="'$(LibrariesSharedFrameworkBinArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'pkg', '$(NetCoreAppCurrent)', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)'))</LibrariesSharedFrameworkBinArtifactsPath>
-    <LibrariesAllBinArtifactsPath Condition="'$(LibrariesAllBinArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'runtime', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)'))</LibrariesAllBinArtifactsPath>
-    <LibrariesNativeArtifactsPath Condition="'$(LibrariesNativeArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'native', '$(NetCoreAppCurrent)-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)'))</LibrariesNativeArtifactsPath>
+    <LibrariesSharedFrameworkBinArtifactsPath Condition="'$(LibrariesSharedFrameworkBinArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'pkg', '$(NetCoreAppCurrent)', 'runtime'))$(LibrariesOSGroupConfigurationArchitecture)\</LibrariesSharedFrameworkBinArtifactsPath>
+    <LibrariesAllBinArtifactsPath Condition="'$(LibrariesAllBinArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'runtime'))$(NetCoreAppCurrent)-$(LibrariesOSGroupConfigurationArchitecture)\</LibrariesAllBinArtifactsPath>
+    <LibrariesNativeArtifactsPath Condition="'$(LibrariesNativeArtifactsPath)' == ''">$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'bin', 'native'))$(NetCoreAppCurrent)-$(LibrariesOSGroupConfigurationArchitecture)\</LibrariesNativeArtifactsPath>
+
     <CoreCLRCrossTargetComponentDirName Condition="'$(TargetArchitecture)' == 'arm64' and '$(BuildArchitecture)' != 'arm64'">x64</CoreCLRCrossTargetComponentDirName>
     <CoreCLRCrossTargetComponentDirName Condition="'$(TargetArchitecture)' == 'arm' and '$(BuildArchitecture)' != 'arm' and '$(TargetsWindows)' == 'true'">x86</CoreCLRCrossTargetComponentDirName>
     <CoreCLRCrossTargetComponentDirName Condition="'$(TargetArchitecture)' == 'arm' and '$(BuildArchitecture)' != 'arm' and '$(TargetsLinux)' == 'true'">x64</CoreCLRCrossTargetComponentDirName>
@@ -81,12 +126,26 @@
     <Error Condition="'@(CoreCLRFiles)' == ''" Text="The CoreCLR subset category must be built before building this project." />
   </Target>
 
-  <Target Name="ResolveLibrariesFromLocalBuild">
+  <Target Name="EnsureLocalArtifactsExist">
     <Error Condition="!Exists('$(LibrariesSharedFrameworkRefArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesSharedFrameworkRefArtifactsPath)" />
     <Error Condition="!Exists('$(LibrariesAllRefArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesAllRefArtifactsPath)" />
+  </Target>
+
+  <!--
+    Ensure artifacts exist for the more advanced paths. If the configuration is '*', don't emit
+    these errors: it isn't a local dev scenario.
+  -->
+  <Target Name="EnsureLocalOSGroupConfigurationArchitectureSpecificArtifactsExist"
+          Condition="'$(LibrariesOSGroupConfigurationArchitecture)' != '*'">
     <Error Condition="!Exists('$(LibrariesSharedFrameworkBinArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesSharedFrameworkBinArtifactsPath)" />
     <Error Condition="!Exists('$(LibrariesAllBinArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesAllBinArtifactsPath)" />
     <Error Condition="!Exists('$(LibrariesNativeArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesNativeArtifactsPath)" />
+  </Target>
+
+  <Target Name="ResolveLibrariesFromLocalBuild"
+          DependsOnTargets="
+            EnsureLocalArtifactsExist;
+            EnsureLocalOSGroupConfigurationArchitectureSpecificArtifactsExist">
     <ItemGroup>
       <LibrariesRefAssemblies Condition="'$(IncludeOOBLibraries)' != 'true'" Include="$(LibrariesSharedFrameworkRefArtifactsPath)*.dll;$(LibrariesSharedFrameworkRefArtifactsPath)*.pdb" />
       <LibrariesRefAssemblies Condition="'$(IncludeOOBLibraries)' == 'true'" Include="$(LibrariesAllRefArtifactsPath)*.dll;$(LibrariesAllRefArtifactsPath)*.pdb" />
@@ -101,6 +160,7 @@
       <LibrariesRuntimeFiles Include="
         $(LibrariesNativeArtifactsPath)*.dll;
         $(LibrariesNativeArtifactsPath)*.dylib;
+        $(LibrariesNativeArtifactsPath)*.a;
         $(LibrariesNativeArtifactsPath)*.so;
         $(LibrariesNativeArtifactsPath)*.dbg;
         $(LibrariesNativeArtifactsPath)*.dwarf;

eng/pipelines/installer/jobs/base-job.yml

@@ -29,6 +29,7 @@ parameters:
 
   liveCoreClrBuildConfig: ''
   liveLibrariesBuildConfig: ''
+  upstreamPlatforms: []
 
 jobs:
 - job: ${{ format('installer_{0}', coalesce(parameters.name, parameters.platform)) }}
@@ -70,23 +71,28 @@ jobs:
 
   - name: LiveOverridePathArgs
     value: >-
-      $(CoreCLRArtifactsPathArg)
+      $(CoreCLRArtifactsArgs)
       $(LibrariesConfigurationArg)
       $(LibrariesAllConfigurationsArtifactsPathArg)
+      $(AllArtifactsArgs)
 
-  - name: CoreCLRArtifactsPathArg
+  - name: CoreCLRArtifactsArgs
     value: ''
   - name: LibrariesConfigurationArg
     value: ''
   - name: LibrariesAllConfigurationsArtifactsPathArg
     value: ''
+  - name: AllArtifactsArgs
+    value: ''
 
   - name: CoreClrDownloadPath
     value: ''
   - name: LibrariesDownloadPath
     value: ''
   - name: LibrariesDownloadPathAllConfigurations
     value: ''
+  - name: AllArtifactsDownloadPath
+    value: ''
 
   - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}:
     - name: liveCoreClrLegName
@@ -97,8 +103,10 @@ jobs:
         parameters.liveCoreClrBuildConfig) }}
     - name: CoreClrDownloadPath
       value: 'artifacts/transport/coreclr'
-    - name: CoreCLRArtifactsPathArg
-      value: /p:CoreCLROverridePath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath)
+    - name: CoreCLRArtifactsArgs
+      value: >-
+        /p:CoreCLROverridePath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath)
+        /p:CoreCLRConfiguration=${{ parameters.liveCoreClrBuildConfig }}
     - name: CoreClrArtifactName
       value: CoreCLRProduct_$(liveCoreClrLegName)
 
@@ -126,6 +134,16 @@ jobs:
     - name: LibrariesArtifactNameAllConfigurations
       value: libraries_bin_official_allconfigurations
 
+  # Download all upstream platforms and incorporate in the platform manifest. Use 'each' as a
+  # makeshift condition because there is no documented way to evaluate this and duplication is ok.
+  - ${{ each platform in parameters.upstreamPlatforms }}:
+    - name: AllArtifactsDownloadPath
+      value: 'artifacts/transport/AllArtifacts'
+    - name: AllArtifactsArgs
+      value: >-
+        /p:AllArtifactsDownloadPath=${{ parameters.buildCommandSourcesDirectory }}$(AllArtifactsDownloadPath)
+        /p:BuildFullPlatformManifest=true
+
   dependsOn:
   - checkout
   - ${{ parameters.dependsOn }}
@@ -143,6 +161,9 @@ jobs:
         parameters.liveLibrariesBuildConfig) }}
   - ${{ if eq(parameters.useOfficialAllConfigurations, true) }}:
     - libraries_build_allconfigurations_Windows_NT_x64_Release
+  - ${{ each platform in parameters.upstreamPlatforms }}:
+    - coreclr_product_build_${{ platform }}_${{ parameters.liveCoreClrBuildConfig }}
+    - libraries_build_netcoreapp_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}
 
   steps:
 
@@ -168,6 +189,33 @@ jobs:
 
   - template: /eng/pipelines/common/clone-checkout-bundle-step.yml
 
+  # Download and extract artifacts for earlier subsets.
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifacts for all platforms'
+    inputs:
+      buildType: current
+      downloadType: specific
+      downloadPath: '$(Build.SourcesDirectory)/__download__/AllPlatforms/'
+      itemPattern: |
+        CoreCLRProduct_*/**
+        libraries_bin_*/**
+
+  - ${{ each platform in parameters.upstreamPlatforms }}:
+    - task: ExtractFiles@1
+      displayName: 'Unzip CoreCLR artifacts: ${{ platform }}'
+      inputs:
+        archiveFilePatterns: |
+          $(Build.SourcesDirectory)/__download__/AllPlatforms/CoreCLRProduct_${{ platform }}_${{ parameters.liveCoreClrBuildConfig }}.*
+        destinationFolder: $(AllArtifactsDownloadPath)/CoreCLRProduct_${{ platform }}_${{ parameters.liveCoreClrBuildConfig }}/
+        cleanUnpackFolder: false
+    - task: ExtractFiles@1
+      displayName: 'Unzip Libraries artifacts: ${{ platform }}'
+      inputs:
+        archiveFilePatterns: |
+          $(Build.SourcesDirectory)/__download__/AllPlatforms/libraries_bin_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}.*
+        destinationFolder: $(AllArtifactsDownloadPath)/libraries_bin_${{ platform }}_${{ parameters.liveLibrariesBuildConfig }}/
+        cleanUnpackFolder: false
+
   - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}:
     - template: /eng/pipelines/common/download-artifact-step.yml
       parameters:

eng/pipelines/runtime-official.yml

@@ -106,6 +106,17 @@ stages:
       liveLibrariesBuildConfig: Release
       isOfficialBuild: ${{ variables.isOfficialBuild }}
       useOfficialAllConfigurations: true
+      upstreamPlatforms:
+      - OSX_x64
+      - Linux_x64
+      - Linux_arm
+      - Linux_arm64
+      - Linux_musl_x64
+      - Linux_musl_arm64
+      - Windows_NT_x86
+      - Windows_NT_x64
+      - Windows_NT_arm
+      - Windows_NT_arm64
 
 - ${{ if eq(variables.isOfficialBuild, true) }}:
   - template: /eng/pipelines/official/stages/publish.yml

src/installer/pkg/projects/netcoreappRIDs.props

@@ -8,7 +8,6 @@
   <ItemGroup>
     <OfficialBuildRID Include="linux-x64" />
     <OfficialBuildRID Include="linux-musl-x64" />
-    <OfficialBuildRID Include="rhel.6-x64" />
     <OfficialBuildRID Include="osx-x64" />
     <!-- Not currently built by CoreFX. -->
     <!-- <OfficialBuildRID Include="freebsd-x64" /> -->