diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
new file mode 100644
index 000000000000..5f3105f21114
--- /dev/null
+++ b/eng/common/SetupNugetSources.ps1
@@ -0,0 +1,167 @@
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under for each Maestro managed private feed. Two additional credential
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+#
+# - task: PowerShell@2
+# displayName: Setup Private Feeds Credentials
+# condition: eq(variables['Agent.OS'], 'Windows_NT')
+# inputs:
+# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+# env:
+# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+[CmdletBinding()]
+param (
+ [Parameter(Mandatory = $true)][string]$ConfigFile,
+ [Parameter(Mandatory = $true)][string]$Password
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+. $PSScriptRoot\tools.ps1
+
+# Add source entry to PackageSources
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+ $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
+
+ if ($packageSource -eq $null)
+ {
+ $packageSource = $doc.CreateElement("add")
+ $packageSource.SetAttribute("key", $SourceName)
+ $packageSource.SetAttribute("value", $SourceEndPoint)
+ $sources.AppendChild($packageSource) | Out-Null
+ }
+ else {
+ Write-Host "Package source $SourceName already present."
+ }
+
+ AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+}
+
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $password) {
+ # Looks for credential configuration for the given SourceName. Create it if none is found.
+ $sourceElement = $creds.SelectSingleNode($Source)
+ if ($sourceElement -eq $null)
+ {
+ $sourceElement = $doc.CreateElement($Source)
+ $creds.AppendChild($sourceElement) | Out-Null
+ }
+
+ # Add the node to the credential if none is found.
+ $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+ if ($usernameElement -eq $null)
+ {
+ $usernameElement = $doc.CreateElement("add")
+ $usernameElement.SetAttribute("key", "Username")
+ $sourceElement.AppendChild($usernameElement) | Out-Null
+ }
+ $usernameElement.SetAttribute("value", $Username)
+
+ # Add the to the credential if none is found.
+ # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+ # -> https://github.com/NuGet/Home/issues/5526
+ $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+ if ($passwordElement -eq $null)
+ {
+ $passwordElement = $doc.CreateElement("add")
+ $passwordElement.SetAttribute("key", "ClearTextPassword")
+ $sourceElement.AppendChild($passwordElement) | Out-Null
+ }
+ $passwordElement.SetAttribute("value", $Password)
+}
+
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
+ $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+
+ Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+
+ ForEach ($PackageSource in $maestroPrivateSources) {
+ Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+ AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+ }
+}
+
+function EnablePrivatePackageSources($DisabledPackageSources) {
+ $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+ ForEach ($DisabledPackageSource in $maestroPrivateSources) {
+ Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+ # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
+ $DisabledPackageSources.RemoveChild($DisabledPackageSource)
+ }
+}
+
+if (!(Test-Path $ConfigFile -PathType Leaf)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+ ExitWithExitCode 1
+}
+
+if (!$Password) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
+ ExitWithExitCode 1
+}
+
+# Load NuGet.config
+$doc = New-Object System.Xml.XmlDocument
+$filename = (Get-Item $ConfigFile).FullName
+$doc.Load($filename)
+
+# Get reference to or create one if none exist already
+$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
+if ($sources -eq $null) {
+ $sources = $doc.CreateElement("packageSources")
+ $doc.DocumentElement.AppendChild($sources) | Out-Null
+}
+
+# Looks for a node. Create it if none is found.
+$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+if ($creds -eq $null) {
+ $creds = $doc.CreateElement("packageSourceCredentials")
+ $doc.DocumentElement.AppendChild($creds) | Out-Null
+}
+
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
+if ($disabledSources -ne $null) {
+ Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+ EnablePrivatePackageSources -DisabledPackageSources $disabledSources
+}
+
+$userName = "dn-bot"
+
+# Insert credential nodes for Maestro's private feeds
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
+
+$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
+if ($dotnet31Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']")
+if ($dotnet5Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
+if ($dotnet6Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$doc.Save($filename)
\ No newline at end of file
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
new file mode 100644
index 000000000000..30964e5b9500
--- /dev/null
+++ b/eng/common/SetupNugetSources.sh
@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under for each Maestro's managed private feed. Two additional credential
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+#
+# - task: Bash@3
+# displayName: Setup Private Feeds Credentials
+# inputs:
+# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+# arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+# condition: ne(variables['Agent.OS'], 'Windows_NT')
+# env:
+# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+ConfigFile=$1
+CredToken=$2
+NL='\n'
+TB=' '
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ ! -f "$ConfigFile" ]; then
+ Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+ ExitWithExitCode 1
+fi
+
+if [ -z "$CredToken" ]; then
+ Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
+ ExitWithExitCode 1
+fi
+
+if [[ `uname -s` == "Darwin" ]]; then
+ NL=$'\\\n'
+ TB=''
+fi
+
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
+if [ "$?" != "0" ]; then
+ echo "Adding ... section."
+ ConfigNodeHeader=""
+ PackageSourcesTemplate="${TB}${NL}${TB}"
+
+ sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
+fi
+
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
+if [ "$?" != "0" ]; then
+ echo "Adding ... section."
+
+ PackageSourcesNodeFooter=""
+ PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+fi
+
+PackageSources=()
+
+# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet3.1-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet3.1-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet3.1-internal-transport')
+fi
+
+# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet5-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet5-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet5-internal-transport')
+fi
+
+# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet6-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet6-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet6-internal-transport')
+fi
+
+# I want things split line by line
+PrevIFS=$IFS
+IFS=$'\n'
+PackageSources+="$IFS"
+PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
+IFS=$PrevIFS
+
+for FeedName in ${PackageSources[@]} ; do
+ # Check if there is no existing credential for this FeedName
+ grep -i "<$FeedName>" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding credentials for $FeedName."
+
+ PackageSourceCredentialsNodeFooter=""
+ NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}"
+
+ sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+ fi
+done
+
+# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
+grep -i "" $ConfigFile
+if [ "$?" == "0" ]; then
+ DisabledDarcIntSources=()
+ echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
+ DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"')
+ for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+ if [[ $DisabledSourceName == darc-int* ]]
+ then
+ OldDisableValue=""
+ NewDisableValue=""
+ sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
+ echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
+ fi
+ done
+fi
\ No newline at end of file
diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1
new file mode 100644
index 000000000000..28d7823299eb
--- /dev/null
+++ b/eng/common/pipeline-logging-functions.ps1
@@ -0,0 +1,260 @@
+# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified.
+
+# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1
+
+$script:loggingCommandPrefix = '##vso['
+$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"?
+ New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' }
+ New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' }
+ New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' }
+ New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' }
+)
+# TODO: BUG: Escape % ???
+# TODO: Add test to verify don't need to escape "=".
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTelemetryError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Category,
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force)
+
+ $PSBoundParameters.Remove('Category') | Out-Null
+
+ if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+ $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+ }
+ $PSBoundParameters.Remove('Message') | Out-Null
+ $PSBoundParameters.Add('Message', $Message)
+ Write-PipelineTaskError @PSBoundParameters
+}
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTaskError {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Message,
+ [Parameter(Mandatory = $false)]
+ [string]$Type = 'error',
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput,
+ [switch]$Force
+ )
+
+ if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+ if ($Type -eq 'error') {
+ Write-Host $Message -ForegroundColor Red
+ return
+ }
+ elseif ($Type -eq 'warning') {
+ Write-Host $Message -ForegroundColor Yellow
+ return
+ }
+ }
+
+ if (($Type -ne 'error') -and ($Type -ne 'warning')) {
+ Write-Host $Message
+ return
+ }
+ $PSBoundParameters.Remove('Force') | Out-Null
+ if (-not $PSBoundParameters.ContainsKey('Type')) {
+ $PSBoundParameters.Add('Type', 'error')
+ }
+ Write-LogIssue @PSBoundParameters
+}
+
+function Write-PipelineSetVariable {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Name,
+ [string]$Value,
+ [switch]$Secret,
+ [switch]$AsOutput,
+ [bool]$IsMultiJobVariable = $true)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
+ 'variable' = $Name
+ 'isSecret' = $Secret
+ 'isOutput' = $IsMultiJobVariable
+ } -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelinePrependPath {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Path,
+ [switch]$AsOutput)
+
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
+ }
+}
+
+function Write-PipelineSetResult {
+ [CmdletBinding()]
+ param(
+ [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+ [Parameter(Mandatory = $true)]
+ [string]$Result,
+ [string]$Message)
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+ 'result' = $Result
+ }
+ }
+}
+
+<########################################
+# Private functions.
+########################################>
+function Format-LoggingCommandData {
+ [CmdletBinding()]
+ param([string]$Value, [switch]$Reverse)
+
+ if (!$Value) {
+ return ''
+ }
+
+ if (!$Reverse) {
+ foreach ($mapping in $script:loggingCommandEscapeMappings) {
+ $Value = $Value.Replace($mapping.Token, $mapping.Replacement)
+ }
+ }
+ else {
+ for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
+ $mapping = $script:loggingCommandEscapeMappings[$i]
+ $Value = $Value.Replace($mapping.Replacement, $mapping.Token)
+ }
+ }
+
+ return $Value
+}
+
+function Format-LoggingCommand {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory = $true)]
+ [string]$Area,
+ [Parameter(Mandatory = $true)]
+ [string]$Event,
+ [string]$Data,
+ [hashtable]$Properties)
+
+ # Append the preamble.
+ [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder
+ $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event)
+
+ # Append the properties.
+ if ($Properties) {
+ $first = $true
+ foreach ($key in $Properties.Keys) {
+ [string]$value = Format-LoggingCommandData $Properties[$key]
+ if ($value) {
+ if ($first) {
+ $null = $sb.Append(' ')
+ $first = $false
+ }
+ else {
+ $null = $sb.Append(';')
+ }
+
+ $null = $sb.Append("$key=$value")
+ }
+ }
+ }
+
+ # Append the tail and output the value.
+ $Data = Format-LoggingCommandData $Data
+ $sb.Append(']').Append($Data).ToString()
+}
+
+function Write-LoggingCommand {
+ [CmdletBinding(DefaultParameterSetName = 'Parameters')]
+ param(
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Area,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+ [string]$Event,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [string]$Data,
+ [Parameter(ParameterSetName = 'Parameters')]
+ [hashtable]$Properties,
+ [Parameter(Mandatory = $true, ParameterSetName = 'Object')]
+ $Command,
+ [switch]$AsOutput)
+
+ if ($PSCmdlet.ParameterSetName -eq 'Object') {
+ Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput
+ return
+ }
+
+ $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
+ if ($AsOutput) {
+ $command
+ }
+ else {
+ Write-Host $command
+ }
+}
+
+function Write-LogIssue {
+ [CmdletBinding()]
+ param(
+ [ValidateSet('warning', 'error')]
+ [Parameter(Mandatory = $true)]
+ [string]$Type,
+ [string]$Message,
+ [string]$ErrCode,
+ [string]$SourcePath,
+ [string]$LineNumber,
+ [string]$ColumnNumber,
+ [switch]$AsOutput)
+
+ $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
+ 'type' = $Type
+ 'code' = $ErrCode
+ 'sourcepath' = $SourcePath
+ 'linenumber' = $LineNumber
+ 'columnnumber' = $ColumnNumber
+ }
+ if ($AsOutput) {
+ return $command
+ }
+
+ if ($Type -eq 'error') {
+ $foregroundColor = $host.PrivateData.ErrorForegroundColor
+ $backgroundColor = $host.PrivateData.ErrorBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Red
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+ else {
+ $foregroundColor = $host.PrivateData.WarningForegroundColor
+ $backgroundColor = $host.PrivateData.WarningBackgroundColor
+ if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+ $foregroundColor = [System.ConsoleColor]::Yellow
+ $backgroundColor = [System.ConsoleColor]::Black
+ }
+ }
+
+ Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor
+}
\ No newline at end of file
diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
new file mode 100644
index 000000000000..2eaafa8f323b
--- /dev/null
+++ b/eng/common/pipeline-logging-functions.sh
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+
+function Write-PipelineTelemetryError {
+ local telemetry_category=''
+ local force=false
+ local function_args=()
+ local message=''
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -category|-c)
+ telemetry_category=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ -*)
+ function_args+=("$1 $2")
+ shift
+ ;;
+ *)
+ message=$*
+ ;;
+ esac
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$message" >&2
+ return
+ fi
+
+ if [[ $force == true ]]; then
+ function_args+=("-force")
+ fi
+ message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
+ function_args+=("$message")
+ Write-PipelineTaskError ${function_args[@]}
+}
+
+function Write-PipelineTaskError {
+ local message_type="error"
+ local sourcepath=''
+ local linenumber=''
+ local columnnumber=''
+ local error_code=''
+ local force=false
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -type|-t)
+ message_type=$2
+ shift
+ ;;
+ -sourcepath|-s)
+ sourcepath=$2
+ shift
+ ;;
+ -linenumber|-ln)
+ linenumber=$2
+ shift
+ ;;
+ -columnnumber|-cn)
+ columnnumber=$2
+ shift
+ ;;
+ -errcode|-e)
+ error_code=$2
+ shift
+ ;;
+ -force|-f)
+ force=true
+ ;;
+ *)
+ break
+ ;;
+ esac
+
+ shift
+ done
+
+ if [[ $force != true ]] && [[ "$ci" != true ]]; then
+ echo "$@" >&2
+ return
+ fi
+
+ local message="##vso[task.logissue"
+
+ message="$message type=$message_type"
+
+ if [ -n "$sourcepath" ]; then
+ message="$message;sourcepath=$sourcepath"
+ fi
+
+ if [ -n "$linenumber" ]; then
+ message="$message;linenumber=$linenumber"
+ fi
+
+ if [ -n "$columnnumber" ]; then
+ message="$message;columnnumber=$columnnumber"
+ fi
+
+ if [ -n "$error_code" ]; then
+ message="$message;code=$error_code"
+ fi
+
+ message="$message]$*"
+ echo "$message"
+}
+
+function Write-PipelineSetVariable {
+ if [[ "$ci" != true ]]; then
+ return
+ fi
+
+ local name=''
+ local value=''
+ local secret=false
+ local as_output=false
+ local is_multi_job_variable=true
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -name|-n)
+ name=$2
+ shift
+ ;;
+ -value|-v)
+ value=$2
+ shift
+ ;;
+ -secret|-s)
+ secret=true
+ ;;
+ -as_output|-a)
+ as_output=true
+ ;;
+ -is_multi_job_variable|-i)
+ is_multi_job_variable=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ value=${value/;/%3B}
+ value=${value/\\r/%0D}
+ value=${value/\\n/%0A}
+ value=${value/]/%5D}
+
+ local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+ if [[ "$as_output" == true ]]; then
+ $message
+ else
+ echo "$message"
+ fi
+}
+
+function Write-PipelinePrependPath {
+ local prepend_path=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -path|-p)
+ prepend_path=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ export PATH="$prepend_path:$PATH"
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.prependpath]$prepend_path"
+ fi
+}
+
+function Write-PipelineSetResult {
+ local result=''
+ local message=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -result|-r)
+ result=$2
+ shift
+ ;;
+ -message|-m)
+ message=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.complete result=$result;]$message"
+ fi
+}
\ No newline at end of file
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
new file mode 100644
index 000000000000..a4745b344456
--- /dev/null
+++ b/eng/common/tools.ps1
@@ -0,0 +1,916 @@
+# Initialize variables if they aren't already defined.
+# These may be defined as parameters of the importing script, or set after importing this script.
+
+# CI mode - set to true on CI server for PR validation build or official build.
+[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
+
+# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
+[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' }
+
+# Set to true to opt out of outputting binary log while running in CI
+[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false }
+
+# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
+[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog }
+
+# Set to true to use the pipelines logger which will enable Azure logging output.
+# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
+# This flag is meant as a temporary opt-opt for the feature while validate it across
+# our consumers. It will be deleted in the future.
+[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci }
+
+# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
+[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
+
+# True to restore toolsets and dependencies.
+[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
+
+# Adjusts msbuild verbosity level.
+[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' }
+
+# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
+
+# Configures warning treatment in msbuild.
+[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true }
+
+# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json).
+[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
+
+# True to attempt using .NET Core already that meets requirements specified in global.json
+# installed on the machine instead of downloading one.
+[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
+
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
+[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
+
+# True to use global NuGet cache instead of restoring packages to repository-local directory.
+[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
+
+# True to exclude prerelease versions Visual Studio during build
+[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false }
+
+# An array of names of processes to stop on script exit if prepareMachine is true.
+$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
+
+$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
+
+set-strictmode -version 2.0
+$ErrorActionPreference = 'Stop'
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
+[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
+# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
+[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
+
+function Create-Directory ([string[]] $path) {
+ New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
+}
+
+function Unzip([string]$zipfile, [string]$outpath) {
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+ [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
+}
+
+# This will exec a process using the console and return it's exit code.
+# This will not throw when the process fails.
+# Returns process exit code.
+function Exec-Process([string]$command, [string]$commandArgs) {
+ $startInfo = New-Object System.Diagnostics.ProcessStartInfo
+ $startInfo.FileName = $command
+ $startInfo.Arguments = $commandArgs
+ $startInfo.UseShellExecute = $false
+ $startInfo.WorkingDirectory = Get-Location
+
+ $process = New-Object System.Diagnostics.Process
+ $process.StartInfo = $startInfo
+ $process.Start() | Out-Null
+
+ $finished = $false
+ try {
+ while (-not $process.WaitForExit(100)) {
+ # Non-blocking loop done to allow ctr-c interrupts
+ }
+
+ $finished = $true
+ return $global:LASTEXITCODE = $process.ExitCode
+ }
+ finally {
+ # If we didn't finish then an error occurred or the user hit ctrl-c. Either
+ # way kill the process
+ if (-not $finished) {
+ $process.Kill()
+ }
+ }
+}
+
+# Take the given block, print it, print what the block probably references from the current set of
+# variables using low-effort string matching, then run the block.
+#
+# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes,
+# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less
+# maintainable and less reliable. It is easy to make a mistake and modify the command without
+# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of
+# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in
+# existing source code can also be difficult, because the strings are not aligned to each other and
+# the line may be 300+ columns long.
+#
+# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues.
+#
+# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically.
+# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it
+# doesn't print any info about the variables being used by the command, which is normally the
+# interesting part to diagnose.
+function Exec-BlockVerbosely([scriptblock] $block) {
+ Write-Host "--- Running script block:"
+ $blockString = $block.ToString().Trim()
+ Write-Host $blockString
+
+ Write-Host "--- List of variables that might be used:"
+ # For each variable x in the environment, check the block for a reference to x via simple "$x" or
+ # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x",
+ # among others). It only catches what this function was originally written for: simple
+ # command-line commands.
+ $variableTable = Get-Variable |
+ Where-Object {
+ $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)")
+ } |
+ Format-Table -AutoSize -HideTableHeaders -Wrap |
+ Out-String
+ Write-Host $variableTable.Trim()
+
+ Write-Host "--- Executing:"
+ & $block
+ Write-Host "--- Done running script block!"
+}
+
+# createSdkLocationFile parameter enables a file being generated under the toolset directory
+# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
+# as dot sourcing isn't possible.
+function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
+ if (Test-Path variable:global:_DotNetInstallDir) {
+ return $global:_DotNetInstallDir
+ }
+
+ # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
+ $env:DOTNET_MULTILEVEL_LOOKUP=0
+
+ # Disable first run since we do not need all ASP.NET packages restored.
+ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+
+ # Disable telemetry on CI.
+ if ($ci) {
+ $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
+
+ # In case of network error, try to log the current IP for reference
+ Try-LogClientIpAddress
+ }
+
+ # Source Build uses DotNetCoreSdkDir variable
+ if ($env:DotNetCoreSdkDir -ne $null) {
+ $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir
+ }
+
+ # Find the first path on %PATH% that contains the dotnet.exe
+ if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
+ $dotnetExecutable = GetExecutableFileName 'dotnet'
+ $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue
+
+ if ($dotnetCmd -ne $null) {
+ $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
+ }
+ }
+
+ $dotnetSdkVersion = $GlobalJson.tools.dotnet
+
+ # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
+ # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
+ if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+ $dotnetRoot = $env:DOTNET_INSTALL_DIR
+ } else {
+ $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+ if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
+ if ($install) {
+ InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
+ ExitWithExitCode 1
+ }
+ }
+
+ $env:DOTNET_INSTALL_DIR = $dotnetRoot
+ }
+
+ # Creates a temporary file under the toolset dir.
+ # The following code block is protecting against concurrent access so that this function can
+ # be called in parallel.
+ if ($createSdkLocationFile) {
+ do {
+ $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName())
+ }
+ until (!(Test-Path $sdkCacheFileTemp))
+ Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
+
+ try {
+ Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
+ } catch {
+ # Somebody beat us
+ Remove-Item -Path $sdkCacheFileTemp
+ }
+ }
+
+ # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+ # build steps from using anything other than what we've downloaded.
+ # It also ensures that VS msbuild will use the downloaded sdk targets.
+ $env:PATH = "$dotnetRoot;$env:PATH"
+
+ # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
+ Write-PipelinePrependPath -Path $dotnetRoot
+
+ Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
+ Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
+
+ return $global:_DotNetInstallDir = $dotnetRoot
+}
+
+function Retry($downloadBlock, $maxRetries = 5) {
+ $retries = 1
+
+ while($true) {
+ try {
+ & $downloadBlock
+ break
+ }
+ catch {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ }
+
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+ break
+ }
+
+ }
+}
+
+function GetDotNetInstallScript([string] $dotnetRoot) {
+ $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
+ if (!(Test-Path $installScript)) {
+ Create-Directory $dotnetRoot
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+ $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
+
+ Retry({
+ Write-Host "GET $uri"
+ Invoke-WebRequest $uri -OutFile $installScript
+ })
+ }
+
+ return $installScript
+}
+
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+ InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
+}
+
+function InstallDotNet([string] $dotnetRoot,
+ [string] $version,
+ [string] $architecture = '',
+ [string] $runtime = '',
+ [bool] $skipNonVersionedFiles = $false,
+ [string] $runtimeSourceFeed = '',
+ [string] $runtimeSourceFeedKey = '',
+ [switch] $noPath) {
+
+ $installScript = GetDotNetInstallScript $dotnetRoot
+ $installParameters = @{
+ Version = $version
+ InstallDir = $dotnetRoot
+ }
+
+ if ($architecture) { $installParameters.Architecture = $architecture }
+ if ($runtime) { $installParameters.Runtime = $runtime }
+ if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+ if ($noPath) { $installParameters.NoPath = $True }
+
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
+ Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
+ if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+
+ if ($runtimeSourceFeedKey) {
+ $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+ $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+ $installParameters.FeedCredential = $decodedString
+ }
+
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
+ ExitWithExitCode 1
+ }
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
+ ExitWithExitCode 1
+ }
+ }
+}
+
+#
+# Locates Visual Studio MSBuild installation.
+# The preference order for MSBuild to use is as follows:
+#
+# 1. MSBuild from an active VS command prompt
+# 2. MSBuild from a compatible VS installation
+# 3. MSBuild from the xcopy tool package
+#
+# Returns full path to msbuild.exe.
+# Throws on failure.
+#
+function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
+ if (-not (IsWindowsPlatform)) {
+ throw "Cannot initialize Visual Studio on non-Windows"
+ }
+
+ if (Test-Path variable:global:_MSBuildExe) {
+ return $global:_MSBuildExe
+ }
+
+ # Minimum VS version to require.
+ $vsMinVersionReqdStr = '16.8'
+ $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
+
+ # If the version of msbuild is going to be xcopied,
+ # use this version. Version matches a package here:
+ # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview
+ $defaultXCopyMSBuildVersion = '16.10.0-preview2'
+
+ if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+ $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
+ $vsMinVersion = [Version]::new($vsMinVersionStr)
+
+ # Try msbuild command available in the environment.
+ if ($env:VSINSTALLDIR -ne $null) {
+ $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
+ if ($msbuildCmd -ne $null) {
+ # Workaround for https://github.com/dotnet/roslyn/issues/35793
+ # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
+ $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0])
+
+ if ($msbuildVersion -ge $vsMinVersion) {
+ return $global:_MSBuildExe = $msbuildCmd.Path
+ }
+
+ # Report error - the developer environment is initialized with incompatible VS version.
+ throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window"
+ }
+ }
+
+ # Locate Visual Studio installation or download x-copy msbuild.
+ $vsInfo = LocateVisualStudio $vsRequirements
+ if ($vsInfo -ne $null) {
+ $vsInstallDir = $vsInfo.installationPath
+ $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
+
+ InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
+ } else {
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
+ $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
+ $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
+ } else {
+ #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
+ if($vsMinVersion -lt $vsMinVersionReqd){
+ Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+ $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
+ }
+ else{
+ # If the VS version IS compatible, look for an xcopy msbuild package
+ # with a version matching VS.
+ # Note: If this version does not exist, then an explicit version of xcopy msbuild
+ # can be specified in global.json. This will be required for pre-release versions of msbuild.
+ $vsMajorVersion = $vsMinVersion.Major
+ $vsMinorVersion = $vsMinVersion.Minor
+ $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
+ }
+ }
+
+ $vsInstallDir = $null
+ if ($xcopyMSBuildVersion.Trim() -ine "none") {
+ $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+ if ($vsInstallDir -eq $null) {
+ throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+ }
+ }
+ if ($vsInstallDir -eq $null) {
+ throw 'Unable to find Visual Studio that has required version and components installed'
+ }
+ }
+
+ $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
+
+ $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin"
+ $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false }
+ if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) {
+ $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe"
+ } else {
+ $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe"
+ }
+
+ return $global:_MSBuildExe
+}
+
+function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
+ $env:VSINSTALLDIR = $vsInstallDir
+ Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
+
+ $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
+ if (Test-Path $vsSdkInstallDir) {
+ Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
+ $env:VSSDKInstall = $vsSdkInstallDir
+ }
+}
+
+function InstallXCopyMSBuild([string]$packageVersion) {
+ return InitializeXCopyMSBuild $packageVersion -install $true
+}
+
+function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
+ $packageName = 'RoslynTools.MSBuild'
+ $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
+ $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
+
+ if (!(Test-Path $packageDir)) {
+ if (!$install) {
+ return $null
+ }
+
+ Create-Directory $packageDir
+
+ Write-Host "Downloading $packageName $packageVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+ Retry({
+ Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
+ })
+
+ Unzip $packagePath $packageDir
+ }
+
+ return Join-Path $packageDir 'tools'
+}
+
+#
+# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
+#
+# The following properties of tools.vs are recognized:
+# "version": "{major}.{minor}"
+# Two part minimal VS version, e.g. "15.9", "16.0", etc.
+# "components": ["componentId1", "componentId2", ...]
+# Array of ids of workload components that must be available in the VS instance.
+# See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
+#
+# Returns JSON describing the located VS instance (same format as returned by vswhere),
+# or $null if no instance meeting the requirements is found on the machine.
+#
+function LocateVisualStudio([object]$vsRequirements = $null){
+ if (-not (IsWindowsPlatform)) {
+ throw "Cannot run vswhere on non-Windows platforms."
+ }
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
+ $vswhereVersion = $GlobalJson.tools.vswhere
+ } else {
+ $vswhereVersion = '2.5.2'
+ }
+
+ $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
+ $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
+
+ if (!(Test-Path $vsWhereExe)) {
+ Create-Directory $vsWhereDir
+ Write-Host 'Downloading vswhere'
+ Retry({
+ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+ })
+ }
+
+ if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+ $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
+
+ if (!$excludePrereleaseVS) {
+ $args += '-prerelease'
+ }
+
+ if (Get-Member -InputObject $vsRequirements -Name 'version') {
+ $args += '-version'
+ $args += $vsRequirements.version
+ }
+
+ if (Get-Member -InputObject $vsRequirements -Name 'components') {
+ foreach ($component in $vsRequirements.components) {
+ $args += '-requires'
+ $args += $component
+ }
+ }
+
+ $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
+
+ if ($lastExitCode -ne 0) {
+ return $null
+ }
+
+ # use first matching instance
+ return $vsInfo[0]
+}
+
+function InitializeBuildTool() {
+ if (Test-Path variable:global:_BuildTool) {
+ # If the requested msbuild parameters do not match, clear the cached variables.
+ if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
+ Remove-Item variable:global:_BuildTool
+ Remove-Item variable:global:_MSBuildExe
+ } else {
+ return $global:_BuildTool
+ }
+ }
+
+ if (-not $msbuildEngine) {
+ $msbuildEngine = GetDefaultMSBuildEngine
+ }
+
+ # Initialize dotnet cli if listed in 'tools'
+ $dotnetRoot = $null
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
+ $dotnetRoot = InitializeDotNetCli -install:$restore
+ }
+
+ if ($msbuildEngine -eq 'dotnet') {
+ if (!$dotnetRoot) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
+ ExitWithExitCode 1
+ }
+ $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
+ } elseif ($msbuildEngine -eq "vs") {
+ try {
+ $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
+ } catch {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ ExitWithExitCode 1
+ }
+
+ $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ } else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
+ ExitWithExitCode 1
+ }
+
+ return $global:_BuildTool = $buildTool
+}
+
+function GetDefaultMSBuildEngine() {
+ # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
+ return 'vs'
+ }
+
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
+ return 'dotnet'
+ }
+
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
+ ExitWithExitCode 1
+}
+
+function GetNuGetPackageCachePath() {
+ if ($env:NUGET_PACKAGES -eq $null) {
+ # Use local cache on CI to ensure deterministic build.
+ # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
+ # use global cache in dev builds to avoid cost of downloading packages.
+ # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
+ if ($useGlobalNuGetCache) {
+ $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
+ } else {
+ $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+ $env:RESTORENOCACHE = $true
+ }
+ }
+
+ return $env:NUGET_PACKAGES
+}
+
+# Returns a full path to an Arcade SDK task project file.
+function GetSdkTaskProject([string]$taskName) {
+ return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj"
+}
+
+function InitializeNativeTools() {
+ if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
+ $nativeArgs= @{}
+ if ($ci) {
+ $nativeArgs = @{
+ InstallDirectory = "$ToolsDir"
+ }
+ }
+ & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
+ }
+}
+
+function InitializeToolset() {
+ if (Test-Path variable:global:_ToolsetBuildProj) {
+ return $global:_ToolsetBuildProj
+ }
+
+ $nugetCache = GetNuGetPackageCachePath
+
+ $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
+ $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
+
+ if (Test-Path $toolsetLocationFile) {
+ $path = Get-Content $toolsetLocationFile -TotalCount 1
+ if (Test-Path $path) {
+ return $global:_ToolsetBuildProj = $path
+ }
+ }
+
+ if (-not $restore) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
+ ExitWithExitCode 1
+ }
+
+ $buildTool = InitializeBuildTool
+
+ $proj = Join-Path $ToolsetDir 'restore.proj'
+ $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
+
+ '' | Set-Content $proj
+
+ MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+
+ $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1
+ if (!(Test-Path $path)) {
+ throw "Invalid toolset path: $path"
+ }
+
+ return $global:_ToolsetBuildProj = $path
+}
+
+function ExitWithExitCode([int] $exitCode) {
+ if ($ci -and $prepareMachine) {
+ Stop-Processes
+ }
+ exit $exitCode
+}
+
+# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for
+# diagnostics, then exit the script with the $LASTEXITCODE.
+function Exit-IfNZEC([string] $category = "General") {
+ Write-Host "Exit code $LASTEXITCODE"
+ if ($LASTEXITCODE -ne 0) {
+ $message = "Last command failed with exit code $LASTEXITCODE."
+ Write-PipelineTelemetryError -Force -Category $category -Message $message
+ ExitWithExitCode $LASTEXITCODE
+ }
+}
+
+function Stop-Processes() {
+ Write-Host 'Killing running build processes...'
+ foreach ($processName in $processesToStopOnExit) {
+ Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
+ }
+}
+
+#
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild() {
+ if ($pipelinesLog) {
+ $buildTool = InitializeBuildTool
+
+ if ($ci -and $buildTool.Tool -eq 'dotnet') {
+ $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
+ $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
+ }
+
+ Enable-Nuget-EnhancedRetry
+
+ $toolsetBuildProject = InitializeToolset
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
+ if (-not $selectedPath) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ ExitWithExitCode 1
+ }
+ $args += "/logger:$selectedPath"
+ }
+
+ MSBuild-Core @args
+}
+
+#
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild-Core() {
+ if ($ci) {
+ if (!$binaryLog -and !$excludeCIBinarylog) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.'
+ ExitWithExitCode 1
+ }
+
+ if ($nodeReuse) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
+ ExitWithExitCode 1
+ }
+ }
+
+ Enable-Nuget-EnhancedRetry
+
+ $buildTool = InitializeBuildTool
+
+ $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
+
+ if ($warnAsError) {
+ $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
+ }
+ else {
+ $cmdArgs += ' /p:TreatWarningsAsErrors=false'
+ }
+
+ foreach ($arg in $args) {
+ if ($null -ne $arg -and $arg.Trim() -ne "") {
+ if ($arg.EndsWith('\')) {
+ $arg = $arg + "\"
+ }
+ $cmdArgs += " `"$arg`""
+ }
+ }
+
+ $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
+
+ $exitCode = Exec-Process $buildTool.Path $cmdArgs
+
+ if ($exitCode -ne 0) {
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
+
+ $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
+ if ($null -ne $buildLog) {
+ Write-Host "See log: $buildLog" -ForegroundColor DarkGray
+ }
+
+ if ($ci) {
+ Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ } else {
+ ExitWithExitCode $exitCode
+ }
+ }
+}
+
+function GetMSBuildBinaryLogCommandLineArgument($arguments) {
+ foreach ($argument in $arguments) {
+ if ($argument -ne $null) {
+ $arg = $argument.Trim()
+ if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
+ return $arg.Substring('/bl:'.Length)
+ }
+
+ if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
+ return $arg.Substring('/binaryLogger:'.Length)
+ }
+ }
+ }
+
+ return $null
+}
+
+function GetExecutableFileName($baseName) {
+ if (IsWindowsPlatform) {
+ return "$baseName.exe"
+ }
+ else {
+ return $baseName
+ }
+}
+
+function IsWindowsPlatform() {
+ return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
+}
+
+function Get-Darc($version) {
+ $darcPath = "$TempDir\darc\$(New-Guid)"
+ if ($version -ne $null) {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
+ } else {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
+ }
+ return "$darcPath\darc.exe"
+}
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\')
+$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
+$ArtifactsDir = Join-Path $RepoRoot 'artifacts'
+$ToolsetDir = Join-Path $ArtifactsDir 'toolset'
+$ToolsDir = Join-Path $RepoRoot '.tools'
+$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
+$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
+$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
+# true if global.json contains a "runtimes" section
+$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
+
+Create-Directory $ToolsetDir
+Create-Directory $TempDir
+Create-Directory $LogDir
+
+Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir
+Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
+Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
+Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
+Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
+
+# Import custom tools configuration, if present in the repo.
+# Note: Import in global scope so that the script set top-level variables without qualification.
+if (!$disableConfigureToolsetImport) {
+ $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
+ if (Test-Path $configureToolsetScript) {
+ . $configureToolsetScript
+ if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
+ if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
+ ExitWithExitCode $LastExitCode
+ }
+ }
+ }
+}
+
+function Try-LogClientIpAddress()
+{
+ Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
+ try
+ {
+ $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
+ $lines = $result.Content.Split([Environment]::NewLine)
+ $socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
+ Write-Host $socketIp
+ $clientIp = $lines | Select-String -Pattern "^Client IP:.*"
+ Write-Host $clientIp
+ }
+ catch
+ {
+ Write-Host "Unable to get this machine's effective IP address for logging: $_"
+ }
+}
+
+#
+# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
+#
+function Enable-Nuget-EnhancedRetry() {
+ if ($ci) {
+ Write-Host "Setting NUGET enhanced retry environment variables"
+ $env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true'
+ $env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6
+ $env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
+ Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true'
+ Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6'
+ Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
+ }
+}
\ No newline at end of file
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
new file mode 100644
index 000000000000..bd89e2a2c952
--- /dev/null
+++ b/eng/common/tools.sh
@@ -0,0 +1,551 @@
+#!/usr/bin/env bash
+
+# Initialize variables if they aren't already defined.
+
+# CI mode - set to true on CI server for PR validation build or official build.
+ci=${ci:-false}
+
+# Set to true to use the pipelines logger which will enable Azure logging output.
+# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
+# This flag is meant as a temporary opt-opt for the feature while validate it across
+# our consumers. It will be deleted in the future.
+if [[ "$ci" == true ]]; then
+ pipelines_log=${pipelines_log:-true}
+else
+ pipelines_log=${pipelines_log:-false}
+fi
+
+# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
+configuration=${configuration:-'Debug'}
+
+# Set to true to opt out of outputting binary log while running in CI
+exclude_ci_binary_log=${exclude_ci_binary_log:-false}
+
+if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then
+ binary_log_default=true
+else
+ binary_log_default=false
+fi
+
+# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
+binary_log=${binary_log:-$binary_log_default}
+
+# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
+prepare_machine=${prepare_machine:-false}
+
+# True to restore toolsets and dependencies.
+restore=${restore:-true}
+
+# Adjusts msbuild verbosity level.
+verbosity=${verbosity:-'minimal'}
+
+# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+if [[ "$ci" == true ]]; then
+ node_reuse=${node_reuse:-false}
+else
+ node_reuse=${node_reuse:-true}
+fi
+
+# Configures warning treatment in msbuild.
+warn_as_error=${warn_as_error:-true}
+
+# True to attempt using .NET Core already that meets requirements specified in global.json
+# installed on the machine instead of downloading one.
+use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
+
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
+dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
+
+# True to use global NuGet cache instead of restoring packages to repository-local directory.
+if [[ "$ci" == true ]]; then
+ use_global_nuget_cache=${use_global_nuget_cache:-false}
+else
+ use_global_nuget_cache=${use_global_nuget_cache:-true}
+fi
+
+# Used when restoring .NET SDK from alternative feeds
+runtime_source_feed=${runtime_source_feed:-''}
+runtime_source_feed_key=${runtime_source_feed_key:-''}
+
+# Resolve any symlinks in the given path.
+function ResolvePath {
+ local path=$1
+
+ while [[ -h $path ]]; do
+ local dir="$( cd -P "$( dirname "$path" )" && pwd )"
+ path="$(readlink "$path")"
+
+ # if $path was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $path != /* ]] && path="$dir/$path"
+ done
+
+ # return value
+ _ResolvePath="$path"
+}
+
+# ReadVersionFromJson [json key]
+function ReadGlobalVersion {
+ local key=$1
+
+ if command -v jq &> /dev/null; then
+ _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+ elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+ _ReadGlobalVersion=${BASH_REMATCH[1]}
+ fi
+
+ if [[ -z "$_ReadGlobalVersion" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
+ ExitWithExitCode 1
+ fi
+}
+
+function InitializeDotNetCli {
+ if [[ -n "${_InitializeDotNetCli:-}" ]]; then
+ return
+ fi
+
+ local install=$1
+
+ # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
+ export DOTNET_MULTILEVEL_LOOKUP=0
+
+ # Disable first run since we want to control all package sources
+ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+
+ # Disable telemetry on CI
+ if [[ $ci == true ]]; then
+ export DOTNET_CLI_TELEMETRY_OPTOUT=1
+ fi
+
+ # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
+ # so it doesn't output warnings to the console.
+ export LTTNG_HOME="$HOME"
+
+ # Source Build uses DotNetCoreSdkDir variable
+ if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
+ export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
+ fi
+
+ # Find the first path on $PATH that contains the dotnet.exe
+ if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
+ local dotnet_path=`command -v dotnet`
+ if [[ -n "$dotnet_path" ]]; then
+ ResolvePath "$dotnet_path"
+ export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
+ fi
+ fi
+
+ ReadGlobalVersion "dotnet"
+ local dotnet_sdk_version=$_ReadGlobalVersion
+ local dotnet_root=""
+
+ # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
+ # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
+ if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+ dotnet_root="$DOTNET_INSTALL_DIR"
+ else
+ dotnet_root="$repo_root/.dotnet"
+
+ export DOTNET_INSTALL_DIR="$dotnet_root"
+
+ if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+ if [[ "$install" == true ]]; then
+ InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
+ else
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
+ ExitWithExitCode 1
+ fi
+ fi
+ fi
+
+ # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+ # build steps from using anything other than what we've downloaded.
+ Write-PipelinePrependPath -path "$dotnet_root"
+
+ Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
+ Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
+
+ # return value
+ _InitializeDotNetCli="$dotnet_root"
+}
+
+function InstallDotNetSdk {
+ local root=$1
+ local version=$2
+ local architecture="unset"
+ if [[ $# -ge 3 ]]; then
+ architecture=$3
+ fi
+ InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key
+}
+
+function InstallDotNet {
+ local root=$1
+ local version=$2
+
+ GetDotNetInstallScript "$root"
+ local install_script=$_GetDotNetInstallScript
+
+ local archArg=''
+ if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
+ archArg="--architecture $3"
+ fi
+ local runtimeArg=''
+ if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
+ runtimeArg="--runtime $4"
+ fi
+ local skipNonVersionedFilesArg=""
+ if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
+ skipNonVersionedFilesArg="--skip-non-versioned-files"
+ fi
+ bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
+ local exit_code=$?
+ echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+
+ local runtimeSourceFeed=''
+ if [[ -n "${6:-}" ]]; then
+ runtimeSourceFeed="--azure-feed $6"
+ fi
+
+ local runtimeSourceFeedKey=''
+ if [[ -n "${7:-}" ]]; then
+ # The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
+ # '-d'. To work around this, do a simple detection and switch the parameter
+ # accordingly.
+ decodeArg="--decode"
+ if base64 --help 2>&1 | grep -q "BusyBox"; then
+ decodeArg="-d"
+ fi
+ decodedFeedKey=`echo $7 | base64 $decodeArg`
+ runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+ fi
+
+ if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
+ bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ else
+ if [[ $exit_code != 0 ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+ fi
+ ExitWithExitCode $exit_code
+ fi
+ }
+}
+
+function with_retries {
+ local maxRetries=5
+ local retries=1
+ echo "Trying to run '$@' for maximum of $maxRetries attempts."
+ while [[ $((retries++)) -le $maxRetries ]]; do
+ "$@"
+
+ if [[ $? == 0 ]]; then
+ echo "Ran '$@' successfully."
+ return 0
+ fi
+
+ timeout=$((3**$retries-1))
+ echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
+ sleep $timeout
+ done
+
+ echo "Failed to execute '$@' for $maxRetries times." 1>&2
+
+ return 1
+}
+
+function GetDotNetInstallScript {
+ local root=$1
+ local install_script="$root/dotnet-install.sh"
+ local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+
+ if [[ ! -a "$install_script" ]]; then
+ mkdir -p "$root"
+
+ echo "Downloading '$install_script_url'"
+
+ # Use curl if available, otherwise use wget
+ if command -v curl > /dev/null; then
+ # first, try directly, if this fails we will retry with verbose logging
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ if command -v openssl &> /dev/null; then
+ echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+ echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ fi
+ echo "Will now retry the same URL with verbose logging."
+ with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ }
+ else
+ with_retries wget -v -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ fi
+ fi
+ # return value
+ _GetDotNetInstallScript="$install_script"
+}
+
+function InitializeBuildTool {
+ if [[ -n "${_InitializeBuildTool:-}" ]]; then
+ return
+ fi
+
+ InitializeDotNetCli $restore
+
+ # return values
+ _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
+ _InitializeBuildToolCommand="msbuild"
+ _InitializeBuildToolFramework="netcoreapp3.1"
+}
+
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
+function GetNuGetPackageCachePath {
+ if [[ -z ${NUGET_PACKAGES:-} ]]; then
+ if [[ "$use_global_nuget_cache" == true ]]; then
+ export NUGET_PACKAGES="$HOME/.nuget/packages"
+ else
+ export NUGET_PACKAGES="$repo_root/.packages"
+ export RESTORENOCACHE=true
+ fi
+ fi
+
+ # return value
+ _GetNuGetPackageCachePath=$NUGET_PACKAGES
+}
+
+function InitializeNativeTools() {
+ if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
+ return
+ fi
+ if grep -Fq "native-tools" $global_json_file
+ then
+ local nativeArgs=""
+ if [[ "$ci" == true ]]; then
+ nativeArgs="--installDirectory $tools_dir"
+ fi
+ "$_script_dir/init-tools-native.sh" $nativeArgs
+ fi
+}
+
+function InitializeToolset {
+ if [[ -n "${_InitializeToolset:-}" ]]; then
+ return
+ fi
+
+ GetNuGetPackageCachePath
+
+ ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
+
+ local toolset_version=$_ReadGlobalVersion
+ local toolset_location_file="$toolset_dir/$toolset_version.txt"
+
+ if [[ -a "$toolset_location_file" ]]; then
+ local path=`cat "$toolset_location_file"`
+ if [[ -a "$path" ]]; then
+ # return value
+ _InitializeToolset="$path"
+ return
+ fi
+ fi
+
+ if [[ "$restore" != true ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored."
+ ExitWithExitCode 2
+ fi
+
+ local proj="$toolset_dir/restore.proj"
+
+ local bl=""
+ if [[ "$binary_log" == true ]]; then
+ bl="/bl:$log_dir/ToolsetRestore.binlog"
+ fi
+
+ echo '' > "$proj"
+ MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
+
+ local toolset_build_proj=`cat "$toolset_location_file"`
+
+ if [[ ! -a "$toolset_build_proj" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
+ ExitWithExitCode 3
+ fi
+
+ # return value
+ _InitializeToolset="$toolset_build_proj"
+}
+
+function ExitWithExitCode {
+ if [[ "$ci" == true && "$prepare_machine" == true ]]; then
+ StopProcesses
+ fi
+ exit $1
+}
+
+function StopProcesses {
+ echo "Killing running build processes..."
+ pkill -9 "dotnet" || true
+ pkill -9 "vbcscompiler" || true
+ return 0
+}
+
+function TryLogClientIpAddress () {
+ echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes'
+ if command -v curl > /dev/null; then
+ curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true
+ fi
+}
+
+function MSBuild {
+ local args=$@
+ if [[ "$pipelines_log" == true ]]; then
+ InitializeBuildTool
+ InitializeToolset
+
+ if [[ "$ci" == true ]]; then
+ export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+ export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
+
+ export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true
+ export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6
+ export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000
+ Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true"
+ Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6"
+ Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000"
+ fi
+
+ local toolset_dir="${_InitializeToolset%/*}"
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
+ if [[ -z "$selectedPath" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ ExitWithExitCode 1
+ fi
+ args+=( "-logger:$selectedPath" )
+ fi
+
+ MSBuild-Core ${args[@]}
+}
+
+function MSBuild-Core {
+ if [[ "$ci" == true ]]; then
+ if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch."
+ ExitWithExitCode 1
+ fi
+
+ if [[ "$node_reuse" == true ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build."
+ ExitWithExitCode 1
+ fi
+ fi
+
+ InitializeBuildTool
+
+ local warnaserror_switch=""
+ if [[ $warn_as_error == true ]]; then
+ warnaserror_switch="/warnaserror"
+ fi
+
+ function RunBuildTool {
+ export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@"
+
+ "$_InitializeBuildTool" "$@" || {
+ local exit_code=$?
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ echo "Build failed with exit code $exit_code. Check errors above."
+ if [[ "$ci" == "true" ]]; then
+ Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ else
+ ExitWithExitCode $exit_code
+ fi
+ }
+ }
+
+ RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
+}
+
+ResolvePath "${BASH_SOURCE[0]}"
+_script_dir=`dirname "$_ResolvePath"`
+
+. "$_script_dir/pipeline-logging-functions.sh"
+
+eng_root=`cd -P "$_script_dir/.." && pwd`
+repo_root=`cd -P "$_script_dir/../.." && pwd`
+repo_root="${repo_root}/"
+artifacts_dir="${repo_root}artifacts"
+toolset_dir="$artifacts_dir/toolset"
+tools_dir="${repo_root}.tools"
+log_dir="$artifacts_dir/log/$configuration"
+temp_dir="$artifacts_dir/tmp/$configuration"
+
+global_json_file="${repo_root}global.json"
+# determine if global.json contains a "runtimes" entry
+global_json_has_runtimes=false
+if command -v jq &> /dev/null; then
+ if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+ global_json_has_runtimes=true
+ fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
+ global_json_has_runtimes=true
+fi
+
+# HOME may not be defined in some scenarios, but it is required by NuGet
+if [[ -z $HOME ]]; then
+ export HOME="${repo_root}artifacts/.home/"
+ mkdir -p "$HOME"
+fi
+
+mkdir -p "$toolset_dir"
+mkdir -p "$temp_dir"
+mkdir -p "$log_dir"
+
+Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
+Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
+Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
+Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
+Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
+
+# Import custom tools configuration, if present in the repo.
+if [ -z "${disable_configure_toolset_import:-}" ]; then
+ configure_toolset_script="$eng_root/configure-toolset.sh"
+ if [[ -a "$configure_toolset_script" ]]; then
+ . "$configure_toolset_script"
+ fi
+fi
+
+# TODO: https://github.com/dotnet/arcade/issues/1468
+# Temporary workaround to avoid breaking change.
+# Remove once repos are updated.
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
+ use_installed_dotnet_cli="$useInstalledDotNetCli"
+fi
\ No newline at end of file