Merge master into release/2.1

Documentation/Samples/CloudTest.Helix.targets.sampleproject

@@ -5,15 +5,15 @@
     <HelixWorkItem Include="Demo Work item #1">
       <Command>call DoStuff.bat</Command>
       <!-- Needs to be a zip file that contains DoStuff.bat, adjust accordingly -->
-      <PayloadFile>Path to zip</PayloadFile>
+      <PayloadFile>Path to local zip file</PayloadFile>
       <WorkItemId>Demo Work item #1</WorkItemId>
       <TimeoutInSeconds>300</TimeoutInSeconds>
     </HelixWorkItem>
 
     <HelixWorkItem Include="Demo Work item #2">
       <Command>call DoStuff.bat</Command>
       <!-- Needs to be a zip file that contains DoStuff.bat, adjust accordingly -->
-      <PayloadFile>Path to zip</PayloadFile>
+      <PayloadFile>Path to some other local zip file</PayloadFile>
       <WorkItemId>Demo Work item #2</WorkItemId>
       <TimeoutInSeconds>200</TimeoutInSeconds>
     </HelixWorkItem>
@@ -27,12 +27,12 @@
 
   <!-- Required properties -->
   <PropertyGroup>
-    <HelixApiEndpoint>https://helix.int-dot.net/api/2016-06-28/jobs</HelixApiEndpoint>
+    <HelixApiEndpoint>https://helix.dot.net/api/2016-06-28/jobs</HelixApiEndpoint>
     <HelixApiAccessKey></HelixApiAccessKey>
     <HelixJobType>unspecified/</HelixJobType>
     <HelixSource>pr/unspecified/</HelixSource>
-    <TargetQueues>Windows.10.Amd64;Windows.7.Amd64;Windows.81.Amd64;Windows.10.Core.Amd64;</TargetQueues>
-    <BuildMoniker>20170301.0000</BuildMoniker>
+    <TargetQueues>Windows.10.Amd64;Windows.10.Core.Amd64;</TargetQueues>
+    <BuildMoniker>20180312.00</BuildMoniker>
     <!-- Use any valid Connection string here --> 
     <CloudDropConnectionString>...</CloudDropConnectionString>
     <CloudResultsConnectionString>...</CloudResultsConnectionString>
@@ -45,7 +45,7 @@
 
   <Target Name="Build">
     <Message Text="Beginning Cloud Build!" />
-    <HelixCloudBuild/>
+    <CallTarget Targets="HelixCloudBuild" />
   </Target>
 
-</Project>
+</Project>

dependencies.props

@@ -89,7 +89,7 @@
     <NuGetDependency Include="NuGet.RuntimeModel"/>
     <NuGetDependency Include="NuGet.Versioning"/>
     <StaticDependency Include="@(NuGetDependency)">
-      <Version>4.3.0</Version>
+      <Version>4.4.0</Version>
     </StaticDependency>
 
     <NETCoreDependency Include="Microsoft.NETCore.App"/>

src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json

@@ -1,7 +1,7 @@
 {
   "dependencies": {
     "Newtonsoft.Json": "9.0.1",
-    "NuGet.Versioning": "4.3.0",
+    "NuGet.Versioning": "4.4.0",
     "System.Reflection.Metadata": "1.4.1",
     "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03"
   },

src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs

@@ -154,7 +154,7 @@ private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Buil
                 }
                 token.ThrowIfCancellationRequested();
 
-                Thread.Sleep(waitFor);
+                Task.Delay(waitFor, token).Wait();
             }
         }
 

src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs

@@ -0,0 +1,243 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Build.CloudTestTasks
+{
+    public class CopyBlobsToLatest : AzureConnectionStringBuildTask
+    {
+        private static readonly string[] DefaultLatestVersionFilenames =
+        {
+            "latest.version"
+        };
+
+        private static readonly Regex VersionRegex = new Regex(
+            @"(?<version>\d+\.\d+\.\d+)(-(?<prerelease>[^-]+-)?(?<major>\d+)-(?<minor>\d+))?");
+
+        [Required]
+        public string ContainerName { get; set; }
+
+        [Required]
+        public string Product { get; set; }
+
+        [Required]
+        public string ProductVersion { get; set; }
+
+        [Required]
+        public string Channel { get; set; }
+
+        [Required]
+        public string Commit { get; set; }
+
+        public string[] LatestVersionFilenames { get; set; }
+
+        /// <summary>
+        /// A list of full version strings that should be converted to "latest" for each blob id,
+        /// in addition to ProductVersion.
+        /// </summary>
+        public string[] FullVersions { get; set; }
+
+        /// <summary>
+        /// If this task is called by multiple build legs that make up one build, enable this option
+        /// to prevent race conditions by dropping a version hint of what version this is. If we see
+        /// this file and it is the same as our version then we know that a race happened where two+
+        /// builds finished at the same time and someone already took care of publishing and we have
+        /// no work to do.
+        /// 
+        /// This is not necessary if the build uses some other mechanism to ensure each build only
+        /// has one finalization attempt, e.g. PipeBuild.
+        /// </summary>
+        public bool EnableVersionHint { get; set; }
+
+        public override bool Execute()
+        {
+            ParseConnectionString();
+
+            if (Log.HasLoggedErrors)
+            {
+                return false;
+            }
+
+            string sourceDir = $"{Product}/{ProductVersion}/";
+            string channelDir = $"{Product}/{Channel}/";
+            string semaphoreBlob = $"{channelDir}publishSemaphore";
+
+            CreateBlobIfNotExists(semaphoreBlob);
+
+            AzureBlobLease blobLease = new AzureBlobLease(
+                AccountName,
+                AccountKey,
+                ConnectionString,
+                ContainerName,
+                semaphoreBlob,
+                Log);
+
+            Log.LogMessage($"Acquiring lease on semaphore blob '{semaphoreBlob}'");
+            blobLease.Acquire();
+
+            try
+            {
+                if (EnableVersionHint)
+                {
+                    string targetVersionFile = $"{channelDir}{ProductVersion}";
+
+                    // Check if this build is already finalized.
+                    if (IsLatestSpecifiedVersion(targetVersionFile))
+                    {
+                        Log.LogMessage(
+                            MessageImportance.High,
+                            $"Version '{ProductVersion}' is already published. " +
+                                $"Skipping finalization. Hint file: '{targetVersionFile}'");
+
+                        return true;
+                    }
+
+                    // Delete old version files.
+                    GetBlobList(channelDir)
+                        .Select(s => s.Replace($"/{ContainerName}/", ""))
+                        .Where(w => VersionRegex.Replace(Path.GetFileName(w), "") == "")
+                        .ToList()
+                        .ForEach(f => TryDeleteBlob(f));
+
+                    // Drop the version file signaling such for any race-condition builds.
+                    CreateBlobIfNotExists(targetVersionFile);
+                }
+
+                CopyBlobs(sourceDir, channelDir);
+
+                // Generate the latest version text file
+                string versionText =
+                    $"{Commit}{Environment.NewLine}" +
+                    $"{ProductVersion}{Environment.NewLine}";
+
+                if (LatestVersionFilenames?.Any() != true)
+                {
+                    LatestVersionFilenames = DefaultLatestVersionFilenames;
+                }
+
+                foreach (string latestFilename in LatestVersionFilenames)
+                {
+                    PublishStringToBlob(
+                        ContainerName,
+                        $"{channelDir}{latestFilename}",
+                        versionText,
+                        "text/plain");
+                }
+            }
+            finally
+            {
+                Log.LogMessage($"Releasing lease on semaphore blob '{semaphoreBlob}'");
+                blobLease.Release();
+            }
+
+            return !Log.HasLoggedErrors;
+        }
+
+        private bool CopyBlobs(string sourceFolder, string destinationFolder)
+        {
+            // List of versions that need to be replaced with "latest" when copying blobs.
+            var versions = new List<string> { ProductVersion };
+            if (FullVersions != null)
+            {
+                versions.AddRange(FullVersions);
+            }
+
+            bool returnStatus = true;
+            List<Task<bool>> copyTasks = new List<Task<bool>>();
+            string[] blobs = GetBlobList(sourceFolder);
+            foreach (string blob in blobs)
+            {
+                string targetName = versions.Aggregate(
+                    Path.GetFileName(blob),
+                    (agg, version) => agg.Replace(version, "latest"));
+
+                string sourceBlob = blob.Replace($"/{ContainerName}/", "");
+                string destinationBlob = $"{destinationFolder}{targetName}";
+                Log.LogMessage($"Copying blob '{sourceBlob}' to '{destinationBlob}'");
+                copyTasks.Add(CopyBlobAsync(sourceBlob, destinationBlob));
+            }
+            Task.WaitAll(copyTasks.ToArray());
+            copyTasks.ForEach(c => returnStatus &= c.Result);
+            return returnStatus;
+        }
+
+        private bool TryDeleteBlob(string path)
+        {
+            return DeleteBlob(ContainerName, path);
+        }
+
+        private void CreateBlobIfNotExists(string path)
+        {
+            var blobList = GetBlobList(path);
+            if (blobList.Count() == 0)
+            {
+                PublishStringToBlob(ContainerName, path, DateTime.Now.ToString());
+            }
+        }
+
+        private bool IsLatestSpecifiedVersion(string versionFile)
+        {
+            var blobList = GetBlobList(versionFile);
+            return blobList.Count() != 0;
+        }
+
+        private bool DeleteBlob(string container, string blob)
+        {
+            return DeleteAzureBlob.Execute(
+                AccountName,
+                AccountKey,
+                ConnectionString,
+                container,
+                blob,
+                BuildEngine,
+                HostObject);
+        }
+
+        private Task<bool> CopyBlobAsync(string sourceBlobName, string destinationBlobName)
+        {
+            return CopyAzureBlobToBlob.ExecuteAsync(
+                AccountName,
+                AccountKey,
+                ConnectionString,
+                ContainerName,
+                sourceBlobName,
+                destinationBlobName,
+                BuildEngine,
+                HostObject);
+        }
+
+        private string[] GetBlobList(string path)
+        {
+            return ListAzureBlobs.Execute(
+                AccountName,
+                AccountKey,
+                ConnectionString,
+                ContainerName,
+                path,
+                BuildEngine,
+                HostObject);
+        }
+
+        private bool PublishStringToBlob(string container, string blob, string contents, string contentType = null)
+        {
+            return PublishStringToAzureBlob.Execute(
+                AccountName,
+                AccountKey,
+                ConnectionString,
+                container,
+                blob,
+                contents,
+                contentType,
+                BuildEngine,
+                HostObject);
+        }
+    }
+}

src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs

@@ -119,7 +119,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim
                 {
                     client.Timeout = TimeSpan.FromMinutes(10);
                     Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
-                    string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
+                    string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
                     filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob));
 
                     if (!DownloadFlatFiles)
@@ -155,7 +155,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim
                         filename = Path.Combine(downloadBlobDirectory, blobFilename);
                     }
 
-                    var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey);
+                    var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey);
 
                     using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
                     {

src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj

@@ -29,6 +29,7 @@
     <Compile Include="AzureConnectionStringBuildTask.cs" />
     <Compile Include="AzureBlobLease.cs" />
     <Compile Include="AzureHelper.cs" />
+    <Compile Include="CopyBlobsToLatest.cs" />
     <Compile Include="CopyAzureBlobToBlob.cs" />
     <Compile Include="CreateAzureContainer.cs" />
     <Compile Include="DeleteAzureBlob.cs" />

src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/DumplingHelper.py

@@ -1,6 +1,7 @@
 import os
 import platform
 import urllib
+import urllib2
 import glob
 import time
 import sys
@@ -16,10 +17,19 @@ def install_dumpling():
       url = "https://dumpling.azurewebsites.net/api/client/dumpling.py"
       scriptPath = os.path.dirname(os.path.realpath(__file__))
       downloadLocation = scriptPath + "/dumpling.py"
-      urllib.urlretrieve(url, downloadLocation)
-      subprocess.call([sys.executable, downloadLocation, "install", "--update"])
+      response = urllib2.urlopen(url)
+      if response.getcode() == 200:
+        with open(downloadLocation, 'w') as f:
+          f.write(response.read())
+        subprocess.call([sys.executable, downloadLocation, "install", "--update"])
+      else:
+        raise urllib2.URLError("HTTP Status Code" + str(result.getcode()))
 
     subprocess.call([sys.executable, dumplingPath, "install"])
+  except urllib2.HTTPError, e:
+    print("Dumpling cannot be installed due to: " + str(e).replace(':', '')) # Remove : to avoid looking like error format
+  except  urllib2.URLError, e:
+    print(e.reason)
   except:
     print("An unexpected error was encountered while installing dumpling.py: " + sys.exc_info()[0])