diff --git a/Documentation/Samples/CloudTest.Helix.targets.sampleproject b/Documentation/Samples/CloudTest.Helix.targets.sampleproject index bfeb3f52fe..cbddc9b78e 100644 --- a/Documentation/Samples/CloudTest.Helix.targets.sampleproject +++ b/Documentation/Samples/CloudTest.Helix.targets.sampleproject @@ -5,7 +5,7 @@ call DoStuff.bat - Path to zip + Path to local zip file Demo Work item #1 300 @@ -13,7 +13,7 @@ call DoStuff.bat - Path to zip + Path to some other local zip file Demo Work item #2 200 @@ -27,12 +27,12 @@ - https://helix.int-dot.net/api/2016-06-28/jobs + https://helix.dot.net/api/2016-06-28/jobs unspecified/ pr/unspecified/ - Windows.10.Amd64;Windows.7.Amd64;Windows.81.Amd64;Windows.10.Core.Amd64; - 20170301.0000 + Windows.10.Amd64;Windows.10.Core.Amd64; + 20180312.00 ... ... @@ -45,7 +45,7 @@ - + - \ No newline at end of file + diff --git a/dependencies.props b/dependencies.props index cddfb87d08..718471a888 100644 --- a/dependencies.props +++ b/dependencies.props @@ -89,7 +89,7 @@ - 4.3.0 + 4.4.0 diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json b/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json index 38a9984c10..2498fde577 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json +++ b/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json @@ -1,7 +1,7 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", + "NuGet.Versioning": "4.4.0", "System.Reflection.Metadata": "1.4.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs index 6428f3516b..d604988c89 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs @@ -154,7 +154,7 @@ private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Buil } token.ThrowIfCancellationRequested(); - Thread.Sleep(waitFor); + Task.Delay(waitFor, token).Wait(); } } diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs new file mode 100644 index 0000000000..7ddd578fd8 --- /dev/null +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs @@ -0,0 +1,243 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Build.CloudTestTasks +{ + public class CopyBlobsToLatest : AzureConnectionStringBuildTask + { + private static readonly string[] DefaultLatestVersionFilenames = + { + "latest.version" + }; + + private static readonly Regex VersionRegex = new Regex( + @"(?\d+\.\d+\.\d+)(-(?[^-]+-)?(?\d+)-(?\d+))?"); + + [Required] + public string ContainerName { get; set; } + + [Required] + public string Product { get; set; } + + [Required] + public string ProductVersion { get; set; } + + [Required] + public string Channel { get; set; } + + [Required] + public string Commit { get; set; } + + public string[] LatestVersionFilenames { get; set; } + + /// + /// A list of full version strings that should be converted to "latest" for each blob id, + /// in addition to ProductVersion. + /// + public string[] FullVersions { get; set; } + + /// + /// If this task is called by multiple build legs that make up one build, enable this option + /// to prevent race conditions by dropping a version hint of what version this is. If we see + /// this file and it is the same as our version then we know that a race happened where two+ + /// builds finished at the same time and someone already took care of publishing and we have + /// no work to do. + /// + /// This is not necessary if the build uses some other mechanism to ensure each build only + /// has one finalization attempt, e.g. PipeBuild. + /// + public bool EnableVersionHint { get; set; } + + public override bool Execute() + { + ParseConnectionString(); + + if (Log.HasLoggedErrors) + { + return false; + } + + string sourceDir = $"{Product}/{ProductVersion}/"; + string channelDir = $"{Product}/{Channel}/"; + string semaphoreBlob = $"{channelDir}publishSemaphore"; + + CreateBlobIfNotExists(semaphoreBlob); + + AzureBlobLease blobLease = new AzureBlobLease( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + semaphoreBlob, + Log); + + Log.LogMessage($"Acquiring lease on semaphore blob '{semaphoreBlob}'"); + blobLease.Acquire(); + + try + { + if (EnableVersionHint) + { + string targetVersionFile = $"{channelDir}{ProductVersion}"; + + // Check if this build is already finalized. + if (IsLatestSpecifiedVersion(targetVersionFile)) + { + Log.LogMessage( + MessageImportance.High, + $"Version '{ProductVersion}' is already published. " + + $"Skipping finalization. Hint file: '{targetVersionFile}'"); + + return true; + } + + // Delete old version files. + GetBlobList(channelDir) + .Select(s => s.Replace($"/{ContainerName}/", "")) + .Where(w => VersionRegex.Replace(Path.GetFileName(w), "") == "") + .ToList() + .ForEach(f => TryDeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds. + CreateBlobIfNotExists(targetVersionFile); + } + + CopyBlobs(sourceDir, channelDir); + + // Generate the latest version text file + string versionText = + $"{Commit}{Environment.NewLine}" + + $"{ProductVersion}{Environment.NewLine}"; + + if (LatestVersionFilenames?.Any() != true) + { + LatestVersionFilenames = DefaultLatestVersionFilenames; + } + + foreach (string latestFilename in LatestVersionFilenames) + { + PublishStringToBlob( + ContainerName, + $"{channelDir}{latestFilename}", + versionText, + "text/plain"); + } + } + finally + { + Log.LogMessage($"Releasing lease on semaphore blob '{semaphoreBlob}'"); + blobLease.Release(); + } + + return !Log.HasLoggedErrors; + } + + private bool CopyBlobs(string sourceFolder, string destinationFolder) + { + // List of versions that need to be replaced with "latest" when copying blobs. + var versions = new List { ProductVersion }; + if (FullVersions != null) + { + versions.AddRange(FullVersions); + } + + bool returnStatus = true; + List> copyTasks = new List>(); + string[] blobs = GetBlobList(sourceFolder); + foreach (string blob in blobs) + { + string targetName = versions.Aggregate( + Path.GetFileName(blob), + (agg, version) => agg.Replace(version, "latest")); + + string sourceBlob = blob.Replace($"/{ContainerName}/", ""); + string destinationBlob = $"{destinationFolder}{targetName}"; + Log.LogMessage($"Copying blob '{sourceBlob}' to '{destinationBlob}'"); + copyTasks.Add(CopyBlobAsync(sourceBlob, destinationBlob)); + } + Task.WaitAll(copyTasks.ToArray()); + copyTasks.ForEach(c => returnStatus &= c.Result); + return returnStatus; + } + + private bool TryDeleteBlob(string path) + { + return DeleteBlob(ContainerName, path); + } + + private void CreateBlobIfNotExists(string path) + { + var blobList = GetBlobList(path); + if (blobList.Count() == 0) + { + PublishStringToBlob(ContainerName, path, DateTime.Now.ToString()); + } + } + + private bool IsLatestSpecifiedVersion(string versionFile) + { + var blobList = GetBlobList(versionFile); + return blobList.Count() != 0; + } + + private bool DeleteBlob(string container, string blob) + { + return DeleteAzureBlob.Execute( + AccountName, + AccountKey, + ConnectionString, + container, + blob, + BuildEngine, + HostObject); + } + + private Task CopyBlobAsync(string sourceBlobName, string destinationBlobName) + { + return CopyAzureBlobToBlob.ExecuteAsync( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + sourceBlobName, + destinationBlobName, + BuildEngine, + HostObject); + } + + private string[] GetBlobList(string path) + { + return ListAzureBlobs.Execute( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + path, + BuildEngine, + HostObject); + } + + private bool PublishStringToBlob(string container, string blob, string contents, string contentType = null) + { + return PublishStringToAzureBlob.Execute( + AccountName, + AccountKey, + ConnectionString, + container, + blob, + contents, + contentType, + BuildEngine, + HostObject); + } + } +} diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs index 69473a2fc5..58e4b1a2a0 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs @@ -119,7 +119,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim { client.Timeout = TimeSpan.FromMinutes(10); Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); - string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); + string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob)); if (!DownloadFlatFiles) @@ -155,7 +155,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim filename = Path.Combine(downloadBlobDirectory, blobFilename); } - var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj b/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj index 3ce0310e8e..29f0cf3f58 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/DumplingHelper.py b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/DumplingHelper.py index 264552e112..7b9258433d 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/DumplingHelper.py +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/DumplingHelper.py @@ -1,6 +1,7 @@ import os import platform import urllib +import urllib2 import glob import time import sys @@ -16,10 +17,19 @@ def install_dumpling(): url = "https://dumpling.azurewebsites.net/api/client/dumpling.py" scriptPath = os.path.dirname(os.path.realpath(__file__)) downloadLocation = scriptPath + "/dumpling.py" - urllib.urlretrieve(url, downloadLocation) - subprocess.call([sys.executable, downloadLocation, "install", "--update"]) + response = urllib2.urlopen(url) + if response.getcode() == 200: + with open(downloadLocation, 'w') as f: + f.write(response.read()) + subprocess.call([sys.executable, downloadLocation, "install", "--update"]) + else: + raise urllib2.URLError("HTTP Status Code" + str(result.getcode())) subprocess.call([sys.executable, dumplingPath, "install"]) + except urllib2.HTTPError, e: + print("Dumpling cannot be installed due to: " + str(e).replace(':', '')) # Remove : to avoid looking like error format + except urllib2.URLError, e: + print(e.reason) except: print("An unexpected error was encountered while installing dumpling.py: " + sys.exc_info()[0]) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs index b30c7eb571..ce31e8d787 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs @@ -8,10 +8,12 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Text; using System.Threading; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.DotNet.Build.CloudTestTasks @@ -61,7 +63,7 @@ public async Task UploadBlockBlobAsync( List blockIds = new List(); int numberOfBlocks = (size / blockSize) + 1; int countForId = 0; - using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open)) + using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read)) { int offset = 0; @@ -208,6 +210,57 @@ public async Task UploadBlockBlobAsync( } } } + + public async Task FileEqualsExistingBlobAsync( + string accountName, + string accountKey, + string containerName, + string filePath, + string destinationBlob, + int uploadTimeout) + { + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(uploadTimeout) + }) + { + log.LogMessage( + MessageImportance.Low, + $"Downloading blob {destinationBlob} to check if identical."); + + string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( + log, + client, + createRequest)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Failed to retrieve existing blob {destinationBlob}, " + + $"status code {response.StatusCode}."); + } + + byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); + byte[] localBytes = File.ReadAllBytes(filePath); + + bool equal = localBytes.SequenceEqual(existingBytes); + + if (equal) + { + log.LogMessage( + MessageImportance.Normal, + "Item exists in blob storage, and is verified to be identical. " + + $"File: '{filePath}' Blob: '{destinationBlob}'"); + } + + return equal; + } + } + } + private string DetermineContentTypeBasedOnFileExtension(string filename) { if (Path.GetExtension(filename) == ".svg") diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs index 68343748f3..568eda1964 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs @@ -42,6 +42,16 @@ public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask /// public bool Overwrite { get; set; } = false; + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + /// /// Specifies the maximum number of clients to concurrently upload blobs to azure /// @@ -137,8 +147,18 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha if (!File.Exists(item.ItemSpec)) throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); + UploadClient uploadClient = new UploadClient(Log); + if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) + { + if (PassIfExistingItemIdentical && + await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) + { + return; + } + throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); + } string contentType = item.GetMetadata("ContentType"); @@ -147,7 +167,6 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha try { Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); - UploadClient uploadClient = new UploadClient(Log); await uploadClient.UploadBlockBlobAsync( ct, @@ -164,5 +183,28 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha clientThrottle.Release(); } } + + private async Task ItemEqualsExistingBlobAsync( + ITaskItem item, + string relativeBlobPath, + UploadClient client, + SemaphoreSlim clientThrottle) + { + await clientThrottle.WaitAsync(); + try + { + return await client.FileEqualsExistingBlobAsync( + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj index 824505547f..8c7d9420dd 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj @@ -13,6 +13,10 @@ .NETFramework,Version=v4.5 + + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json index 404e925be2..c2ef6b79ae 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json @@ -1,8 +1,8 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", - "NuGet.Packaging": "4.3.0", + "NuGet.Versioning": "4.4.0", + "NuGet.Packaging": "4.4.0", "System.Reflection.Metadata": "1.4.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03", "sleetlib": "2.2.24" diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs index 4763854ef3..a7a49b78e7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs @@ -41,7 +41,7 @@ public BlobFeed(string accountName, string accountKey, string containerName, str public string FeedContainerUrl => AzureHelper.GetContainerRestUrl(AccountName, ContainerName); - public async Task CheckIfBlobExists(string blobPath) + public async Task CheckIfBlobExistsAsync(string blobPath) { string url = $"{FeedContainerUrl}/{blobPath}?comp=metadata"; using (HttpClient client = new HttpClient()) @@ -67,21 +67,38 @@ public async Task CheckIfBlobExists(string blobPath) } } - public async Task DownloadBlobAsString(string blobPath) + public async Task DownloadBlobAsStringAsync(string blobPath) + { + using (HttpResponseMessage response = await DownloadBlobAsync(blobPath)) + { + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsStringAsync(); + } + return null; + } + } + + public async Task DownloadBlobAsBytesAsync(string blobPath) + { + using (HttpResponseMessage response = await DownloadBlobAsync(blobPath)) + { + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsByteArrayAsync(); + } + return null; + } + } + + private async Task DownloadBlobAsync(string blobPath) { string url = $"{FeedContainerUrl}/{blobPath}"; using (HttpClient client = new HttpClient()) { client.DefaultRequestHeaders.Clear(); var request = AzureHelper.RequestMessage("GET", url, AccountName, AccountKey)(); - using (HttpResponseMessage response = await client.SendAsync(request)) - { - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadAsStringAsync(); - } - return null; - } + return await client.SendAsync(request); } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs index 3f3e915373..c30bec932b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs @@ -6,6 +6,8 @@ using Microsoft.DotNet.Build.CloudTestTasks; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json.Linq; +using NuGet.Packaging; +using NuGet.Packaging.Core; using Sleet; using System; using System.Collections.Generic; @@ -14,9 +16,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using NuGet.Packaging.Core; using MSBuild = Microsoft.Build.Utilities; -using CloudTestTasks = Microsoft.DotNet.Build.CloudTestTasks; namespace Microsoft.DotNet.Build.Tasks.Feed { @@ -65,7 +65,9 @@ public BlobFeedAction(string expectedFeedUrl, string accountKey, MSBuild.TaskLog } } - public async Task PushToFeed(IEnumerable items, bool allowOverwrite = false) + public async Task PushToFeedAsync( + IEnumerable items, + PushOptions options) { if (IsSanityChecked(items)) { @@ -75,13 +77,15 @@ public async Task PushToFeed(IEnumerable items, bool allowOverwrit CancellationToken.ThrowIfCancellationRequested(); } - await PushItemsToFeedAsync(items, allowOverwrite); + await PushItemsToFeedAsync(items, options); } return !Log.HasLoggedErrors; } - public async Task PushItemsToFeedAsync(IEnumerable items, bool allowOverwrite) + public async Task PushItemsToFeedAsync( + IEnumerable items, + PushOptions options) { Log.LogMessage(MessageImportance.Low, $"START pushing items to feed"); @@ -93,7 +97,7 @@ public async Task PushItemsToFeedAsync(IEnumerable items, bool all try { - bool result = await PushAsync(items, allowOverwrite); + bool result = await PushAsync(items, options); return result; } catch (Exception e) @@ -104,7 +108,11 @@ public async Task PushItemsToFeedAsync(IEnumerable items, bool all return !Log.HasLoggedErrors; } - public async Task UploadAssets(ITaskItem item, SemaphoreSlim clientThrottle, int uploadTimeout, bool allowOverwrite = false) + public async Task UploadAssetAsync( + ITaskItem item, + SemaphoreSlim clientThrottle, + int uploadTimeout, + PushOptions options) { string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); @@ -133,17 +141,33 @@ public async Task UploadAssets(ITaskItem item, SemaphoreSlim clientThrottle, int try { - bool blobExists = false; + UploadClient uploadClient = new UploadClient(Log); - if (!allowOverwrite) + if (!options.AllowOverwrite && await feed.CheckIfBlobExistsAsync(relativeBlobPath)) { - blobExists = await feed.CheckIfBlobExists(relativeBlobPath); + if (options.PassIfExistingItemIdentical) + { + if (!await uploadClient.FileEqualsExistingBlobAsync( + feed.AccountName, + feed.AccountKey, + feed.ContainerName, + item.ItemSpec, + relativeBlobPath, + uploadTimeout)) + { + Log.LogError( + $"Item '{item}' already exists with different contents " + + $"at '{relativeBlobPath}'"); + } + } + else + { + Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); + } } - - if (allowOverwrite || !blobExists) + else { Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); - UploadClient uploadClient = new UploadClient(Log); await uploadClient.UploadBlockBlobAsync( CancellationToken, feed.AccountName, @@ -154,10 +178,6 @@ await uploadClient.UploadBlockBlobAsync( contentType, uploadTimeout); } - else - { - Log.LogError($"Item '{item}' already exists in {relativeBlobPath}."); - } } catch (Exception exc) { @@ -252,6 +272,35 @@ private bool IsSanityChecked(IEnumerable items) return true; } + private async Task IsPackageIdenticalOnFeedAsync( + string item, + PackageIndex packageIndex, + ISleetFileSystem source, + FlatContainer flatContainer, + SleetLogger log) + { + using (var package = new PackageArchiveReader(item)) + { + var id = await package.GetIdentityAsync(CancellationToken); + if (await packageIndex.Exists(id)) + { + using (Stream remoteStream = await source + .Get(flatContainer.GetNupkgPath(id)) + .GetStream(log, CancellationToken)) + using (var remote = new MemoryStream()) + { + await remoteStream.CopyToAsync(remote); + + byte[] existingBytes = remote.ToArray(); + byte[] localBytes = File.ReadAllBytes(item); + + return existingBytes.SequenceEqual(localBytes); + } + } + return null; + } + } + private LocalSettings GetSettings() { SleetSettings sleetSettings = new SleetSettings() @@ -277,12 +326,82 @@ private AzureFileSystem GetAzureFileSystem() return fileSystem; } - private async Task PushAsync(IEnumerable items, bool allowOverwrite) + private async Task PushAsync( + IEnumerable items, + PushOptions options) { LocalSettings settings = GetSettings(); AzureFileSystem fileSystem = GetAzureFileSystem(); - bool result = await PushCommand.RunAsync(settings, fileSystem, items.ToList(), allowOverwrite, skipExisting: false, log: new SleetLogger(Log)); - return result; + SleetLogger log = new SleetLogger(Log); + + var packagesToPush = items.ToList(); + + if (!options.AllowOverwrite && options.PassIfExistingItemIdentical) + { + var context = new SleetContext + { + LocalSettings = settings, + Log = log, + Source = fileSystem, + Token = CancellationToken + }; + context.SourceSettings = await FeedSettingsUtility.GetSettingsOrDefault( + context.Source, + context.Log, + context.Token); + + var flatContainer = new FlatContainer(context); + + var packageIndex = new PackageIndex(context); + + // Check packages sequentially: Task.WhenAll caused IO exceptions in Sleet. + for (int i = packagesToPush.Count - 1; i >= 0; i--) + { + string item = packagesToPush[i]; + + bool? identical = await IsPackageIdenticalOnFeedAsync( + item, + packageIndex, + context.Source, + flatContainer, + log); + + if (identical == null) + { + continue; + } + + packagesToPush.RemoveAt(i); + + if (identical == true) + { + Log.LogMessage( + MessageImportance.Normal, + "Package exists on the feed, and is verified to be identical. " + + $"Skipping upload: '{item}'"); + } + else + { + Log.LogError( + "Package exists on the feed, but contents are different. " + + $"Upload failed: '{item}'"); + } + } + + if (!packagesToPush.Any()) + { + Log.LogMessage("After skipping idempotent uploads, no items need pushing."); + return true; + } + } + + return await PushCommand.RunAsync( + settings, + fileSystem, + packagesToPush, + options.AllowOverwrite, + skipExisting: false, + log: log); } private async Task InitAsync() diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs index 99f099e383..ee3906adda 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs @@ -15,8 +15,10 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class FetchOrchestratedBuildManifestInfo : Task + public class FetchOrchestratedBuildManifestInfo : BuildTask { + private const string IdentitySummaryMetadataName = "IdentitySummary"; + [Required] public string VersionsRepoPath { get; set; } @@ -30,19 +32,16 @@ public class FetchOrchestratedBuildManifestInfo : Task public string VersionsRepoRef { get; set; } [Output] - public string OrchestratedBuildId { get; set; } - - [Output] - public string OrchestratedIdentity { get; set; } + public ITaskItem OrchestratedBuild { get; set; } [Output] public ITaskItem[] OrchestratedBlobFeed { get; set; } [Output] public ITaskItem[] OrchestratedBlobFeedArtifacts { get; set; } - + [Output] - public ITaskItem[] OrchestratedBuilds { get; set; } + public ITaskItem[] OrchestratedBuildConstituents { get; set; } public override bool Execute() { @@ -61,8 +60,7 @@ public override bool Execute() VersionsRepoPath) .Result; - OrchestratedBuildId = manifest.Identity.BuildId; - OrchestratedIdentity = manifest.Identity.ToString(); + OrchestratedBuild = CreateItem(manifest.Identity); EndpointModel[] orchestratedFeeds = manifest.Endpoints .Where(e => e.IsOrchestratedBlobFeed) @@ -85,7 +83,7 @@ public override bool Execute() IEnumerable buildItems = manifest.Builds.Select(CreateItem); - OrchestratedBuilds = buildItems.ToArray(); + OrchestratedBuildConstituents = buildItems.ToArray(); } return !Log.HasLoggedErrors; @@ -103,9 +101,13 @@ private ITaskItem CreateItem(PackageArtifactModel model) private ITaskItem CreateItem(BuildIdentity model) { - return new TaskItem( + var item = new TaskItem( model.Name, ArtifactMetadata(model.ToXmlBuildElement(), model.Attributes)); + + item.SetMetadata(IdentitySummaryMetadataName, model.ToString()); + + return item; } private Dictionary ArtifactMetadata( diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs index 639c31fd7e..188eb975c5 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs @@ -3,18 +3,18 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.Automation; using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using Microsoft.DotNet.VersionTools.BuildManifest; using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Xml.Linq; namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class PushOrchestratedBuildManifest : Task + public class PushOrchestratedBuildManifest : BuildTask { [Required] public string ManifestFile { get; set; } @@ -36,7 +36,9 @@ public class PushOrchestratedBuildManifest : Task /// /// %(Identity): A file to upload to the versions repo. - /// %(RelativePath): Optional path to upload the file to, relative to VersionsRepoPath. + /// %(RelativePath): Optional path to upload the file to, relative to VersionsRepoPath. + /// If it begins with '/', it is treated as an absolute path within the versions repo. + /// '\' is automatically converted to '/'. /// public ITaskItem[] SupplementaryFiles { get; set; } @@ -55,33 +57,39 @@ public override bool Execute() { var client = new BuildManifestClient(gitHubClient); - SupplementaryUploadRequest[] supplementaryUploads = SupplementaryFiles - ?.Select(i => - { - string path = i.GetMetadata("RelativePath"); - if (string.IsNullOrEmpty(path)) - { - path = Path.GetFileName(i.ItemSpec); - } - return new SupplementaryUploadRequest - { - Contents = File.ReadAllText(i.ItemSpec), - Path = path - }; - }) - .ToArray(); - - var pushTask = client.PushNewBuildAsync( + var location = new BuildManifestLocation( new GitHubProject(VersionsRepo, VersionsRepoOwner), $"heads/{VersionsRepoBranch}", - VersionsRepoPath, + VersionsRepoPath); + + var pushTask = client.PushNewBuildAsync( + location, model, - supplementaryUploads, + CreateUploadRequests(SupplementaryFiles), CommitMessage); pushTask.Wait(); } return !Log.HasLoggedErrors; } + + public static SupplementaryUploadRequest[] CreateUploadRequests(IEnumerable items) + { + return items + ?.Select(i => + { + string path = i.GetMetadata("RelativePath")?.Replace('\\', '/'); + if (string.IsNullOrEmpty(path)) + { + path = Path.GetFileName(i.ItemSpec); + } + return new SupplementaryUploadRequest + { + Contents = File.ReadAllText(i.ItemSpec), + Path = path + }; + }) + .ToArray(); + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs index 7865d71fad..ae5f92fd73 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.Automation; using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using Microsoft.DotNet.VersionTools.BuildManifest; @@ -15,7 +14,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class UpdateOrchestratedBuildManifest : Task + public class UpdateOrchestratedBuildManifest : BuildTask { private enum UpdateType { @@ -29,6 +28,7 @@ private enum UpdateType } public const string XmlMetadataName = "Xml"; + private const string JoinSemaphorePathMetadataName = "JoinSemaphorePath"; /// /// Updates to perform on the manifest. The metadata 'UpdateType' selects a type of update, @@ -60,14 +60,31 @@ private enum UpdateType public string CommitMessage { get; set; } - public string OrchestratedIdentity { get; set; } + public string OrchestratedIdentitySummary { get; set; } + + /// + /// %(Identity): A file to upload to the versions repo. + /// %(RelativePath): Optional path to upload the file to, relative to VersionsRepoPath. + /// If it begins with '/', it is treated as an absolute path within the versions repo. + /// '\' is automatically converted to '/'. + /// + public ITaskItem[] SupplementaryFiles { get; set; } + + /// + /// "Join semaphore" groups. A join semaphore is created when all semaphores in the group + /// are complete for a certain build. + /// + /// %(Identity): A semaphore name that is part of a join semaphore group. + /// %(JoinSemaphorePath): The name of the join semaphore, created when the parallel work joins. + /// + public ITaskItem[] JoinSemaphoreGroups { get; set; } public override bool Execute() { if (string.IsNullOrEmpty(CommitMessage)) { string semaphores = string.Join(", ", SemaphoreNames); - string identity = OrchestratedIdentity ?? VersionsRepoPath; + string identity = OrchestratedIdentitySummary ?? VersionsRepoPath; CommitMessage = $"Update {identity}: {semaphores}"; } @@ -84,21 +101,45 @@ private async System.Threading.Tasks.Task PushChangeAsync(BuildManifestClient cl { try { - await client.PushChangeAsync( + var location = new BuildManifestLocation( new GitHubProject(VersionsRepo, VersionsRepoOwner), $"heads/{VersionsRepoBranch}", - VersionsRepoPath, + VersionsRepoPath); + + IEnumerable joinGroups = JoinSemaphoreGroups + .Select(item => new + { + ParallelPartPath = item.ItemSpec, + JoinSemaphorePath = item.GetMetadata(JoinSemaphorePathMetadataName) + }) + .GroupBy(j => j.JoinSemaphorePath, j => j.ParallelPartPath) + .Select(g => new JoinSemaphoreGroup + { + JoinSemaphorePath = g.Key, + ParallelSemaphorePaths = g + }); + + SupplementaryUploadRequest[] supplementaryUploads = + PushOrchestratedBuildManifest.CreateUploadRequests(SupplementaryFiles); + + var change = new BuildManifestChange( + location, + CommitMessage, OrchestratedBuildId, + SemaphoreNames, manifest => { foreach (var update in ManifestUpdates ?? Enumerable.Empty()) { ApplyUpdate(manifest, update); } - }, - SemaphoreNames, - null, - CommitMessage); + }) + { + SupplementaryUploads = supplementaryUploads, + JoinSemaphoreGroups = joinGroups, + }; + + await client.PushChangeAsync(change); } catch (ManifestChangeOutOfDateException e) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs index 76cfad4a0d..1d6a73b19a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.BuildManifest.Model; using System.Linq; using System.Text; @@ -11,7 +10,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class WriteOrchestratedBuildManifestSummaryToFile : Task + public class WriteOrchestratedBuildManifestSummaryToFile : BuildTask { [Required] public string File { get; set; } @@ -35,19 +34,15 @@ public override bool Execute() string sdkProductVersion = model.Builds .FirstOrDefault(b => b.Name == "cli") - ?.BuildId; - - string runtimeProductVersion = blobFeed.Artifacts.Blobs - .FirstOrDefault(b => - b.Id.StartsWith("Runtime/") && - b.Id.EndsWith("/Microsoft.NET.CoreRuntime.2.1.appx")) - ?.Id.Split('/')[1]; - - string aspnetProductVersion = blobFeed.Artifacts.Blobs - .FirstOrDefault(b => - b.Id.StartsWith("Runtime/") && - b.Id.EndsWith("/aspnetcore_base_runtime.version")) - ?.Id.Split('/')[1]; + ?.ProductVersion; + + string runtimeProductVersion = model.Builds + .FirstOrDefault(b => b.Name == "core-setup") + ?.ProductVersion; + + string aspnetProductVersion = model.Builds + .FirstOrDefault(b => b.Name == "aspnet") + ?.ProductVersion; var builder = new StringBuilder(); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs index 5cdca452b0..bb38e74543 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs @@ -2,15 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using System.Collections.Generic; using System.Xml.Linq; namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class WriteOrchestratedBuildManifestToFile : Task + public class WriteOrchestratedBuildManifestToFile : BuildTask { [Required] public string File { get; set; } @@ -63,6 +62,12 @@ public override bool Execute() string contents = System.IO.File.ReadAllText(buildManifestFile.ItemSpec); BuildModel build = BuildModel.Parse(XElement.Parse(contents)); + + foreach (PackageArtifactModel package in build.Artifacts.Packages) + { + package.OriginBuildName = build.Identity.Name; + } + orchestratedBuild.AddParticipantBuild(build); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs index e66cf17fed..88f01bb97c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs @@ -2,14 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Microsoft.Build.Utilities; -using System.IO; using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; using System; +using System.IO; namespace Microsoft.DotNet.Build.Tasks.Feed { - public class ConfigureInputFeed : Task + public class ConfigureInputFeed : BuildTask { [Required] public ITaskItem[] EnableFeeds { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs index c80ef39ad6..1510b668ea 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs @@ -3,17 +3,16 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using System.Collections.Generic; -using MSBuild = Microsoft.Build.Utilities; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace Microsoft.DotNet.Build.Tasks.Feed { - public sealed class CopyBlobDirectory : MSBuild.Task + public sealed class CopyBlobDirectory : BuildTask { [Required] public string SourceBlobDirectory { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs index 3f1cc0391c..c5dd89b14c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs @@ -12,7 +12,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public class GetBlobFeedPackageList : MSBuild.Task + public class GetBlobFeedPackageList : BuildTask { private const string NuGetPackageInfoId = "PackageId"; private const string NuGetPackageInfoVersion = "PackageVersion"; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj index b499ac2d54..ddb477ff4f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj @@ -21,6 +21,9 @@ + + BuildTask.cs + AzureBlobLease.cs @@ -51,6 +54,7 @@ + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets index 3efe7fbfa4..5e924fa333 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets @@ -221,7 +221,11 @@ [Out] $(OrchestratedBuildId): The orchestrated build manifest's build id attribute value. - $(OrchestratedIdentity): The human-readable full identity of the orchestrated build manifest. + $(OrchestratedIdentitySummary): Human-readable full identity of the orchestrated build manifest. + @(OrchestratedBuild): A single item describing the root manifest element. + %(Identity): The name of the build. + %(Xml): The raw XML string representing the build in the manifest. + %(...): Metadata is created for each attribute on the element. @(OrchestratedBlobFeed): A single item for the orchestrated blob feed Endpoint. %(...): Metadata is created for each attribute on the element. @(ParsedOrchestratedBlobFeed): The result of parsing the OrchestratedBlobFeed url. @@ -232,9 +236,9 @@ %(Identity): 'Package' or 'Blob', matching manifest element name. %(Xml): The raw XML string representing the artifact in the manifest. %(...): Metadata is created for each attribute on the element. - @(OrchestratedBuilds): An item for each Build in the orchestrated build manifest. + @(OrchestratedBuildConstituents): An item for each Build in the orchestrated build manifest. %(Identity): The name of the build. - %(Xml): The raw XML string representing the artifact in the manifest. + %(Xml): The raw XML string representing the build in the manifest. %(...): Metadata is created for each attribute on the element. --> - - + - + + %(OrchestratedBuild.BuildId) + %(OrchestratedBuild.IdentitySummary) %(OrchestratedBlobFeed.Url) @@ -304,7 +309,15 @@ $([System.String]::Copy('%(OrchestratedBlobFeedArtifacts.Version)').ToLowerInvariant()) - + + %(LowercaseId).%(LowercaseVersion).nupkg + + + + $(FinalDownloadDirectory)%(NupkgFile) + + + @@ -343,10 +356,14 @@ $(VersionsRepo): The GitHub repo name. Default: 'versions' $(VersionsRepoOwner): The GitHub repo owner. Default: 'dotnet' $(VersionsRepoBranch): The branch to fetch from. Default: 'master' - $(OrchestratedIdentity): Human-readable identity of the orchestrated build, used to generate - a concise commit message. This is expected to come from FetchOrchestratedBuildManifestInfo. + $(OrchestratedIdentitySummary): Human-readable identity of the orchestrated build, used to + generate a concise commit message. This is expected to come from + FetchOrchestratedBuildManifestInfo. Default: the full VersionsRepoPath is used in the commit message. $(CommitMessage): Overrides the generated commit message. + @(SupplementaryFiles): Uploads supplementary files to the versions repo as part of the update. + For item requirements and behavior, see 'UpdateOrchestratedBuildManifest.cs' in + dotnet/buildtools. --> @@ -368,6 +385,8 @@ VersionsRepoOwner="$(VersionsRepoOwner)" VersionsRepoBranch="$(VersionsRepoBranch)" CommitMessage="$(CommitMessage)" - OrchestratedIdentity="$(OrchestratedIdentity)" /> + OrchestratedIdentitySummary="$(OrchestratedIdentitySummary)" + SupplementaryFiles="@(SupplementaryFiles)" + JoinSemaphoreGroups="@(JoinSemaphoreGroups)" /> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs index d7f4af16f7..fdb54d93c7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs @@ -5,17 +5,10 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using System; -using System.Linq; -using System.Threading.Tasks; -using System.Collections.Generic; -using MSBuild = Microsoft.Build.Utilities; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using System.Collections; namespace Microsoft.DotNet.Build.Tasks.Feed { - public sealed class ParseBlobUrl : MSBuild.Task + public sealed class ParseBlobUrl : BuildTask { [Required] public string BlobUrl { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs new file mode 100644 index 0000000000..729edfecbb --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class PushOptions + { + public bool AllowOverwrite { get; set; } + + public bool PassIfExistingItemIdentical { get; set; } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs index 7ecdbb1c19..042d41231b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs @@ -16,7 +16,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public class PushToBlobFeed : MSBuild.Task + public partial class PushToBlobFeed : BuildTask { private static readonly char[] ManifestDataPairSeparators = { ';' }; private const string DisableManifestPushConfigurationBlob = "disable-manifest-push"; @@ -33,6 +33,16 @@ public class PushToBlobFeed : MSBuild.Task public bool Overwrite { get; set; } + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + public bool PublishFlatContainer { get; set; } public int MaxClients { get; set; } = 8; @@ -108,7 +118,7 @@ public async Task ExecuteAsync() var packagePaths = packageItems.Select(i => i.ItemSpec); - await blobFeedAction.PushToFeed(packagePaths, Overwrite); + await blobFeedAction.PushToFeedAsync(packagePaths, CreatePushOptions()); await PublishToFlatContainerAsync(symbolItems, blobFeedAction); packageArtifacts = ConcatPackageArtifacts(packageArtifacts, packageItems); @@ -134,7 +144,7 @@ private async Task PushBuildManifestAsync( IEnumerable blobArtifacts, IEnumerable packageArtifacts) { - bool disabledByBlob = await blobFeedAction.feed.CheckIfBlobExists( + bool disabledByBlob = await blobFeedAction.feed.CheckIfBlobExistsAsync( $"{blobFeedAction.feed.RelativePath}{DisableManifestPushConfigurationBlob}"); if (disabledByBlob) @@ -147,7 +157,7 @@ private async Task PushBuildManifestAsync( string blobPath = $"{AssetsVirtualDir}{ManifestAssetOutputDir}{ManifestName}.xml"; - string existingStr = await blobFeedAction.feed.DownloadBlobAsString( + string existingStr = await blobFeedAction.feed.DownloadBlobAsStringAsync( $"{blobFeedAction.feed.RelativePath}{blobPath}"); BuildModel buildModel; @@ -186,11 +196,14 @@ private async Task PushBuildManifestAsync( using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) { - await blobFeedAction.UploadAssets( + await blobFeedAction.UploadAssetAsync( item, clientThrottle, UploadTimeoutInMinutes, - allowOverwrite: true); + new PushOptions + { + AllowOverwrite = true + }); } } finally @@ -209,7 +222,12 @@ private async Task PublishToFlatContainerAsync(IEnumerable taskItems, using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) { Log.LogMessage($"Uploading {taskItems.Count()} items..."); - await Task.WhenAll(taskItems.Select(item => blobFeedAction.UploadAssets(item, clientThrottle, UploadTimeoutInMinutes, Overwrite))); + await Task.WhenAll(taskItems.Select( + item => blobFeedAction.UploadAssetAsync( + item, + clientThrottle, + UploadTimeoutInMinutes, + CreatePushOptions()))); } } } @@ -288,5 +306,14 @@ private static Dictionary ParseManifestMetadataString(string dat .Where(pair => pair != null) .ToDictionary(pair => pair.Key, pair => pair.Value); } + + private PushOptions CreatePushOptions() + { + return new PushOptions + { + AllowOverwrite = Overwrite, + PassIfExistingItemIdentical = PassIfExistingItemIdentical + }; + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/project.json b/src/Microsoft.DotNet.Build.Tasks.Feed/project.json index 62186756a3..0def9d910c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/project.json @@ -6,8 +6,8 @@ "System.Reflection.Metadata": "1.3.0", "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", - "NuGet.Packaging": "4.3.0", + "NuGet.Versioning": "4.4.0", + "NuGet.Packaging": "4.4.0", "SleetLib": "2.2.24", "WindowsAzure.Storage": "8.5.0" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json index e79a09eef8..8aea195359 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json @@ -2,9 +2,9 @@ "dependencies": { "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", "System.Reflection.Metadata": "1.0.22", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json index 99fd0ee0fa..4a6676e771 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json @@ -4,9 +4,9 @@ "Microsoft.Build.Framework": "0.1.0-preview-00022", "Microsoft.Build.Utilities.Core": "0.1.0-preview-00022", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", "NETStandard.Library": "1.6.0" }, "frameworks": { diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json index 0edfa8ff07..c2e72dec1e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json @@ -2,7 +2,7 @@ "dependencies": { "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", + "NuGet.Packaging": "4.4.0", "System.Reflection.Metadata": "1.3.0", "xunit": "2.1.0", "xunit.runner.visualstudio": "2.1.0" diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json index 5ab29d24c9..5d752d4121 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json @@ -4,7 +4,7 @@ "Microsoft.Build.Framework": "0.1.0-preview-00022", "Microsoft.Build.Utilities.Core": "0.1.0-preview-00022", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", + "NuGet.Packaging": "4.4.0", "NETStandard.Library": "1.6.0", "Microsoft.NETCore.Targets": "1.0.2", "xunit": "2.1.0", diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj index 827abcf5fb..6f10e25ef7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj @@ -15,11 +15,11 @@ .NETFramework,Version=v4.6 + - diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/project.json b/src/Microsoft.DotNet.Build.Tasks.net45/project.json index 6ba3bdf0b7..685ae3fded 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.net45/project.json @@ -4,10 +4,10 @@ "Microsoft.DotNet.PlatformAbstractions": "1.2.0-beta-001090", "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Collections.Immutable": "1.3.1", "System.Reflection.Metadata": "1.4.2", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" diff --git a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs index dfa6a62c0b..1804748408 100644 --- a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs +++ b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs @@ -334,7 +334,7 @@ private async System.Threading.Tasks.Task CleanupDirsAsync() cleanupTasks.Clear(); foreach (string additionalCleanupDirectory in _additionalCleanupDirectories) { - cleanupTasks.Add(CleanupDirectoryAsync(additionalCleanupDirectory, 0, true)); + cleanupTasks.Add(CleanupDirectoryAsync(additionalCleanupDirectory, 0, true, true)); } System.Threading.Tasks.Task.WaitAll(cleanupTasks.ToArray()); @@ -348,7 +348,7 @@ private async System.Threading.Tasks.Task CleanupAgentAsync(string workDir return returnStatus; } - private async System.Threading.Tasks.Task CleanupDirectoryAsync(string directory, int attempts = 0, bool ignoreExceptions = false) + private async System.Threading.Tasks.Task CleanupDirectoryAsync(string directory, int attempts = 0, bool ignoreExceptions = false, bool isAdditionalDirectory = false) { try { @@ -392,6 +392,13 @@ private async System.Threading.Tasks.Task CleanupDirectoryAsync(string dir } #endif Log.LogMessage("Success"); + + // For additional directories we want only the contents deleted but not the root. The easiest is just to recreate the root folder + // after the whole folder has been created. Since this just applies to additional ('known') folders we should not worry about long paths. + if (isAdditionalDirectory && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } } return true; } @@ -595,7 +602,7 @@ private string[] GetFilesByAttributes(string directory, FileAttributes attribute IntPtr hFile = FindFirstFile(directory + "\\*", out findData); int error = Marshal.GetLastWin32Error(); - if (hFile.ToInt32() != -1) + if (hFile != IntPtr.Zero && hFile != new IntPtr(-1)) { do { diff --git a/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs b/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs index 0c27c0df32..97cd4e418c 100644 --- a/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs +++ b/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Net; using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.Build.Tasks @@ -37,6 +38,11 @@ public sealed class DownloadFilesFromUrl : BuildTask public ITaskItem[] FilesCreated { get; set; } public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async Task ExecuteAsync() { if (Items == null || Items.Length <= 0) { @@ -102,11 +108,11 @@ public override bool Execute() Log.LogMessage(MessageImportance.Normal, $"Downloading {downloadSource} -> {destinationFullPath}"); - using (Stream responseStream = client.GetStreamAsync(downloadUri).GetAwaiter().GetResult()) + using (Stream responseStream = await client.GetStreamAsync(downloadUri)) { using (Stream destinationStream = File.OpenWrite(destinationFullPath)) { - responseStream.CopyToAsync(destinationStream).GetAwaiter().GetResult(); + await responseStream.CopyToAsync(destinationStream); TaskItem createdItem = new TaskItem(destinationFullPath); item.CopyMetadataTo(createdItem); filesCreated.Add(createdItem); diff --git a/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs b/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs index 72a933add7..f34cd0e57f 100644 --- a/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs +++ b/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs @@ -145,7 +145,9 @@ private RestoreSummaryRequest Create( restoreContext.ApplyStandardProperties(request); - var summaryRequest = new RestoreSummaryRequest(request, inputPath, settings, sources); + IEnumerable configFiles = SettingsUtility.GetConfigFilePaths(settings); + + var summaryRequest = new RestoreSummaryRequest(request, inputPath, configFiles, sources); return summaryRequest; } diff --git a/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs b/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs index a1af57e1bf..112ef3e7ab 100644 --- a/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs +++ b/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs @@ -4,9 +4,12 @@ using Microsoft.Build.Framework; using Microsoft.Build.Tasks; +using Microsoft.DotNet.VersionTools.Automation; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -53,6 +56,19 @@ public class ExecWithRetriesForNuGetPush : BuildTask /// public ITaskItem[] IgnoredErrorMessagesWithConditional { get; set; } + /// + /// Package file that is pushed by the given command. Required if PassIfIdenticalV2Feed + /// is set: it is read to compare against the copy of the package on the feed. + /// + public string PackageFile { get; set; } + + /// + /// If this property specifies a v2 feed endpoint, for example + /// "https://dotnet.myget.org/F/dotnet-core/api/v2", all errors are ignored if the feed + /// contains the package and it's byte-for-byte identical to the one being pushed. + /// + public string PassIfIdenticalV2Feed { get; set; } + private CancellationTokenSource _cancelTokenSource = new CancellationTokenSource(); private Exec _runningExec; @@ -96,7 +112,7 @@ public override bool Execute() } int exitCode = _runningExec.ExitCode; - if (exitCode == 0) + if (exitCode == 0 || FeedContainsIdenticalPackage()) { return true; } @@ -150,6 +166,61 @@ public override bool Execute() } return false; } + + private bool FeedContainsIdenticalPackage() + { + if (string.IsNullOrEmpty(PassIfIdenticalV2Feed) || + string.IsNullOrEmpty(PackageFile)) + { + return false; + } + + var packageInfo = new NupkgInfo(PackageFile); + string packageUrl = + $"{PassIfIdenticalV2Feed}/package/{packageInfo.Id}/{packageInfo.Version}"; + + byte[] localBytes = File.ReadAllBytes(PackageFile); + + bool identical = false; + + try + { + Log.LogMessage( + MessageImportance.High, + $"Downloading package from '{packageUrl}' " + + $"to check if identical to '{PackageFile}'"); + + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(10) + }) + using (var response = client.GetAsync(packageUrl).Result) + { + byte[] remoteBytes = response.Content.ReadAsByteArrayAsync().Result; + + identical = localBytes.SequenceEqual(remoteBytes); + } + } + catch (Exception e) + { + Log.LogWarningFromException(e, true); + } + + if (identical) + { + Log.LogMessage( + MessageImportance.High, + $"Package '{PackageFile}' is identical to feed download: ignoring push error."); + } + else + { + Log.LogMessage( + MessageImportance.High, + $"Package '{PackageFile}' is different from feed download."); + } + + return identical; + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj index 1bd51ce5b0..f5e5c9aaa0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj +++ b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj @@ -103,7 +103,9 @@ - + + BuildTask.cs + diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets index eef26c9097..f24f4930dc 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets @@ -10,14 +10,16 @@ $(MSBuildProjectDirectory)\ApiCompatBaseline.$(TargetGroup).txt $(MSBuildProjectDirectory)\ApiCompatBaseline.txt + $(MSBuildProjectDirectory)\MatchingRefApiCompatBaseline.$(TargetGroup).txt + $(MSBuildProjectDirectory)\MatchingRefApiCompatBaseline.txt + true - - false + + $(RunApiCompatForSrc) true $(TargetsTriggeredByCompilation);ValidateApiCompatForSrc - $(TargetsTriggeredByCompilation);ValidateApiCompatForRef + $(TargetsTriggeredByCompilation);RunMatchingRefApiCompat @@ -31,7 +33,9 @@ <_DependencyDirectoriesTemp Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> - <_DependencyDirectories Include="%(_DependencyDirectoriesTemp.Identity)" /> + + <_DependencyDirectories Condition="'%(_DependencyDirectoriesTemp.ReferenceSourceTarget)'=='ProjectReference'" Include="%(_DependencyDirectoriesTemp.Identity)" /> + <_DependencyDirectories Condition="'%(_DependencyDirectoriesTemp.ReferenceSourceTarget)'!='ProjectReference'" Include="%(_DependencyDirectoriesTemp.Identity)" /> <_ContractDependencyDirectories Include="@(ResolvedMatchingContract->'%(RootDir)%(Directory)')" /> <_ContractDependencyDirectories Include="$(ContractOutputPath)" /> @@ -68,55 +72,54 @@ - - - - - - - + + - - $(ReferenceAssemblyOutputPath)$(AssemblyName)\$(PreviousContractVersion)\$(AssemblyName).dll + + @(IntermediateAssembly) - <_CurrentContractDependencies Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> - - <_PreviousContractDependencyDirectories Include="@(_CurrentContractDependencies)" /> + <_ContractDependencyDirectoriesTemp Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> + + + <_ContractDependencyDirectories Condition="'%(_ContractDependencyDirectoriesTemp.ReferenceSourceTarget)'=='ProjectReference'" Include="%(_ContractDependencyDirectoriesTemp.Identity)" /> + <_ContractDependencyDirectories Condition="'%(_ContractDependencyDirectoriesTemp.ReferenceSourceTarget)'!='ProjectReference'" Include="%(_ContractDependencyDirectoriesTemp.Identity)" /> + <_ImplementationDependencyDirectories Include="@(ResolvedMatchingContract->'%(RootDir)%(Directory)')" /> + <_ImplementationDependencyDirectories Include="$(ContractOutputPath)" /> - <_ApiCompatCmd>$(ToolHostCmd) "$(ToolsDir)ApiCompat.exe" "$(PreviousContractAssembly)" - <_ApiCompatCmd>$(_ApiCompatCmd) -contractDepends:"@(_PreviousContractDependencyDirectories);" - <_ApiCompatCmd>$(_ApiCompatCmd) -implDirs:"$(IntermediateOutputPath);@(_CurrentContractDependencies);" - <_ApiCompatCmd Condition="Exists('$(ApiCompatBaseline)')">$(_ApiCompatCmd) -baseline:"$(ApiCompatBaseline)" - 0 + $(MatchingRefApiCompatArgs) "$(ImplemetnationAssemblyAsContract)" + $(MatchingRefApiCompatArgs) -contractDepends:"@(_ContractDependencyDirectories, ',')," + $(MatchingRefApiCompatArgs) -implDirs:"@(_ImplementationDependencyDirectories, ',')," + $(MatchingRefApiCompatArgs) -baseline:"$(MatchingRefApiCompatBaseline)" + > $(MatchingRefApiCompatBaseline) + + 0 + + $(IntermediateOutputPath)MatchingRefApiCompat_verifyexactref.rsp + $(ToolHostCmd) "$(ToolsDir)ApiCompat.exe" - + + + - + - - + + diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.props b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.props index 905f5ce008..5e83f4ccd7 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.props +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.props @@ -16,7 +16,7 @@ - $(MSBuildThisFileDirectory)checksum.rsp + SHA256 diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets index ed8df90bc4..01d5e79e3d 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets @@ -7,6 +7,9 @@ + + $(BasicCoreTargetsPath) + $(ToolsDir) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets index 4a920295de..225bef6185 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets @@ -106,9 +106,6 @@ - - - $(PackagesDir)/$(RoslynPackageName).$(RoslynVersion)/ + Microsoft.NETCore.Compilers + $(PackagesDir)/$(RoslynPackageName.ToLower())/$(RoslynVersion)/ $(RoslynPackageDir)build/$(RoslynPackageName).props - + $(SymbolPackageExtractDir)**\*.pdb - - - $(SymbolPackageExtractDir)WindowsPDB/ - - - <_ConvertPdbCommand>"$(MSBuildBinPath)\msbuild.exe" - <_ConvertPdbCommand>$(_ConvertPdbCommand) /v:Detailed - <_ConvertPdbCommand>$(_ConvertPdbCommand) "$(MSBuildProjectFullPath)" - <_ConvertPdbCommand>$(_ConvertPdbCommand) /t:CreateWindowsPdbsFromPortablePdbs - <_ConvertPdbCommand>$(_ConvertPdbCommand) "/p:PortablePdbToConvertGlob=$(PortablePdbToConvertGlob)" - <_ConvertPdbCommand>$(_ConvertPdbCommand) "/p:WindowsPdbConversionTargetPath=$(WindowsPdbConversionTargetPath)" + $(SymbolPackageExtractDir)WindowsPDB\ - - - - + - + NormalizeAssemblyName; @@ -87,7 +87,7 @@ - @@ -113,6 +113,8 @@ true FxResources.$(AssemblyName).SR.resources FxResources.$(AssemblyName).SR + + MSBuild:GenerateResourcesSource diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props new file mode 100644 index 0000000000..a6fdf8ec91 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props @@ -0,0 +1,6 @@ + + + diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets new file mode 100644 index 0000000000..68a1e6ac70 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets index 1d586d5f79..5d09a3a764 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets @@ -262,8 +262,13 @@ + + + + + 0 @@ -365,6 +370,8 @@ $(BuiltByString) %40BuiltBy: $(VersionUserName)-$(VersionHostName) + + $(BuiltByString) %40Branch: $(GitBranchName) $(BuiltByString) %40SrcCode: $(GitHubRepositoryUrl)/tree/$(LatestCommit) diff --git a/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs b/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs index 5b42a58b8a..8f419b1946 100644 --- a/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs +++ b/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs @@ -20,16 +20,47 @@ public class LocalUpdatePublishedVersions : BuildTask [Required] public string VersionsRepoPath { get; set; } + public string GitHubAuthToken { get; set; } + public string GitHubUser { get; set; } + + /// + /// If specified, create the local build-infos based on the information available in the + /// versions repo. Specifically, Latest_Packages.txt will contain the latest version of each + /// package, even if this build didn't produce that certain package. Useful when servicing, + /// where a subset of packages are built. + /// + public string VersionsRepo { get; set; } + public string VersionsRepoOwner { get; set; } + public string VersionsRepoBranch { get; set; } = "master"; + public override bool Execute() { Trace.Listeners.MsBuildListenedInvoke(Log, () => { var updater = new LocalVersionsRepoUpdater(); - updater.UpdateBuildInfoLatestPackages( - ShippedNuGetPackage.Select(item => item.ItemSpec), - VersionsRepoLocalBaseDir, - VersionsRepoPath); + if (!string.IsNullOrEmpty(GitHubAuthToken)) + { + updater.GitHubAuth = new GitHubAuth(GitHubAuthToken, GitHubUser); + } + + GitHubBranch branch = null; + if (!string.IsNullOrEmpty(VersionsRepo)) + { + branch = new GitHubBranch( + VersionsRepoBranch, + new GitHubProject( + VersionsRepo, + VersionsRepoOwner)); + } + + updater + .UpdateBuildInfoFilesAsync( + ShippedNuGetPackage.Select(i => i.ItemSpec), + VersionsRepoLocalBaseDir, + VersionsRepoPath, + branch) + .Wait(); }); return true; } diff --git a/src/Microsoft.DotNet.Build.Tasks/project.json b/src/Microsoft.DotNet.Build.Tasks/project.json index b4a27ee02b..e1b18e95e0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/project.json +++ b/src/Microsoft.DotNet.Build.Tasks/project.json @@ -12,10 +12,10 @@ }, "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Diagnostics.TraceSource": "4.0.0", "System.Dynamic.Runtime": "4.0.11", "System.IO.UnmanagedMemoryStream": "4.3.0", diff --git a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs index 7be6b2825a..ab0e262235 100644 --- a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs +++ b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs @@ -83,9 +83,7 @@ public async Task TestPushNewBuildAsync() .ReturnsAsync(() => null); await client.PushNewBuildAsync( - proj, - @ref, - basePath, + new BuildManifestLocation(proj, @ref, basePath), build, null, message); @@ -149,14 +147,12 @@ public async Task TestPushChangeSemaphoreAsync() .ReturnsAsync(() => null); await client.PushChangeAsync( - proj, - @ref, - basePath, - fakeExistingBuild.Identity.BuildId, - _ => { }, - new[] { addSemaphorePath }, - null, - message); + new BuildManifestChange( + new BuildManifestLocation(proj, @ref, basePath), + message, + fakeExistingBuild.Identity.BuildId, + new[] { addSemaphorePath }, + _ => { })); mockGitHub.VerifyAll(); } @@ -190,14 +186,13 @@ public async Task TestPushConflictingChangeAsync() await Assert.ThrowsAsync( async () => await client.PushChangeAsync( - proj, - @ref, - basePath, - fakeExistingBuild.Identity.BuildId, - _ => { }, - new[] { addSemaphorePath }, - null, - message)); + new BuildManifestChange( + new BuildManifestLocation(proj, @ref, basePath), + message, + fakeExistingBuild.Identity.BuildId, + new[] { addSemaphorePath }, + _ => { } + ))); mockGitHub.VerifyAll(); } @@ -245,9 +240,7 @@ public async Task TestPushConflictAsync() .ThrowsAsync(new NotFastForwardUpdateException("Testing non-fast-forward update.")); await client.PushNewBuildAsync( - proj, - @ref, - basePath, + new BuildManifestLocation(proj, @ref, basePath), build, null, message); diff --git a/src/Microsoft.DotNet.VersionTools.net45/project.json b/src/Microsoft.DotNet.VersionTools.net45/project.json index c721fa1f7d..305167f85e 100644 --- a/src/Microsoft.DotNet.VersionTools.net45/project.json +++ b/src/Microsoft.DotNet.VersionTools.net45/project.json @@ -1,8 +1,8 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Packaging": "4.4.0", + "NuGet.Versioning": "4.4.0", "Microsoft.NETCore.Platforms": "1.0.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs index 4710e27c7c..a238b11ad6 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; using System.Net.Http; using System.Threading.Tasks; @@ -67,10 +66,7 @@ public async Task UpdateBuildInfoAsync( NupkgInfo[] packages = CreatePackageInfos(packagePaths).ToArray(); - string prereleaseVersion = packages - .Select(t => t.Prerelease) - .FirstOrDefault(prerelease => !string.IsNullOrEmpty(prerelease)) - ?? "stable"; + string prereleaseVersion = GetPrereleaseVersion(packages); Dictionary packageDictionary = CreatePackageInfoDictionary(packages); @@ -91,7 +87,7 @@ public async Task UpdateBuildInfoAsync( { objects.Add(new GitObject { - Path = $"{versionsRepoPath}/Last_Build_Packages.txt", + Path = $"{versionsRepoPath}/{BuildInfo.LastBuildPackagesTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = CreatePackageListContent(packageDictionary) @@ -100,36 +96,20 @@ public async Task UpdateBuildInfoAsync( if (updateLatestPackageList) { - string latestPackagesPath = $"{versionsRepoPath}/Latest_Packages.txt"; - var allPackages = new Dictionary(packageDictionary); if (updateLastBuildPackageList) { - Dictionary existingPackages = await GetPackagesAsync(client, latestPackagesPath); - - if (existingPackages == null) - { - Trace.TraceInformation( - "No exising Latest_Packages file found; one will be " + - $"created in '{versionsRepoPath}'"); - } - else - { - // Add each existing package if there isn't a new package with the same id. - foreach (var package in existingPackages) - { - if (!allPackages.ContainsKey(package.Key)) - { - allPackages[package.Key] = package.Value; - } - } - } + await AddExistingPackages( + client, + new GitHubBranch("master", _project), + versionsRepoPath, + allPackages); } objects.Add(new GitObject { - Path = latestPackagesPath, + Path = $"{versionsRepoPath}/{BuildInfo.LatestPackagesTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = CreatePackageListContent(allPackages) @@ -140,7 +120,7 @@ public async Task UpdateBuildInfoAsync( { objects.Add(new GitObject { - Path = $"{versionsRepoPath}/Latest.txt", + Path = $"{versionsRepoPath}/{BuildInfo.LatestTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = prereleaseVersion @@ -184,22 +164,5 @@ public async Task UpdateBuildInfoAsync( } } } - - private async Task> GetPackagesAsync(GitHubClient client, string path) - { - string latestPackages = await client.GetGitHubFileContentsAsync( - path, - new GitHubBranch("master", _project)); - - if (latestPackages == null) - { - return null; - } - - using (var reader = new StringReader(latestPackages)) - { - return await BuildInfo.ReadPackageListAsync(reader); - } - } } } diff --git a/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs index 420f62d54b..3df58c520a 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs @@ -2,21 +2,27 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace Microsoft.DotNet.VersionTools.Automation { public class LocalVersionsRepoUpdater : VersionsRepoUpdater { + public GitHubAuth GitHubAuth { get; set; } + /// - /// Updates only the Latest_Packages file in the specified on-disk versions repository dir. + /// Create Latest_Packages, and if versionsRepoBranch is passed, Last_Build_Packages. /// - public void UpdateBuildInfoLatestPackages( + public async Task UpdateBuildInfoFilesAsync( IEnumerable packagePaths, string localBaseDir, - string versionsRepoPath) + string versionsRepoPath, + GitHubBranch versionsRepoBranch) { if (packagePaths == null) { @@ -31,17 +37,39 @@ public void UpdateBuildInfoLatestPackages( throw new ArgumentException(nameof(versionsRepoPath)); } - Dictionary packages = CreatePackageInfoDictionary(CreatePackageInfos(packagePaths)); - string latestPackagesDir = Path.Combine( localBaseDir, versionsRepoPath); Directory.CreateDirectory(latestPackagesDir); + NupkgInfo[] packages = CreatePackageInfos(packagePaths).ToArray(); + + Dictionary packageDictionary = CreatePackageInfoDictionary(packages); + + if (versionsRepoBranch != null) + { + File.WriteAllText( + Path.Combine(latestPackagesDir, BuildInfo.LastBuildPackagesTxtFilename), + CreatePackageListContent(packageDictionary)); + + using (var client = new GitHubClient(GitHubAuth)) + { + await AddExistingPackages( + client, + versionsRepoBranch, + versionsRepoPath, + packageDictionary); + } + } + + File.WriteAllText( + Path.Combine(latestPackagesDir, BuildInfo.LatestTxtFilename), + GetPrereleaseVersion(packages)); + File.WriteAllText( Path.Combine(latestPackagesDir, BuildInfo.LatestPackagesTxtFilename), - CreatePackageListContent(packages)); + CreatePackageListContent(packageDictionary)); } } } diff --git a/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs index cb34456c42..993a0329d5 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs @@ -2,9 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; +using System.Threading.Tasks; namespace Microsoft.DotNet.VersionTools.Automation { @@ -31,5 +35,61 @@ protected static string CreatePackageListContent(Dictionary pack .OrderBy(t => t.Key) .Select(t => $"{t.Key} {t.Value}")); } + + protected static async Task AddExistingPackages( + GitHubClient client, + GitHubBranch branch, + string versionsRepoPath, + Dictionary packages) + { + Dictionary existingPackages = await GetPackagesAsync( + client, + branch, + $"{versionsRepoPath}/{BuildInfo.LatestPackagesTxtFilename}"); + + if (existingPackages == null) + { + Trace.TraceInformation( + "No exising Latest_Packages file found; one will be " + + $"created in '{versionsRepoPath}'"); + } + else + { + // Add each existing package if there isn't a new package with the same id. + foreach (var package in existingPackages) + { + if (!packages.ContainsKey(package.Key)) + { + packages[package.Key] = package.Value; + } + } + } + } + + private static async Task> GetPackagesAsync( + GitHubClient client, + GitHubBranch branch, + string path) + { + string latestPackages = await client.GetGitHubFileContentsAsync(path, branch); + + if (latestPackages == null) + { + return null; + } + + using (var reader = new StringReader(latestPackages)) + { + return await BuildInfo.ReadPackageListAsync(reader); + } + } + + protected static string GetPrereleaseVersion(NupkgInfo[] packages) + { + return packages + .Select(t => t.Prerelease) + .FirstOrDefault(prerelease => !string.IsNullOrEmpty(prerelease)) + ?? "stable"; + } } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs new file mode 100644 index 0000000000..04a90ca3a7 --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using System; +using System.Collections.Generic; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class BuildManifestChange + { + public BuildManifestLocation Location { get; } + + public string CommitMessage { get; } + + public string OrchestratedBuildId { get; } + + public IEnumerable SemaphorePaths { get; } + + public Action ApplyModelChanges { get; } + + public IEnumerable JoinSemaphoreGroups { get; set; } + + public IEnumerable SupplementaryUploads { get; set; } + + public BuildManifestChange( + BuildManifestLocation location, + string commitMessage, + string orchestratedBuildId, + IEnumerable semaphorePaths, + Action applyModelChanges) + { + if (location == null) + { + throw new ArgumentNullException(nameof(location)); + } + + if (string.IsNullOrEmpty(commitMessage)) + { + throw new ArgumentException(nameof(commitMessage)); + } + + if (string.IsNullOrEmpty(orchestratedBuildId)) + { + throw new ArgumentException(nameof(orchestratedBuildId)); + } + + if (applyModelChanges == null) + { + throw new ArgumentNullException(nameof(applyModelChanges)); + } + + if (semaphorePaths == null) + { + throw new ArgumentNullException(nameof(semaphorePaths)); + } + + Location = location; + CommitMessage = commitMessage; + OrchestratedBuildId = orchestratedBuildId; + SemaphorePaths = semaphorePaths; + ApplyModelChanges = applyModelChanges; + } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs index 7716ab3f9d..8cd09ac110 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs @@ -47,20 +47,27 @@ public async Task FetchSemaphoreAsync( project, @ref); + if (contents == null) + { + return null; + } + return SemaphoreModel.Parse(semaphorePath, contents); } public async Task PushNewBuildAsync( - GitHubProject project, - string @ref, - string basePath, + BuildManifestLocation location, OrchestratedBuildModel build, IEnumerable supplementaryUploads, string message) { await Retry.RunAsync(async attempt => { - string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha; + GitReference remoteRef = await _github.GetReferenceAsync( + location.GitHubProject, + location.GitHubRef); + + string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); @@ -83,63 +90,86 @@ await Retry.RunAsync(async attempt => }) .ToArray(); - return await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads); + return await PushUploadsAsync(location, message, remoteCommit, uploads); }); } - public async Task PushChangeAsync( - GitHubProject project, - string @ref, - string basePath, - string orchestratedBuildId, - Action changeModel, - IEnumerable semaphorePaths, - IEnumerable supplementaryUploads, - string message) + public async Task PushChangeAsync(BuildManifestChange change) { await Retry.RunAsync(async attempt => { + BuildManifestLocation location = change.Location; + // Get the current commit. Use this throughout to ensure a clean transaction. - string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha; + GitReference remoteRef = await _github.GetReferenceAsync( + location.GitHubProject, + location.GitHubRef); + + string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); - // This is a subsequent publish step: check to make sure the build id matches. - XElement remoteModelXml = await FetchModelXmlAsync(project, remoteCommit, basePath); + XElement remoteModelXml = await FetchModelXmlAsync( + location.GitHubProject, + remoteCommit, + location.GitHubBasePath); OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml); - if (orchestratedBuildId != remoteModel.Identity.BuildId) + // This is a subsequent publish step: make sure a new build hasn't happened already. + if (change.OrchestratedBuildId != remoteModel.Identity.BuildId) { throw new ManifestChangeOutOfDateException( - orchestratedBuildId, + change.OrchestratedBuildId, remoteModel.Identity.BuildId); } OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml); - changeModel(modifiedModel); + change.ApplyModelChanges(modifiedModel); - if (modifiedModel.Identity.BuildId != orchestratedBuildId) + if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId) { throw new ArgumentException( "Change action shouldn't modify BuildId. Changed from " + - $"'{orchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.", - nameof(changeModel)); + $"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.", + nameof(change)); } XElement modifiedModelXml = modifiedModel.ToXml(); - IEnumerable uploads = semaphorePaths.NullAsEmpty() + string[] changedSemaphorePaths = change.SemaphorePaths.ToArray(); + + // Check if any join groups are completed by this change. + var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty() + .Select(async g => new + { + Group = g, + Joinable = await IsGroupJoinableAsync( + location, + remoteCommit, + change.OrchestratedBuildId, + changedSemaphorePaths, + g) + }); + + var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks)) + .Where(g => g.Joinable) + .Select(g => g.Group.JoinSemaphorePath) + .ToArray(); + + IEnumerable semaphoreUploads = completeJoinedSemaphores + .Concat(changedSemaphorePaths) .Select(p => new SupplementaryUploadRequest { Path = p, Contents = new SemaphoreModel { - BuildId = orchestratedBuildId + BuildId = change.OrchestratedBuildId }.ToFileContent() - }) - .Concat(supplementaryUploads.NullAsEmpty()) - .ToArray(); + }); + + IEnumerable uploads = + semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty()); if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml)) { @@ -153,7 +183,11 @@ await Retry.RunAsync(async attempt => }); } - return await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads); + return await PushUploadsAsync( + location, + change.CommitMessage, + remoteCommit, + uploads); }); } @@ -171,9 +205,7 @@ private async Task FetchModelXmlAsync( } private async Task PushUploadsAsync( - GitHubProject project, - string @ref, - string basePath, + BuildManifestLocation location, string message, string remoteCommit, IEnumerable uploads) @@ -181,17 +213,21 @@ private async Task PushUploadsAsync( GitObject[] objects = uploads .Select(upload => new GitObject { - Path = $"{basePath}/{upload.Path}", + Path = upload.GetAbsolutePath(location.GitHubBasePath), Mode = GitObject.ModeFile, Type = GitObject.TypeBlob, - Content = upload.Contents + // Always upload files using LF to avoid bad dev scenarios with Git autocrlf. + Content = upload.Contents.Replace("\r\n", "\n") }) .ToArray(); - GitTree tree = await _github.PostTreeAsync(project, remoteCommit, objects); + GitTree tree = await _github.PostTreeAsync( + location.GitHubProject, + remoteCommit, + objects); GitCommit commit = await _github.PostCommitAsync( - project, + location.GitHubProject, message, tree.Sha, new[] { remoteCommit }); @@ -199,7 +235,11 @@ private async Task PushUploadsAsync( try { // Only fast-forward. Don't overwrite other changes: throw exception instead. - await _github.PatchReferenceAsync(project, @ref, commit.Sha, force: false); + await _github.PatchReferenceAsync( + location.GitHubProject, + location.GitHubRef, + commit.Sha, + force: false); } catch (NotFastForwardUpdateException e) { @@ -210,5 +250,40 @@ private async Task PushUploadsAsync( return true; } + + private async Task IsGroupJoinableAsync( + BuildManifestLocation location, + string commit, + string buildId, + IEnumerable changedSemaphorePaths, + JoinSemaphoreGroup joinGroup) + { + string[] remainingSemaphores = joinGroup + .ParallelSemaphorePaths + .Except(changedSemaphorePaths) + .ToArray(); + + if (remainingSemaphores.Length == joinGroup.ParallelSemaphorePaths.Count()) + { + // No semaphores in this group are changing: it can't be joinable by this update. + return false; + } + + // TODO: Avoid redundant fetches if multiple groups share a semaphore. https://github.com/dotnet/buildtools/issues/1910 + bool[] remainingSemaphoreIsComplete = await Task.WhenAll( + remainingSemaphores.Select( + async path => + { + SemaphoreModel semaphore = await FetchSemaphoreAsync( + location.GitHubProject, + commit, + location.GitHubBasePath, + path); + + return semaphore?.BuildId == buildId; + })); + + return remainingSemaphoreIsComplete.All(x => x); + } } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs new file mode 100644 index 0000000000..3fd32eecf2 --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.VersionTools.Automation; +using System; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class BuildManifestLocation + { + public GitHubProject GitHubProject { get; } + + public string GitHubRef { get; } + + public string GitHubBasePath { get; } + + public BuildManifestLocation( + GitHubProject gitHubProject, + string gitHubRef, + string gitHubBasePath) + { + if (gitHubProject == null) + { + throw new ArgumentNullException(nameof(gitHubProject)); + } + + if (string.IsNullOrEmpty(gitHubRef)) + { + throw new ArgumentException(nameof(gitHubRef)); + } + + if (string.IsNullOrEmpty(gitHubBasePath)) + { + throw new ArgumentException(nameof(gitHubBasePath)); + } + + GitHubProject = gitHubProject; + GitHubRef = gitHubRef; + GitHubBasePath = gitHubBasePath; + } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs new file mode 100644 index 0000000000..7491a19dbc --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class JoinSemaphoreGroup + { + public string JoinSemaphorePath { get; set; } + + /// + /// Names of the semaphores that must all complete to update the join semaphore. + /// + public IEnumerable ParallelSemaphorePaths { get; set; } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs index 970f07115a..c0c412119f 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs @@ -4,18 +4,24 @@ using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; +using System.Linq; using System.Xml.Linq; namespace Microsoft.DotNet.VersionTools.BuildManifest.Model { public class PackageArtifactModel { - private static readonly string[] AttributeOrder = + private static readonly string[] RequiredAttributes = { nameof(Id), nameof(Version) }; + private static readonly string[] AttributeOrder = RequiredAttributes.Concat(new[] + { + nameof(OriginBuildName) + }).ToArray(); + public IDictionary Attributes { get; set; } = new Dictionary(); public string Id @@ -30,19 +36,25 @@ public string Version set { Attributes[nameof(Version)] = value; } } + public string OriginBuildName + { + get { return Attributes.GetOrDefault(nameof(OriginBuildName)); } + set { Attributes[nameof(OriginBuildName)] = value; } + } + public override string ToString() => $"Package {Id} {Version}"; public XElement ToXml() => new XElement( "Package", Attributes - .ThrowIfMissingAttributes(AttributeOrder) + .ThrowIfMissingAttributes(RequiredAttributes) .CreateXmlAttributes(AttributeOrder)); public static PackageArtifactModel Parse(XElement xml) => new PackageArtifactModel { Attributes = xml .CreateAttributeDictionary() - .ThrowIfMissingAttributes(AttributeOrder) + .ThrowIfMissingAttributes(RequiredAttributes) }; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs index 8cf43c2bea..77b2b74454 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs @@ -6,8 +6,32 @@ namespace Microsoft.DotNet.VersionTools.BuildManifest { public class SupplementaryUploadRequest { + /// + /// Path, relative to the primary upload dir or absolute with a leading '/'. + /// public string Path { get; set; } public string Contents { get; set; } + + /// + /// Combine currentPath and Path into the absolute path of this request. The result is + /// compatible with GitHub paths. This is similar to Path.Combine except: + /// + /// If Path is absolute (begins with '/'), the leading '/' is trimmed from the result. + /// + /// If Path is relative, the path is always joined with '/'. (Never '\'.) + /// + /// + /// The absolute path of a dir that Path should be made relative to, if Path isn't already + /// an absolute path. Can't start or end in '/'. + /// + public string GetAbsolutePath(string currentPath) + { + if (Path.StartsWith("/")) + { + return Path.Substring(1); + } + return $"{currentPath}/{Path}"; + } } } diff --git a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj index c1fe1017e8..a0df9f95c9 100644 --- a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj +++ b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj @@ -11,6 +11,7 @@ false 1.0.0.0 .NETStandard,Version=v1.5 + 6 @@ -30,7 +31,9 @@ + + @@ -41,6 +44,7 @@ + diff --git a/src/Microsoft.DotNet.VersionTools/project.json b/src/Microsoft.DotNet.VersionTools/project.json index 46195abfd3..28a6c09a2a 100644 --- a/src/Microsoft.DotNet.VersionTools/project.json +++ b/src/Microsoft.DotNet.VersionTools/project.json @@ -2,8 +2,8 @@ "dependencies": { "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Packaging": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Diagnostics.Process": "4.1.0", "System.Diagnostics.TraceSource": "4.0.0" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/BuildTask.Desktop.cs b/src/common/BuildTask.Desktop.cs similarity index 100% rename from src/Microsoft.DotNet.Build.Tasks.net45/BuildTask.Desktop.cs rename to src/common/BuildTask.Desktop.cs diff --git a/src/Microsoft.DotNet.Build.Tasks/BuildTask.cs b/src/common/BuildTask.cs similarity index 100% rename from src/Microsoft.DotNet.Build.Tasks/BuildTask.cs rename to src/common/BuildTask.cs diff --git a/src/nuget/Microsoft.DotNet.VersionTools.nuspec b/src/nuget/Microsoft.DotNet.VersionTools.nuspec index 66beda1ac3..7470a606a7 100644 --- a/src/nuget/Microsoft.DotNet.VersionTools.nuspec +++ b/src/nuget/Microsoft.DotNet.VersionTools.nuspec @@ -15,11 +15,13 @@ Copyright © Microsoft Corporation + + + + + - - - @@ -27,7 +29,7 @@ - + diff --git a/src/packages.builds b/src/packages.builds index 5c26aebef0..9382224796 100644 --- a/src/packages.builds +++ b/src/packages.builds @@ -13,7 +13,7 @@ true - 2.1.0-preview1-$(BuildNumberMajor)-$(BuildNumberMinor) + 2.1.0-preview2-$(BuildNumberMajor)-$(BuildNumberMinor)