From b76c4b6e773f85efa9566b7a0090a8bca98cb57c Mon Sep 17 00:00:00 2001 From: Jose Perez Rodriguez Date: Mon, 29 Jan 2018 16:12:53 -0800 Subject: [PATCH 01/49] Changes required in order to build corefx on linux using CLI's MSBuild (#1879) * Changes required in order to build corefx on linux using CLI's MSBuild * Adding NoWarn to crossgen project for package downgrades detected * Clean up FrameworkTargeting.targets * Workaround roslyn props bug for VB * Fix crossgen project for core-setup repo since they force the warnings to be treated as errors --- .../PackageFiles/Build.Common.targets | 3 +++ .../PackageFiles/FrameworkTargeting.targets | 16 ++++++++++++---- .../PackageFiles/Roslyn.Common.props | 3 ++- .../PackageFiles/crossgen.sh | 2 +- .../PackageFiles/init-tools.sh | 2 ++ .../PackageFiles/msbuild.sh | 2 +- 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets index ed8df90bc4..01d5e79e3d 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Build.Common.targets @@ -7,6 +7,9 @@ + + $(BasicCoreTargetsPath) + $(ToolsDir) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets index 4a920295de..225bef6185 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/FrameworkTargeting.targets @@ -106,9 +106,6 @@ - - - $(PackagesDir)/$(RoslynPackageName).$(RoslynVersion)/ + Microsoft.NETCore.Compilers + $(PackagesDir)/$(RoslynPackageName.ToLower())/$(RoslynVersion)/ $(RoslynPackageDir)build/$(RoslynPackageName).props @@ -335,12 +337,14 @@ $(BuildMoniker) $(HelixJobType) $(HelixSource) + $(MaxRetryCount) + From cb76b2fd323dbdc7fa46991e565669fc7fec08aa Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 30 Jan 2018 11:28:39 -0600 Subject: [PATCH 03/49] Allow custom attributes in build manifest 'BuildIdentity' (#1882) * Allow custom attributes in BuildIdentity Also improve the required attribute mechanism to avoid redundant writes and verify dictionary contents during ToXml. --- .../FetchOrchestratedBuildManifestInfo.cs | 19 +++- .../WriteOrchestratedBuildManifestToFile.cs | 14 ++- .../Microsoft.DotNet.Build.Tasks.Feed.targets | 7 ++ .../PushToBlobFeed.cs | 30 +++-- .../BuildManifest/BuildManifestClientTests.cs | 10 +- .../BuildManifest/ManifestModelTests.cs | 17 ++- .../BuildManifest/Model/BlobArtifactModel.cs | 14 ++- .../BuildManifest/Model/BuildIdentity.cs | 105 +++++++++++------- .../BuildManifest/Model/BuildModel.cs | 2 +- .../BuildManifest/Model/EndpointModel.cs | 7 +- .../Model/OrchestratedBuildModel.cs | 4 +- .../Model/PackageArtifactModel.cs | 17 +-- .../Model/XElementParsingExtensions.cs | 17 ++- .../Util/EnumerableExtensions.cs | 9 ++ 14 files changed, 186 insertions(+), 86 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs index 3d11c485ff..99f099e383 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs @@ -40,6 +40,9 @@ public class FetchOrchestratedBuildManifestInfo : Task [Output] public ITaskItem[] OrchestratedBlobFeedArtifacts { get; set; } + + [Output] + public ITaskItem[] OrchestratedBuilds { get; set; } public override bool Execute() { @@ -58,6 +61,9 @@ public override bool Execute() VersionsRepoPath) .Result; + OrchestratedBuildId = manifest.Identity.BuildId; + OrchestratedIdentity = manifest.Identity.ToString(); + EndpointModel[] orchestratedFeeds = manifest.Endpoints .Where(e => e.IsOrchestratedBlobFeed) .ToArray(); @@ -74,10 +80,12 @@ public override bool Execute() IEnumerable packageItems = feed.Artifacts.Packages.Select(CreateItem); IEnumerable blobItems = feed.Artifacts.Blobs.Select(CreateItem); - OrchestratedBuildId = manifest.Identity.BuildId; - OrchestratedIdentity = manifest.Identity.ToString(); OrchestratedBlobFeed = new[] { new TaskItem("Endpoint", feed.Attributes) }; OrchestratedBlobFeedArtifacts = packageItems.Concat(blobItems).ToArray(); + + IEnumerable buildItems = manifest.Builds.Select(CreateItem); + + OrchestratedBuilds = buildItems.ToArray(); } return !Log.HasLoggedErrors; @@ -93,6 +101,13 @@ private ITaskItem CreateItem(PackageArtifactModel model) return new TaskItem("Package", ArtifactMetadata(model.ToXml(), model.Attributes)); } + private ITaskItem CreateItem(BuildIdentity model) + { + return new TaskItem( + model.Name, + ArtifactMetadata(model.ToXmlBuildElement(), model.Attributes)); + } + private Dictionary ArtifactMetadata( XElement artifactXml, IDictionary attributes) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs index d62ab9b375..5cdca452b0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs @@ -42,11 +42,15 @@ public override bool Execute() ManifestCommit = null; } - var orchestratedBuild = new OrchestratedBuildModel(new BuildIdentity( - ManifestName, - ManifestBuildId, - ManifestBranch, - ManifestCommit)) + var identity = new BuildIdentity + { + Name = ManifestName, + BuildId = ManifestBuildId, + Branch = ManifestBranch, + Commit = ManifestCommit + }; + + var orchestratedBuild = new OrchestratedBuildModel(identity) { Endpoints = new List { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets index 7e734e6e22..3efe7fbfa4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets @@ -85,6 +85,7 @@ ManifestBuildId="$(ManifestBuildId)" ManifestBranch="$(ManifestBranch)" ManifestCommit="$(ManifestCommit)" + ManifestBuildData="$(ManifestBuildData)" SkipCreateManifest="$(SkipCreateManifest)" /> @@ -115,6 +116,7 @@ ManifestBuildId="$(ManifestBuildId)" ManifestBranch="$(ManifestBranch)" ManifestCommit="$(ManifestCommit)" + ManifestBuildData="$(ManifestBuildData)" SkipCreateManifest="$(SkipCreateManifest)" /> @@ -230,6 +232,10 @@ %(Identity): 'Package' or 'Blob', matching manifest element name. %(Xml): The raw XML string representing the artifact in the manifest. %(...): Metadata is created for each attribute on the element. + @(OrchestratedBuilds): An item for each Build in the orchestrated build manifest. + %(Identity): The name of the build. + %(Xml): The raw XML string representing the artifact in the manifest. + %(...): Metadata is created for each attribute on the element. --> @@ -251,6 +257,7 @@ + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs index 2fb491a75d..7ecdbb1c19 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs @@ -47,6 +47,7 @@ public class PushToBlobFeed : MSBuild.Task public string ManifestBuildId { get; set; } = "no build id provided"; public string ManifestBranch { get; set; } public string ManifestCommit { get; set; } + public string ManifestBuildData { get; set; } /// /// When publishing build outputs to an orchestrated blob feed, do not change this property. @@ -158,11 +159,14 @@ private async Task PushBuildManifestAsync( else { buildModel = new BuildModel( - new BuildIdentity( - ManifestName, - ManifestBuildId, - ManifestBranch, - ManifestCommit)); + new BuildIdentity + { + Attributes = ParseManifestMetadataString(ManifestBuildData), + Name = ManifestName, + BuildId = ManifestBuildId, + Branch = ManifestBranch, + Commit = ManifestCommit + }); } buildModel.Artifacts.Blobs.AddRange(blobArtifacts); @@ -257,9 +261,17 @@ private static BlobArtifactModel CreateBlobArtifactModel(ITaskItem item) private static Dictionary ParseCustomAttributes(ITaskItem item) { - Dictionary customAttributes = item - .GetMetadata("ManifestArtifactData") - ?.Split(ManifestDataPairSeparators, StringSplitOptions.RemoveEmptyEntries) + return ParseManifestMetadataString(item.GetMetadata("ManifestArtifactData")); + } + + private static Dictionary ParseManifestMetadataString(string data) + { + if (string.IsNullOrEmpty(data)) + { + return new Dictionary(); + } + + return data.Split(ManifestDataPairSeparators, StringSplitOptions.RemoveEmptyEntries) .Select(pair => { int keyValueSeparatorIndex = pair.IndexOf('='); @@ -275,8 +287,6 @@ private static Dictionary ParseCustomAttributes(ITaskItem item) }) .Where(pair => pair != null) .ToDictionary(pair => pair.Key, pair => pair.Value); - - return customAttributes ?? new Dictionary(); } } } diff --git a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs index f467814d80..7be6b2825a 100644 --- a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs +++ b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs @@ -40,7 +40,7 @@ public async Task TestPushNewBuildAsync() var mockGitHub = new Mock(MockBehavior.Strict); var client = new BuildManifestClient(mockGitHub.Object); - var build = new OrchestratedBuildModel(new BuildIdentity("orch", "123")); + var build = new OrchestratedBuildModel(new BuildIdentity { Name = "orch", BuildId = "123"}); var proj = new GitHubProject("versions", "dotnet"); string @ref = "heads/master"; string basePath = "build-info/dotnet/product/cli/master"; @@ -105,7 +105,7 @@ public async Task TestPushChangeSemaphoreAsync() string message = "Test change manifest commit"; string addSemaphorePath = "add-identity.semaphore"; - var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity("orch", "123")); + var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity { Name = "orch", BuildId = "123"}); string fakeExistingBuildString = fakeExistingBuild.ToXml().ToString(); string fakeCommitHash = "fakeCommitHash"; string fakeTreeHash = "fakeTreeHash"; @@ -173,8 +173,8 @@ public async Task TestPushConflictingChangeAsync() string message = "Test change manifest commit"; string addSemaphorePath = "add-identity.semaphore"; - var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity("orch", "123")); - var fakeNewExistingBuild = new OrchestratedBuildModel(new BuildIdentity("orch", "456")); + var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity { Name = "orch", BuildId = "123" }); + var fakeNewExistingBuild = new OrchestratedBuildModel(new BuildIdentity { Name = "orch", BuildId = "456" }); string fakeCommitHash = "fakeCommitHash"; mockGitHub @@ -208,7 +208,7 @@ public async Task TestPushConflictAsync() var mockGitHub = new Mock(MockBehavior.Strict); var client = new BuildManifestClient(mockGitHub.Object); - var build = new OrchestratedBuildModel(new BuildIdentity("orch", "123")); + var build = new OrchestratedBuildModel(new BuildIdentity { Name = "orch", BuildId = "123" }); var proj = new GitHubProject("versions", "dotnet"); string @ref = "heads/master"; string basePath = "build-info/dotnet/product/cli/master"; diff --git a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/ManifestModelTests.cs b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/ManifestModelTests.cs index d5b5fad593..e512434846 100644 --- a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/ManifestModelTests.cs +++ b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/ManifestModelTests.cs @@ -43,6 +43,19 @@ public void TestExampleOrchestratedBuildManifestRoundtrip() "Model failed to output the parsed XML."); } + [Fact] + public void TestExampleCustomBuildIdentityRoundtrip() + { + XElement xml = XElement.Parse( + @""); + var model = BuildModel.Parse(xml); + XElement modelXml = model.ToXml(); + + Assert.True( + XNode.DeepEquals(xml, modelXml), + "Model failed to output the parsed XML."); + } + [Fact] public void TestPackageOnlyBuildManifest() { @@ -56,7 +69,7 @@ public void TestPackageOnlyBuildManifest() [Fact] public void TestMergeBuildManifests() { - var orchestratedModel = new OrchestratedBuildModel(new BuildIdentity("Orchestrated", "123")) + var orchestratedModel = new OrchestratedBuildModel(new BuildIdentity { Name = "Orchestrated", BuildId = "123" }) { Endpoints = new List { @@ -87,7 +100,7 @@ public void TestMergeBuildManifests() private BuildModel CreatePackageOnlyBuildManifestModel() { - return new BuildModel(new BuildIdentity("SimpleBuildManifest", "123")) + return new BuildModel(new BuildIdentity { Name = "SimpleBuildManifest", BuildId = "123" }) { Artifacts = new ArtifactSet { diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BlobArtifactModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BlobArtifactModel.cs index 871595f1d1..95e952003f 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BlobArtifactModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BlobArtifactModel.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; using System.Xml.Linq; @@ -14,11 +15,11 @@ public class BlobArtifactModel nameof(Id) }; - public Dictionary Attributes { get; set; } = new Dictionary(); + public IDictionary Attributes { get; set; } = new Dictionary(); public string Id { - get { return Attributes[nameof(Id)]; } + get { return Attributes.GetOrDefault(nameof(Id)); } set { Attributes[nameof(Id)] = value; } } @@ -26,12 +27,15 @@ public string Id public XElement ToXml() => new XElement( "Blob", - Attributes.CreateXmlAttributes(AttributeOrder)); + Attributes + .ThrowIfMissingAttributes(AttributeOrder) + .CreateXmlAttributes(AttributeOrder)); public static BlobArtifactModel Parse(XElement xml) => new BlobArtifactModel { - Id = xml.GetRequiredAttribute(nameof(Id)), - Attributes = xml.CreateAttributeDictionary() + Attributes = xml + .CreateAttributeDictionary() + .ThrowIfMissingAttributes(AttributeOrder) }; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildIdentity.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildIdentity.cs index a9d70e7bfc..463b96c834 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildIdentity.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildIdentity.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; +using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; using System.Xml.Linq; @@ -10,34 +10,59 @@ namespace Microsoft.DotNet.VersionTools.BuildManifest.Model { public class BuildIdentity { - public BuildIdentity( - string name, - string buildId, - string branch = null, - string commit = null) + private static readonly string[] AttributeOrder = { - if (string.IsNullOrEmpty(name)) - { - throw new ArgumentException("Expected a non-empty string.", nameof(name)); - } - Name = name; - if (string.IsNullOrEmpty(buildId)) - { - throw new ArgumentException("Expected a non-empty string.", nameof(buildId)); - } - BuildId = buildId; - Branch = branch; - Commit = commit; + nameof(Name), + nameof(BuildId), + nameof(ProductVersion), + nameof(Branch), + nameof(Commit) + }; + + private static readonly string[] RequiredAttributes = + { + nameof(Name) + }; + + public IDictionary Attributes { get; set; } = new Dictionary(); + + public string Name + { + get { return Attributes.GetOrDefault(nameof(Name)); } + set { Attributes[nameof(Name)] = value; } + } + + public string BuildId + { + get { return Attributes.GetOrDefault(nameof(BuildId)); } + set { Attributes[nameof(BuildId)] = value; } + } + + public string ProductVersion + { + get { return Attributes.GetOrDefault(nameof(ProductVersion)); } + set { Attributes[nameof(ProductVersion)] = value; } + } + + public string Branch + { + get { return Attributes.GetOrDefault(nameof(Branch)); } + set { Attributes[nameof(Branch)] = value; } } - public string Name { get; } - public string BuildId { get; } - public string Branch { get; } - public string Commit { get; } + public string Commit + { + get { return Attributes.GetOrDefault(nameof(Commit)); } + set { Attributes[nameof(Commit)] = value; } + } public override string ToString() { - string s = $"{Name} '{BuildId}'"; + string s = Name; + if (!string.IsNullOrEmpty(ProductVersion)) + { + s += $" {ProductVersion}"; + } if (!string.IsNullOrEmpty(Branch)) { s += $" on '{Branch}'"; @@ -46,30 +71,24 @@ public override string ToString() { s += $" ({Commit})"; } - return s; - } - - public IEnumerable ToXml() - { - yield return new XAttribute(nameof(Name), Name); - yield return new XAttribute(nameof(BuildId), BuildId); - if (!string.IsNullOrEmpty(Branch)) - { - yield return new XAttribute(nameof(Branch), Branch); - } - if (!string.IsNullOrEmpty(Commit)) + if (!string.IsNullOrEmpty(BuildId)) { - yield return new XAttribute(nameof(Commit), Commit); + s += $" build {BuildId}"; } + return s; } - public static BuildIdentity Parse(XElement xml) + public IEnumerable ToXmlAttributes() => Attributes + .ThrowIfMissingAttributes(RequiredAttributes) + .CreateXmlAttributes(AttributeOrder); + + public XElement ToXmlBuildElement() => new XElement("Build", ToXmlAttributes()); + + public static BuildIdentity Parse(XElement xml) => new BuildIdentity { - return new BuildIdentity( - xml.GetRequiredAttribute(nameof(Name)), - xml.GetRequiredAttribute(nameof(BuildId)), - xml.Attribute(nameof(Branch))?.Value, - xml.Attribute(nameof(Commit))?.Value); - } + Attributes = xml + .CreateAttributeDictionary() + .ThrowIfMissingAttributes(RequiredAttributes) + }; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildModel.cs index dbcf504a36..257008b8fe 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/BuildModel.cs @@ -26,7 +26,7 @@ public BuildModel(BuildIdentity identity) public XElement ToXml() => new XElement( "Build", - Identity.ToXml(), + Identity.ToXmlAttributes(), Artifacts.ToXml()); public static BuildModel Parse(XElement xml) => new BuildModel(BuildIdentity.Parse(xml)) diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/EndpointModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/EndpointModel.cs index aac54c9a27..635febddbf 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/EndpointModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/EndpointModel.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; using System.Xml.Linq; @@ -24,19 +25,19 @@ public class EndpointModel public string Id { - get { return Attributes[nameof(Id)]; } + get { return Attributes.GetOrDefault(nameof(Id)); } set { Attributes[nameof(Id)] = value; } } public string Type { - get { return Attributes[nameof(Type)]; } + get { return Attributes.GetOrDefault(nameof(Type)); } set { Attributes[nameof(Type)] = value; } } public string Url { - get { return Attributes[nameof(Url)]; } + get { return Attributes.GetOrDefault(nameof(Url)); } set { Attributes[nameof(Url)] = value; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/OrchestratedBuildModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/OrchestratedBuildModel.cs index 92cf6c5638..a3734dff77 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/OrchestratedBuildModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/OrchestratedBuildModel.cs @@ -42,9 +42,9 @@ public void AddParticipantBuild(BuildModel build) public XElement ToXml() => new XElement( "OrchestratedBuild", - Identity.ToXml(), + Identity.ToXmlAttributes(), Endpoints.Select(x => x.ToXml()), - Builds.Select(x => new XElement("Build", x.ToXml()))); + Builds.Select(x => x.ToXmlBuildElement())); public static OrchestratedBuildModel Parse(XElement xml) => new OrchestratedBuildModel(BuildIdentity.Parse(xml)) { diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs index 9aa2079093..970f07115a 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; using System.Xml.Linq; @@ -15,17 +16,17 @@ public class PackageArtifactModel nameof(Version) }; - public Dictionary Attributes { get; set; } = new Dictionary(); + public IDictionary Attributes { get; set; } = new Dictionary(); public string Id { - get { return Attributes[nameof(Id)]; } + get { return Attributes.GetOrDefault(nameof(Id)); } set { Attributes[nameof(Id)] = value; } } public string Version { - get { return Attributes[nameof(Version)]; } + get { return Attributes.GetOrDefault(nameof(Version)); } set { Attributes[nameof(Version)] = value; } } @@ -33,13 +34,15 @@ public string Version public XElement ToXml() => new XElement( "Package", - Attributes.CreateXmlAttributes(AttributeOrder)); + Attributes + .ThrowIfMissingAttributes(AttributeOrder) + .CreateXmlAttributes(AttributeOrder)); public static PackageArtifactModel Parse(XElement xml) => new PackageArtifactModel { - Id = xml.GetRequiredAttribute(nameof(Id)), - Version = xml.GetRequiredAttribute(nameof(Version)), - Attributes = xml.CreateAttributeDictionary() + Attributes = xml + .CreateAttributeDictionary() + .ThrowIfMissingAttributes(AttributeOrder) }; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/XElementParsingExtensions.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/XElementParsingExtensions.cs index bf5d1ae5f8..10e65b6249 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/XElementParsingExtensions.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/XElementParsingExtensions.cs @@ -28,14 +28,29 @@ public static Dictionary CreateAttributeDictionary(this XElement } public static XAttribute[] CreateXmlAttributes( - this Dictionary attributes, + this IDictionary attributes, string[] keySortOrder) { return attributes + .Where(pair => pair.Value != null) .OrderBy(pair => keySortOrder.TakeWhile(o => pair.Key != o).Count()) .ThenBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase) .Select(pair => new XAttribute(pair.Key, pair.Value)) .ToArray(); } + + public static IDictionary ThrowIfMissingAttributes( + this IDictionary attributes, + IEnumerable requiredAttributes) + { + var missing = requiredAttributes?.Where(r => !attributes.ContainsKey(r)).ToArray(); + if (missing?.Any() == true) + { + throw new ArgumentException( + $"Required attribute(s) missing: {string.Join(", ", missing)}"); + } + + return attributes; + } } } diff --git a/src/Microsoft.DotNet.VersionTools/Util/EnumerableExtensions.cs b/src/Microsoft.DotNet.VersionTools/Util/EnumerableExtensions.cs index c71b6141e2..da43f5883c 100644 --- a/src/Microsoft.DotNet.VersionTools/Util/EnumerableExtensions.cs +++ b/src/Microsoft.DotNet.VersionTools/Util/EnumerableExtensions.cs @@ -13,5 +13,14 @@ public static IEnumerable NullAsEmpty(this IEnumerable source) { return source ?? Enumerable.Empty(); } + + public static TValue GetOrDefault( + this IDictionary attributes, + TKey key) + { + TValue value; + attributes.TryGetValue(key, out value); + return value; + } } } From a65f13fbb73db6cf9ea4da276db44f6eb271deeb Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 30 Jan 2018 11:34:08 -0600 Subject: [PATCH 04/49] Use project.json dep versions in VersionTools pkg (#1884) --- src/nuget/Microsoft.DotNet.VersionTools.nuspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nuget/Microsoft.DotNet.VersionTools.nuspec b/src/nuget/Microsoft.DotNet.VersionTools.nuspec index e77a1ad2a8..66beda1ac3 100644 --- a/src/nuget/Microsoft.DotNet.VersionTools.nuspec +++ b/src/nuget/Microsoft.DotNet.VersionTools.nuspec @@ -18,8 +18,8 @@ - - + + From 069862f02fec281676ca71266f54ecf8e41df3b4 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 30 Jan 2018 14:59:26 -0600 Subject: [PATCH 05/49] Produce 2.2.0-prerelease-* packages (#1888) --- src/packages.builds | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages.builds b/src/packages.builds index c041512546..f5e80acf9d 100644 --- a/src/packages.builds +++ b/src/packages.builds @@ -13,7 +13,7 @@ true - 2.1.0-prerelease-$(BuildNumberMajor)-$(BuildNumberMinor) + 2.2.0-prerelease-$(BuildNumberMajor)-$(BuildNumberMinor) From 89f91fdc99d48788ebe40efbc7ffc2b8bc2055a9 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 30 Jan 2018 19:56:12 -0600 Subject: [PATCH 06/49] Produce 2.1.0-preview2-* packages (#1889) --- src/packages.builds | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages.builds b/src/packages.builds index f5e80acf9d..9382224796 100644 --- a/src/packages.builds +++ b/src/packages.builds @@ -13,7 +13,7 @@ true - 2.2.0-prerelease-$(BuildNumberMajor)-$(BuildNumberMinor) + 2.1.0-preview2-$(BuildNumberMajor)-$(BuildNumberMinor) From 2031290b12420327c2159d3476f3f1e8e4688cf4 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 31 Jan 2018 12:18:08 -0600 Subject: [PATCH 07/49] Add Finalize/Copy to Latest task for final publish step (#1871) * Move FinalizeBuild from Core-Setup * Rename to CopyBlobsToLatest * Generalize "copy to latest" for final publish Include blob lease fix: avoid an up to one minute wait per lease release by flowing the cancellation token into the delay. * Make version hinting optional Instead of ForcePublish, add EnableVersionHint that can be disabled. * Accept custom latest version filenames This accounts for scenarios like CLI's coherent builds and ASP.NET Core's runtime which is published into the same virtual dir as an existing latest version file. --- .../AzureBlobLease.cs | 2 +- .../CopyBlobsToLatest.cs | 243 ++++++++++++++++++ ...crosoft.DotNet.Build.CloudTestTasks.csproj | 1 + 3 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs index 6428f3516b..d604988c89 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/AzureBlobLease.cs @@ -154,7 +154,7 @@ private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Buil } token.ThrowIfCancellationRequested(); - Thread.Sleep(waitFor); + Task.Delay(waitFor, token).Wait(); } } diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs new file mode 100644 index 0000000000..7ddd578fd8 --- /dev/null +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/CopyBlobsToLatest.cs @@ -0,0 +1,243 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Build.CloudTestTasks +{ + public class CopyBlobsToLatest : AzureConnectionStringBuildTask + { + private static readonly string[] DefaultLatestVersionFilenames = + { + "latest.version" + }; + + private static readonly Regex VersionRegex = new Regex( + @"(?\d+\.\d+\.\d+)(-(?[^-]+-)?(?\d+)-(?\d+))?"); + + [Required] + public string ContainerName { get; set; } + + [Required] + public string Product { get; set; } + + [Required] + public string ProductVersion { get; set; } + + [Required] + public string Channel { get; set; } + + [Required] + public string Commit { get; set; } + + public string[] LatestVersionFilenames { get; set; } + + /// + /// A list of full version strings that should be converted to "latest" for each blob id, + /// in addition to ProductVersion. + /// + public string[] FullVersions { get; set; } + + /// + /// If this task is called by multiple build legs that make up one build, enable this option + /// to prevent race conditions by dropping a version hint of what version this is. If we see + /// this file and it is the same as our version then we know that a race happened where two+ + /// builds finished at the same time and someone already took care of publishing and we have + /// no work to do. + /// + /// This is not necessary if the build uses some other mechanism to ensure each build only + /// has one finalization attempt, e.g. PipeBuild. + /// + public bool EnableVersionHint { get; set; } + + public override bool Execute() + { + ParseConnectionString(); + + if (Log.HasLoggedErrors) + { + return false; + } + + string sourceDir = $"{Product}/{ProductVersion}/"; + string channelDir = $"{Product}/{Channel}/"; + string semaphoreBlob = $"{channelDir}publishSemaphore"; + + CreateBlobIfNotExists(semaphoreBlob); + + AzureBlobLease blobLease = new AzureBlobLease( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + semaphoreBlob, + Log); + + Log.LogMessage($"Acquiring lease on semaphore blob '{semaphoreBlob}'"); + blobLease.Acquire(); + + try + { + if (EnableVersionHint) + { + string targetVersionFile = $"{channelDir}{ProductVersion}"; + + // Check if this build is already finalized. + if (IsLatestSpecifiedVersion(targetVersionFile)) + { + Log.LogMessage( + MessageImportance.High, + $"Version '{ProductVersion}' is already published. " + + $"Skipping finalization. Hint file: '{targetVersionFile}'"); + + return true; + } + + // Delete old version files. + GetBlobList(channelDir) + .Select(s => s.Replace($"/{ContainerName}/", "")) + .Where(w => VersionRegex.Replace(Path.GetFileName(w), "") == "") + .ToList() + .ForEach(f => TryDeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds. + CreateBlobIfNotExists(targetVersionFile); + } + + CopyBlobs(sourceDir, channelDir); + + // Generate the latest version text file + string versionText = + $"{Commit}{Environment.NewLine}" + + $"{ProductVersion}{Environment.NewLine}"; + + if (LatestVersionFilenames?.Any() != true) + { + LatestVersionFilenames = DefaultLatestVersionFilenames; + } + + foreach (string latestFilename in LatestVersionFilenames) + { + PublishStringToBlob( + ContainerName, + $"{channelDir}{latestFilename}", + versionText, + "text/plain"); + } + } + finally + { + Log.LogMessage($"Releasing lease on semaphore blob '{semaphoreBlob}'"); + blobLease.Release(); + } + + return !Log.HasLoggedErrors; + } + + private bool CopyBlobs(string sourceFolder, string destinationFolder) + { + // List of versions that need to be replaced with "latest" when copying blobs. + var versions = new List { ProductVersion }; + if (FullVersions != null) + { + versions.AddRange(FullVersions); + } + + bool returnStatus = true; + List> copyTasks = new List>(); + string[] blobs = GetBlobList(sourceFolder); + foreach (string blob in blobs) + { + string targetName = versions.Aggregate( + Path.GetFileName(blob), + (agg, version) => agg.Replace(version, "latest")); + + string sourceBlob = blob.Replace($"/{ContainerName}/", ""); + string destinationBlob = $"{destinationFolder}{targetName}"; + Log.LogMessage($"Copying blob '{sourceBlob}' to '{destinationBlob}'"); + copyTasks.Add(CopyBlobAsync(sourceBlob, destinationBlob)); + } + Task.WaitAll(copyTasks.ToArray()); + copyTasks.ForEach(c => returnStatus &= c.Result); + return returnStatus; + } + + private bool TryDeleteBlob(string path) + { + return DeleteBlob(ContainerName, path); + } + + private void CreateBlobIfNotExists(string path) + { + var blobList = GetBlobList(path); + if (blobList.Count() == 0) + { + PublishStringToBlob(ContainerName, path, DateTime.Now.ToString()); + } + } + + private bool IsLatestSpecifiedVersion(string versionFile) + { + var blobList = GetBlobList(versionFile); + return blobList.Count() != 0; + } + + private bool DeleteBlob(string container, string blob) + { + return DeleteAzureBlob.Execute( + AccountName, + AccountKey, + ConnectionString, + container, + blob, + BuildEngine, + HostObject); + } + + private Task CopyBlobAsync(string sourceBlobName, string destinationBlobName) + { + return CopyAzureBlobToBlob.ExecuteAsync( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + sourceBlobName, + destinationBlobName, + BuildEngine, + HostObject); + } + + private string[] GetBlobList(string path) + { + return ListAzureBlobs.Execute( + AccountName, + AccountKey, + ConnectionString, + ContainerName, + path, + BuildEngine, + HostObject); + } + + private bool PublishStringToBlob(string container, string blob, string contents, string contentType = null) + { + return PublishStringToAzureBlob.Execute( + AccountName, + AccountKey, + ConnectionString, + container, + blob, + contents, + contentType, + BuildEngine, + HostObject); + } + } +} diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj b/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj index 3ce0310e8e..29f0cf3f58 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/Microsoft.DotNet.Build.CloudTestTasks.csproj @@ -29,6 +29,7 @@ + From 2404f8e2dbed817687190c386acb037ab828b721 Mon Sep 17 00:00:00 2001 From: Jose Perez Rodriguez Date: Wed, 31 Jan 2018 11:11:02 -0800 Subject: [PATCH 08/49] Fixing crossgen on core-setup by forcing the implicit NuGet fallback folder (#1892) --- src/Microsoft.DotNet.Build.Tasks/PackageFiles/crossgen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/crossgen.sh b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/crossgen.sh index ac24c6c8e0..6e3579a206 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/crossgen.sh +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/crossgen.sh @@ -18,7 +18,7 @@ restore_crossgen() __pjDir=$__toolsDir/crossgen mkdir -p $__pjDir - echo "false$(NoWarn);NU1605;NU1103netcoreapp2.0true$__packageRid" > "$__pjDir/crossgen.csproj" + echo "falsefalse$(NoWarn);NU1605;NU1103netcoreapp2.0true$__packageRid" > "$__pjDir/crossgen.csproj" $__dotnet restore $__pjDir/crossgen.csproj --packages $__packagesDir --source $__MyGetFeed __crossgen=$__packagesDir/runtime.$__packageRid.microsoft.netcore.app/$__sharedFxVersion/tools/crossgen if [ ! -e $__crossgen ]; then From 2a9486a85cb385042e26ada50da9370f996f110d Mon Sep 17 00:00:00 2001 From: Vance Morrison Date: Fri, 2 Feb 2018 12:54:32 -0800 Subject: [PATCH 09/49] It is useful to know the branch where a particular build was made. Add that to the extended version information (#1895) --- .../PackageFiles/versioning.targets | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets index 1d586d5f79..5d09a3a764 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/versioning.targets @@ -262,8 +262,13 @@ + + + + + 0 @@ -365,6 +370,8 @@ $(BuiltByString) %40BuiltBy: $(VersionUserName)-$(VersionHostName) + + $(BuiltByString) %40Branch: $(GitBranchName) $(BuiltByString) %40SrcCode: $(GitHubRepositoryUrl)/tree/$(LatestCommit) From a4da379c014274e977f94ba4b037d18a90658050 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 7 Feb 2018 12:05:19 -0600 Subject: [PATCH 10/49] Add idempotency options to publish tasks (#1894) --- .../DownloadFromAzure.cs | 4 +- .../UploadClient.cs | 53 ++++++ .../UploadToAzure.cs | 44 ++++- .../BlobFeed.cs | 37 ++-- .../BlobFeedAction.cs | 159 +++++++++++++++--- .../Microsoft.DotNet.Build.Tasks.Feed.csproj | 1 + .../PushOptions.cs | 13 ++ .../PushToBlobFeed.cs | 39 ++++- .../ExecWithRetriesForNuGetPush.cs | 73 +++++++- .../PackageFiles/PublishProduct.targets | 8 +- 10 files changed, 389 insertions(+), 42 deletions(-) create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs index 69473a2fc5..58e4b1a2a0 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/DownloadFromAzure.cs @@ -119,7 +119,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim { client.Timeout = TimeSpan.FromMinutes(10); Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); - string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); + string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob)); if (!DownloadFlatFiles) @@ -155,7 +155,7 @@ private async Task DownloadItem(CancellationToken ct, string blob, SemaphoreSlim filename = Path.Combine(downloadBlobDirectory, blobFilename); } - var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs index b30c7eb571..a8e71cec8c 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadClient.cs @@ -8,10 +8,12 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Text; using System.Threading; +using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; namespace Microsoft.DotNet.Build.CloudTestTasks @@ -208,6 +210,57 @@ public async Task UploadBlockBlobAsync( } } } + + public async Task FileEqualsExistingBlobAsync( + string accountName, + string accountKey, + string containerName, + string filePath, + string destinationBlob, + int uploadTimeout) + { + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(uploadTimeout) + }) + { + log.LogMessage( + MessageImportance.Low, + $"Downloading blob {destinationBlob} to check if identical."); + + string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( + log, + client, + createRequest)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Failed to retrieve existing blob {destinationBlob}, " + + $"status code {response.StatusCode}."); + } + + byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); + byte[] localBytes = File.ReadAllBytes(filePath); + + bool equal = localBytes.SequenceEqual(existingBytes); + + if (equal) + { + log.LogMessage( + MessageImportance.Normal, + "Item exists in blob storage, and is verified to be identical. " + + $"File: '{filePath}' Blob: '{destinationBlob}'"); + } + + return equal; + } + } + } + private string DetermineContentTypeBasedOnFileExtension(string filename) { if (Path.GetExtension(filename) == ".svg") diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs index 68343748f3..568eda1964 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs @@ -42,6 +42,16 @@ public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask /// public bool Overwrite { get; set; } = false; + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + /// /// Specifies the maximum number of clients to concurrently upload blobs to azure /// @@ -137,8 +147,18 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha if (!File.Exists(item.ItemSpec)) throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); + UploadClient uploadClient = new UploadClient(Log); + if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) + { + if (PassIfExistingItemIdentical && + await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) + { + return; + } + throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); + } string contentType = item.GetMetadata("ContentType"); @@ -147,7 +167,6 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha try { Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); - UploadClient uploadClient = new UploadClient(Log); await uploadClient.UploadBlockBlobAsync( ct, @@ -164,5 +183,28 @@ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, Ha clientThrottle.Release(); } } + + private async Task ItemEqualsExistingBlobAsync( + ITaskItem item, + string relativeBlobPath, + UploadClient client, + SemaphoreSlim clientThrottle) + { + await clientThrottle.WaitAsync(); + try + { + return await client.FileEqualsExistingBlobAsync( + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs index 4763854ef3..a7a49b78e7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeed.cs @@ -41,7 +41,7 @@ public BlobFeed(string accountName, string accountKey, string containerName, str public string FeedContainerUrl => AzureHelper.GetContainerRestUrl(AccountName, ContainerName); - public async Task CheckIfBlobExists(string blobPath) + public async Task CheckIfBlobExistsAsync(string blobPath) { string url = $"{FeedContainerUrl}/{blobPath}?comp=metadata"; using (HttpClient client = new HttpClient()) @@ -67,21 +67,38 @@ public async Task CheckIfBlobExists(string blobPath) } } - public async Task DownloadBlobAsString(string blobPath) + public async Task DownloadBlobAsStringAsync(string blobPath) + { + using (HttpResponseMessage response = await DownloadBlobAsync(blobPath)) + { + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsStringAsync(); + } + return null; + } + } + + public async Task DownloadBlobAsBytesAsync(string blobPath) + { + using (HttpResponseMessage response = await DownloadBlobAsync(blobPath)) + { + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsByteArrayAsync(); + } + return null; + } + } + + private async Task DownloadBlobAsync(string blobPath) { string url = $"{FeedContainerUrl}/{blobPath}"; using (HttpClient client = new HttpClient()) { client.DefaultRequestHeaders.Clear(); var request = AzureHelper.RequestMessage("GET", url, AccountName, AccountKey)(); - using (HttpResponseMessage response = await client.SendAsync(request)) - { - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadAsStringAsync(); - } - return null; - } + return await client.SendAsync(request); } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs index 3f3e915373..c30bec932b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BlobFeedAction.cs @@ -6,6 +6,8 @@ using Microsoft.DotNet.Build.CloudTestTasks; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json.Linq; +using NuGet.Packaging; +using NuGet.Packaging.Core; using Sleet; using System; using System.Collections.Generic; @@ -14,9 +16,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using NuGet.Packaging.Core; using MSBuild = Microsoft.Build.Utilities; -using CloudTestTasks = Microsoft.DotNet.Build.CloudTestTasks; namespace Microsoft.DotNet.Build.Tasks.Feed { @@ -65,7 +65,9 @@ public BlobFeedAction(string expectedFeedUrl, string accountKey, MSBuild.TaskLog } } - public async Task PushToFeed(IEnumerable items, bool allowOverwrite = false) + public async Task PushToFeedAsync( + IEnumerable items, + PushOptions options) { if (IsSanityChecked(items)) { @@ -75,13 +77,15 @@ public async Task PushToFeed(IEnumerable items, bool allowOverwrit CancellationToken.ThrowIfCancellationRequested(); } - await PushItemsToFeedAsync(items, allowOverwrite); + await PushItemsToFeedAsync(items, options); } return !Log.HasLoggedErrors; } - public async Task PushItemsToFeedAsync(IEnumerable items, bool allowOverwrite) + public async Task PushItemsToFeedAsync( + IEnumerable items, + PushOptions options) { Log.LogMessage(MessageImportance.Low, $"START pushing items to feed"); @@ -93,7 +97,7 @@ public async Task PushItemsToFeedAsync(IEnumerable items, bool all try { - bool result = await PushAsync(items, allowOverwrite); + bool result = await PushAsync(items, options); return result; } catch (Exception e) @@ -104,7 +108,11 @@ public async Task PushItemsToFeedAsync(IEnumerable items, bool all return !Log.HasLoggedErrors; } - public async Task UploadAssets(ITaskItem item, SemaphoreSlim clientThrottle, int uploadTimeout, bool allowOverwrite = false) + public async Task UploadAssetAsync( + ITaskItem item, + SemaphoreSlim clientThrottle, + int uploadTimeout, + PushOptions options) { string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); @@ -133,17 +141,33 @@ public async Task UploadAssets(ITaskItem item, SemaphoreSlim clientThrottle, int try { - bool blobExists = false; + UploadClient uploadClient = new UploadClient(Log); - if (!allowOverwrite) + if (!options.AllowOverwrite && await feed.CheckIfBlobExistsAsync(relativeBlobPath)) { - blobExists = await feed.CheckIfBlobExists(relativeBlobPath); + if (options.PassIfExistingItemIdentical) + { + if (!await uploadClient.FileEqualsExistingBlobAsync( + feed.AccountName, + feed.AccountKey, + feed.ContainerName, + item.ItemSpec, + relativeBlobPath, + uploadTimeout)) + { + Log.LogError( + $"Item '{item}' already exists with different contents " + + $"at '{relativeBlobPath}'"); + } + } + else + { + Log.LogError($"Item '{item}' already exists at '{relativeBlobPath}'"); + } } - - if (allowOverwrite || !blobExists) + else { Log.LogMessage($"Uploading {item} to {relativeBlobPath}."); - UploadClient uploadClient = new UploadClient(Log); await uploadClient.UploadBlockBlobAsync( CancellationToken, feed.AccountName, @@ -154,10 +178,6 @@ await uploadClient.UploadBlockBlobAsync( contentType, uploadTimeout); } - else - { - Log.LogError($"Item '{item}' already exists in {relativeBlobPath}."); - } } catch (Exception exc) { @@ -252,6 +272,35 @@ private bool IsSanityChecked(IEnumerable items) return true; } + private async Task IsPackageIdenticalOnFeedAsync( + string item, + PackageIndex packageIndex, + ISleetFileSystem source, + FlatContainer flatContainer, + SleetLogger log) + { + using (var package = new PackageArchiveReader(item)) + { + var id = await package.GetIdentityAsync(CancellationToken); + if (await packageIndex.Exists(id)) + { + using (Stream remoteStream = await source + .Get(flatContainer.GetNupkgPath(id)) + .GetStream(log, CancellationToken)) + using (var remote = new MemoryStream()) + { + await remoteStream.CopyToAsync(remote); + + byte[] existingBytes = remote.ToArray(); + byte[] localBytes = File.ReadAllBytes(item); + + return existingBytes.SequenceEqual(localBytes); + } + } + return null; + } + } + private LocalSettings GetSettings() { SleetSettings sleetSettings = new SleetSettings() @@ -277,12 +326,82 @@ private AzureFileSystem GetAzureFileSystem() return fileSystem; } - private async Task PushAsync(IEnumerable items, bool allowOverwrite) + private async Task PushAsync( + IEnumerable items, + PushOptions options) { LocalSettings settings = GetSettings(); AzureFileSystem fileSystem = GetAzureFileSystem(); - bool result = await PushCommand.RunAsync(settings, fileSystem, items.ToList(), allowOverwrite, skipExisting: false, log: new SleetLogger(Log)); - return result; + SleetLogger log = new SleetLogger(Log); + + var packagesToPush = items.ToList(); + + if (!options.AllowOverwrite && options.PassIfExistingItemIdentical) + { + var context = new SleetContext + { + LocalSettings = settings, + Log = log, + Source = fileSystem, + Token = CancellationToken + }; + context.SourceSettings = await FeedSettingsUtility.GetSettingsOrDefault( + context.Source, + context.Log, + context.Token); + + var flatContainer = new FlatContainer(context); + + var packageIndex = new PackageIndex(context); + + // Check packages sequentially: Task.WhenAll caused IO exceptions in Sleet. + for (int i = packagesToPush.Count - 1; i >= 0; i--) + { + string item = packagesToPush[i]; + + bool? identical = await IsPackageIdenticalOnFeedAsync( + item, + packageIndex, + context.Source, + flatContainer, + log); + + if (identical == null) + { + continue; + } + + packagesToPush.RemoveAt(i); + + if (identical == true) + { + Log.LogMessage( + MessageImportance.Normal, + "Package exists on the feed, and is verified to be identical. " + + $"Skipping upload: '{item}'"); + } + else + { + Log.LogError( + "Package exists on the feed, but contents are different. " + + $"Upload failed: '{item}'"); + } + } + + if (!packagesToPush.Any()) + { + Log.LogMessage("After skipping idempotent uploads, no items need pushing."); + return true; + } + } + + return await PushCommand.RunAsync( + settings, + fileSystem, + packagesToPush, + options.AllowOverwrite, + skipExisting: false, + log: log); } private async Task InitAsync() diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj index b499ac2d54..7152f87ee7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj @@ -51,6 +51,7 @@ + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs new file mode 100644 index 0000000000..729edfecbb --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class PushOptions + { + public bool AllowOverwrite { get; set; } + + public bool PassIfExistingItemIdentical { get; set; } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs index 7ecdbb1c19..05c9b0c3c5 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs @@ -33,6 +33,16 @@ public class PushToBlobFeed : MSBuild.Task public bool Overwrite { get; set; } + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + public bool PublishFlatContainer { get; set; } public int MaxClients { get; set; } = 8; @@ -108,7 +118,7 @@ public async Task ExecuteAsync() var packagePaths = packageItems.Select(i => i.ItemSpec); - await blobFeedAction.PushToFeed(packagePaths, Overwrite); + await blobFeedAction.PushToFeedAsync(packagePaths, CreatePushOptions()); await PublishToFlatContainerAsync(symbolItems, blobFeedAction); packageArtifacts = ConcatPackageArtifacts(packageArtifacts, packageItems); @@ -134,7 +144,7 @@ private async Task PushBuildManifestAsync( IEnumerable blobArtifacts, IEnumerable packageArtifacts) { - bool disabledByBlob = await blobFeedAction.feed.CheckIfBlobExists( + bool disabledByBlob = await blobFeedAction.feed.CheckIfBlobExistsAsync( $"{blobFeedAction.feed.RelativePath}{DisableManifestPushConfigurationBlob}"); if (disabledByBlob) @@ -147,7 +157,7 @@ private async Task PushBuildManifestAsync( string blobPath = $"{AssetsVirtualDir}{ManifestAssetOutputDir}{ManifestName}.xml"; - string existingStr = await blobFeedAction.feed.DownloadBlobAsString( + string existingStr = await blobFeedAction.feed.DownloadBlobAsStringAsync( $"{blobFeedAction.feed.RelativePath}{blobPath}"); BuildModel buildModel; @@ -186,11 +196,14 @@ private async Task PushBuildManifestAsync( using (var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients)) { - await blobFeedAction.UploadAssets( + await blobFeedAction.UploadAssetAsync( item, clientThrottle, UploadTimeoutInMinutes, - allowOverwrite: true); + new PushOptions + { + AllowOverwrite = true + }); } } finally @@ -209,7 +222,12 @@ private async Task PublishToFlatContainerAsync(IEnumerable taskItems, using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) { Log.LogMessage($"Uploading {taskItems.Count()} items..."); - await Task.WhenAll(taskItems.Select(item => blobFeedAction.UploadAssets(item, clientThrottle, UploadTimeoutInMinutes, Overwrite))); + await Task.WhenAll(taskItems.Select( + item => blobFeedAction.UploadAssetAsync( + item, + clientThrottle, + UploadTimeoutInMinutes, + CreatePushOptions()))); } } } @@ -288,5 +306,14 @@ private static Dictionary ParseManifestMetadataString(string dat .Where(pair => pair != null) .ToDictionary(pair => pair.Key, pair => pair.Value); } + + private PushOptions CreatePushOptions() + { + return new PushOptions + { + AllowOverwrite = Overwrite, + PassIfExistingItemIdentical = PassIfExistingItemIdentical + }; + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs b/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs index a1af57e1bf..112ef3e7ab 100644 --- a/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs +++ b/src/Microsoft.DotNet.Build.Tasks/ExecWithRetriesForNuGetPush.cs @@ -4,9 +4,12 @@ using Microsoft.Build.Framework; using Microsoft.Build.Tasks; +using Microsoft.DotNet.VersionTools.Automation; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -53,6 +56,19 @@ public class ExecWithRetriesForNuGetPush : BuildTask /// public ITaskItem[] IgnoredErrorMessagesWithConditional { get; set; } + /// + /// Package file that is pushed by the given command. Required if PassIfIdenticalV2Feed + /// is set: it is read to compare against the copy of the package on the feed. + /// + public string PackageFile { get; set; } + + /// + /// If this property specifies a v2 feed endpoint, for example + /// "https://dotnet.myget.org/F/dotnet-core/api/v2", all errors are ignored if the feed + /// contains the package and it's byte-for-byte identical to the one being pushed. + /// + public string PassIfIdenticalV2Feed { get; set; } + private CancellationTokenSource _cancelTokenSource = new CancellationTokenSource(); private Exec _runningExec; @@ -96,7 +112,7 @@ public override bool Execute() } int exitCode = _runningExec.ExitCode; - if (exitCode == 0) + if (exitCode == 0 || FeedContainsIdenticalPackage()) { return true; } @@ -150,6 +166,61 @@ public override bool Execute() } return false; } + + private bool FeedContainsIdenticalPackage() + { + if (string.IsNullOrEmpty(PassIfIdenticalV2Feed) || + string.IsNullOrEmpty(PackageFile)) + { + return false; + } + + var packageInfo = new NupkgInfo(PackageFile); + string packageUrl = + $"{PassIfIdenticalV2Feed}/package/{packageInfo.Id}/{packageInfo.Version}"; + + byte[] localBytes = File.ReadAllBytes(PackageFile); + + bool identical = false; + + try + { + Log.LogMessage( + MessageImportance.High, + $"Downloading package from '{packageUrl}' " + + $"to check if identical to '{PackageFile}'"); + + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(10) + }) + using (var response = client.GetAsync(packageUrl).Result) + { + byte[] remoteBytes = response.Content.ReadAsByteArrayAsync().Result; + + identical = localBytes.SequenceEqual(remoteBytes); + } + } + catch (Exception e) + { + Log.LogWarningFromException(e, true); + } + + if (identical) + { + Log.LogMessage( + MessageImportance.High, + $"Package '{PackageFile}' is identical to feed download: ignoring push error."); + } + else + { + Log.LogMessage( + MessageImportance.High, + $"Package '{PackageFile}' is different from feed download."); + } + + return identical; + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/PublishProduct.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/PublishProduct.targets index ec4d7596ed..621e948ed7 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/PublishProduct.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/PublishProduct.targets @@ -31,7 +31,9 @@ - + + %(Identity) + - - + - + + %(OrchestratedBuild.BuildId) + %(OrchestratedBuild.IdentitySummary) %(OrchestratedBlobFeed.Url) @@ -304,7 +309,15 @@ $([System.String]::Copy('%(OrchestratedBlobFeedArtifacts.Version)').ToLowerInvariant()) - + + %(LowercaseId).%(LowercaseVersion).nupkg + + + + $(FinalDownloadDirectory)%(NupkgFile) + + + @@ -343,10 +356,14 @@ $(VersionsRepo): The GitHub repo name. Default: 'versions' $(VersionsRepoOwner): The GitHub repo owner. Default: 'dotnet' $(VersionsRepoBranch): The branch to fetch from. Default: 'master' - $(OrchestratedIdentity): Human-readable identity of the orchestrated build, used to generate - a concise commit message. This is expected to come from FetchOrchestratedBuildManifestInfo. + $(OrchestratedIdentitySummary): Human-readable identity of the orchestrated build, used to + generate a concise commit message. This is expected to come from + FetchOrchestratedBuildManifestInfo. Default: the full VersionsRepoPath is used in the commit message. $(CommitMessage): Overrides the generated commit message. + @(SupplementaryFiles): Uploads supplementary files to the versions repo as part of the update. + For item requirements and behavior, see 'UpdateOrchestratedBuildManifest.cs' in + dotnet/buildtools. --> @@ -368,6 +385,7 @@ VersionsRepoOwner="$(VersionsRepoOwner)" VersionsRepoBranch="$(VersionsRepoBranch)" CommitMessage="$(CommitMessage)" - OrchestratedIdentity="$(OrchestratedIdentity)" /> + OrchestratedIdentitySummary="$(OrchestratedIdentitySummary)" + SupplementaryFiles="@(SupplementaryFiles)" /> diff --git a/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs b/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs index 5b42a58b8a..8f419b1946 100644 --- a/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs +++ b/src/Microsoft.DotNet.Build.Tasks/VersionTools/LocalUpdatePublishedVersions.cs @@ -20,16 +20,47 @@ public class LocalUpdatePublishedVersions : BuildTask [Required] public string VersionsRepoPath { get; set; } + public string GitHubAuthToken { get; set; } + public string GitHubUser { get; set; } + + /// + /// If specified, create the local build-infos based on the information available in the + /// versions repo. Specifically, Latest_Packages.txt will contain the latest version of each + /// package, even if this build didn't produce that certain package. Useful when servicing, + /// where a subset of packages are built. + /// + public string VersionsRepo { get; set; } + public string VersionsRepoOwner { get; set; } + public string VersionsRepoBranch { get; set; } = "master"; + public override bool Execute() { Trace.Listeners.MsBuildListenedInvoke(Log, () => { var updater = new LocalVersionsRepoUpdater(); - updater.UpdateBuildInfoLatestPackages( - ShippedNuGetPackage.Select(item => item.ItemSpec), - VersionsRepoLocalBaseDir, - VersionsRepoPath); + if (!string.IsNullOrEmpty(GitHubAuthToken)) + { + updater.GitHubAuth = new GitHubAuth(GitHubAuthToken, GitHubUser); + } + + GitHubBranch branch = null; + if (!string.IsNullOrEmpty(VersionsRepo)) + { + branch = new GitHubBranch( + VersionsRepoBranch, + new GitHubProject( + VersionsRepo, + VersionsRepoOwner)); + } + + updater + .UpdateBuildInfoFilesAsync( + ShippedNuGetPackage.Select(i => i.ItemSpec), + VersionsRepoLocalBaseDir, + VersionsRepoPath, + branch) + .Wait(); }); return true; } diff --git a/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs index 4710e27c7c..a238b11ad6 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/GitHubVersionsRepoUpdater.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; using System.Net.Http; using System.Threading.Tasks; @@ -67,10 +66,7 @@ public async Task UpdateBuildInfoAsync( NupkgInfo[] packages = CreatePackageInfos(packagePaths).ToArray(); - string prereleaseVersion = packages - .Select(t => t.Prerelease) - .FirstOrDefault(prerelease => !string.IsNullOrEmpty(prerelease)) - ?? "stable"; + string prereleaseVersion = GetPrereleaseVersion(packages); Dictionary packageDictionary = CreatePackageInfoDictionary(packages); @@ -91,7 +87,7 @@ public async Task UpdateBuildInfoAsync( { objects.Add(new GitObject { - Path = $"{versionsRepoPath}/Last_Build_Packages.txt", + Path = $"{versionsRepoPath}/{BuildInfo.LastBuildPackagesTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = CreatePackageListContent(packageDictionary) @@ -100,36 +96,20 @@ public async Task UpdateBuildInfoAsync( if (updateLatestPackageList) { - string latestPackagesPath = $"{versionsRepoPath}/Latest_Packages.txt"; - var allPackages = new Dictionary(packageDictionary); if (updateLastBuildPackageList) { - Dictionary existingPackages = await GetPackagesAsync(client, latestPackagesPath); - - if (existingPackages == null) - { - Trace.TraceInformation( - "No exising Latest_Packages file found; one will be " + - $"created in '{versionsRepoPath}'"); - } - else - { - // Add each existing package if there isn't a new package with the same id. - foreach (var package in existingPackages) - { - if (!allPackages.ContainsKey(package.Key)) - { - allPackages[package.Key] = package.Value; - } - } - } + await AddExistingPackages( + client, + new GitHubBranch("master", _project), + versionsRepoPath, + allPackages); } objects.Add(new GitObject { - Path = latestPackagesPath, + Path = $"{versionsRepoPath}/{BuildInfo.LatestPackagesTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = CreatePackageListContent(allPackages) @@ -140,7 +120,7 @@ public async Task UpdateBuildInfoAsync( { objects.Add(new GitObject { - Path = $"{versionsRepoPath}/Latest.txt", + Path = $"{versionsRepoPath}/{BuildInfo.LatestTxtFilename}", Type = GitObject.TypeBlob, Mode = GitObject.ModeFile, Content = prereleaseVersion @@ -184,22 +164,5 @@ public async Task UpdateBuildInfoAsync( } } } - - private async Task> GetPackagesAsync(GitHubClient client, string path) - { - string latestPackages = await client.GetGitHubFileContentsAsync( - path, - new GitHubBranch("master", _project)); - - if (latestPackages == null) - { - return null; - } - - using (var reader = new StringReader(latestPackages)) - { - return await BuildInfo.ReadPackageListAsync(reader); - } - } } } diff --git a/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs index 420f62d54b..3df58c520a 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/LocalVersionsRepoUpdater.cs @@ -2,21 +2,27 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace Microsoft.DotNet.VersionTools.Automation { public class LocalVersionsRepoUpdater : VersionsRepoUpdater { + public GitHubAuth GitHubAuth { get; set; } + /// - /// Updates only the Latest_Packages file in the specified on-disk versions repository dir. + /// Create Latest_Packages, and if versionsRepoBranch is passed, Last_Build_Packages. /// - public void UpdateBuildInfoLatestPackages( + public async Task UpdateBuildInfoFilesAsync( IEnumerable packagePaths, string localBaseDir, - string versionsRepoPath) + string versionsRepoPath, + GitHubBranch versionsRepoBranch) { if (packagePaths == null) { @@ -31,17 +37,39 @@ public void UpdateBuildInfoLatestPackages( throw new ArgumentException(nameof(versionsRepoPath)); } - Dictionary packages = CreatePackageInfoDictionary(CreatePackageInfos(packagePaths)); - string latestPackagesDir = Path.Combine( localBaseDir, versionsRepoPath); Directory.CreateDirectory(latestPackagesDir); + NupkgInfo[] packages = CreatePackageInfos(packagePaths).ToArray(); + + Dictionary packageDictionary = CreatePackageInfoDictionary(packages); + + if (versionsRepoBranch != null) + { + File.WriteAllText( + Path.Combine(latestPackagesDir, BuildInfo.LastBuildPackagesTxtFilename), + CreatePackageListContent(packageDictionary)); + + using (var client = new GitHubClient(GitHubAuth)) + { + await AddExistingPackages( + client, + versionsRepoBranch, + versionsRepoPath, + packageDictionary); + } + } + + File.WriteAllText( + Path.Combine(latestPackagesDir, BuildInfo.LatestTxtFilename), + GetPrereleaseVersion(packages)); + File.WriteAllText( Path.Combine(latestPackagesDir, BuildInfo.LatestPackagesTxtFilename), - CreatePackageListContent(packages)); + CreatePackageListContent(packageDictionary)); } } } diff --git a/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs b/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs index cb34456c42..993a0329d5 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/VersionsRepoUpdater.cs @@ -2,9 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; +using System.Threading.Tasks; namespace Microsoft.DotNet.VersionTools.Automation { @@ -31,5 +35,61 @@ protected static string CreatePackageListContent(Dictionary pack .OrderBy(t => t.Key) .Select(t => $"{t.Key} {t.Value}")); } + + protected static async Task AddExistingPackages( + GitHubClient client, + GitHubBranch branch, + string versionsRepoPath, + Dictionary packages) + { + Dictionary existingPackages = await GetPackagesAsync( + client, + branch, + $"{versionsRepoPath}/{BuildInfo.LatestPackagesTxtFilename}"); + + if (existingPackages == null) + { + Trace.TraceInformation( + "No exising Latest_Packages file found; one will be " + + $"created in '{versionsRepoPath}'"); + } + else + { + // Add each existing package if there isn't a new package with the same id. + foreach (var package in existingPackages) + { + if (!packages.ContainsKey(package.Key)) + { + packages[package.Key] = package.Value; + } + } + } + } + + private static async Task> GetPackagesAsync( + GitHubClient client, + GitHubBranch branch, + string path) + { + string latestPackages = await client.GetGitHubFileContentsAsync(path, branch); + + if (latestPackages == null) + { + return null; + } + + using (var reader = new StringReader(latestPackages)) + { + return await BuildInfo.ReadPackageListAsync(reader); + } + } + + protected static string GetPrereleaseVersion(NupkgInfo[] packages) + { + return packages + .Select(t => t.Prerelease) + .FirstOrDefault(prerelease => !string.IsNullOrEmpty(prerelease)) + ?? "stable"; + } } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs index 7716ab3f9d..84c128a65e 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs @@ -181,10 +181,11 @@ private async Task PushUploadsAsync( GitObject[] objects = uploads .Select(upload => new GitObject { - Path = $"{basePath}/{upload.Path}", + Path = upload.GetAbsolutePath(basePath), Mode = GitObject.ModeFile, Type = GitObject.TypeBlob, - Content = upload.Contents + // Always upload files using LF to avoid bad dev scenarios with Git autocrlf. + Content = upload.Contents.Replace("\r\n", "\n") }) .ToArray(); diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs index 970f07115a..c0c412119f 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs @@ -4,18 +4,24 @@ using Microsoft.DotNet.VersionTools.Util; using System.Collections.Generic; +using System.Linq; using System.Xml.Linq; namespace Microsoft.DotNet.VersionTools.BuildManifest.Model { public class PackageArtifactModel { - private static readonly string[] AttributeOrder = + private static readonly string[] RequiredAttributes = { nameof(Id), nameof(Version) }; + private static readonly string[] AttributeOrder = RequiredAttributes.Concat(new[] + { + nameof(OriginBuildName) + }).ToArray(); + public IDictionary Attributes { get; set; } = new Dictionary(); public string Id @@ -30,19 +36,25 @@ public string Version set { Attributes[nameof(Version)] = value; } } + public string OriginBuildName + { + get { return Attributes.GetOrDefault(nameof(OriginBuildName)); } + set { Attributes[nameof(OriginBuildName)] = value; } + } + public override string ToString() => $"Package {Id} {Version}"; public XElement ToXml() => new XElement( "Package", Attributes - .ThrowIfMissingAttributes(AttributeOrder) + .ThrowIfMissingAttributes(RequiredAttributes) .CreateXmlAttributes(AttributeOrder)); public static PackageArtifactModel Parse(XElement xml) => new PackageArtifactModel { Attributes = xml .CreateAttributeDictionary() - .ThrowIfMissingAttributes(AttributeOrder) + .ThrowIfMissingAttributes(RequiredAttributes) }; } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs index 8cf43c2bea..77b2b74454 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/SupplementaryUploadRequest.cs @@ -6,8 +6,32 @@ namespace Microsoft.DotNet.VersionTools.BuildManifest { public class SupplementaryUploadRequest { + /// + /// Path, relative to the primary upload dir or absolute with a leading '/'. + /// public string Path { get; set; } public string Contents { get; set; } + + /// + /// Combine currentPath and Path into the absolute path of this request. The result is + /// compatible with GitHub paths. This is similar to Path.Combine except: + /// + /// If Path is absolute (begins with '/'), the leading '/' is trimmed from the result. + /// + /// If Path is relative, the path is always joined with '/'. (Never '\'.) + /// + /// + /// The absolute path of a dir that Path should be made relative to, if Path isn't already + /// an absolute path. Can't start or end in '/'. + /// + public string GetAbsolutePath(string currentPath) + { + if (Path.StartsWith("/")) + { + return Path.Substring(1); + } + return $"{currentPath}/{Path}"; + } } } From ecacba8b94100c1dfd4d99a96e1a5bff58f3ea65 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 13 Feb 2018 15:37:31 -0600 Subject: [PATCH 17/49] Implement "join semaphore" groups --- .../PushOrchestratedBuildManifest.cs | 7 +- .../UpdateOrchestratedBuildManifest.cs | 46 +++++- .../Microsoft.DotNet.Build.Tasks.Feed.targets | 3 +- .../BuildManifest/BuildManifestClientTests.cs | 37 ++--- .../BuildManifest/BuildManifestChange.cs | 66 ++++++++ .../BuildManifest/BuildManifestClient.cs | 146 +++++++++++++----- .../BuildManifest/BuildManifestLocation.cs | 43 ++++++ .../BuildManifest/JoinSemaphoreGroup.cs | 18 +++ .../Microsoft.DotNet.VersionTools.csproj | 3 + 9 files changed, 302 insertions(+), 67 deletions(-) create mode 100644 src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs create mode 100644 src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs create mode 100644 src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs index 52495bb5e0..a7ab830f55 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs @@ -58,10 +58,13 @@ public override bool Execute() { var client = new BuildManifestClient(gitHubClient); - var pushTask = client.PushNewBuildAsync( + var location = new BuildManifestLocation( new GitHubProject(VersionsRepo, VersionsRepoOwner), $"heads/{VersionsRepoBranch}", - VersionsRepoPath, + VersionsRepoPath); + + var pushTask = client.PushNewBuildAsync( + location, model, CreateUploadRequests(SupplementaryFiles), CommitMessage); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs index 38c25727b0..17a9081541 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs @@ -29,6 +29,7 @@ private enum UpdateType } public const string XmlMetadataName = "Xml"; + private const string JoinSemaphorePathMetadataName = "JoinSemaphorePath"; /// /// Updates to perform on the manifest. The metadata 'UpdateType' selects a type of update, @@ -70,6 +71,15 @@ private enum UpdateType /// public ITaskItem[] SupplementaryFiles { get; set; } + /// + /// "Join semaphore" groups. A join semaphore is created when all semaphores in the group + /// are complete for a certain build. + /// + /// %(Identity): A semaphore name that is part of a join semaphore group. + /// %(JoinSemaphorePath): The name of the join semaphore, created when the parallel work joins. + /// + public ITaskItem[] JoinSemaphoreGroups { get; set; } + public override bool Execute() { if (string.IsNullOrEmpty(CommitMessage)) @@ -92,21 +102,45 @@ private async System.Threading.Tasks.Task PushChangeAsync(BuildManifestClient cl { try { - await client.PushChangeAsync( + var location = new BuildManifestLocation( new GitHubProject(VersionsRepo, VersionsRepoOwner), $"heads/{VersionsRepoBranch}", - VersionsRepoPath, + VersionsRepoPath); + + IEnumerable joinGroups = JoinSemaphoreGroups + .Select(item => new + { + ParallelPartPath = item.ItemSpec, + JoinSemaphorePath = item.GetMetadata(JoinSemaphorePathMetadataName) + }) + .GroupBy(j => j.JoinSemaphorePath, j => j.ParallelPartPath) + .Select(g => new JoinSemaphoreGroup + { + JoinSemaphorePath = g.Key, + ParallelSemaphorePaths = g + }); + + SupplementaryUploadRequest[] supplementaryUploads = + PushOrchestratedBuildManifest.CreateUploadRequests(SupplementaryFiles); + + var change = new BuildManifestChange( + location, + CommitMessage, OrchestratedBuildId, + SemaphoreNames, manifest => { foreach (var update in ManifestUpdates ?? Enumerable.Empty()) { ApplyUpdate(manifest, update); } - }, - SemaphoreNames, - PushOrchestratedBuildManifest.CreateUploadRequests(SupplementaryFiles), - CommitMessage); + }) + { + SupplementaryUploads = supplementaryUploads, + JoinSemaphoreGroups = joinGroups, + }; + + await client.PushChangeAsync(change); } catch (ManifestChangeOutOfDateException e) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets index df42ee742b..5e924fa333 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PackageFiles/Microsoft.DotNet.Build.Tasks.Feed.targets @@ -386,6 +386,7 @@ VersionsRepoBranch="$(VersionsRepoBranch)" CommitMessage="$(CommitMessage)" OrchestratedIdentitySummary="$(OrchestratedIdentitySummary)" - SupplementaryFiles="@(SupplementaryFiles)" /> + SupplementaryFiles="@(SupplementaryFiles)" + JoinSemaphoreGroups="@(JoinSemaphoreGroups)" /> diff --git a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs index 7be6b2825a..ab0e262235 100644 --- a/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs +++ b/src/Microsoft.DotNet.VersionTools.Tests/BuildManifest/BuildManifestClientTests.cs @@ -83,9 +83,7 @@ public async Task TestPushNewBuildAsync() .ReturnsAsync(() => null); await client.PushNewBuildAsync( - proj, - @ref, - basePath, + new BuildManifestLocation(proj, @ref, basePath), build, null, message); @@ -149,14 +147,12 @@ public async Task TestPushChangeSemaphoreAsync() .ReturnsAsync(() => null); await client.PushChangeAsync( - proj, - @ref, - basePath, - fakeExistingBuild.Identity.BuildId, - _ => { }, - new[] { addSemaphorePath }, - null, - message); + new BuildManifestChange( + new BuildManifestLocation(proj, @ref, basePath), + message, + fakeExistingBuild.Identity.BuildId, + new[] { addSemaphorePath }, + _ => { })); mockGitHub.VerifyAll(); } @@ -190,14 +186,13 @@ public async Task TestPushConflictingChangeAsync() await Assert.ThrowsAsync( async () => await client.PushChangeAsync( - proj, - @ref, - basePath, - fakeExistingBuild.Identity.BuildId, - _ => { }, - new[] { addSemaphorePath }, - null, - message)); + new BuildManifestChange( + new BuildManifestLocation(proj, @ref, basePath), + message, + fakeExistingBuild.Identity.BuildId, + new[] { addSemaphorePath }, + _ => { } + ))); mockGitHub.VerifyAll(); } @@ -245,9 +240,7 @@ public async Task TestPushConflictAsync() .ThrowsAsync(new NotFastForwardUpdateException("Testing non-fast-forward update.")); await client.PushNewBuildAsync( - proj, - @ref, - basePath, + new BuildManifestLocation(proj, @ref, basePath), build, null, message); diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs new file mode 100644 index 0000000000..04a90ca3a7 --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestChange.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using System; +using System.Collections.Generic; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class BuildManifestChange + { + public BuildManifestLocation Location { get; } + + public string CommitMessage { get; } + + public string OrchestratedBuildId { get; } + + public IEnumerable SemaphorePaths { get; } + + public Action ApplyModelChanges { get; } + + public IEnumerable JoinSemaphoreGroups { get; set; } + + public IEnumerable SupplementaryUploads { get; set; } + + public BuildManifestChange( + BuildManifestLocation location, + string commitMessage, + string orchestratedBuildId, + IEnumerable semaphorePaths, + Action applyModelChanges) + { + if (location == null) + { + throw new ArgumentNullException(nameof(location)); + } + + if (string.IsNullOrEmpty(commitMessage)) + { + throw new ArgumentException(nameof(commitMessage)); + } + + if (string.IsNullOrEmpty(orchestratedBuildId)) + { + throw new ArgumentException(nameof(orchestratedBuildId)); + } + + if (applyModelChanges == null) + { + throw new ArgumentNullException(nameof(applyModelChanges)); + } + + if (semaphorePaths == null) + { + throw new ArgumentNullException(nameof(semaphorePaths)); + } + + Location = location; + CommitMessage = commitMessage; + OrchestratedBuildId = orchestratedBuildId; + SemaphorePaths = semaphorePaths; + ApplyModelChanges = applyModelChanges; + } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs index 84c128a65e..8cd09ac110 100644 --- a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestClient.cs @@ -47,20 +47,27 @@ public async Task FetchSemaphoreAsync( project, @ref); + if (contents == null) + { + return null; + } + return SemaphoreModel.Parse(semaphorePath, contents); } public async Task PushNewBuildAsync( - GitHubProject project, - string @ref, - string basePath, + BuildManifestLocation location, OrchestratedBuildModel build, IEnumerable supplementaryUploads, string message) { await Retry.RunAsync(async attempt => { - string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha; + GitReference remoteRef = await _github.GetReferenceAsync( + location.GitHubProject, + location.GitHubRef); + + string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); @@ -83,63 +90,86 @@ await Retry.RunAsync(async attempt => }) .ToArray(); - return await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads); + return await PushUploadsAsync(location, message, remoteCommit, uploads); }); } - public async Task PushChangeAsync( - GitHubProject project, - string @ref, - string basePath, - string orchestratedBuildId, - Action changeModel, - IEnumerable semaphorePaths, - IEnumerable supplementaryUploads, - string message) + public async Task PushChangeAsync(BuildManifestChange change) { await Retry.RunAsync(async attempt => { + BuildManifestLocation location = change.Location; + // Get the current commit. Use this throughout to ensure a clean transaction. - string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha; + GitReference remoteRef = await _github.GetReferenceAsync( + location.GitHubProject, + location.GitHubRef); + + string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); - // This is a subsequent publish step: check to make sure the build id matches. - XElement remoteModelXml = await FetchModelXmlAsync(project, remoteCommit, basePath); + XElement remoteModelXml = await FetchModelXmlAsync( + location.GitHubProject, + remoteCommit, + location.GitHubBasePath); OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml); - if (orchestratedBuildId != remoteModel.Identity.BuildId) + // This is a subsequent publish step: make sure a new build hasn't happened already. + if (change.OrchestratedBuildId != remoteModel.Identity.BuildId) { throw new ManifestChangeOutOfDateException( - orchestratedBuildId, + change.OrchestratedBuildId, remoteModel.Identity.BuildId); } OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml); - changeModel(modifiedModel); + change.ApplyModelChanges(modifiedModel); - if (modifiedModel.Identity.BuildId != orchestratedBuildId) + if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId) { throw new ArgumentException( "Change action shouldn't modify BuildId. Changed from " + - $"'{orchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.", - nameof(changeModel)); + $"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.", + nameof(change)); } XElement modifiedModelXml = modifiedModel.ToXml(); - IEnumerable uploads = semaphorePaths.NullAsEmpty() + string[] changedSemaphorePaths = change.SemaphorePaths.ToArray(); + + // Check if any join groups are completed by this change. + var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty() + .Select(async g => new + { + Group = g, + Joinable = await IsGroupJoinableAsync( + location, + remoteCommit, + change.OrchestratedBuildId, + changedSemaphorePaths, + g) + }); + + var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks)) + .Where(g => g.Joinable) + .Select(g => g.Group.JoinSemaphorePath) + .ToArray(); + + IEnumerable semaphoreUploads = completeJoinedSemaphores + .Concat(changedSemaphorePaths) .Select(p => new SupplementaryUploadRequest { Path = p, Contents = new SemaphoreModel { - BuildId = orchestratedBuildId + BuildId = change.OrchestratedBuildId }.ToFileContent() - }) - .Concat(supplementaryUploads.NullAsEmpty()) - .ToArray(); + }); + + IEnumerable uploads = + semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty()); if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml)) { @@ -153,7 +183,11 @@ await Retry.RunAsync(async attempt => }); } - return await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads); + return await PushUploadsAsync( + location, + change.CommitMessage, + remoteCommit, + uploads); }); } @@ -171,9 +205,7 @@ private async Task FetchModelXmlAsync( } private async Task PushUploadsAsync( - GitHubProject project, - string @ref, - string basePath, + BuildManifestLocation location, string message, string remoteCommit, IEnumerable uploads) @@ -181,7 +213,7 @@ private async Task PushUploadsAsync( GitObject[] objects = uploads .Select(upload => new GitObject { - Path = upload.GetAbsolutePath(basePath), + Path = upload.GetAbsolutePath(location.GitHubBasePath), Mode = GitObject.ModeFile, Type = GitObject.TypeBlob, // Always upload files using LF to avoid bad dev scenarios with Git autocrlf. @@ -189,10 +221,13 @@ private async Task PushUploadsAsync( }) .ToArray(); - GitTree tree = await _github.PostTreeAsync(project, remoteCommit, objects); + GitTree tree = await _github.PostTreeAsync( + location.GitHubProject, + remoteCommit, + objects); GitCommit commit = await _github.PostCommitAsync( - project, + location.GitHubProject, message, tree.Sha, new[] { remoteCommit }); @@ -200,7 +235,11 @@ private async Task PushUploadsAsync( try { // Only fast-forward. Don't overwrite other changes: throw exception instead. - await _github.PatchReferenceAsync(project, @ref, commit.Sha, force: false); + await _github.PatchReferenceAsync( + location.GitHubProject, + location.GitHubRef, + commit.Sha, + force: false); } catch (NotFastForwardUpdateException e) { @@ -211,5 +250,40 @@ private async Task PushUploadsAsync( return true; } + + private async Task IsGroupJoinableAsync( + BuildManifestLocation location, + string commit, + string buildId, + IEnumerable changedSemaphorePaths, + JoinSemaphoreGroup joinGroup) + { + string[] remainingSemaphores = joinGroup + .ParallelSemaphorePaths + .Except(changedSemaphorePaths) + .ToArray(); + + if (remainingSemaphores.Length == joinGroup.ParallelSemaphorePaths.Count()) + { + // No semaphores in this group are changing: it can't be joinable by this update. + return false; + } + + // TODO: Avoid redundant fetches if multiple groups share a semaphore. https://github.com/dotnet/buildtools/issues/1910 + bool[] remainingSemaphoreIsComplete = await Task.WhenAll( + remainingSemaphores.Select( + async path => + { + SemaphoreModel semaphore = await FetchSemaphoreAsync( + location.GitHubProject, + commit, + location.GitHubBasePath, + path); + + return semaphore?.BuildId == buildId; + })); + + return remainingSemaphoreIsComplete.All(x => x); + } } } diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs new file mode 100644 index 0000000000..3fd32eecf2 --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/BuildManifestLocation.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.VersionTools.Automation; +using System; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class BuildManifestLocation + { + public GitHubProject GitHubProject { get; } + + public string GitHubRef { get; } + + public string GitHubBasePath { get; } + + public BuildManifestLocation( + GitHubProject gitHubProject, + string gitHubRef, + string gitHubBasePath) + { + if (gitHubProject == null) + { + throw new ArgumentNullException(nameof(gitHubProject)); + } + + if (string.IsNullOrEmpty(gitHubRef)) + { + throw new ArgumentException(nameof(gitHubRef)); + } + + if (string.IsNullOrEmpty(gitHubBasePath)) + { + throw new ArgumentException(nameof(gitHubBasePath)); + } + + GitHubProject = gitHubProject; + GitHubRef = gitHubRef; + GitHubBasePath = gitHubBasePath; + } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs b/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs new file mode 100644 index 0000000000..7491a19dbc --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools/BuildManifest/JoinSemaphoreGroup.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.DotNet.VersionTools.BuildManifest +{ + public class JoinSemaphoreGroup + { + public string JoinSemaphorePath { get; set; } + + /// + /// Names of the semaphores that must all complete to update the join semaphore. + /// + public IEnumerable ParallelSemaphorePaths { get; set; } + } +} diff --git a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj index c1fe1017e8..94b753ff76 100644 --- a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj +++ b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj @@ -30,7 +30,9 @@ + + @@ -41,6 +43,7 @@ + From b131ddcdf60ad35ef827fe1619dc6bb8e239c27e Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 13 Feb 2018 15:38:38 -0600 Subject: [PATCH 18/49] Set VersionTools LangVersion to 6 This reflects the capabilities of the build system and lets tooling avoid invalid changes. --- .../Microsoft.DotNet.VersionTools.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj index 94b753ff76..a0df9f95c9 100644 --- a/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj +++ b/src/Microsoft.DotNet.VersionTools/Microsoft.DotNet.VersionTools.csproj @@ -11,6 +11,7 @@ false 1.0.0.0 .NETStandard,Version=v1.5 + 6 From 9afe1cd4750794ea031a3fa1284ee1c1ebef5a3e Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Thu, 15 Feb 2018 13:48:28 -0800 Subject: [PATCH 19/49] Improve unit test output to show how to repro directly (#1913) * Improve unix template * Improve windows template * Briefer --- .../PackageFiles/RunnerTemplate.Unix.txt | 10 ++++++---- .../PackageFiles/RunnerTemplate.Windows.txt | 8 ++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt index ed00d57972..bb73630c89 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt @@ -45,7 +45,6 @@ then echo error: RUNTIME_PATH is not defined. Usage: $0 RUNTIME_PATH exit -1 fi -echo Using $RUNTIME_DIR as the test runtime folder. # ========================= BEGIN Core File Setup ============================ if [ "$(uname -s)" == "Darwin" ]; then @@ -72,15 +71,18 @@ fi # ========================= END Core File Setup ============================== # ========================= BEGIN Test Execution ============================= -echo Running tests... Start time: $(date +"%T") -echo Commands: +echo ----- start $(date +"%T") =============== To repro directly: ===================================================== +echo pushd $EXECUTION_DIR [[TestRunCommandsEcho]] +echo popd +echo =========================================================================================================== pushd $EXECUTION_DIR [[TestRunCommands]] test_exitcode=$? popd -echo Finished running tests. End time=$(date +"%T"). Return value was $test_exitcode +echo ----- end $(date +"%T") ----- exit code $test_exitcode ---------------------------------------------------------- # ========================= END Test Execution =============================== + # ======================= BEGIN Core File Inspection ========================= if [ "$(uname -s)" == "Linux" ]; then # Depending on distro/configuration, the core files may either be named "core" diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Windows.txt b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Windows.txt index 22689cba30..5a8d9ae266 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Windows.txt +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Windows.txt @@ -11,13 +11,17 @@ set EXECUTION_DIR=%~dp0 echo Executing in %EXECUTION_DIR% :: ========================= BEGIN Test Execution ============================= -echo Running tests... Start time: %TIME% +echo ----- start %TIME% =============== To repro directly: ===================================================== +echo pushd %EXECUTION_DIR% + [[TestRunCommandsEcho]] +echo popd +echo =========================================================================================================== pushd %EXECUTION_DIR% echo on [[TestRunCommands]] @echo off popd -echo Finished running tests. End time=%TIME%, Exit code = %ERRORLEVEL% +echo ----- end %TIME% ----- exit code %ERRORLEVEL% ---------------------------------------------------------- EXIT /B %ERRORLEVEL% :: ========================= END Test Execution ================================= From 83a46469e8e62e1b0c1f3ed6ea5f87a14d614117 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 15 Feb 2018 16:28:47 -0800 Subject: [PATCH 20/49] Revert "Use continuation marker in UploadToAzure" --- .../UploadToAzure.cs | 36 +++++-------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs index e3c0889b76..568eda1964 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/UploadToAzure.cs @@ -79,9 +79,9 @@ public async Task ExecuteAsync(CancellationToken ct) } Log.LogMessage( - MessageImportance.Normal, - "Begin uploading blobs to Azure account {0} in container {1}.", - AccountName, + MessageImportance.Normal, + "Begin uploading blobs to Azure account {0} in container {1}.", + AccountName, ContainerName); if (Items.Length == 0) @@ -91,7 +91,7 @@ public async Task ExecuteAsync(CancellationToken ct) } // first check what blobs are present - string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list&maxresults=3000"; + string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -99,15 +99,14 @@ public async Task ExecuteAsync(CancellationToken ct) { using (HttpClient client = new HttpClient()) { - string nextMarker = string.Empty; var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); - Log.LogMessage(MessageImportance.Low, "Sending request(s) to enumerate existing blobs"); + Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { var doc = new XmlDocument(); - string rawXml = await response.Content.ReadAsStringAsync(); - doc.LoadXml(rawXml); + doc.LoadXml(await response.Content.ReadAsStringAsync()); + XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); foreach (XmlNode node in nodes) @@ -115,27 +114,8 @@ public async Task ExecuteAsync(CancellationToken ct) blobsPresent.Add(node["Name"].InnerText); } - nextMarker = doc.DocumentElement.GetElementsByTagName("NextMarker").Cast().FirstOrDefault()?.InnerText; - } - // Quick implementation avoiding refactoring to make this work for too many blobs while awaiting - // reimplementation via Azure SDK Client Libraries. - while (!string.IsNullOrEmpty(nextMarker)) - { - var continuationRequest = AzureHelper.RequestMessage("GET", $"{checkListUrl}&marker={nextMarker}", AccountName, AccountKey); - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, continuationRequest)) - { - var doc = new XmlDocument(); - string rawXml = await response.Content.ReadAsStringAsync(); - doc.LoadXml(rawXml); - XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); - foreach (XmlNode node in nodes) - { - blobsPresent.Add(node["Name"].InnerText); - } - nextMarker = doc.DocumentElement.GetElementsByTagName("NextMarker").Cast().FirstOrDefault()?.InnerText; - } + Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); } - Log.LogMessage(MessageImportance.Low, $"Found {blobsPresent.Count} blob(s) in {ContainerName}"); } using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) From a0c661753ac1fdd21311b00ed76ff40a33208b24 Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Thu, 15 Feb 2018 18:47:23 -0800 Subject: [PATCH 21/49] Enforce order of references passed to ApiCompat (#1916) ApiCompat depends on the order of the references, this change ensures that project references come first. --- .../PackageFiles/ApiCompat.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets index eef26c9097..3ef602cbe0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets @@ -31,7 +31,9 @@ <_DependencyDirectoriesTemp Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> - <_DependencyDirectories Include="%(_DependencyDirectoriesTemp.Identity)" /> + + <_DependencyDirectories Condition="'%(_DependencyDirectoriesTemp.ReferenceSourceTarget)'=='ProjectReference'" Include="%(_DependencyDirectoriesTemp.Identity)" /> + <_DependencyDirectories Condition="'%(_DependencyDirectoriesTemp.ReferenceSourceTarget)'!='ProjectReference'" Include="%(_DependencyDirectoriesTemp.Identity)" /> <_ContractDependencyDirectories Include="@(ResolvedMatchingContract->'%(RootDir)%(Directory)')" /> <_ContractDependencyDirectories Include="$(ContractOutputPath)" /> From 3c7cf6f5adc80d643b12a4f8cb231a6097b55a1f Mon Sep 17 00:00:00 2001 From: Tomas Weinfurt Date: Fri, 16 Feb 2018 15:03:36 -0800 Subject: [PATCH 22/49] use "$@" instead of $* to properly preserve arguments with spaces --- src/Microsoft.DotNet.Build.Tasks/PackageFiles/msbuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/msbuild.sh b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/msbuild.sh index a0ebae2e21..a095d0c11c 100755 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/msbuild.sh +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/msbuild.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$working_tree_root/dotnetcli/dotnet msbuild $* +$working_tree_root/dotnetcli/dotnet msbuild "$@" exit $? From 4c3c7333645c99b03ef39aa0a51b66895f661ffb Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 20 Feb 2018 11:22:14 -0800 Subject: [PATCH 23/49] Fix default MaxRetryCount Because my initial in-head implementation had this as "MaxDeliveryCount" I made an off by one error in this. This explains the hurried fixes we had to crank out to deal with this behavior. --- .../PackageFiles/CloudTest.Helix.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index f0acbd51db..d7e544789a 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -50,7 +50,7 @@ HelixJobProperties- Must be JSON. String describing Helix MC-specific metadata. Default: HelixArchLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: HelixConfigLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: - MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 1 + MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 0 (no retry) ********************************************************************************************************************************** Re-queuing Properties: @@ -84,7 +84,7 @@ $(ArchivesRoot)$(SupplementalPayloadFilename) false https://helix.dot.net/api/2016-06-28/jobs - 1 + 0 From bee934ecff4004758945fd0f0835a6a814efeb89 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 20 Feb 2018 16:21:00 -0600 Subject: [PATCH 24/49] Run PPDB conversion in the same MSBuild process The AccessViolationException being worked around should no longer occur. This allows properties such as SkipCreateWindowsPdbsFromPortablePdbs to flow to CreateWindowsPdbsFromPortablePdbs without specifically being passed. --- .../PackageFiles/Symbols.targets | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Symbols.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Symbols.targets index 8e96fb92d7..06ee7466b1 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Symbols.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/Symbols.targets @@ -238,38 +238,19 @@ - + $(SymbolPackageExtractDir)**\*.pdb - - - $(SymbolPackageExtractDir)WindowsPDB/ - - - <_ConvertPdbCommand>"$(MSBuildBinPath)\msbuild.exe" - <_ConvertPdbCommand>$(_ConvertPdbCommand) /v:Detailed - <_ConvertPdbCommand>$(_ConvertPdbCommand) "$(MSBuildProjectFullPath)" - <_ConvertPdbCommand>$(_ConvertPdbCommand) /t:CreateWindowsPdbsFromPortablePdbs - <_ConvertPdbCommand>$(_ConvertPdbCommand) "/p:PortablePdbToConvertGlob=$(PortablePdbToConvertGlob)" - <_ConvertPdbCommand>$(_ConvertPdbCommand) "/p:WindowsPdbConversionTargetPath=$(WindowsPdbConversionTargetPath)" + $(SymbolPackageExtractDir)WindowsPDB\ - - - - + - + NormalizeAssemblyName; @@ -87,7 +87,7 @@ - @@ -114,6 +114,11 @@ FxResources.$(AssemblyName).SR.resources FxResources.$(AssemblyName).SR + + + + MSBuild:GenerateResourcesSource + From e008d7f790edd19e4dbb6509b9f6eeeb7ca987f2 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 21 Feb 2018 13:53:20 -0800 Subject: [PATCH 29/49] Revert "Allow generating resources cs code at design time (#1921)" (#1924) This reverts commit 5812d960f73ed3b2f021d5a4af5a3571ed7993ad. --- .../PackageFiles/resources.targets | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets index efb5d8b926..1e1b6c7376 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets @@ -44,8 +44,8 @@ - @@ -58,9 +58,9 @@ - + - + NormalizeAssemblyName; @@ -87,7 +87,7 @@ - @@ -114,11 +114,6 @@ FxResources.$(AssemblyName).SR.resources FxResources.$(AssemblyName).SR - - - - MSBuild:GenerateResourcesSource - From 63db91f2fe1a7aec78ff190e7307dbdc0b62757d Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 21 Feb 2018 17:28:12 -0800 Subject: [PATCH 30/49] Enable Design Time Resource Code Generation (#1926) --- .../PackageFiles/resources.targets | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets index 1e1b6c7376..89a96fa1e2 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/resources.targets @@ -44,8 +44,8 @@ - @@ -58,9 +58,9 @@ - + - + NormalizeAssemblyName; @@ -87,7 +87,7 @@ - @@ -113,6 +113,8 @@ true FxResources.$(AssemblyName).SR.resources FxResources.$(AssemblyName).SR + + MSBuild:GenerateResourcesSource From ccf477726ca4afdebc6ef5710d0d01a3db03e8e5 Mon Sep 17 00:00:00 2001 From: Chad Nedzlek Date: Wed, 21 Feb 2018 18:30:33 -0800 Subject: [PATCH 31/49] Add HelixAttempt support to helix job sending --- .../PackageFiles/CloudTest.Helix.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index d7e544789a..a04aaf7fd1 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -338,6 +338,7 @@ $(HelixJobType) $(HelixSource) $(MaxRetryCount) + $(HelixAttempt) From 173f96eecd754d5176892599d62b133b13fa046a Mon Sep 17 00:00:00 2001 From: Chad Nedzlek Date: Wed, 21 Feb 2018 18:57:01 -0800 Subject: [PATCH 32/49] Add some documentation of HelixAttempt --- .../PackageFiles/CloudTest.Helix.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index a04aaf7fd1..1d4115c382 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -51,6 +51,7 @@ HelixArchLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: HelixConfigLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 0 (no retry) + HelixAttempt - A lexically monotonic id distiquishing overriding job executions Default: ********************************************************************************************************************************** Re-queuing Properties: From 4fb0060e4e242cbaf4cc5a99d95fca270678459c Mon Sep 17 00:00:00 2001 From: Chad Nedzlek Date: Wed, 21 Feb 2018 18:59:19 -0800 Subject: [PATCH 33/49] Make documentation a bit more explicit --- .../PackageFiles/CloudTest.Helix.targets | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index 1d4115c382..5e677b03cb 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -51,7 +51,8 @@ HelixArchLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: HelixConfigLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 0 (no retry) - HelixAttempt - A lexically monotonic id distiquishing overriding job executions Default: + HelixAttempt - A lexically monotonic increasing string distiquishing Default: + jobs whose results should override previous executions. ********************************************************************************************************************************** Re-queuing Properties: From 92bd8486db627660500b878c1836a8eff5228467 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 22 Feb 2018 16:28:21 -0600 Subject: [PATCH 34/49] Use TLS 1.2 security protocol in GitHubClient (#1925) --- .../GitHubApi/GitHubClient.Desktop.cs | 17 +++++++++++++++++ .../Microsoft.DotNet.VersionTools.net45.csproj | 3 +++ .../Automation/GitHubApi/GitHubClient.cs | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.DotNet.VersionTools.net45/Automation/GitHubApi/GitHubClient.Desktop.cs diff --git a/src/Microsoft.DotNet.VersionTools.net45/Automation/GitHubApi/GitHubClient.Desktop.cs b/src/Microsoft.DotNet.VersionTools.net45/Automation/GitHubApi/GitHubClient.Desktop.cs new file mode 100644 index 0000000000..f987535640 --- /dev/null +++ b/src/Microsoft.DotNet.VersionTools.net45/Automation/GitHubApi/GitHubClient.Desktop.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net; + +namespace Microsoft.DotNet.VersionTools.Automation.GitHubApi +{ + public partial class GitHubClient : IGitHubClient, IDisposable + { + static GitHubClient() + { + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + } + } +} diff --git a/src/Microsoft.DotNet.VersionTools.net45/Microsoft.DotNet.VersionTools.net45.csproj b/src/Microsoft.DotNet.VersionTools.net45/Microsoft.DotNet.VersionTools.net45.csproj index 6aa012f39c..ae59217288 100644 --- a/src/Microsoft.DotNet.VersionTools.net45/Microsoft.DotNet.VersionTools.net45.csproj +++ b/src/Microsoft.DotNet.VersionTools.net45/Microsoft.DotNet.VersionTools.net45.csproj @@ -13,6 +13,9 @@ .NETFramework,Version=v4.5 + + + diff --git a/src/Microsoft.DotNet.VersionTools/Automation/GitHubApi/GitHubClient.cs b/src/Microsoft.DotNet.VersionTools/Automation/GitHubApi/GitHubClient.cs index 0f1da46a95..0d4b10e6a9 100644 --- a/src/Microsoft.DotNet.VersionTools/Automation/GitHubApi/GitHubClient.cs +++ b/src/Microsoft.DotNet.VersionTools/Automation/GitHubApi/GitHubClient.cs @@ -16,7 +16,7 @@ namespace Microsoft.DotNet.VersionTools.Automation.GitHubApi { - public class GitHubClient : IGitHubClient, IDisposable + public partial class GitHubClient : IGitHubClient, IDisposable { /// /// A default user agent to use if none is provided to the constructor. GitHub always From d5e58559ea1265daa6c6aacd72e6c40ed0030741 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Mon, 26 Feb 2018 19:06:30 -0800 Subject: [PATCH 35/49] Add SafeFindHandle to fix OverflowException (#1930) * Simplest change * Typo --- src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs index dfa6a62c0b..81d36f1f48 100644 --- a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs +++ b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs @@ -595,7 +595,7 @@ private string[] GetFilesByAttributes(string directory, FileAttributes attribute IntPtr hFile = FindFirstFile(directory + "\\*", out findData); int error = Marshal.GetLastWin32Error(); - if (hFile.ToInt32() != -1) + if (hFile != IntPtr.Zero && hFile != new IntPtr(-1)) { do { From f8444e2bfdd036a2de0fcc7041b12ccc22c09168 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 27 Feb 2018 21:27:50 -0600 Subject: [PATCH 36/49] Use the built tasks to update dotnet/versions This allows us to use the TLS 1.2 UpdatePublishedVersions fix without upgrading the BuildTools toolset version. --- build.proj | 11 +++++++++++ dir.props | 10 ++++++++++ .../Microsoft.DotNet.Build.Tasks.net45.csproj | 1 + .../Microsoft.DotNet.Build.Tasks.csproj | 3 +-- src/nuget/Microsoft.DotNet.BuildTools.nuspec | 1 - 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/build.proj b/build.proj index 5bfa60c559..130c1dbb59 100644 --- a/build.proj +++ b/build.proj @@ -12,6 +12,17 @@ true + + + diff --git a/dir.props b/dir.props index cec15e8a06..e731e2f282 100644 --- a/dir.props +++ b/dir.props @@ -210,6 +210,16 @@ $(BinDir)$(OSPlatformConfig) + + + .net45 + $(PackagesBasePath)\Microsoft.DotNet.Build.Tasks$(BuildToolsOutputProjectSuffix)\Microsoft.DotNet.Build.Tasks.dll + + true diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj index 827abcf5fb..c88080caf8 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj @@ -9,6 +9,7 @@ {B3331D88-7569-42D5-919B-F267DA011911} net45 + .net45 diff --git a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj index 1bd51ce5b0..14a5584092 100644 --- a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj +++ b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj @@ -134,10 +134,9 @@ {2179f9b5-1dba-4563-9402-a94de75ea9fa} Microsoft.Cci.Extensions - + {8d524fa5-a8c5-4ebd-ba8b-2a4fed03ee58} Microsoft.DotNet.VersionTools - False diff --git a/src/nuget/Microsoft.DotNet.BuildTools.nuspec b/src/nuget/Microsoft.DotNet.BuildTools.nuspec index 238333c650..231aa29e38 100644 --- a/src/nuget/Microsoft.DotNet.BuildTools.nuspec +++ b/src/nuget/Microsoft.DotNet.BuildTools.nuspec @@ -51,7 +51,6 @@ - From 4d76b216f52193e6da3d1f6f4e2a829c0690f171 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 28 Feb 2018 13:42:54 -0600 Subject: [PATCH 37/49] Revert "Use the built tasks to update dotnet/versions" This reverts commit f8444e2bfdd036a2de0fcc7041b12ccc22c09168. --- build.proj | 11 ----------- dir.props | 10 ---------- .../Microsoft.DotNet.Build.Tasks.net45.csproj | 1 - .../Microsoft.DotNet.Build.Tasks.csproj | 3 ++- src/nuget/Microsoft.DotNet.BuildTools.nuspec | 1 + 5 files changed, 3 insertions(+), 23 deletions(-) diff --git a/build.proj b/build.proj index 130c1dbb59..5bfa60c559 100644 --- a/build.proj +++ b/build.proj @@ -12,17 +12,6 @@ true - - - diff --git a/dir.props b/dir.props index e731e2f282..cec15e8a06 100644 --- a/dir.props +++ b/dir.props @@ -210,16 +210,6 @@ $(BinDir)$(OSPlatformConfig) - - - .net45 - $(PackagesBasePath)\Microsoft.DotNet.Build.Tasks$(BuildToolsOutputProjectSuffix)\Microsoft.DotNet.Build.Tasks.dll - - true diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj index c88080caf8..827abcf5fb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj @@ -9,7 +9,6 @@ {B3331D88-7569-42D5-919B-F267DA011911} net45 - .net45 diff --git a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj index 14a5584092..1bd51ce5b0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj +++ b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj @@ -134,9 +134,10 @@ {2179f9b5-1dba-4563-9402-a94de75ea9fa} Microsoft.Cci.Extensions - + {8d524fa5-a8c5-4ebd-ba8b-2a4fed03ee58} Microsoft.DotNet.VersionTools + False diff --git a/src/nuget/Microsoft.DotNet.BuildTools.nuspec b/src/nuget/Microsoft.DotNet.BuildTools.nuspec index 231aa29e38..238333c650 100644 --- a/src/nuget/Microsoft.DotNet.BuildTools.nuspec +++ b/src/nuget/Microsoft.DotNet.BuildTools.nuspec @@ -51,6 +51,7 @@ + From 9cc52a0ca48f96fa46a1bad63bc6a18cfb57c868 Mon Sep 17 00:00:00 2001 From: Jose Perez Rodriguez Date: Fri, 2 Mar 2018 10:27:45 -0800 Subject: [PATCH 38/49] Updating NuGet mappings --- dependencies.props | 2 +- .../project.json | 2 +- src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json | 4 ++-- src/Microsoft.DotNet.Build.Tasks.Feed/project.json | 4 ++-- .../src.Desktop/project.json | 6 +++--- .../src/project.json | 6 +++--- .../test.Desktop/project.json | 2 +- .../test/project.json | 2 +- src/Microsoft.DotNet.Build.Tasks.net45/project.json | 8 ++++---- .../EncryptedConfigNuGetRestore.cs | 4 +++- src/Microsoft.DotNet.Build.Tasks/project.json | 8 ++++---- src/Microsoft.DotNet.VersionTools.net45/project.json | 4 ++-- src/Microsoft.DotNet.VersionTools/project.json | 4 ++-- src/nuget/Microsoft.DotNet.VersionTools.nuspec | 4 ++-- 14 files changed, 31 insertions(+), 29 deletions(-) diff --git a/dependencies.props b/dependencies.props index cddfb87d08..718471a888 100644 --- a/dependencies.props +++ b/dependencies.props @@ -89,7 +89,7 @@ - 4.3.0 + 4.4.0 diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json b/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json index 38a9984c10..2498fde577 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json +++ b/src/Microsoft.DotNet.Build.CloudTestTasks.net45/project.json @@ -1,7 +1,7 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", + "NuGet.Versioning": "4.4.0", "System.Reflection.Metadata": "1.4.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json index 404e925be2..c2ef6b79ae 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/project.json @@ -1,8 +1,8 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", - "NuGet.Packaging": "4.3.0", + "NuGet.Versioning": "4.4.0", + "NuGet.Packaging": "4.4.0", "System.Reflection.Metadata": "1.4.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03", "sleetlib": "2.2.24" diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/project.json b/src/Microsoft.DotNet.Build.Tasks.Feed/project.json index 62186756a3..0def9d910c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/project.json @@ -6,8 +6,8 @@ "System.Reflection.Metadata": "1.3.0", "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Versioning": "4.3.0", - "NuGet.Packaging": "4.3.0", + "NuGet.Versioning": "4.4.0", + "NuGet.Packaging": "4.4.0", "SleetLib": "2.2.24", "WindowsAzure.Storage": "8.5.0" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json index e79a09eef8..8aea195359 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src.Desktop/project.json @@ -2,9 +2,9 @@ "dependencies": { "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", "System.Reflection.Metadata": "1.0.22", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json index 99fd0ee0fa..4a6676e771 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/project.json @@ -4,9 +4,9 @@ "Microsoft.Build.Framework": "0.1.0-preview-00022", "Microsoft.Build.Utilities.Core": "0.1.0-preview-00022", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", "NETStandard.Library": "1.6.0" }, "frameworks": { diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json index 0edfa8ff07..c2e72dec1e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/test.Desktop/project.json @@ -2,7 +2,7 @@ "dependencies": { "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", + "NuGet.Packaging": "4.4.0", "System.Reflection.Metadata": "1.3.0", "xunit": "2.1.0", "xunit.runner.visualstudio": "2.1.0" diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json index 5ab29d24c9..5d752d4121 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/project.json @@ -4,7 +4,7 @@ "Microsoft.Build.Framework": "0.1.0-preview-00022", "Microsoft.Build.Utilities.Core": "0.1.0-preview-00022", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", + "NuGet.Packaging": "4.4.0", "NETStandard.Library": "1.6.0", "Microsoft.NETCore.Targets": "1.0.2", "xunit": "2.1.0", diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/project.json b/src/Microsoft.DotNet.Build.Tasks.net45/project.json index 6ba3bdf0b7..685ae3fded 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/project.json +++ b/src/Microsoft.DotNet.Build.Tasks.net45/project.json @@ -4,10 +4,10 @@ "Microsoft.DotNet.PlatformAbstractions": "1.2.0-beta-001090", "Microsoft.NETCore.Platforms": "1.0.1", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Collections.Immutable": "1.3.1", "System.Reflection.Metadata": "1.4.2", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" diff --git a/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs b/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs index 72a933add7..f34cd0e57f 100644 --- a/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs +++ b/src/Microsoft.DotNet.Build.Tasks/EncryptedConfigNuGetRestore.cs @@ -145,7 +145,9 @@ private RestoreSummaryRequest Create( restoreContext.ApplyStandardProperties(request); - var summaryRequest = new RestoreSummaryRequest(request, inputPath, settings, sources); + IEnumerable configFiles = SettingsUtility.GetConfigFilePaths(settings); + + var summaryRequest = new RestoreSummaryRequest(request, inputPath, configFiles, sources); return summaryRequest; } diff --git a/src/Microsoft.DotNet.Build.Tasks/project.json b/src/Microsoft.DotNet.Build.Tasks/project.json index b4a27ee02b..e1b18e95e0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/project.json +++ b/src/Microsoft.DotNet.Build.Tasks/project.json @@ -12,10 +12,10 @@ }, "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Commands": "4.3.0", - "NuGet.Packaging": "4.3.0", - "NuGet.ProjectModel": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Commands": "4.4.0", + "NuGet.Packaging": "4.4.0", + "NuGet.ProjectModel": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Diagnostics.TraceSource": "4.0.0", "System.Dynamic.Runtime": "4.0.11", "System.IO.UnmanagedMemoryStream": "4.3.0", diff --git a/src/Microsoft.DotNet.VersionTools.net45/project.json b/src/Microsoft.DotNet.VersionTools.net45/project.json index c721fa1f7d..305167f85e 100644 --- a/src/Microsoft.DotNet.VersionTools.net45/project.json +++ b/src/Microsoft.DotNet.VersionTools.net45/project.json @@ -1,8 +1,8 @@ { "dependencies": { "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Packaging": "4.4.0", + "NuGet.Versioning": "4.4.0", "Microsoft.NETCore.Platforms": "1.0.1", "System.Runtime.InteropServices.RuntimeInformation": "4.4.0-beta-24813-03" }, diff --git a/src/Microsoft.DotNet.VersionTools/project.json b/src/Microsoft.DotNet.VersionTools/project.json index 46195abfd3..28a6c09a2a 100644 --- a/src/Microsoft.DotNet.VersionTools/project.json +++ b/src/Microsoft.DotNet.VersionTools/project.json @@ -2,8 +2,8 @@ "dependencies": { "NETStandard.Library": "1.6.0", "Newtonsoft.Json": "9.0.1", - "NuGet.Packaging": "4.3.0", - "NuGet.Versioning": "4.3.0", + "NuGet.Packaging": "4.4.0", + "NuGet.Versioning": "4.4.0", "System.Diagnostics.Process": "4.1.0", "System.Diagnostics.TraceSource": "4.0.0" }, diff --git a/src/nuget/Microsoft.DotNet.VersionTools.nuspec b/src/nuget/Microsoft.DotNet.VersionTools.nuspec index 536aee9c3a..7470a606a7 100644 --- a/src/nuget/Microsoft.DotNet.VersionTools.nuspec +++ b/src/nuget/Microsoft.DotNet.VersionTools.nuspec @@ -17,8 +17,8 @@ - - + + From 1733178aaf16618b5e5e43ebc5a93b25f1776c3a Mon Sep 17 00:00:00 2001 From: Chad Nedzlek Date: Fri, 2 Mar 2018 13:17:07 -0800 Subject: [PATCH 39/49] Fix typo --- .../PackageFiles/CloudTest.Helix.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index 5e677b03cb..9d2e2fb990 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -51,7 +51,7 @@ HelixArchLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: HelixConfigLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 0 (no retry) - HelixAttempt - A lexically monotonic increasing string distiquishing Default: + HelixAttempt - A lexically monotonic increasing string distinquishing Default: jobs whose results should override previous executions. ********************************************************************************************************************************** From 2dbd445ffd6c5049392412f187b7c71f7674d3f2 Mon Sep 17 00:00:00 2001 From: Chad Nedzlek Date: Fri, 2 Mar 2018 14:08:05 -0800 Subject: [PATCH 40/49] Even more typos --- .../PackageFiles/CloudTest.Helix.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets index 9d2e2fb990..54d953944b 100644 --- a/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets +++ b/src/Microsoft.DotNet.Build.CloudTestTasks/PackageFiles/CloudTest.Helix.targets @@ -51,7 +51,7 @@ HelixArchLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: HelixConfigLabel - If HelixJobProperties is not set, we'll use this to fill it out Default: MaxRetryCount - Max automatic retry of workitems which do not return 0 Default: 0 (no retry) - HelixAttempt - A lexically monotonic increasing string distinquishing Default: + HelixAttempt - A lexically monotonic increasing string distinguishing Default: jobs whose results should override previous executions. ********************************************************************************************************************************** From f0f57a872b537f3b97da00cfe356bf737fc9c17d Mon Sep 17 00:00:00 2001 From: Juan Carlos Aguilera Mendez Date: Mon, 5 Mar 2018 15:35:13 -0800 Subject: [PATCH 41/49] Don't delete additional directory root --- src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs index 81d36f1f48..1804748408 100644 --- a/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs +++ b/src/Microsoft.DotNet.Build.Tasks/CleanupVSTSAgent.cs @@ -334,7 +334,7 @@ private async System.Threading.Tasks.Task CleanupDirsAsync() cleanupTasks.Clear(); foreach (string additionalCleanupDirectory in _additionalCleanupDirectories) { - cleanupTasks.Add(CleanupDirectoryAsync(additionalCleanupDirectory, 0, true)); + cleanupTasks.Add(CleanupDirectoryAsync(additionalCleanupDirectory, 0, true, true)); } System.Threading.Tasks.Task.WaitAll(cleanupTasks.ToArray()); @@ -348,7 +348,7 @@ private async System.Threading.Tasks.Task CleanupAgentAsync(string workDir return returnStatus; } - private async System.Threading.Tasks.Task CleanupDirectoryAsync(string directory, int attempts = 0, bool ignoreExceptions = false) + private async System.Threading.Tasks.Task CleanupDirectoryAsync(string directory, int attempts = 0, bool ignoreExceptions = false, bool isAdditionalDirectory = false) { try { @@ -392,6 +392,13 @@ private async System.Threading.Tasks.Task CleanupDirectoryAsync(string dir } #endif Log.LogMessage("Success"); + + // For additional directories we want only the contents deleted but not the root. The easiest is just to recreate the root folder + // after the whole folder has been created. Since this just applies to additional ('known') folders we should not worry about long paths. + if (isAdditionalDirectory && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } } return true; } From f78c7c067026255ba171886c7af1516f54da70d0 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Mon, 5 Mar 2018 18:01:12 -0600 Subject: [PATCH 42/49] Add empty Directory.Build.props and targets files so the tool-runtime project doesn't pick up the Directory.Build.props and targets from the calling repo. --- .../PackageFiles/tool-runtime/Directory.Build.props | 6 ++++++ .../PackageFiles/tool-runtime/Directory.Build.targets | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props create mode 100644 src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props new file mode 100644 index 0000000000..a6fdf8ec91 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.props @@ -0,0 +1,6 @@ + + + diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets new file mode 100644 index 0000000000..68a1e6ac70 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/tool-runtime/Directory.Build.targets @@ -0,0 +1,6 @@ + + + \ No newline at end of file From e911bc7868472909e884a641af03626b1ca9036e Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 6 Mar 2018 13:43:32 -0600 Subject: [PATCH 43/49] Use AssemblyResolver during PushToBlobFeed Fixes failure to resolve NuGet.Common, Version=4.3.0.5 from Sleet dependency. --- ...icrosoft.DotNet.Build.Tasks.Feed.net45.csproj | 4 ++++ .../PushToBlobFeed.Desktop.cs | 16 ++++++++++++++++ .../PushToBlobFeed.cs | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj index 824505547f..c2ba541669 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj @@ -13,6 +13,10 @@ .NETFramework,Version=v4.5 + + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs new file mode 100644 index 0000000000..9c6788fd34 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.Build.Common.Desktop; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public partial class PushToBlobFeed + { + static PushToBlobFeed() + { + AssemblyResolver.Enable(); + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs index 05c9b0c3c5..cd11f40102 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs @@ -16,7 +16,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public class PushToBlobFeed : MSBuild.Task + public partial class PushToBlobFeed : MSBuild.Task { private static readonly char[] ManifestDataPairSeparators = { ';' }; private const string DisableManifestPushConfigurationBlob = "disable-manifest-push"; From feeafb8e1c92821c2cc9190912e38c928e4e9053 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 6 Mar 2018 15:06:11 -0600 Subject: [PATCH 44/49] Make BuildTask common, use in Feed project --- ...icrosoft.DotNet.Build.Tasks.Feed.net45.csproj | 2 +- .../PushToBlobFeed.Desktop.cs | 16 ---------------- .../FetchOrchestratedBuildManifestInfo.cs | 2 +- .../PushOrchestratedBuildManifest.cs | 3 +-- .../UpdateOrchestratedBuildManifest.cs | 3 +-- ...riteOrchestratedBuildManifestSummaryToFile.cs | 3 +-- .../WriteOrchestratedBuildManifestToFile.cs | 5 ++--- .../ConfigureInputFeed.cs | 6 +++--- .../CopyBlobDirectory.cs | 9 ++++----- .../GetBlobFeedPackageList.cs | 2 +- .../Microsoft.DotNet.Build.Tasks.Feed.csproj | 3 +++ .../ParseBlobUrl.cs | 9 +-------- .../PushToBlobFeed.cs | 2 +- .../Microsoft.DotNet.Build.Tasks.net45.csproj | 2 +- .../Microsoft.DotNet.Build.Tasks.csproj | 4 +++- .../BuildTask.Desktop.cs | 0 .../BuildTask.cs | 0 17 files changed, 24 insertions(+), 47 deletions(-) delete mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs rename src/{Microsoft.DotNet.Build.Tasks.net45 => common}/BuildTask.Desktop.cs (100%) rename src/{Microsoft.DotNet.Build.Tasks => common}/BuildTask.cs (100%) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj index c2ba541669..8c7d9420dd 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/Microsoft.DotNet.Build.Tasks.Feed.net45.csproj @@ -14,8 +14,8 @@ .NETFramework,Version=v4.5 + - diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs deleted file mode 100644 index 9c6788fd34..0000000000 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.net45/PushToBlobFeed.Desktop.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.DotNet.Build.Common.Desktop; - -namespace Microsoft.DotNet.Build.Tasks.Feed -{ - public partial class PushToBlobFeed - { - static PushToBlobFeed() - { - AssemblyResolver.Enable(); - } - } -} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs index 285a693358..ee3906adda 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/FetchOrchestratedBuildManifestInfo.cs @@ -15,7 +15,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class FetchOrchestratedBuildManifestInfo : Task + public class FetchOrchestratedBuildManifestInfo : BuildTask { private const string IdentitySummaryMetadataName = "IdentitySummary"; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs index a7ab830f55..188eb975c5 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/PushOrchestratedBuildManifest.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.Automation; using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using Microsoft.DotNet.VersionTools.BuildManifest; @@ -15,7 +14,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class PushOrchestratedBuildManifest : Task + public class PushOrchestratedBuildManifest : BuildTask { [Required] public string ManifestFile { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs index 17a9081541..ae5f92fd73 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/UpdateOrchestratedBuildManifest.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.Automation; using Microsoft.DotNet.VersionTools.Automation.GitHubApi; using Microsoft.DotNet.VersionTools.BuildManifest; @@ -15,7 +14,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class UpdateOrchestratedBuildManifest : Task + public class UpdateOrchestratedBuildManifest : BuildTask { private enum UpdateType { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs index 720858ebf0..1d6a73b19a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestSummaryToFile.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.BuildManifest.Model; using System.Linq; using System.Text; @@ -11,7 +10,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class WriteOrchestratedBuildManifestSummaryToFile : Task + public class WriteOrchestratedBuildManifestSummaryToFile : BuildTask { [Required] public string File { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs index 1a55029be8..bb38e74543 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/BuildManifest/WriteOrchestratedBuildManifestToFile.cs @@ -2,15 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Microsoft.DotNet.VersionTools.BuildManifest.Model; +using System.Collections.Generic; using System.Xml.Linq; namespace Microsoft.DotNet.Build.Tasks.Feed.BuildManifest { - public class WriteOrchestratedBuildManifestToFile : Task + public class WriteOrchestratedBuildManifestToFile : BuildTask { [Required] public string File { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs index e66cf17fed..88f01bb97c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/ConfigureInputFeed.cs @@ -2,14 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Microsoft.Build.Utilities; -using System.IO; using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; using System; +using System.IO; namespace Microsoft.DotNet.Build.Tasks.Feed { - public class ConfigureInputFeed : Task + public class ConfigureInputFeed : BuildTask { [Required] public ITaskItem[] EnableFeeds { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs index c80ef39ad6..1510b668ea 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/CopyBlobDirectory.cs @@ -3,17 +3,16 @@ // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using System.Collections.Generic; -using MSBuild = Microsoft.Build.Utilities; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace Microsoft.DotNet.Build.Tasks.Feed { - public sealed class CopyBlobDirectory : MSBuild.Task + public sealed class CopyBlobDirectory : BuildTask { [Required] public string SourceBlobDirectory { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs index 3f1cc0391c..c5dd89b14c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/GetBlobFeedPackageList.cs @@ -12,7 +12,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public class GetBlobFeedPackageList : MSBuild.Task + public class GetBlobFeedPackageList : BuildTask { private const string NuGetPackageInfoId = "PackageId"; private const string NuGetPackageInfoVersion = "PackageVersion"; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj index 7152f87ee7..ddb477ff4f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj @@ -21,6 +21,9 @@ + + BuildTask.cs + AzureBlobLease.cs diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs index d7f4af16f7..fdb54d93c7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/ParseBlobUrl.cs @@ -5,17 +5,10 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using System; -using System.Linq; -using System.Threading.Tasks; -using System.Collections.Generic; -using MSBuild = Microsoft.Build.Utilities; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using System.Collections; namespace Microsoft.DotNet.Build.Tasks.Feed { - public sealed class ParseBlobUrl : MSBuild.Task + public sealed class ParseBlobUrl : BuildTask { [Required] public string BlobUrl { get; set; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs index cd11f40102..042d41231b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/PushToBlobFeed.cs @@ -16,7 +16,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public partial class PushToBlobFeed : MSBuild.Task + public partial class PushToBlobFeed : BuildTask { private static readonly char[] ManifestDataPairSeparators = { ';' }; private const string DisableManifestPushConfigurationBlob = "disable-manifest-push"; diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj index 827abcf5fb..6f10e25ef7 100644 --- a/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.net45/Microsoft.DotNet.Build.Tasks.net45.csproj @@ -15,11 +15,11 @@ .NETFramework,Version=v4.6 + - diff --git a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj index 1bd51ce5b0..f5e5c9aaa0 100644 --- a/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj +++ b/src/Microsoft.DotNet.Build.Tasks/Microsoft.DotNet.Build.Tasks.csproj @@ -103,7 +103,9 @@ - + + BuildTask.cs + diff --git a/src/Microsoft.DotNet.Build.Tasks.net45/BuildTask.Desktop.cs b/src/common/BuildTask.Desktop.cs similarity index 100% rename from src/Microsoft.DotNet.Build.Tasks.net45/BuildTask.Desktop.cs rename to src/common/BuildTask.Desktop.cs diff --git a/src/Microsoft.DotNet.Build.Tasks/BuildTask.cs b/src/common/BuildTask.cs similarity index 100% rename from src/Microsoft.DotNet.Build.Tasks/BuildTask.cs rename to src/common/BuildTask.cs From 8bfc0315aed0f2011e47b378d388ce7ad7298e1f Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Wed, 7 Mar 2018 15:41:41 -0800 Subject: [PATCH 45/49] Fix dumping stack from test coredumps on Linux (#1948) * Fix stack dumping to console on Linux * Hide pushd --- .../PackageFiles/RunnerTemplate.Unix.txt | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt index bb73630c89..0f2839d458 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt @@ -5,7 +5,7 @@ export EXECUTION_DIR=$(dirname "$0") function print_info_from_core_file { local core_file_name=$1 - local executable_name=$2 + local executable_name=$RUNTIME_PATH/$2 if ! [ -e $executable_name ]; then echo "Unable to find executable $executable_name" @@ -33,11 +33,10 @@ function copy_core_file_to_temp_location { # Create the directory (this shouldn't fail even if it already exists). mkdir -p $storage_location - # Only copy the file over if the directory is empty. Otherwise, do nothing. - if [ ! "$(ls -A $storage_location)" ]; then - echo "Copying core file $core_file_name to $storage_location" - cp $core_file_name $storage_location - fi + local new_location=$storage_location/core.$RANDOM + + echo "Copying core file $core_file_name to $new_location in case you need it." + cp $core_file_name $new_location } if [ "$RUNTIME_PATH" == "" ] @@ -84,7 +83,9 @@ echo ----- end $(date +"%T") ----- exit code $test_exitcode -------------------- # ========================= END Test Execution =============================== # ======================= BEGIN Core File Inspection ========================= +pushd $EXECUTION_DIR > nul if [ "$(uname -s)" == "Linux" ]; then + echo Looking around for any Linux dump... # Depending on distro/configuration, the core files may either be named "core" # or "core." by default. We read /proc/sys/kernel/core_uses_pid to # determine which it is. @@ -96,14 +97,18 @@ if [ "$(uname -s)" == "Linux" ]; then if [ $core_name_uses_pid == "1" ]; then # We don't know what the PID of the process was, so let's look at all core # files whose name matches core.NUMBER + echo Looking for files matching core.* ... for f in core.*; do - [[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" "corerun" && copy_core_file_to_temp_location "$f" && rm "$f" + [[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" "dotnet" && copy_core_file_to_temp_location "$f" && rm "$f" done elif [ -f core ]; then - print_info_from_core_file "core" "corerun" + print_info_from_core_file "core" "dotnet" copy_core_file_to_temp_location "core" rm "core" + else + echo ... found no dump in $PWD fi fi +popd > nul # ======================== END Core File Inspection ========================== exit $test_exitcode From 39f99fe0c4a9669b025d4478d50beda332c336ce Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 8 Mar 2018 16:22:41 -0800 Subject: [PATCH 46/49] Add target reverse APICompat check for libraries There are a lot of libraries that have a 1:1 ref and implementation which want to keep the API surface in sync. In order to enforce that we have a reverse APICompat check that will verify that the ref has a matching compatibile set of APIs that the implementation has. This target is called RunMatchingRefApiCompat. It is only enabled if RunApiCompat=true and there are no ReferenceFromRuntime references. It is skipped for projects with runtime references because those are typically many implementation to one ref which doesn't work as well for verifying the APIs match. It can be individually controlled per-project: 1) RunMatchingRefApiCompat = false to disable the check completely 2) Add a baseline file MatchingRefApiCompatBaseline[.$(TargetGroup)]txt --- .../PackageFiles/ApiCompat.targets | 73 ++++++++++--------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets index 3ef602cbe0..f24f4930dc 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/ApiCompat.targets @@ -10,14 +10,16 @@ $(MSBuildProjectDirectory)\ApiCompatBaseline.$(TargetGroup).txt $(MSBuildProjectDirectory)\ApiCompatBaseline.txt + $(MSBuildProjectDirectory)\MatchingRefApiCompatBaseline.$(TargetGroup).txt + $(MSBuildProjectDirectory)\MatchingRefApiCompatBaseline.txt + true - - false + + $(RunApiCompatForSrc) true $(TargetsTriggeredByCompilation);ValidateApiCompatForSrc - $(TargetsTriggeredByCompilation);ValidateApiCompatForRef + $(TargetsTriggeredByCompilation);RunMatchingRefApiCompat @@ -70,55 +72,54 @@ - - - - - - - + + - - $(ReferenceAssemblyOutputPath)$(AssemblyName)\$(PreviousContractVersion)\$(AssemblyName).dll + + @(IntermediateAssembly) - <_CurrentContractDependencies Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> - - <_PreviousContractDependencyDirectories Include="@(_CurrentContractDependencies)" /> + <_ContractDependencyDirectoriesTemp Include="@(ReferencePath->'%(RootDir)%(Directory)')" /> + + + <_ContractDependencyDirectories Condition="'%(_ContractDependencyDirectoriesTemp.ReferenceSourceTarget)'=='ProjectReference'" Include="%(_ContractDependencyDirectoriesTemp.Identity)" /> + <_ContractDependencyDirectories Condition="'%(_ContractDependencyDirectoriesTemp.ReferenceSourceTarget)'!='ProjectReference'" Include="%(_ContractDependencyDirectoriesTemp.Identity)" /> + <_ImplementationDependencyDirectories Include="@(ResolvedMatchingContract->'%(RootDir)%(Directory)')" /> + <_ImplementationDependencyDirectories Include="$(ContractOutputPath)" /> - <_ApiCompatCmd>$(ToolHostCmd) "$(ToolsDir)ApiCompat.exe" "$(PreviousContractAssembly)" - <_ApiCompatCmd>$(_ApiCompatCmd) -contractDepends:"@(_PreviousContractDependencyDirectories);" - <_ApiCompatCmd>$(_ApiCompatCmd) -implDirs:"$(IntermediateOutputPath);@(_CurrentContractDependencies);" - <_ApiCompatCmd Condition="Exists('$(ApiCompatBaseline)')">$(_ApiCompatCmd) -baseline:"$(ApiCompatBaseline)" - 0 + $(MatchingRefApiCompatArgs) "$(ImplemetnationAssemblyAsContract)" + $(MatchingRefApiCompatArgs) -contractDepends:"@(_ContractDependencyDirectories, ',')," + $(MatchingRefApiCompatArgs) -implDirs:"@(_ImplementationDependencyDirectories, ',')," + $(MatchingRefApiCompatArgs) -baseline:"$(MatchingRefApiCompatBaseline)" + > $(MatchingRefApiCompatBaseline) + + 0 + + $(IntermediateOutputPath)MatchingRefApiCompat_verifyexactref.rsp + $(ToolHostCmd) "$(ToolsDir)ApiCompat.exe" - + + + - + - - + + From 0e715457c63a4f3c0a9d56d2809511aab09e935e Mon Sep 17 00:00:00 2001 From: Ricardo Arenas Date: Fri, 9 Mar 2018 09:55:51 -0800 Subject: [PATCH 47/49] Extract async task to avoid swallowing of TaskCanceledException from HttpClient.GetStreamAsync --- .../DownloadFilesFromUrl.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs b/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs index 0c27c0df32..97cd4e418c 100644 --- a/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs +++ b/src/Microsoft.DotNet.Build.Tasks/DownloadFilesFromUrl.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Net; using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.Build.Tasks @@ -37,6 +38,11 @@ public sealed class DownloadFilesFromUrl : BuildTask public ITaskItem[] FilesCreated { get; set; } public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async Task ExecuteAsync() { if (Items == null || Items.Length <= 0) { @@ -102,11 +108,11 @@ public override bool Execute() Log.LogMessage(MessageImportance.Normal, $"Downloading {downloadSource} -> {destinationFullPath}"); - using (Stream responseStream = client.GetStreamAsync(downloadUri).GetAwaiter().GetResult()) + using (Stream responseStream = await client.GetStreamAsync(downloadUri)) { using (Stream destinationStream = File.OpenWrite(destinationFullPath)) { - responseStream.CopyToAsync(destinationStream).GetAwaiter().GetResult(); + await responseStream.CopyToAsync(destinationStream); TaskItem createdItem = new TaskItem(destinationFullPath); item.CopyMetadataTo(createdItem); filesCreated.Add(createdItem); From b0cfbfcf2f23ecbd1e1f68de5c3766eaf08bb10e Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sun, 11 Mar 2018 19:47:26 -0700 Subject: [PATCH 48/49] /dev/null not nul on Unix (#1953) --- .../PackageFiles/RunnerTemplate.Unix.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt index 0f2839d458..b11028b3b7 100644 --- a/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt +++ b/src/Microsoft.DotNet.Build.Tasks/PackageFiles/RunnerTemplate.Unix.txt @@ -83,7 +83,7 @@ echo ----- end $(date +"%T") ----- exit code $test_exitcode -------------------- # ========================= END Test Execution =============================== # ======================= BEGIN Core File Inspection ========================= -pushd $EXECUTION_DIR > nul +pushd $EXECUTION_DIR >/dev/null if [ "$(uname -s)" == "Linux" ]; then echo Looking around for any Linux dump... # Depending on distro/configuration, the core files may either be named "core" @@ -109,6 +109,6 @@ if [ "$(uname -s)" == "Linux" ]; then echo ... found no dump in $PWD fi fi -popd > nul +popd >/dev/null # ======================== END Core File Inspection ========================== exit $test_exitcode From 5274e0734eb4f89cfee4127e128288b395553bb7 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Mon, 12 Mar 2018 16:05:15 -0700 Subject: [PATCH 49/49] Switch to using CallTarget some minor fixups for the sample as the above does not work on latest msbuild. --- .../Samples/CloudTest.Helix.targets.sampleproject | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Documentation/Samples/CloudTest.Helix.targets.sampleproject b/Documentation/Samples/CloudTest.Helix.targets.sampleproject index bfeb3f52fe..cbddc9b78e 100644 --- a/Documentation/Samples/CloudTest.Helix.targets.sampleproject +++ b/Documentation/Samples/CloudTest.Helix.targets.sampleproject @@ -5,7 +5,7 @@ call DoStuff.bat - Path to zip + Path to local zip file Demo Work item #1 300 @@ -13,7 +13,7 @@ call DoStuff.bat - Path to zip + Path to some other local zip file Demo Work item #2 200 @@ -27,12 +27,12 @@ - https://helix.int-dot.net/api/2016-06-28/jobs + https://helix.dot.net/api/2016-06-28/jobs unspecified/ pr/unspecified/ - Windows.10.Amd64;Windows.7.Amd64;Windows.81.Amd64;Windows.10.Core.Amd64; - 20170301.0000 + Windows.10.Amd64;Windows.10.Core.Amd64; + 20180312.00 ... ... @@ -45,7 +45,7 @@ - + - \ No newline at end of file +