From ad6dfc5d95a9d215f17a977f5592500e0d3d5e1d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 10:16:09 +0100 Subject: [PATCH 01/21] update the integration test --- tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index a7a40ed96a..c50e6a0e3a 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -24,7 +24,7 @@ public void LatestCoreRtVersionIsSupported() .AddJob(Job.Dry .WithRuntime(CoreRtRuntime.GetCurrentVersion()) .WithToolchain(CoreRtToolchain.CreateBuilder() - .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-*") // we test against latest version to make sure we support latest version and avoid issues like #1055 + .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-*") // we test against latest version to make sure we support latest version and avoid issues like #1055 .ToToolchain())); CanExecute(config); From 27b72ad8ef55fa052f8e564de57ad73cecf15ee4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 10:17:43 +0100 Subject: [PATCH 02/21] update CoreRtToolchain to use new CoreRT NuGet feed and compiler version --- .../Toolchains/CoreRt/CoreRtToolchain.cs | 14 +++++++------- .../Toolchains/CoreRt/CoreRtToolchainBuilder.cs | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs index b9d32fb8d9..8bded809d9 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs @@ -8,31 +8,31 @@ namespace BenchmarkDotNet.Toolchains.CoreRt public class CoreRtToolchain : Toolchain { /// - /// compiled as netcoreapp2.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); /// - /// compiled as netcoreapp2.1, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.1, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); /// - /// compiled as netcoreapp2.2, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.2, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); /// - /// compiled as netcoreapp3.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp3.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); /// - /// compiled as netcoreapp3.1, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp3.1, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); /// - /// compiled as net5.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core50 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net5.0").ToToolchain(); /// - /// compiled as net6.0, targets latest (1.0.0-alpha-*) CoreRT build from https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as net6.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core60 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net6.0").ToToolchain(); diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs index 739c7ea8b0..0ce12a1dbe 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs @@ -23,12 +23,12 @@ public class CoreRtToolchainBuilder : CustomDotNetCliToolchainBuilder /// /// creates a CoreRT toolchain targeting NuGet build of CoreRT - /// Based on https://github.com/dotnet/corert/blob/7f902d4d8b1c3280e60f5e06c71951a60da173fb/samples/HelloWorld/README.md#add-corert-to-your-project + /// Based on https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project /// - /// the version of Microsoft.DotNet.ILCompiler which should be used. The default is: "1.0.0-alpha-*" - /// url to NuGet CoreRT feed, The default is: "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" + /// the version of Microsoft.DotNet.ILCompiler which should be used. The default is: "6.0.0-*" + /// url to NuGet CoreRT feed, The default is: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json" [PublicAPI] - public CoreRtToolchainBuilder UseCoreRtNuGet(string microsoftDotNetILCompilerVersion = "1.0.0-alpha-*", string nuGetFeedUrl = "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json") + public CoreRtToolchainBuilder UseCoreRtNuGet(string microsoftDotNetILCompilerVersion = "6.0.0-*", string nuGetFeedUrl = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json") { coreRtVersion = microsoftDotNetILCompilerVersion ?? throw new ArgumentNullException(nameof(microsoftDotNetILCompilerVersion)); From 7edde6567579c9c21354145ce70b84c6533aa3e8 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 10:18:22 +0100 Subject: [PATCH 03/21] update docs --- docs/articles/configs/toolchains.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 38957a6f71..84eb7570ab 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -190,18 +190,18 @@ Example: `dotnet run -c Release -- --coreRun "C:\Projects\corefx\bin\testhost\ne ## CoreRT -BenchmarkDotNet supports [CoreRT](https://github.com/dotnet/corert)! However, you might want to know how it works to get a better understanding of the results that you get. +BenchmarkDotNet supports [CoreRT](https://github.com/dotnet/runtimelab/tree/feature/NativeAOT)! However, you might want to know how it works to get a better understanding of the results that you get. * CoreRT is a flavor of .NET Core. Which means that: - * you have to target .NET Core to be able to build CoreRT benchmarks (`netcoreapp2.1` in the .csproj file) - * you have to specify the CoreRT runtime in an explicit way, either by using `[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(CoreRtRuntime.$version)` - * to run CoreRT benchmark you run the app as a .NET Core/.NET process (`dotnet run -c Release -f netcoreapp2.1`) and BenchmarkDotNet does all the CoreRT compilation for you. If you want to check what files are generated you need to apply `[KeepBenchmarkFiles]` attribute to the class which defines benchmarks. + * you have to target .NET Core to be able to build CoreRT benchmarks (example: `net5.0` in the .csproj file) + * you have to specify the CoreRT runtime in an explicit way, either by using `[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(CoreRtRuntime.$version)` or console line arguments `--runtimes corert50` + * to run CoreRT benchmark you run the app as a .NET Core/.NET process (example: `dotnet run -c Release -f net5.01`) and BenchmarkDotNet does all the CoreRT compilation for you. If you want to check what files are generated you need to apply `[KeepBenchmarkFiles]` attribute to the class which defines benchmarks. -By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the CoreRT benchmark according to [this instructions](https://github.com/dotnet/corert/tree/7f902d4d8b1c3280e60f5e06c71951a60da173fb/samples/HelloWorld#add-corert-to-your-project). +By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the CoreRT benchmark according to [this instructions](https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project). ```cs var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRtRuntime.CoreRt21)); // compiles the benchmarks as netcoreapp2.1 and uses the latest CoreRT to build a native app + .With(Job.Default.With(CoreRtRuntime.CoreRt50)); // compiles the benchmarks as net5.0 and uses the latest CoreRT to build a native app BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -209,7 +209,7 @@ BenchmarkSwitcher ``` ```cs -[SimpleJob(RuntimeMoniker.CoreRt21)] // compiles the benchmarks as netcoreapp2.1 and uses the latest CoreRT to build a native app +[SimpleJob(RuntimeMoniker.CoreRt50)] // compiles the benchmarks as net5.0 and uses the latest CoreRT to build a native app public class TheTypeWithBenchmarks { [Benchmark] // the benchmarks go here @@ -218,15 +218,17 @@ public class TheTypeWithBenchmarks **Note**: BenchmarkDotNet is going to run `dotnet restore` on the auto-generated project. The first time it does so, it's going to take a **LOT** of time to download all the dependencies (few minutes). Just give it some time and don't press `Ctrl+C` too fast ;) -If you want to benchmark some particular version of CoreRT you have to specify it in an explicit way: +If you want to benchmark some particular version of CoreRT (or from a different NuGet feed) you have to specify it in an explicit way: ```cs var config = DefaultConfig.Instance .With(Job.ShortRun .With(CoreRtToolchain.CreateBuilder() - .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-26412-02") // the version goes here + .UseCoreRtNuGet( + microsoftDotNetILCompilerVersion: "6.0.0-*", // the version goes here + nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json") // this address might change over time .DisplayName("CoreRT NuGet") - .TargetFrameworkMoniker("netcoreapp2.1") + .TargetFrameworkMoniker("net5.0") .ToToolchain())); ``` From 97a6291060cff4902ddf2958c645cf3111ba8cdb Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 10:52:03 +0100 Subject: [PATCH 04/21] CoreRtToolchain for TFMs prior to net5.0 should keep using the old CoreRT compiler and old NuGet feed, only .NET5+ should use the new version --- .../Toolchains/CoreRt/CoreRtToolchain.cs | 27 ++++++++++--------- .../MemoryDiagnoserTests.cs | 11 ++++++-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs index 8bded809d9..e89c6b2c7e 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs @@ -7,32 +7,35 @@ namespace BenchmarkDotNet.Toolchains.CoreRt { public class CoreRtToolchain : Toolchain { + private const string OldCoreRtVersion = "1.0.0-alpha-*"; + private const string OldCoreRtFeed = "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"; + /// - /// compiled as netcoreapp2.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as netcoreapp2.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); + public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); /// - /// compiled as netcoreapp2.1, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as netcoreapp2.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); + public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); /// - /// compiled as netcoreapp2.2, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as netcoreapp2.2, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); + public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); /// - /// compiled as netcoreapp3.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as netcoreapp3.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); + public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); /// - /// compiled as netcoreapp3.1, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as netcoreapp3.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); + public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); /// - /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core50 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net5.0").ToToolchain(); /// - /// compiled as net6.0, targets latest (6.0.0-*) CoreRT build from https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json + /// compiled as net6.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// public static readonly IToolchain Core60 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("net6.0").ToToolchain(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index d7b5259f23..1b6813ae7a 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -38,10 +38,17 @@ public static IEnumerable GetToolchains() { new object[] { Job.Default.GetToolchain() }, new object[] { InProcessEmitToolchain.Instance }, -#if NETCOREAPP2_1 +#if NETCOREAPP2_1 || NETCOREAPP3_1 // we don't want to test CoreRT twice (for .NET 4.6 and Core 2.1) when running the integration tests (these tests take a lot of time) // we test against specific version to keep this test stable - new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-27408-02").ToToolchain() } + new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet( + microsoftDotNetILCompilerVersion: "1.0.0-alpha-27408-02", + nuGetFeedUrl: "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json").ToToolchain() } +#elif NET5_0 || NET6_0 + // for .NET 5.0+ we use the newer version of CoreRT (it does not work with older tfms like netcoreapp2.1) + new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet( + microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1", + nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json").ToToolchain() } #endif }; From 115b2abe54ec76d60b5dbae0f3480feb23b4be7c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 14:59:37 +0100 Subject: [PATCH 05/21] more places to update --- .../ConsoleArguments/ConfigParser.cs | 2 ++ .../Toolchains/CoreRt/CoreRtToolchain.cs | 18 ++++++++++-------- .../CoreRt/CoreRtToolchainBuilder.cs | 7 +++++++ .../CoreRtTests.cs | 6 +----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index a18822cc12..5970a4325f 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -368,6 +368,8 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma builder.UseCoreRtLocal(options.CoreRtPath.FullName); else if (!string.IsNullOrEmpty(options.CoreRtVersion)) builder.UseCoreRtNuGet(options.CoreRtVersion); + else if (runtimeMoniker <= RuntimeMoniker.CoreRt31) + builder.UseOldCoreRtNuGet(); else builder.UseCoreRtNuGet(); diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs index e89c6b2c7e..7a1b61162e 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs @@ -7,29 +7,31 @@ namespace BenchmarkDotNet.Toolchains.CoreRt { public class CoreRtToolchain : Toolchain { - private const string OldCoreRtVersion = "1.0.0-alpha-*"; - private const string OldCoreRtFeed = "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"; - /// /// compiled as netcoreapp2.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + public static readonly IToolchain Core20 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); /// /// compiled as netcoreapp2.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + public static readonly IToolchain Core21 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); /// /// compiled as netcoreapp2.2, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + public static readonly IToolchain Core22 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); /// /// compiled as netcoreapp3.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + public static readonly IToolchain Core30 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); /// /// compiled as netcoreapp3.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /// - public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet(OldCoreRtVersion, OldCoreRtFeed).TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + public static readonly IToolchain Core31 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); /// /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs index 0ce12a1dbe..1354b7d71b 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs @@ -21,6 +21,13 @@ public class CoreRtToolchainBuilder : CustomDotNetCliToolchainBuilder private bool isCoreRtConfigured; + /// + /// uses an OLD version and NuGet feed of CoreRT that was used prior to .NET 5 + /// + /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version + [PublicAPI] + public CoreRtToolchainBuilder UseOldCoreRtNuGet() => UseCoreRtNuGet("1.0.0-alpha-*", "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"); + /// /// creates a CoreRT toolchain targeting NuGet build of CoreRT /// Based on https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index c50e6a0e3a..7947e3f7b1 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -5,7 +5,6 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.CoreRt; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests @@ -22,10 +21,7 @@ public void LatestCoreRtVersionIsSupported() var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(CoreRtRuntime.GetCurrentVersion()) - .WithToolchain(CoreRtToolchain.CreateBuilder() - .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-*") // we test against latest version to make sure we support latest version and avoid issues like #1055 - .ToToolchain())); + .WithRuntime(CoreRtRuntime.GetCurrentVersion())); // we test against latest version for current TFM to make sure we avoid issues like #1055 CanExecute(config); } From 30ef63f690c9b000255d08ee2f8ca84f78bab808 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 18:08:45 +0100 Subject: [PATCH 06/21] add workaround that was not needed before --- src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs index fb23206343..bf6e26fd09 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs @@ -136,6 +136,7 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif {rootAllApplicationAssemblies} {ilcGenerateCompleteTypeMetadata} {ilcGenerateStackTraceData} + false {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} From 965c9a7bcb25830a9925cb1e0b024d76ce1ab6c9 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 18:09:17 +0100 Subject: [PATCH 07/21] change CoreRT version detection to use .NET Core version detection code (it's well tested) --- .../Environments/Runtimes/CoreRtRuntime.cs | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs index 32049c9488..9616aa5223 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs @@ -10,62 +10,60 @@ public class CoreRtRuntime : Runtime /// /// CoreRT compiled as netcoreapp2.0 /// - public static readonly CoreRtRuntime CoreRt20 = new CoreRtRuntime(RuntimeMoniker.CoreRt20, "netcoreapp2.0", "CoreRt 2.0"); + public static readonly CoreRtRuntime CoreRt20 = new CoreRtRuntime(RuntimeMoniker.CoreRt20, "netcoreapp2.0", "CoreRT 2.0"); /// /// CoreRT compiled as netcoreapp2.1 /// - public static readonly CoreRtRuntime CoreRt21 = new CoreRtRuntime(RuntimeMoniker.CoreRt21, "netcoreapp2.1", "CoreRt 2.1"); + public static readonly CoreRtRuntime CoreRt21 = new CoreRtRuntime(RuntimeMoniker.CoreRt21, "netcoreapp2.1", "CoreRT 2.1"); /// /// CoreRT compiled as netcoreapp2.2 /// - public static readonly CoreRtRuntime CoreRt22 = new CoreRtRuntime(RuntimeMoniker.CoreRt22, "netcoreapp2.2", "CoreRt 2.2"); + public static readonly CoreRtRuntime CoreRt22 = new CoreRtRuntime(RuntimeMoniker.CoreRt22, "netcoreapp2.2", "CoreRT 2.2"); /// /// CoreRT compiled as netcoreapp3.0 /// - public static readonly CoreRtRuntime CoreRt30 = new CoreRtRuntime(RuntimeMoniker.CoreRt30, "netcoreapp3.0", "CoreRt 3.0"); + public static readonly CoreRtRuntime CoreRt30 = new CoreRtRuntime(RuntimeMoniker.CoreRt30, "netcoreapp3.0", "CoreRT 3.0"); /// /// CoreRT compiled as netcoreapp3.1 /// - public static readonly CoreRtRuntime CoreRt31 = new CoreRtRuntime(RuntimeMoniker.CoreRt31, "netcoreapp3.1", "CoreRt 3.1"); + public static readonly CoreRtRuntime CoreRt31 = new CoreRtRuntime(RuntimeMoniker.CoreRt31, "netcoreapp3.1", "CoreRT 3.1"); /// /// CoreRT compiled as net5.0 /// - public static readonly CoreRtRuntime CoreRt50 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net5.0", "CoreRt 5.0"); + public static readonly CoreRtRuntime CoreRt50 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net5.0", "CoreRT 5.0"); /// /// CoreRT compiled as net6.0 /// - public static readonly CoreRtRuntime CoreRt60 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net6.0", "CoreRt 6.0"); + public static readonly CoreRtRuntime CoreRt60 = new CoreRtRuntime(RuntimeMoniker.CoreRt50, "net6.0", "CoreRT 6.0"); private CoreRtRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) : base(runtimeMoniker, msBuildMoniker, displayName) { } - internal static CoreRtRuntime GetCurrentVersion() + public static CoreRtRuntime GetCurrentVersion() { if (!RuntimeInformation.IsNetCore && !RuntimeInformation.IsCoreRT) { throw new NotSupportedException("It's impossible to reliably detect the version of CoreRT if the process is not a .NET Core or CoreRT process!"); } - var frameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; - string netCoreAppVersion = new string(frameworkDescription.SkipWhile(c => !char.IsDigit(c)).ToArray()); - string[] versionNumbers = netCoreAppVersion.Split('.'); - string msBuildMoniker = int.Parse(versionNumbers[0]) >= 5 ? $"net{versionNumbers[0]}.{versionNumbers[1]}" : $"netcoreapp{versionNumbers[0]}.{versionNumbers[1]}"; - string displayName = $"CoreRT {versionNumbers[0]}.{versionNumbers[1]}"; + if (!CoreRuntime.TryGetVersion(out var version)) + { + throw new NotSupportedException("Failed to recognize CoreRT version"); + } - switch (msBuildMoniker) + switch (version) { - case "netcoreapp2.0": return CoreRt20; - case "netcoreapp2.1": return CoreRt21; - case "netcoreapp2.2": return CoreRt22; - case "netcoreapp3.0": return CoreRt30; - case "netcoreapp3.1": return CoreRt31; - case "net5.0": - case "netcoreapp5.0": return CoreRt50; - case "net6.0": return CoreRt60; - default: // support future version of CoreRT - return new CoreRtRuntime(RuntimeMoniker.NotRecognized, msBuildMoniker, displayName); + case Version v when v.Major == 2 && v.Minor == 0: return CoreRt20; + case Version v when v.Major == 2 && v.Minor == 1: return CoreRt21; + case Version v when v.Major == 2 && v.Minor == 2: return CoreRt22; + case Version v when v.Major == 3 && v.Minor == 0: return CoreRt30; + case Version v when v.Major == 3 && v.Minor == 1: return CoreRt31; + case Version v when v.Major == 5 && v.Minor == 0: return CoreRt50; + case Version v when v.Major == 6 && v.Minor == 0: return CoreRt50; + default: + return new CoreRtRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"CoreRT {version.Major}.{version.Minor}"); } } } From 112106c7ab5a17d3b90efa72fe50a43a13199c3c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 18:15:36 +0100 Subject: [PATCH 08/21] always use latest CoreRT compiler --- .../ConsoleArguments/ConfigParser.cs | 2 -- .../Toolchains/CoreRt/CoreRtToolchain.cs | 25 ++++++++----------- .../CoreRt/CoreRtToolchainBuilder.cs | 7 ------ 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 5970a4325f..a18822cc12 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -368,8 +368,6 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma builder.UseCoreRtLocal(options.CoreRtPath.FullName); else if (!string.IsNullOrEmpty(options.CoreRtVersion)) builder.UseCoreRtNuGet(options.CoreRtVersion); - else if (runtimeMoniker <= RuntimeMoniker.CoreRt31) - builder.UseOldCoreRtNuGet(); else builder.UseCoreRtNuGet(); diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs index 7a1b61162e..982ff9a001 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchain.cs @@ -8,30 +8,25 @@ namespace BenchmarkDotNet.Toolchains.CoreRt public class CoreRtToolchain : Toolchain { /// - /// compiled as netcoreapp2.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - public static readonly IToolchain Core20 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); + public static readonly IToolchain Core20 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.0").ToToolchain(); /// - /// compiled as netcoreapp2.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.1, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - public static readonly IToolchain Core21 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); + public static readonly IToolchain Core21 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.1").ToToolchain(); /// - /// compiled as netcoreapp2.2, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp2.2, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - public static readonly IToolchain Core22 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); + public static readonly IToolchain Core22 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp2.2").ToToolchain(); /// - /// compiled as netcoreapp3.0, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp3.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - public static readonly IToolchain Core30 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); + public static readonly IToolchain Core30 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.0").ToToolchain(); /// - /// compiled as netcoreapp3.1, targets an OLD (1.0.0-alpha-*) CoreRT build from the old feed: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /// compiled as netcoreapp3.1, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - public static readonly IToolchain Core31 = CreateBuilder().UseOldCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); + public static readonly IToolchain Core31 = CreateBuilder().UseCoreRtNuGet().TargetFrameworkMoniker("netcoreapp3.1").ToToolchain(); /// /// compiled as net5.0, targets latest (6.0.0-*) CoreRT build from the new feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json /// diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs index 1354b7d71b..0ce12a1dbe 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/CoreRtToolchainBuilder.cs @@ -21,13 +21,6 @@ public class CoreRtToolchainBuilder : CustomDotNetCliToolchainBuilder private bool isCoreRtConfigured; - /// - /// uses an OLD version and NuGet feed of CoreRT that was used prior to .NET 5 - /// - /// it's not guaranteed to work (the feed will be removed one day), you better test against .NET 5.0+ version - [PublicAPI] - public CoreRtToolchainBuilder UseOldCoreRtNuGet() => UseCoreRtNuGet("1.0.0-alpha-*", "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"); - /// /// creates a CoreRT toolchain targeting NuGet build of CoreRT /// Based on https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project From e3828bac94039793a84d664fb6c1f0fbdb6e2734 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Dec 2020 18:15:55 +0100 Subject: [PATCH 09/21] use 6.0+ version for testing MemoryDiagnoser --- .../MemoryDiagnoserTests.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 1b6813ae7a..e96ff191a6 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -38,17 +38,10 @@ public static IEnumerable GetToolchains() { new object[] { Job.Default.GetToolchain() }, new object[] { InProcessEmitToolchain.Instance }, -#if NETCOREAPP2_1 || NETCOREAPP3_1 +#if !NETFRAMEWORK // we don't want to test CoreRT twice (for .NET 4.6 and Core 2.1) when running the integration tests (these tests take a lot of time) // we test against specific version to keep this test stable - new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet( - microsoftDotNetILCompilerVersion: "1.0.0-alpha-27408-02", - nuGetFeedUrl: "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json").ToToolchain() } -#elif NET5_0 || NET6_0 - // for .NET 5.0+ we use the newer version of CoreRT (it does not work with older tfms like netcoreapp2.1) - new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet( - microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1", - nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json").ToToolchain() } + new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1").ToToolchain() } #endif }; From 083a4c03c6b97ed099b26542328e3f4169bc952d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 20 Jan 2021 17:46:07 +0100 Subject: [PATCH 10/21] re-enable CoreRT(s) test which is possible after the CI got updated to .NET 5 SDK --- tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs | 5 +---- .../ThreadingDiagnoserTests.cs | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index ec043bd681..7947e3f7b1 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -5,8 +5,6 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.CoreRt; -using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests @@ -15,8 +13,7 @@ public class CoreRtTests : BenchmarkTestExecutor { public CoreRtTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [Fact(Skip = "Disabled until #1606 gets merged with CoreRT toolchain update")] - //[FactDotNetCoreOnly("It's impossible to reliably detect the version of CoreRT if the process is not a .NET Core or CoreRT process")] + [FactDotNetCoreOnly("It's impossible to reliably detect the version of CoreRT if the process is not a .NET Core or CoreRT process")] public void LatestCoreRtVersionIsSupported() { if (!RuntimeInformation.Is64BitPlatform()) // CoreRT does not support 32bit yet diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index c2daec17e2..efe3369d28 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -29,7 +29,7 @@ public class ThreadingDiagnoserTests public static IEnumerable GetToolchains() => new[] { new object[] { Job.Default.GetToolchain() }, - // new object[] { CoreRtToolchain.Core50 }, Disabled until #1606 gets merged with CoreRT toolchain update + new object[] { CoreRtToolchain.Core50 }, // new object[] { InProcessEmitToolchain.Instance }, }; From d0f7182fb3b6b08c74a2aa2c9155dfb3642959e4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 20 Jan 2021 17:52:45 +0100 Subject: [PATCH 11/21] 2.1 -> 5.0 change for CoreRT tests --- .../LocalCoreRtToolchainTests.cs | 4 ++-- tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs | 4 ++-- .../BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalCoreRtToolchainTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalCoreRtToolchainTests.cs index 88d5f96e44..e98d7a9a2c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalCoreRtToolchainTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalCoreRtToolchainTests.cs @@ -29,7 +29,7 @@ public void CanBenchmarkLocalCoreRtUsingRyuJit() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(CoreRtRuntime.CoreRt21) + .WithRuntime(CoreRtRuntime.CoreRt50) .WithToolchain( CoreRtToolchain.CreateBuilder() .UseCoreRtLocal(IlcPath) @@ -43,7 +43,7 @@ public void CanBenchmarkLocalCoreRtUsingCppCodeGen() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(CoreRtRuntime.CoreRt21) + .WithRuntime(CoreRtRuntime.CoreRt50) .WithToolchain( CoreRtToolchain.CreateBuilder() .UseCoreRtLocal(IlcPath) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index da451f8528..b1489f4b3f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -26,9 +26,9 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(CoreRtRuntime.CoreRt21) + .WithRuntime(CoreRtRuntime.CoreRt50) .WithToolchain(CoreRtToolchain.CreateBuilder() - .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-27408-02") // we test against specific version to keep this test stable + .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1") // we test against specific version to keep this test stable .Timeout(timeout) .ToToolchain())); diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 48d276fdf2..1fe4719d8b 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -39,7 +39,7 @@ public static IEnumerable GetToolchains() new object[] { Job.Default.GetToolchain() }, new object[] { InProcessEmitToolchain.Instance }, #if !NETFRAMEWORK - // we don't want to test CoreRT twice (for .NET 4.6 and Core 2.1) when running the integration tests (these tests take a lot of time) + // we don't want to test CoreRT twice (for .NET 4.6 and 5.0) when running the integration tests (these tests take a lot of time) // we test against specific version to keep this test stable new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-alpha.1.20602.1").ToToolchain() } #endif From 9a2e5cde7a3c242583af7d88c96b7fd8cbb0af5f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 20 Jan 2021 17:55:18 +0100 Subject: [PATCH 12/21] fix a bug --- src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs index 9616aa5223..8af91d00ba 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs @@ -61,7 +61,7 @@ public static CoreRtRuntime GetCurrentVersion() case Version v when v.Major == 3 && v.Minor == 0: return CoreRt30; case Version v when v.Major == 3 && v.Minor == 1: return CoreRt31; case Version v when v.Major == 5 && v.Minor == 0: return CoreRt50; - case Version v when v.Major == 6 && v.Minor == 0: return CoreRt50; + case Version v when v.Major == 6 && v.Minor == 0: return CoreRt60; default: return new CoreRtRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"CoreRT {version.Major}.{version.Minor}"); } From e9aa3515510f24a9e7d5d7e0aa7111d0b65c6913 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 20 Jan 2021 20:27:01 +0100 Subject: [PATCH 13/21] update AzureDevOps VM image to windows-2019 which contains VS 2019 and might help with CoreRT build errors --- azure-pipelines.Windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.Windows.yml b/azure-pipelines.Windows.yml index 63b680a064..071abcab84 100755 --- a/azure-pipelines.Windows.yml +++ b/azure-pipelines.Windows.yml @@ -11,5 +11,5 @@ jobs: - template: build/azure-pipelines.job.template.yml parameters: name: Windows - vmImage: 'vs2017-win2016' + vmImage: 'windows-2019' scriptFileName: .\build.ps1 \ No newline at end of file From 8b8e9d16abc54af415fa6f210002a1b178853644 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 12:42:08 +0100 Subject: [PATCH 14/21] bump ubuntu 16 to 18 --- azure-pipelines.Ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index 518a40de11..2ef95be924 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -11,7 +11,7 @@ jobs: - template: build/azure-pipelines.job.template.yml parameters: name: Ubuntu - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-18.04' scriptFileName: ./build.sh initialization: - bash: | From b87006167a0d0b643fca5e4296a4921e8c632634 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 14:13:24 +0100 Subject: [PATCH 15/21] xenial -> bionic --- azure-pipelines.Ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index 2ef95be924..6ddea48b97 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -15,7 +15,7 @@ jobs: scriptFileName: ./build.sh initialization: - bash: | - echo "deb https://llvm.org/apt/xenial/ llvm-toolchain-xenial-3.9 main" | sudo tee /etc/apt/sources.list.d/llvm.list + echo "deb https://llvm.org/apt/bionic/ llvm-toolchain-bionic-3.9 main" | sudo tee /etc/apt/sources.list.d/llvm.list wget -O - https://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-get update - bash: | From 690e96e0950b176d6473f329ab0f31ba017a6d20 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 14:31:56 +0100 Subject: [PATCH 16/21] bump clang from 3.9 to 6.0 --- azure-pipelines.Ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index 6ddea48b97..5616d0ae1f 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -15,8 +15,8 @@ jobs: scriptFileName: ./build.sh initialization: - bash: | - echo "deb https://llvm.org/apt/bionic/ llvm-toolchain-bionic-3.9 main" | sudo tee /etc/apt/sources.list.d/llvm.list + echo "deb https://llvm.org/apt/bionic/ llvm-toolchain-bionic-6.0 main" | sudo tee /etc/apt/sources.list.d/llvm.list wget -O - https://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-get update - bash: | - sudo apt-get install cmake clang-3.9 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev + sudo apt-get install cmake clang-6.0 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev From 9bec38c5b1237acb7773432b41cf8b4cd84f4101 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 14:37:30 +0100 Subject: [PATCH 17/21] don't ask --- azure-pipelines.Ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index 5616d0ae1f..f94f794a8f 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -15,8 +15,8 @@ jobs: scriptFileName: ./build.sh initialization: - bash: | - echo "deb https://llvm.org/apt/bionic/ llvm-toolchain-bionic-6.0 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -O - https://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add - + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main" sudo apt-get update - bash: | sudo apt-get install cmake clang-6.0 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev From 580ebeb5e4860a816d1fbaba9e2916d115e8763b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 14:44:10 +0100 Subject: [PATCH 18/21] update libicu 55 -> 60 --- azure-pipelines.Ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index f94f794a8f..261e56eef7 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -19,4 +19,4 @@ jobs: sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main" sudo apt-get update - bash: | - sudo apt-get install cmake clang-6.0 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev + sudo apt-get install cmake clang-6.0 libicu60 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev From f68267e2d5c89eee313932d64f3f70a62c1cc196 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 15:02:10 +0100 Subject: [PATCH 19/21] downgrade, as CoreRT toolchain seems to not like clang 6.0 --- azure-pipelines.Ubuntu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.Ubuntu.yml b/azure-pipelines.Ubuntu.yml index 261e56eef7..1610fd7bd6 100755 --- a/azure-pipelines.Ubuntu.yml +++ b/azure-pipelines.Ubuntu.yml @@ -11,12 +11,12 @@ jobs: - template: build/azure-pipelines.job.template.yml parameters: name: Ubuntu - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-16.04' scriptFileName: ./build.sh initialization: - bash: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main" + sudo apt-add-repository "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" sudo apt-get update - bash: | - sudo apt-get install cmake clang-6.0 libicu60 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev + sudo apt-get install cmake clang-3.9 libicu55 uuid-dev libcurl4-openssl-dev zlib1g-dev libkrb5-dev From 93c8d4a7389c73780317fbd26de497fbe9e98249 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 15:58:55 +0100 Subject: [PATCH 20/21] try suggestion from Michal --- src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs index bf6e26fd09..5260a0e103 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs @@ -149,6 +149,9 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif + + + "; private string GenerateProjectForLocalBuild(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) => $@" @@ -183,6 +186,9 @@ private string GenerateProjectForLocalBuild(BuildPartition buildPartition, Artif + + + "; /// From 2f9a6c95ca7122bef1c29c267c087f28f2da3cf1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 21 Jan 2021 17:31:11 +0100 Subject: [PATCH 21/21] previous fix helped Linux, but broke macOS ;) --- src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs index 5260a0e103..5d33639c42 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRt/Generator.cs @@ -149,7 +149,7 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif - + "; @@ -186,7 +186,7 @@ private string GenerateProjectForLocalBuild(BuildPartition buildPartition, Artif - + ";