From 3780ced751bfa884311f1878ae75f7a7ef04d698 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 13:19:35 +0100 Subject: [PATCH 01/13] remove the extra overload that most probably nobody uses but increases the complexity --- src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs index 85fdbc6296..1a10c6494d 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs @@ -29,13 +29,6 @@ public static Summary Run(Type type, IConfig config = null) return RunWithDirtyAssemblyResolveHelper(type, config); } - [PublicAPI] - public static Summary Run(Type type, MethodInfo[] methods, IConfig config = null) - { - using (DirtyAssemblyResolveHelper.Create()) - return RunWithDirtyAssemblyResolveHelper(type, methods, config); - } - [PublicAPI] public static Summary[] Run(Assembly assembly, IConfig config = null) { @@ -75,10 +68,6 @@ public static Summary RunSource(string source, IConfig config = null) private static Summary RunWithDirtyAssemblyResolveHelper(Type type, IConfig config) => BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(type, config) }).Single(); - [MethodImpl(MethodImplOptions.NoInlining)] - private static Summary RunWithDirtyAssemblyResolveHelper(Type type, MethodInfo[] methods, IConfig config = null) - => BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }).Single(); - [MethodImpl(MethodImplOptions.NoInlining)] private static Summary[] RunWithDirtyAssemblyResolveHelper(Assembly assembly, IConfig config = null) => BenchmarkRunnerClean.Run(assembly.GetRunnableBenchmarks().Select(type => BenchmarkConverter.TypeToBenchmarks(type, config)).ToArray()); From 1732cddcd326d5812ffcceb5420c0efad184a146 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 13:21:13 +0100 Subject: [PATCH 02/13] the test code should call a public method without using internal surface if possible --- tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs index d4eea24ce3..53c241c5b8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs @@ -60,7 +60,7 @@ protected Reports.Summary CanExecute(Type type, IConfig config = null, bool full config = config.With(DefaultColumnProviders.Instance); // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = BenchmarkRunner.Run(type, BenchmarkConverter.GetFullConfig(type, config)); + var summary = BenchmarkRunner.Run(type, config); if (fullValidation) { From 6bb01613a20d5930e28e71a15660ef544e0da6d4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 13:21:35 +0100 Subject: [PATCH 03/13] allow for config per method --- .../Running/BenchmarkConverter.cs | 90 +++++++++---------- .../Running/BenchmarkPartitioner.cs | 2 +- 2 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index 72ca3c69df..423f3a47c0 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Parameters; namespace BenchmarkDotNet.Running @@ -22,41 +23,24 @@ public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null // We should check all methods including private to notify users about private methods with the [Benchmark] attribute var bindingFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; - - var fullConfig = GetFullConfig(type, config); - var allMethods = type.GetMethods(bindingFlags); - return MethodsToBenchmarksWithFullConfig(type, allMethods, fullConfig); - } - - public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null) - { - var fullConfig = GetFullConfig(containingType, config); - - return MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, fullConfig); - } + var benchmarkMethods = type.GetMethods(bindingFlags).Where(method => method.HasAttribute()).ToArray(); - private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containingType, MethodInfo[] benchmarkMethods, ImmutableConfig immutableConfig) - { - if (immutableConfig == null) - throw new ArgumentNullException(nameof(immutableConfig)); - - var helperMethods = containingType.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564 - - var globalSetupMethods = GetAttributedMethods(helperMethods, "GlobalSetup"); - var globalCleanupMethods = GetAttributedMethods(helperMethods, "GlobalCleanup"); - var iterationSetupMethods = GetAttributedMethods(helperMethods, "IterationSetup"); - var iterationCleanupMethods = GetAttributedMethods(helperMethods, "IterationCleanup"); + var allPublicMethods = type.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564 - var targetMethods = benchmarkMethods.Where(method => method.HasAttribute()).ToArray(); + var globalSetupMethods = GetAttributedMethods(allPublicMethods, "GlobalSetup"); + var globalCleanupMethods = GetAttributedMethods(allPublicMethods, "GlobalCleanup"); + var iterationSetupMethods = GetAttributedMethods(allPublicMethods, "IterationSetup"); + var iterationCleanupMethods = GetAttributedMethods(allPublicMethods, "IterationCleanup"); - var parameterDefinitions = GetParameterDefinitions(containingType); + var parameterDefinitions = GetParameterDefinitions(type); var parameterInstancesList = parameterDefinitions.Expand(); - var jobs = immutableConfig.GetJobs(); + var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray(); - var targets = GetTargets(targetMethods, containingType, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray(); + var configPerType = GetFullTypeConfig(type, config); var benchmarks = new List(); + foreach (var target in targets) { var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type).ToArray(); @@ -66,38 +50,48 @@ private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containin from argumentDefinition in argumentsDefinitions select new ParameterInstances(parameterInstance.Items.Concat(argumentDefinition.Items).ToArray())).ToArray(); - benchmarks.AddRange( - from job in jobs + var configPerMethod = GetFullMethodConfig(target.WorkloadMethod, configPerType); + + var benchmarksForTarget = + from job in configPerMethod.GetJobs() from parameterInstance in parameterInstances - select BenchmarkCase.Create(target, job, parameterInstance, immutableConfig) - ); + select BenchmarkCase.Create(target, job, parameterInstance, configPerMethod); + + benchmarks.AddRange(GetFilteredBenchmarks(benchmarksForTarget, configPerMethod.GetFilters())); } - var filters = immutableConfig.GetFilters().ToArray(); - var filteredBenchmarks = GetFilteredBenchmarks(benchmarks, filters); - var orderedBenchmarks = immutableConfig.Orderer.GetExecutionOrder(filteredBenchmarks).ToArray(); + var orderedBenchmarks = configPerType.Orderer.GetExecutionOrder(benchmarks.ToImmutableArray()).ToArray(); - return new BenchmarkRunInfo(orderedBenchmarks, containingType, immutableConfig); + return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType); } - public static ImmutableConfig GetFullConfig(Type type, IConfig config) + private static ImmutableConfig GetFullTypeConfig(Type type, IConfig config) { config = config ?? DefaultConfig.Instance; - if (type != null) - { - var typeAttributes = type.GetTypeInfo().GetCustomAttributes(true).OfType(); - var assemblyAttributes = type.GetTypeInfo().Assembly.GetCustomAttributes().OfType(); - var allAttributes = typeAttributes.Concat(assemblyAttributes); - var configs = allAttributes.Select(attribute => attribute.Config) - .OrderBy(c => c.GetJobs().Count(job => job.Meta.IsMutator)); // configs with mutators must be the ones applied at the end - - foreach (var configFromAttribute in configs) - config = ManualConfig.Union(config, configFromAttribute); - } + + var typeAttributes = type.GetCustomAttributes(true).OfType(); + var assemblyAttributes = type.Assembly.GetCustomAttributes().OfType(); + + foreach (var configFromAttribute in typeAttributes.Concat(assemblyAttributes)) + config = ManualConfig.Union(config, configFromAttribute.Config); return ImmutableConfigBuilder.Create(config); } + private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableConfig typeConfig) + { + var methodAttributes = method.GetCustomAttributes(true).OfType(); + + if (!methodAttributes.Any()) // the most common case + return typeConfig; + + var config = ManualConfig.Create(typeConfig); + foreach (var configFromAttribute in methodAttributes) + config = ManualConfig.Union(config, configFromAttribute.Config); + + return ImmutableConfigBuilder.Create(config); + } + private static IEnumerable GetTargets( MethodInfo[] targetMethods, Type type, @@ -238,7 +232,7 @@ private static string[] GetCategories(MethodInfo method) return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); } - private static ImmutableArray GetFilteredBenchmarks(IList benchmarks, IList filters) + private static ImmutableArray GetFilteredBenchmarks(IEnumerable benchmarks, IEnumerable filters) => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray(); private static void AssertMethodHasCorrectSignature(string methodType, MethodInfo methodInfo) diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index e7add20f96..8f14f8d87d 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -13,7 +13,7 @@ public static class BenchmarkPartitioner { public static BuildPartition[] CreateForBuild(BenchmarkRunInfo[] supportedBenchmarks, IResolver resolver) => supportedBenchmarks - .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, info.Config))) + .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, benchmark.Config))) .GroupBy(tuple => tuple.benchmark, BenchmarkRuntimePropertiesComparer.Instance) .Select(group => new BuildPartition(group.Select((item, index) => new BenchmarkBuildInfo(item.benchmark, item.Config, index)).ToArray(), resolver)) .ToArray(); From 75d7a9c6e27ea89a01ecfccfd84b31fa3e114dde Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 13:40:21 +0100 Subject: [PATCH 04/13] allow the FilterConfigBaseAttribute to be applied per method --- .../Filters/FilterConfigBaseAttribute.cs | 2 +- .../Configs/ConfigPerMethodTests.cs | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs diff --git a/src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs index aa07914ca9..28dcaf6e81 100644 --- a/src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs @@ -4,7 +4,7 @@ namespace BenchmarkDotNet.Attributes { - [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] public abstract class FilterConfigBaseAttribute : Attribute, IConfigSource { // CLS-Compliant Code requires a constructor without an array in the argument list diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs new file mode 100644 index 0000000000..6ca3543c28 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -0,0 +1,41 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Running; +using Xunit; + +namespace BenchmarkDotNet.Tests.Configs +{ + public class ConfigPerMethodTests + { + [Fact] + public void PetMethodConfigsAreRespected() + { + var never = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkThatShouldNeverRun)); + + Assert.Empty(never.BenchmarksCases); + + var always = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkThatShouldAlwaysRun)); + + Assert.NotEmpty(always.BenchmarksCases); + } + + public class WithBenchmarkThatShouldNeverRun + { + [Benchmark] + [ConditionalRun(false)] + public void Method() { } + } + + public class WithBenchmarkThatShouldAlwaysRun + { + [Benchmark] + [ConditionalRun(true)] + public void Method() { } + } + + public class ConditionalRun : FilterConfigBaseAttribute + { + public ConditionalRun(bool value) : base(new SimpleFilter(_ => value)) { } + } + } +} \ No newline at end of file From 59fa565482497ada8e382660c0c357f553c7f90d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 14:05:55 +0100 Subject: [PATCH 05/13] introduce OperatingSystemsFilterAttribute --- .../OperatingSystemsFilterAttribute.cs | 45 +++++++++++++++++++ .../Configs/ConfigPerMethodTests.cs | 40 +++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs diff --git a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs new file mode 100644 index 0000000000..e117bd7bfb --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs @@ -0,0 +1,45 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Filters; +using JetBrains.Annotations; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Attributes +{ + [PublicAPI] + public class OperatingSystemsFilterAttribute : FilterConfigBaseAttribute + { + // CLS-Compliant Code requires a constructor without an array in the argument list + public OperatingSystemsFilterAttribute() { } + + /// if set to true, the OSes beloning to platforms are enabled, if set to false, disabled + public OperatingSystemsFilterAttribute(bool allowed, params PlatformID[] platforms) + : base(new SimpleFilter(_ => + { + return allowed + ? platforms.Any(platform => RuntimeInformation.IsOSPlatform(Map(platform))) + : platforms.All(platform => !RuntimeInformation.IsOSPlatform(Map(platform))); + })) + { + } + + // OSPlatform is a struct so it can not be used as attribute argument and this is why we use PlatformID enum + private static OSPlatform Map(PlatformID platform) + { + switch (platform) + { + case PlatformID.MacOSX: + return OSPlatform.OSX; + case PlatformID.Unix: + return OSPlatform.Linux; + case PlatformID.Win32NT: + case PlatformID.Win32S: + case PlatformID.Win32Windows: + case PlatformID.WinCE: + return OSPlatform.Windows; + default: + throw new NotSupportedException($"Platform {platform} is not supported"); + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs index 6ca3543c28..81cc5e06c3 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -1,4 +1,6 @@ -using BenchmarkDotNet.Attributes; +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Running; using Xunit; @@ -19,6 +21,11 @@ public void PetMethodConfigsAreRespected() Assert.NotEmpty(always.BenchmarksCases); } + public class ConditionalRun : FilterConfigBaseAttribute + { + public ConditionalRun(bool value) : base(new SimpleFilter(_ => value)) { } + } + public class WithBenchmarkThatShouldNeverRun { [Benchmark] @@ -33,9 +40,36 @@ public class WithBenchmarkThatShouldAlwaysRun public void Method() { } } - public class ConditionalRun : FilterConfigBaseAttribute + [Fact] + public void CanEnableOrDisableTheBenchmarkPerOperatingSystem() { - public ConditionalRun(bool value) : base(new SimpleFilter(_ => value)) { } + var allowedForWindows = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkAllowedForWindows)); + var notAllowedForWindows = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkNotAllowedForWindows)); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Assert.NotEmpty(allowedForWindows.BenchmarksCases); + Assert.Empty(notAllowedForWindows.BenchmarksCases); + } + else + { + Assert.Empty(allowedForWindows.BenchmarksCases); + Assert.NotEmpty(notAllowedForWindows.BenchmarksCases); + } + } + + public class WithBenchmarkAllowedForWindows + { + [Benchmark] + [OperatingSystemsFilter(allowed: true, PlatformID.Win32NT)] + public void Method() { } + } + + public class WithBenchmarkNotAllowedForWindows + { + [Benchmark] + [OperatingSystemsFilter(allowed: false, PlatformID.Win32NT)] + public void Method() { } } } } \ No newline at end of file From dcd8c526f29cfc027d976142b7ea36244b01309f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 14:13:06 +0100 Subject: [PATCH 06/13] introduce OperatingSystemsArchitectureFilterAttribute --- ...atingSystemsArchitectureFilterAttribute.cs | 25 +++++++++++++++ .../Configs/ConfigPerMethodTests.cs | 32 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs diff --git a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs new file mode 100644 index 0000000000..5b7419a227 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs @@ -0,0 +1,25 @@ +using System.Linq; +using BenchmarkDotNet.Filters; +using JetBrains.Annotations; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Attributes +{ + [PublicAPI] + public class OperatingSystemsArchitectureFilterAttribute : FilterConfigBaseAttribute + { + // CLS-Compliant Code requires a constructor without an array in the argument list + public OperatingSystemsArchitectureFilterAttribute() { } + + /// if set to true, the architectures are enabled, if set to false, disabled + public OperatingSystemsArchitectureFilterAttribute(bool allowed, params Architecture[] architectures) + : base(new SimpleFilter(_ => + { + return allowed + ? architectures.Any(architecture => RuntimeInformation.OSArchitecture == architecture) + : architectures.All(architecture => RuntimeInformation.OSArchitecture != architecture); + })) + { + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs index 81cc5e06c3..bb938994a4 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -71,5 +71,37 @@ public class WithBenchmarkNotAllowedForWindows [OperatingSystemsFilter(allowed: false, PlatformID.Win32NT)] public void Method() { } } + + [Fact] + public void CanEnableOrDisableTheBenchmarkPerOperatingSystemArchitecture() + { + var allowed = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkAllowedForX64)); + var notallowed = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkNotAllowedForX64)); + + if (RuntimeInformation.OSArchitecture == Architecture.X64) + { + Assert.NotEmpty(allowed.BenchmarksCases); + Assert.Empty(notallowed.BenchmarksCases); + } + else + { + Assert.Empty(allowed.BenchmarksCases); + Assert.NotEmpty(notallowed.BenchmarksCases); + } + } + + public class WithBenchmarkAllowedForX64 + { + [Benchmark] + [OperatingSystemsArchitectureFilter(allowed: true, Architecture.X64)] + public void Method() { } + } + + public class WithBenchmarkNotAllowedForX64 + { + [Benchmark] + [OperatingSystemsArchitectureFilter(allowed: false, Architecture.X64)] + public void Method() { } + } } } \ No newline at end of file From 2422eb7f47b06ea8498f18349376c50811e96031 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 10 Mar 2019 14:18:01 +0100 Subject: [PATCH 07/13] typo --- tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs index bb938994a4..ca1ab29214 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -10,7 +10,7 @@ namespace BenchmarkDotNet.Tests.Configs public class ConfigPerMethodTests { [Fact] - public void PetMethodConfigsAreRespected() + public void PerMethodConfigsAreRespected() { var never = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkThatShouldNeverRun)); From 103ed216e6386be0e9a244cb2670490a717e7d20 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:05:02 +0100 Subject: [PATCH 08/13] fix warnings --- src/BenchmarkDotNet/Running/BenchmarkConverter.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index 1129e230bc..1e51b51697 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -53,7 +53,7 @@ from argumentDefinition in argumentsDefinitions var configPerMethod = GetFullMethodConfig(target.WorkloadMethod, configPerType); - var benchmarksForTarget = + var benchmarksForTarget = from job in configPerMethod.GetJobs() from parameterInstance in parameterInstances select BenchmarkCase.Create(target, job, parameterInstance, configPerMethod); @@ -72,10 +72,10 @@ private static ImmutableConfig GetFullTypeConfig(Type type, IConfig config) var typeAttributes = type.GetCustomAttributes(true).OfType(); var assemblyAttributes = type.Assembly.GetCustomAttributes().OfType(); - + foreach (var configFromAttribute in typeAttributes.Concat(assemblyAttributes)) config = ManualConfig.Union(config, configFromAttribute.Config); - + return ImmutableConfigBuilder.Create(config); } @@ -245,7 +245,7 @@ private static string[] GetCategories(MethodInfo method) return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); } - private static ImmutableArray GetFilteredBenchmarks(IEnumerable benchmarks, IEnumerable filters) + private static ImmutableArray GetFilteredBenchmarks(IEnumerable benchmarks, IEnumerable filters) => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray(); private static void AssertMethodHasCorrectSignature(string methodType, MethodInfo methodInfo) From 0edd9af944f4dc75eb26fba0bf45e2414d4d22b3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:12:42 +0100 Subject: [PATCH 09/13] fix compilation errors --- .../Running/BenchmarkConverter.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index 1e51b51697..fdca381fc8 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -9,7 +9,6 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Filters; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; @@ -26,25 +25,32 @@ public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null var bindingFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; var benchmarkMethods = type.GetMethods(bindingFlags).Where(method => method.HasAttribute()).ToArray(); + return MethodsToBenchmarksWithFullConfig(type, benchmarkMethods, config); + } + + public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null) + => MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, config); + + private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, MethodInfo[] benchmarkMethods, IConfig config) + { var allPublicMethods = type.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564 + var configPerType = GetFullTypeConfig(type, config); var globalSetupMethods = GetAttributedMethods(allPublicMethods, "GlobalSetup"); var globalCleanupMethods = GetAttributedMethods(allPublicMethods, "GlobalCleanup"); var iterationSetupMethods = GetAttributedMethods(allPublicMethods, "IterationSetup"); var iterationCleanupMethods = GetAttributedMethods(allPublicMethods, "IterationCleanup"); - var parameterDefinitions = GetParameterDefinitions(type); - var parameterInstancesList = parameterDefinitions.Expand(immutableConfig.SummaryStyle); - var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray(); - var configPerType = GetFullTypeConfig(type, config); + var parameterDefinitions = GetParameterDefinitions(type); + var parameterInstancesList = parameterDefinitions.Expand(configPerType.SummaryStyle); var benchmarks = new List(); foreach (var target in targets) { - var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type, immutableConfig.SummaryStyle).ToArray(); + var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type, configPerType.SummaryStyle).ToArray(); var parameterInstances = (from parameterInstance in parameterInstancesList From de97fca29245460fe3e495faad292861387ee21b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:30:52 +0100 Subject: [PATCH 10/13] introduce our own OS enum, add Browser (WebAssembly) as an option --- .../OperatingSystemsFilterAttribute.cs | 32 +++++++++++++------ .../Configs/ConfigPerMethodTests.cs | 4 +-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs index e117bd7bfb..203d25094e 100644 --- a/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs @@ -6,14 +6,27 @@ namespace BenchmarkDotNet.Attributes { + public enum OS : byte + { + Windows, + Linux, + macOS, + /// + /// WebAssembly + /// + Browser + } + [PublicAPI] public class OperatingSystemsFilterAttribute : FilterConfigBaseAttribute { + private static readonly OSPlatform browser = OSPlatform.Create("BROWSER"); + // CLS-Compliant Code requires a constructor without an array in the argument list public OperatingSystemsFilterAttribute() { } /// if set to true, the OSes beloning to platforms are enabled, if set to false, disabled - public OperatingSystemsFilterAttribute(bool allowed, params PlatformID[] platforms) + public OperatingSystemsFilterAttribute(bool allowed, params OS[] platforms) : base(new SimpleFilter(_ => { return allowed @@ -24,19 +37,18 @@ public OperatingSystemsFilterAttribute(bool allowed, params PlatformID[] platfor } // OSPlatform is a struct so it can not be used as attribute argument and this is why we use PlatformID enum - private static OSPlatform Map(PlatformID platform) + private static OSPlatform Map(OS platform) { switch (platform) { - case PlatformID.MacOSX: - return OSPlatform.OSX; - case PlatformID.Unix: - return OSPlatform.Linux; - case PlatformID.Win32NT: - case PlatformID.Win32S: - case PlatformID.Win32Windows: - case PlatformID.WinCE: + case OS.Windows: return OSPlatform.Windows; + case OS.Linux: + return OSPlatform.Linux; + case OS.macOS: + return OSPlatform.OSX; + case OS.Browser: + return browser; default: throw new NotSupportedException($"Platform {platform} is not supported"); } diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs index ca1ab29214..da25dd8c74 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -61,14 +61,14 @@ public void CanEnableOrDisableTheBenchmarkPerOperatingSystem() public class WithBenchmarkAllowedForWindows { [Benchmark] - [OperatingSystemsFilter(allowed: true, PlatformID.Win32NT)] + [OperatingSystemsFilter(allowed: true, OS.Windows)] public void Method() { } } public class WithBenchmarkNotAllowedForWindows { [Benchmark] - [OperatingSystemsFilter(allowed: false, PlatformID.Win32NT)] + [OperatingSystemsFilter(allowed: false, OS.Windows)] public void Method() { } } From 5140e34639f179af3cff4bca9188f22e2a3b4139 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:40:59 +0100 Subject: [PATCH 11/13] add test --- .../Configs/ConfigPerMethodTests.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs index da25dd8c74..f11fbb7e42 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Filters; @@ -103,5 +104,27 @@ public class WithBenchmarkNotAllowedForX64 [OperatingSystemsArchitectureFilter(allowed: false, Architecture.X64)] public void Method() { } } + + [Fact] + public void CanEnableOrDisableMemoryRandomizationPerMethod() + { + var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(WithMemoryRandomization)).BenchmarksCases; + + Assert.Equal(2, benchmarks.Length); + var disabled = benchmarks.Single(benchmark => benchmark.Descriptor.WorkloadMethod.Name == nameof(WithMemoryRandomization.DisabledByDefault)); + Assert.False(disabled.Job.Run.MemoryRandomization); + var enabled = benchmarks.Single(benchmark => benchmark.Descriptor.WorkloadMethod.Name == nameof(WithMemoryRandomization.EnabledWithAttributeOnMethod)); + Assert.True(enabled.Job.Run.MemoryRandomization); + } + + public class WithMemoryRandomization + { + [Benchmark] + public void DisabledByDefault() { } + + [Benchmark] + [MemoryRandomization(true)] + public void EnabledWithAttributeOnMethod() { } + } } } \ No newline at end of file From e23327990272a39e2545d8e704a8a32e45ed2497 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:42:54 +0100 Subject: [PATCH 12/13] allow setting mutator job attributes per method --- .../Attributes/Mutators/JobMutatorConfigBaseAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Attributes/Mutators/JobMutatorConfigBaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Mutators/JobMutatorConfigBaseAttribute.cs index 518ae6e7b5..a4a4102f49 100644 --- a/src/BenchmarkDotNet/Attributes/Mutators/JobMutatorConfigBaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Mutators/JobMutatorConfigBaseAttribute.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Attributes { - [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] // users must not be able to define given mutator attribute more than once per type + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false)] // users must not be able to define given mutator attribute more than once per type public class JobMutatorConfigBaseAttribute : Attribute, IConfigSource { // CLS-Compliant Code requires a constructor which use only CLS-compliant types From a5c857a9fcde8c82f9d4f084d67c66086ac9ddab Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 27 Jan 2021 14:49:43 +0100 Subject: [PATCH 13/13] update randomization sample --- .../IntroMemoryRandomization.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs index 03d55574ec..5b9e417398 100644 --- a/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs +++ b/samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs @@ -1,9 +1,5 @@ using BenchmarkDotNet.Attributes; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BenchmarkDotNet.Samples { @@ -23,6 +19,12 @@ public void Setup() } [Benchmark] - public void Array() => System.Array.Copy(_array, _destination, Size); + [MemoryRandomization(false)] + public void Array_RandomizationDisabled() => Array.Copy(_array, _destination, Size); + + [Benchmark] + [MemoryRandomization(true)] + [MaxIterationCount(40)] // the benchmark becomes multimodal and need a lower limit of max iterations than the default + public void Array_RandomizationEnabled() => Array.Copy(_array, _destination, Size); } }