From 8ca5b35097bf7da534484a9a161c8d7c0eafd234 Mon Sep 17 00:00:00 2001 From: Michael Blake Date: Thu, 19 Mar 2020 15:59:09 +0000 Subject: [PATCH 1/3] Spec Filtering --- .../Configuration/When_adapter_runs_tests.cs | 2 +- ...n_unhandled_error_during_test_discovery.cs | 2 +- ...n_unhandled_error_during_test_execution.cs | 4 +- .../Execution/ISpecificationFilterProvider.cs | 11 ++ .../Execution/SpecificationFilterProvider.cs | 101 ++++++++++++++++++ .../MSpecTestAdapter.cs | 56 ++++++++++ .../MSpecTestAdapterDiscoverer.cs | 71 +++++------- .../MspecTestAdapterExecutor.cs | 85 ++++++--------- 8 files changed, 232 insertions(+), 100 deletions(-) create mode 100644 src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationFilterProvider.cs create mode 100644 src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs create mode 100644 src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapter.cs diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs index aefafec..3df22eb 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs @@ -32,7 +32,7 @@ public class When_adapter_runs_tests : WithFakes .WhenToldTo(context => context.RunSettings) .Return(The()); - adapter = new MSpecTestAdapter(An(), The()); + adapter = new MSpecTestAdapter(An(), The(), An()); }; Because of = () => diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_discovery.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_discovery.cs index 473bb33..3ff1c8b 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_discovery.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_discovery.cs @@ -19,7 +19,7 @@ public class When_there_is_an_unhandled_error_during_test_discovery : WithFakes throw new InvalidOperationException(); }); - Adapter = new MSpecTestAdapter(The(), An()); + Adapter = new MSpecTestAdapter(The(), An(), An()); }; Because of = () => { diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs index f738cdb..94f9d57 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs @@ -20,11 +20,11 @@ public class When_there_is_an_unhandled_error_during_test_execution : WithFakes Param.IsAnything, Param.IsAnything)) .Throw(new InvalidOperationException()); - Adapter = new MSpecTestAdapter(An(), The()); + Adapter = new MSpecTestAdapter(An(), The(), An()); }; Because of = () => { Adapter.RunTests(new[] {"bla"}, An(), The()); }; - + It should_send_an_error_notification_to_visual_studio = () => { The() diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationFilterProvider.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationFilterProvider.cs new file mode 100644 index 0000000..0f2c686 --- /dev/null +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationFilterProvider.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + +namespace Machine.VSTestAdapter.Execution +{ + public interface ISpecificationFilterProvider + { + IEnumerable FilteredTests(IEnumerable testCases, IRunContext runContext, IFrameworkHandle frameworkHandle); + } +} diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs new file mode 100644 index 0000000..db72b11 --- /dev/null +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Machine.Specifications.Model; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Machine.VSTestAdapter.Execution +{ + public class SpecificationFilterProvider : ISpecificationFilterProvider + { + static readonly TestProperty TagProperty = TestProperty.Register(nameof(Tag), nameof(Tag), typeof(string), typeof(TestCase)); + static readonly TestProperty SubjectProperty = TestProperty.Register(nameof(Subject), nameof(Subject), typeof(string), typeof(TestCase)); + + readonly Dictionary testCaseProperties = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [TestCaseProperties.FullyQualifiedName.Id] = TestCaseProperties.FullyQualifiedName, + [TestCaseProperties.DisplayName.Id] = TestCaseProperties.DisplayName + }; + + readonly Dictionary traitProperties = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [TagProperty.Id] = TagProperty, + [SubjectProperty.Id] = SubjectProperty + }; + + readonly string[] supportedProperties; + + public SpecificationFilterProvider() + { + supportedProperties = testCaseProperties.Keys + .Concat(traitProperties.Keys) + .ToArray(); + } + + + public IEnumerable FilteredTests(IEnumerable testCases, IRunContext runContext, IFrameworkHandle handle) + { + var filterExpression = runContext.GetTestCaseFilter(supportedProperties, propertyName => + { + if (testCaseProperties.TryGetValue(propertyName, out var testProperty)) + { + return testProperty; + } + if (traitProperties.TryGetValue(propertyName, out var traitProperty)) + { + return traitProperty; + } + return null; + }); + + var filteredTests = testCases + .Where(testCase => filterExpression + .MatchTestCase(testCase, propertyName => + { + var value = GetPropertyValue(propertyName, testCase); + handle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter -Filter property '{propertyName}' for test '{testCase.Id}' returned '{value}'"); + return value; + })); + + return filteredTests; + } + + object GetPropertyValue(string propertyName, TestCase testCase) + { + if (testCaseProperties.TryGetValue(propertyName, out var testProperty)) + { + if (testCase.Properties.Contains(testProperty)) + { + return testCase.GetPropertyValue(testProperty); + } + } + + if (traitProperties.TryGetValue(propertyName, out var traitProperty)) + { + var val = TraitContains(testCase, traitProperty.Id); + + if (val.Length == 1) + { + return val[0]; + } + + if (val.Length > 1) + { + return val; + } + } + + return null; + } + + static string[] TraitContains(TestObject testCase, string traitName) + { + return testCase.Traits + .Where(x => x.Name == traitName) + .Select(x => x.Value) + .ToArray(); + } + } +} diff --git a/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapter.cs b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapter.cs new file mode 100644 index 0000000..5c72c6b --- /dev/null +++ b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapter.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using Machine.VSTestAdapter.Configuration; +using Machine.VSTestAdapter.Discovery; +using Machine.VSTestAdapter.Discovery.BuiltIn; +using Machine.VSTestAdapter.Execution; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Machine.VSTestAdapter +{ + [FileExtension(".exe")] + [FileExtension(".dll")] + [ExtensionUri(ExecutorUri)] + [DefaultExecutorUri(ExecutorUri)] + public class MSpecTestAdapter : ITestDiscoverer, ITestExecutor + { + const string ExecutorUri = "executor://machine.vstestadapter"; + public static readonly Uri Uri = new Uri(ExecutorUri); + + readonly MSpecTestAdapterDiscoverer testDiscoverer; + readonly MSpecTestAdapterExecutor mSpecTestAdapterExecutor; + + public MSpecTestAdapter() + : this(new BuiltInSpecificationDiscoverer(), new SpecificationExecutor(), new SpecificationFilterProvider()) + { + } + + public MSpecTestAdapter(ISpecificationDiscoverer discoverer, ISpecificationExecutor executor, ISpecificationFilterProvider specificationFilterProvider) + { + testDiscoverer = new MSpecTestAdapterDiscoverer(discoverer); + mSpecTestAdapterExecutor = new MSpecTestAdapterExecutor(executor, testDiscoverer, specificationFilterProvider); + } + + public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) + { + Settings settings = Settings.Parse(discoveryContext.RunSettings?.SettingsXml); + testDiscoverer.DiscoverTests(sources, settings, logger, discoverySink.SendTestCase); + } + + public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) + { + mSpecTestAdapterExecutor.RunTests(tests, runContext, frameworkHandle); + } + + public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) + { + mSpecTestAdapterExecutor.RunTests(sources, runContext, frameworkHandle); + } + + public void Cancel() + { + } + } +} diff --git a/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs index 9c999e1..08fe7fd 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs @@ -1,54 +1,40 @@ -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; -using System; +using System; using System.Collections.Generic; -using System.Diagnostics; +#if !NETSTANDARD +using System.IO; +#endif using System.Linq; -using System.Reflection; +using Machine.VSTestAdapter.Configuration; using Machine.VSTestAdapter.Discovery; -using Machine.VSTestAdapter.Helpers; using Machine.VSTestAdapter.Discovery.BuiltIn; -using Machine.VSTestAdapter.Execution; -using Machine.VSTestAdapter.Configuration; -using System.IO; +using Machine.VSTestAdapter.Helpers; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; namespace Machine.VSTestAdapter { - [FileExtension(".exe")] - [FileExtension(".dll")] - [ExtensionUri("executor://machine.vstestadapter")] - [DefaultExecutorUri("executor://machine.vstestadapter")] - public partial class MSpecTestAdapter : ITestDiscoverer + public class MSpecTestAdapterDiscoverer { - public const string ExecutorUri = "executor://machine.vstestadapter"; - public const string VSObjectModelAssemblyName = "Microsoft.VisualStudio.TestPlatform.ObjectModel"; - private static Uri uri = new Uri(ExecutorUri); + readonly ISpecificationDiscoverer discoverer; - public MSpecTestAdapter() - : this(new BuiltInSpecificationDiscoverer(), new SpecificationExecutor()) + public MSpecTestAdapterDiscoverer() + : this(new BuiltInSpecificationDiscoverer()) { } - readonly ISpecificationDiscoverer discoverer; - readonly ISpecificationExecutor executor; - - public MSpecTestAdapter(ISpecificationDiscoverer discoverer, ISpecificationExecutor executor) + public MSpecTestAdapterDiscoverer(ISpecificationDiscoverer discoverer) { - this.executor = executor ?? throw new ArgumentNullException(nameof(executor)); - this.discoverer = discoverer ?? throw new ArgumentNullException(nameof(discoverer)); + this.discoverer = discoverer; } - public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) + public void DiscoverTests(IEnumerable sources, Settings settings, IMessageLogger logger, Action discoverySinkAction) { - // indicate start of discovery logger.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Discovering Specifications."); - int discoveredSpecCount = 0; - int sourcesWithSpecs = 0; + var discoveredSpecCount = 0; + var sourcesWithSpecs = 0; - Settings settings = GetSettings(discoveryContext); - - foreach (string assemblyPath in sources.Distinct()) + var sourcesArray = sources.Distinct().ToArray(); + foreach (var assemblyPath in sourcesArray) { try { @@ -59,30 +45,25 @@ public void DiscoverTests(IEnumerable sources, IDiscoveryContext discove sourcesWithSpecs++; - logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovering...looking in {0}", assemblyPath)); + logger.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Discovering...looking in {assemblyPath}"); - List specs = discoverer.DiscoverSpecs(assemblyPath) - .Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri)) + var specs = discoverer.DiscoverSpecs(assemblyPath) + .Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.Uri)) .ToList(); - foreach (TestCase discoveredTest in specs) + foreach (var discoveredTest in specs) { discoveredSpecCount++; - if (discoverySink != null) - { - discoverySink.SendTestCase(discoveredTest); - } + discoverySinkAction(discoveredTest); } } catch (Exception discoverException) { - logger.SendMessage(TestMessageLevel.Error, string.Format("Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {0} - {1}", assemblyPath, discoverException.Message)); + logger.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {assemblyPath} - {discoverException.Message}"); } } - // indicate that we are finished discovering - logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovery Complete - {0} specifications in {2} of {1} assemblies scanned.", discoveredSpecCount, sources.Count(), sourcesWithSpecs)); + logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovery Complete - {0} specifications in {2} of {1} assemblies scanned.", discoveredSpecCount, sourcesArray.Count(), sourcesWithSpecs)); } } - } diff --git a/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs index 08ed747..f20b158 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs @@ -4,92 +4,75 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Diagnostics; -using System.IO; using Machine.VSTestAdapter.Execution; using Machine.VSTestAdapter.Helpers; using Machine.VSTestAdapter.Configuration; namespace Machine.VSTestAdapter { - public partial class MSpecTestAdapter : ITestExecutor + public class MSpecTestAdapterExecutor { - public void Cancel() + readonly ISpecificationExecutor executor; + readonly MSpecTestAdapterDiscoverer discover; + readonly ISpecificationFilterProvider specificationFilterProvider; + + public MSpecTestAdapterExecutor(ISpecificationExecutor executor, MSpecTestAdapterDiscoverer discover, ISpecificationFilterProvider specificationFilterProvider) { - // Not supported + this.executor = executor; + this.discover = discover; + this.specificationFilterProvider = specificationFilterProvider; } public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); - frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Specifications."); - - Settings settings = GetSettings(runContext); - - foreach (string currentAsssembly in sources.Distinct()) - { - try - { -#if !NETSTANDARD - if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(currentAsssembly)), "Machine.Specifications.dll"))) - { - frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine.Specifications.dll not found for {0}", currentAsssembly)); - continue; - } -#endif - - frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine Specifications Visual Studio Test Adapter - Executing tests in {0}", currentAsssembly)); - - this.executor.RunAssembly(currentAsssembly, settings, uri, frameworkHandle); - } - catch (Exception ex) - { - frameworkHandle.SendMessage(TestMessageLevel.Error, String.Format("Machine Specifications Visual Studio Test Adapter - Error while executing specifications in assembly {0} - {1}", currentAsssembly, ex.Message)); - } - } - - frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Complete on {0} assemblies ", sources.Count())); - + frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Source Specifications."); + var testsToRun = new List(); + DiscoverTests(sources, runContext, frameworkHandle, testsToRun); + RunTests(testsToRun, runContext, frameworkHandle); + frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Source Specifications Complete."); } public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); - frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Specifications."); + frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Test Specifications."); + var executedSpecCount = 0; + var settings = Settings.Parse(runContext.RunSettings?.SettingsXml); + var currentAssembly = string.Empty; - int executedSpecCount = 0; - - Settings settings = GetSettings(runContext); - - string currentAsssembly = string.Empty; try { + var testCases = tests.ToArray(); + foreach (var grouping in testCases.GroupBy(x => x.Source)) + { + currentAssembly = grouping.Key; + frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Executing test cases in {currentAssembly}"); - foreach (IGrouping grouping in tests.GroupBy(x => x.Source)) { - currentAsssembly = grouping.Key; - frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Executing tests in {0}", currentAsssembly)); + var filteredTests = specificationFilterProvider.FilteredTests(grouping.AsEnumerable(), runContext, frameworkHandle); - List testsToRun = grouping.Select(test => test.ToVisualStudioTestIdentifier()).ToList(); + var testsToRun = filteredTests + .Select(test => test.ToVisualStudioTestIdentifier()) + .ToList(); - this.executor.RunAssemblySpecifications(currentAsssembly, testsToRun, settings, uri, frameworkHandle); + executor.RunAssemblySpecifications(currentAssembly, testsToRun, settings, MSpecTestAdapter.Uri, frameworkHandle); executedSpecCount += grouping.Count(); } - frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine Specifications Visual Studio Test Adapter - Execution Complete - {0} specifications in {1} assemblies.", executedSpecCount, tests.GroupBy(x => x.Source).Count())); - } catch (Exception ex) - { - frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format("Machine Specifications Visual Studio Test Adapter - Error while executing specifications in assembly {0} - {1}", currentAsssembly, ex.Message)); + frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Execution Complete - {executedSpecCount} specifications in {testCases.GroupBy(x => x.Source).Count()} assemblies."); } - finally + catch (Exception ex) { + frameworkHandle.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while executing specifications in assembly {currentAssembly} - {ex.Message}"); } } - private static Settings GetSettings(IDiscoveryContext runContext) + void DiscoverTests(IEnumerable sources, IRunContext discoveryContext, IMessageLogger logger, List testsToRun) { - return Settings.Parse(runContext?.RunSettings?.SettingsXml); + Settings settings = Settings.Parse(discoveryContext.RunSettings?.SettingsXml); + discover.DiscoverTests(sources, settings, logger, testsToRun.Add); } } } From 7950adbabab48bf40a7f86bc054c42b366acdb0d Mon Sep 17 00:00:00 2001 From: Michael Blake Date: Thu, 19 Mar 2020 18:13:40 +0000 Subject: [PATCH 2/3] MSpecTestAdapter() improvements --- .../Configuration/When_adapter_runs_tests.cs | 18 ++++++++++----- ...figuration_and_mspec_section_is_missing.cs | 1 + .../Execution/With_AssemblyExecutionSetup.cs | 6 ++--- ...n_unhandled_error_during_test_execution.cs | 13 +++++++---- .../Configuration/Settings.cs | 14 +++++++++--- .../Execution/ISpecificationExecutor.cs | 10 +++------ .../Execution/SpecificationExecutor.cs | 22 ++----------------- .../Execution/SpecificationFilterProvider.cs | 15 ++++++++----- .../Execution/TestExecutor.cs | 16 ++------------ .../MSpecTestAdapter.cs | 19 ++++++++-------- .../MSpecTestAdapterDiscoverer.cs | 9 +++++++- .../MspecTestAdapterExecutor.cs | 20 ++++++++--------- 12 files changed, 80 insertions(+), 83 deletions(-) diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs index 3df22eb..495fcb6 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs @@ -4,6 +4,8 @@ using Machine.VSTestAdapter.Configuration; using Machine.VSTestAdapter.Discovery; using Machine.VSTestAdapter.Execution; +using Machine.VSTestAdapter.Helpers; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; namespace Machine.VSTestAdapter.Specs.Configuration @@ -20,7 +22,8 @@ public class When_adapter_runs_tests : WithFakes "; - static MSpecTestAdapter adapter; + static MSpecTestAdapterExecutor adapter; + static MSpecTestAdapterDiscoverer discoverer; Establish establish = () => { @@ -32,24 +35,27 @@ public class When_adapter_runs_tests : WithFakes .WhenToldTo(context => context.RunSettings) .Return(The()); - adapter = new MSpecTestAdapter(An(), The(), An()); + discoverer = new MSpecTestAdapterDiscoverer(An()); + adapter = new MSpecTestAdapterExecutor(The(), discoverer, An()); }; - Because of = () => - adapter.RunTests(new[] { "dll" }, The(), An()); + Because of = () => adapter.RunTests(new[] { new TestCase("a", MSpecTestAdapter.Uri, "dll") }, The(), An()); It should_pick_up_DisableFullTestNameInIDE = () => The() - .WasToldTo(d => d.RunAssembly("dll", + .WasToldTo(d => d.RunAssemblySpecifications("dll", + Param.IsAnything, Param.Matches(s => s.DisableFullTestNameInIDE), Param.IsAnything, Param.IsAnything)); It should_pick_up_DisableFullTestNameInOutput = () => The() - .WasToldTo(d => d.RunAssembly("dll", + .WasToldTo(d => d.RunAssemblySpecifications("dll", + Param.IsAnything, Param.Matches(s => s.DisableFullTestNameInOutput), Param.IsAnything, Param.IsAnything)); + } } diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_parsing_configuration_and_mspec_section_is_missing.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_parsing_configuration_and_mspec_section_is_missing.cs index 435aa57..a14139b 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_parsing_configuration_and_mspec_section_is_missing.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_parsing_configuration_and_mspec_section_is_missing.cs @@ -3,6 +3,7 @@ namespace Machine.VSTestAdapter.Specs.Configuration { + [Tags("Tag1","Tag2")] [Subject(typeof(Settings), "Configuration")] public class When_parsing_configuration_and_mspec_section_is_missing { diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/Execution/With_AssemblyExecutionSetup.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/Execution/With_AssemblyExecutionSetup.cs index d1c72eb..60c8e11 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/Execution/With_AssemblyExecutionSetup.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/Execution/With_AssemblyExecutionSetup.cs @@ -11,21 +11,21 @@ namespace Machine.VSTestAdapter.Specs.Execution { public abstract class With_AssemblyExecutionSetup : WithFakes { - static ISpecificationExecutor Executor; + static MSpecTestAdapter Executor; static CompileContext compiler; static Assembly assembly; Establish context = () => { compiler = new CompileContext(); - Executor = new SpecificationExecutor(); + Executor = new MSpecTestAdapter(); var assemblyPath = compiler.Compile(SampleFixture.Code); assembly = Assembly.LoadFile(assemblyPath); }; Because of = () => - Executor.RunAssembly(assembly.Location, An(), new Uri("bla://executor"), The()); + Executor.RunTests(new[] { assembly.Location }, An(), The()); Cleanup after = () => compiler.Dispose(); diff --git a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs index 94f9d57..87c22ec 100644 --- a/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs +++ b/src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs @@ -1,9 +1,12 @@ using System; +using System.Collections.Generic; using Machine.Fakes; using Machine.Specifications; using Machine.VSTestAdapter.Configuration; using Machine.VSTestAdapter.Discovery; using Machine.VSTestAdapter.Execution; +using Machine.VSTestAdapter.Helpers; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -11,19 +14,21 @@ namespace Machine.VSTestAdapter.Specs { public class When_there_is_an_unhandled_error_during_test_execution : WithFakes { - static MSpecTestAdapter Adapter; + static MSpecTestAdapterExecutor Adapter; Establish context = () => { The() - .WhenToldTo(d => d.RunAssembly(Param.IsAnything, Param.IsNotNull, + .WhenToldTo(d => d.RunAssemblySpecifications(Param.IsAnything, Param.IsAnything, Param.IsNotNull, Param.IsAnything, Param.IsAnything)) .Throw(new InvalidOperationException()); - Adapter = new MSpecTestAdapter(An(), The(), An()); + var adapterDiscoverer = new MSpecTestAdapterDiscoverer(An()); + Adapter = new MSpecTestAdapterExecutor(The(), adapterDiscoverer, An()); }; - Because of = () => { Adapter.RunTests(new[] {"bla"}, An(), The()); }; + Because of = () => + Adapter.RunTests(new[] {new TestCase("a", MSpecTestAdapter.Uri, "dll"), }, An(), The()); It should_send_an_error_notification_to_visual_studio = () => { diff --git a/src/Machine.Specifications.Runner.VisualStudio/Configuration/Settings.cs b/src/Machine.Specifications.Runner.VisualStudio/Configuration/Settings.cs index fcf819b..477dcec 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Configuration/Settings.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Configuration/Settings.cs @@ -18,9 +18,17 @@ public static Settings Parse(string xml) Settings config = new Settings(); XElement mspecConfig = null; - try { - mspecConfig = XDocument.Parse(xml).XPathSelectElement("RunSettings/MSpec"); - } catch { } + try + { + if(!string.IsNullOrEmpty(xml)) + { + mspecConfig = XDocument.Parse(xml).XPathSelectElement("RunSettings/MSpec"); + } + } + catch + { + // ignored + } if (mspecConfig == null) return config; diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs index fa5284d..7dc4846 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs @@ -1,7 +1,5 @@ -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using System; -using System.Collections.Generic; using Machine.VSTestAdapter.Helpers; using Machine.VSTestAdapter.Configuration; @@ -9,8 +7,6 @@ namespace Machine.VSTestAdapter.Execution { public interface ISpecificationExecutor { - void RunAssembly(string assemblyPath, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle); - - void RunAssemblySpecifications(string assemblyPath, IEnumerable specifications, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle); + void RunAssemblySpecifications(string assemblyPath, VisualStudioTestIdentifier[] specifications, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle); } -} \ No newline at end of file +} diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs index 5813630..2768a28 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs @@ -11,26 +11,8 @@ namespace Machine.VSTestAdapter.Execution { public class SpecificationExecutor : ISpecificationExecutor { - public void RunAssembly(string source, Settings settings, Uri executorUri, IFrameworkHandle frameworkHandle) - { - source = Path.GetFullPath(source); - -#if !NETSTANDARD - using (var scope = new IsolatedAppDomainExecutionScope(source)) { - TestExecutor executor = scope.CreateInstance(); -#else - TestExecutor executor = new TestExecutor(); -#endif - - VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(source, frameworkHandle, executorUri, settings); - executor.RunAllTestsInAssembly(source, listener); -#if !NETSTANDARD - } -#endif - } - public void RunAssemblySpecifications(string assemblyPath, - IEnumerable specifications, + VisualStudioTestIdentifier[] specifications, Settings settings, Uri executorUri, IFrameworkHandle frameworkHandle) @@ -51,4 +33,4 @@ public void RunAssemblySpecifications(string assemblyPath, #endif } } -} \ No newline at end of file +} diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs index db72b11..e68137a 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs @@ -50,19 +50,24 @@ public IEnumerable FilteredTests(IEnumerable testCases, IRun return null; }); + if (filterExpression == null) + { + return testCases; + } + + handle?.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Filter property set '{filterExpression.TestCaseFilterValue}'"); + var filteredTests = testCases - .Where(testCase => filterExpression - .MatchTestCase(testCase, propertyName => + .Where(testCase => filterExpression.MatchTestCase(testCase, propertyName => { var value = GetPropertyValue(propertyName, testCase); - handle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter -Filter property '{propertyName}' for test '{testCase.Id}' returned '{value}'"); return value; })); return filteredTests; } - object GetPropertyValue(string propertyName, TestCase testCase) + object GetPropertyValue(string propertyName, TestObject testCase) { if (testCaseProperties.TryGetValue(propertyName, out var testProperty)) { @@ -92,7 +97,7 @@ object GetPropertyValue(string propertyName, TestCase testCase) static string[] TraitContains(TestObject testCase, string traitName) { - return testCase.Traits + return testCase?.Traits? .Where(x => x.Name == traitName) .Select(x => x.Value) .ToArray(); diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/TestExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/TestExecutor.cs index 9a35199..2f95a1e 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/TestExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/TestExecutor.cs @@ -22,18 +22,6 @@ public override object InitializeLifetimeService() } #endif - public TestExecutor() - { - } - - public void RunAllTestsInAssembly(string pathToAssembly, ISpecificationRunListener specificationRunListener) - { - Assembly assemblyToRun = AssemblyHelper.Load(pathToAssembly); - - DefaultRunner mspecRunner = CreateRunner(assemblyToRun, specificationRunListener); - mspecRunner.RunAssembly(assemblyToRun); - } - private DefaultRunner CreateRunner(Assembly assembly,ISpecificationRunListener specificationRunListener) { var listener = new AggregateRunListener(new[] { @@ -48,7 +36,7 @@ public void RunTestsInAssembly(string pathToAssembly, IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { - Settings settings = Settings.Parse(discoveryContext.RunSettings?.SettingsXml); - testDiscoverer.DiscoverTests(sources, settings, logger, discoverySink.SendTestCase); + //Debugger.Launch(); + testAdapterDiscoverer.DiscoverTests(sources, discoveryContext, logger, discoverySink); } public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { - mSpecTestAdapterExecutor.RunTests(tests, runContext, frameworkHandle); + //Debugger.Launch(); + testAdapterExecutor.RunTests(tests, runContext, frameworkHandle); } public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { - mSpecTestAdapterExecutor.RunTests(sources, runContext, frameworkHandle); + //Debugger.Launch(); + testAdapterExecutor.RunTests(sources, runContext, frameworkHandle); } public void Cancel() diff --git a/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs index 08fe7fd..f3b1000 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/MSpecTestAdapterDiscoverer.cs @@ -9,6 +9,7 @@ using Machine.VSTestAdapter.Discovery.BuiltIn; using Machine.VSTestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; namespace Machine.VSTestAdapter @@ -27,6 +28,12 @@ public MSpecTestAdapterDiscoverer(ISpecificationDiscoverer discoverer) this.discoverer = discoverer; } + public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) + { + Settings settings = Settings.Parse(discoveryContext.RunSettings?.SettingsXml); + DiscoverTests(sources, settings, logger, discoverySink.SendTestCase); + } + public void DiscoverTests(IEnumerable sources, Settings settings, IMessageLogger logger, Action discoverySinkAction) { logger.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Discovering Specifications."); @@ -59,7 +66,7 @@ public void DiscoverTests(IEnumerable sources, Settings settings, IMessa } catch (Exception discoverException) { - logger.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {assemblyPath} - {discoverException.Message}"); + logger.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {assemblyPath} - {discoverException}"); } } diff --git a/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs index f20b158..e65f56e 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/MspecTestAdapterExecutor.cs @@ -25,8 +25,6 @@ public MSpecTestAdapterExecutor(ISpecificationExecutor executor, MSpecTestAdapte public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { - //Debugger.Launch(); - frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Source Specifications."); var testsToRun = new List(); DiscoverTests(sources, runContext, frameworkHandle, testsToRun); @@ -36,9 +34,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { - //Debugger.Launch(); - frameworkHandle.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Executing Test Specifications."); + var totalSpecCount = 0; var executedSpecCount = 0; var settings = Settings.Parse(runContext.RunSettings?.SettingsXml); var currentAssembly = string.Empty; @@ -48,24 +45,25 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame var testCases = tests.ToArray(); foreach (var grouping in testCases.GroupBy(x => x.Source)) { - currentAssembly = grouping.Key; - frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Executing test cases in {currentAssembly}"); + totalSpecCount += grouping.Count(); + + frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Executing test cases in {grouping.Key}"); var filteredTests = specificationFilterProvider.FilteredTests(grouping.AsEnumerable(), runContext, frameworkHandle); var testsToRun = filteredTests .Select(test => test.ToVisualStudioTestIdentifier()) - .ToList(); + .ToArray(); - executor.RunAssemblySpecifications(currentAssembly, testsToRun, settings, MSpecTestAdapter.Uri, frameworkHandle); - executedSpecCount += grouping.Count(); + executor.RunAssemblySpecifications(grouping.Key, testsToRun, settings, MSpecTestAdapter.Uri, frameworkHandle); + executedSpecCount += testsToRun.Length; } - frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Execution Complete - {executedSpecCount} specifications in {testCases.GroupBy(x => x.Source).Count()} assemblies."); + frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Execution Complete - {executedSpecCount} of {totalSpecCount} specifications in {testCases.GroupBy(x => x.Source).Count()} assemblies."); } catch (Exception ex) { - frameworkHandle.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while executing specifications in assembly {currentAssembly} - {ex.Message}"); + frameworkHandle.SendMessage(TestMessageLevel.Error, $"Machine Specifications Visual Studio Test Adapter - Error while executing specifications in assembly {currentAssembly} - {ex}"); } } From ac4e26b797f80fb6ac0f337c9aa8969965559839 Mon Sep 17 00:00:00 2001 From: Michael Blake Date: Thu, 19 Mar 2020 18:20:31 +0000 Subject: [PATCH 3/3] VisualStudioTestIdentifier[] to IEnumerable --- .../Execution/ISpecificationExecutor.cs | 3 ++- .../Execution/SpecificationExecutor.cs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs index 7dc4846..ae05121 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using System; +using System.Collections.Generic; using Machine.VSTestAdapter.Helpers; using Machine.VSTestAdapter.Configuration; @@ -7,6 +8,6 @@ namespace Machine.VSTestAdapter.Execution { public interface ISpecificationExecutor { - void RunAssemblySpecifications(string assemblyPath, VisualStudioTestIdentifier[] specifications, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle); + void RunAssemblySpecifications(string assemblyPath, IEnumerable specifications, Settings settings, Uri adapterUri, IFrameworkHandle frameworkHandle); } } diff --git a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs index 2768a28..204fea4 100644 --- a/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs +++ b/src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs @@ -12,9 +12,9 @@ namespace Machine.VSTestAdapter.Execution public class SpecificationExecutor : ISpecificationExecutor { public void RunAssemblySpecifications(string assemblyPath, - VisualStudioTestIdentifier[] specifications, + IEnumerable specifications, Settings settings, - Uri executorUri, + Uri adapterUri, IFrameworkHandle frameworkHandle) { assemblyPath = Path.GetFullPath(assemblyPath); @@ -25,7 +25,7 @@ public void RunAssemblySpecifications(string assemblyPath, #else TestExecutor executor = new TestExecutor(); #endif - VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(assemblyPath, frameworkHandle, executorUri, settings); + VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(assemblyPath, frameworkHandle, adapterUri, settings); executor.RunTestsInAssembly(assemblyPath, specifications, listener); #if !NETSTANDARD