From f51024ea31256379ff9bcffb8c7b160f0387b500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bignon?= Date: Tue, 6 Feb 2018 10:02:01 +0100 Subject: [PATCH 1/4] Fix Multiple results in VS2017 test runner (#336) --- .../Execution/TestMethodRunner.cs | 37 ++++++---- test/E2ETests/Automation.CLI/CLITestBase.cs | 1 + .../CustomTestExecutionExtensibilityTests.cs | 32 +++++++++ .../Smoke.E2E.Tests/Smoke.E2E.Tests.csproj | 1 + .../CustomTestExTests.cs | 69 +++++++++++++++++++ .../FxExtensibilityTestProject.csproj | 1 + .../Execution/TestMethodRunnerTests.cs | 29 +++++++- 7 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs create mode 100644 test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 5559f91338..3e7b45b14b 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -247,22 +247,28 @@ internal UnitTestResult[] RunTestMethod() watch.Start(); this.testContext.SetDataRow(dataRow); - UTF.TestResult currentResult; + UTF.TestResult[] testResults; try { - currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]; + testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { - currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }; + testResults = new[] + { + new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } + }; } - currentResult.DatarowIndex = rowIndex++; watch.Stop(); - currentResult.Duration = watch.Elapsed; + foreach (var testResult in testResults) + { + testResult.DatarowIndex = rowIndex++; + testResult.Duration = watch.Elapsed; + } - results.Add(currentResult); + results.AddRange(testResults); } } finally @@ -293,18 +299,25 @@ internal UnitTestResult[] RunTestMethod() foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) { this.testMethodInfo.SetArguments(data); - UTF.TestResult currentResult; + UTF.TestResult[] testResults; try { - currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]; + testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { - currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }; + testResults = new[] + { + new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } + }; + } + + foreach (var testResult in testResults) + { + testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); } - currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); - results.Add(currentResult); + results.AddRange(testResults); this.testMethodInfo.SetArguments(null); } } @@ -313,7 +326,7 @@ internal UnitTestResult[] RunTestMethod() { try { - results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]); + results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); } catch (Exception ex) { diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 88cd82f668..d09f0d6a64 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -160,6 +160,7 @@ public void ValidatePassedTestsContain(params string[] passedTests) { foreach (var test in passedTests) { + var x = string.Join(", ", this.runEventsHandler.PassedTests.Select(_ => _.DisplayName)); var testFound = this.runEventsHandler.PassedTests.Any( p => test.Equals(p.TestCase?.FullyQualifiedName) || test.Equals(p.DisplayName)); diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs new file mode 100644 index 0000000000..e8b6115cec --- /dev/null +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace MSTestAdapter.Smoke.E2ETests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CustomTestExecutionExtensibilityTests : CLITestBase + { + private const string TestAssembly = "FxExtensibilityTestProject.dll"; + + [TestMethod] + public void ExecuteCustomTestExtensibilityTests() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }); + this.ValidatePassedTestsContain( + "CustomTestMethod1 - Execution number 1", + "CustomTestMethod1 - Execution number 2", + "CustomTestMethod1 - Execution number 4", + "CustomTestMethod1 - Execution number 5", + "CustomTestClass1 - Execution number 1", + "CustomTestClass1 - Execution number 2", + "CustomTestClass1 - Execution number 4", + "CustomTestClass1 - Execution number 5"); + this.ValidateFailedTestsContain( + "CustomTestMethod1 - Execution number 3", + "CustomTestClass1 - Execution number 3"); + } + } +} \ No newline at end of file diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj index 6005ad1c50..8b694c0e14 100644 --- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj +++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj @@ -39,6 +39,7 @@ + diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs new file mode 100644 index 0000000000..71732d55a3 --- /dev/null +++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs @@ -0,0 +1,69 @@ +namespace FxExtensibilityTestProject +{ + using System; + using System.Collections.Generic; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [IterativeTestClass(5)] + public class CustomTestExTests + { + private static int customTestMethod1ExecutionCount; + [IterativeTestMethod(5)] + public void CustomTestMethod1() + { + customTestMethod1ExecutionCount++; + Assert.AreNotEqual(3, customTestMethod1ExecutionCount); + } + + private static int customTestClass1ExecutionCount; + [TestMethod] + public void CustomTestClass1() + { + customTestClass1ExecutionCount++; + Assert.AreNotEqual(3, customTestClass1ExecutionCount); + } + } + + public class IterativeTestMethodAttribute : TestMethodAttribute + { + private readonly int stabilityThreshold; + + public IterativeTestMethodAttribute(int stabilityThreshold) + { + this.stabilityThreshold = stabilityThreshold; + } + + public override TestResult[] Execute(ITestMethod testMethod) + { + var results = new List(); + for (int count = 0; count < this.stabilityThreshold; count++) + { + var testResults = base.Execute(testMethod); + foreach (var testResult in testResults) + { + testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}"; + } + results.AddRange(testResults); + } + + return results.ToArray(); + } + } + + public class IterativeTestClassAttribute : TestClassAttribute + { + private readonly int stabilityThreshold; + + public IterativeTestClassAttribute(int stabilityThreshold) + { + this.stabilityThreshold = stabilityThreshold; + } + + public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute) + { + if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute; + + return new IterativeTestMethodAttribute(this.stabilityThreshold); + } + } +} diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj index db9dc23cba..9e587e801d 100644 --- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj +++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj @@ -44,6 +44,7 @@ + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index 9f66bad592..3dbd30d23e 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test"); } + [TestMethodV1] + public void RunTestMethodForMultipleResultsReturnMultipleResults() + { + var testMethodAttributeMock = new Mock(); + testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] + { + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } + }); + + var localTestMethodOptions = new TestMethodOptions + { + Timeout = 200, + Executor = testMethodAttributeMock.Object, + TestContext = this.testContextImplementation, + ExpectedException = null + }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + var results = testMethodRunner.Execute(); + Assert.AreEqual(2, results.Length); + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + } + [TestMethodV1] public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome() { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed }); + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed }); var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); var results = testMethodRunner.Execute(); From 0b48abb9a581ac25236a47ccf3b9659911ffdbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bignon?= Date: Wed, 7 Feb 2018 10:20:50 +0100 Subject: [PATCH 2/4] Fixed PR comments --- src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs | 3 ++- test/E2ETests/Automation.CLI/CLITestBase.cs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 3e7b45b14b..1ab3be4dd8 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -261,10 +261,11 @@ internal UnitTestResult[] RunTestMethod() }; } + rowIndex++; watch.Stop(); foreach (var testResult in testResults) { - testResult.DatarowIndex = rowIndex++; + testResult.DatarowIndex = rowIndex; testResult.Duration = watch.Elapsed; } diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index d09f0d6a64..88cd82f668 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -160,7 +160,6 @@ public void ValidatePassedTestsContain(params string[] passedTests) { foreach (var test in passedTests) { - var x = string.Join(", ", this.runEventsHandler.PassedTests.Select(_ => _.DisplayName)); var testFound = this.runEventsHandler.PassedTests.Any( p => test.Equals(p.TestCase?.FullyQualifiedName) || test.Equals(p.DisplayName)); From d80a0268c1d4f3ea0269820431ede10c26c14a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bignon?= Date: Wed, 7 Feb 2018 10:21:34 +0100 Subject: [PATCH 3/4] Added tests with data-driven tests --- test/E2ETests/Automation.CLI/CLITestBase.cs | 6 ++++-- .../CustomTestExecutionExtensibilityTests.cs | 19 +++++++++++++++++++ .../CustomTestExTests.cs | 10 +++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 88cd82f668..f846b3e69a 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation using System.Linq; using System.Xml; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; public class CLITestBase @@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "") /// /// List of test assemblies. /// Run settings for execution. - public void InvokeVsTestForExecution(string[] sources, string runSettings = "") + /// Test Case filter for execution. + public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null) { for (var iterator = 0; iterator < sources.Length; iterator++) { @@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "") // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll")); - vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler); + vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler); } /// diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index e8b6115cec..23f7cd81c9 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -25,8 +25,27 @@ public void ExecuteCustomTestExtensibilityTests() "CustomTestClass1 - Execution number 4", "CustomTestClass1 - Execution number 5"); this.ValidateFailedTestsContain( + TestAssembly, "CustomTestMethod1 - Execution number 3", "CustomTestClass1 - Execution number 3"); } + + [TestMethod] + public void ExecuteCustomTestExtensibilityWithTestDataTests() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2"); + this.ValidatePassedTests( + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)"); + this.ValidateFailedTests( + TestAssembly, + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)"); + } } } \ No newline at end of file diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs index 71732d55a3..c42be4a431 100644 --- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs +++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs @@ -1,6 +1,5 @@ namespace FxExtensibilityTestProject { - using System; using System.Collections.Generic; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -15,6 +14,15 @@ public void CustomTestMethod1() Assert.AreNotEqual(3, customTestMethod1ExecutionCount); } + [IterativeTestMethod(3)] + [DataRow("A")] + [DataRow("B")] + [DataRow("C")] + public void CustomTestMethod2(string value) + { + Assert.AreEqual("B", value); + } + private static int customTestClass1ExecutionCount; [TestMethod] public void CustomTestClass1() From 25bea17ec4d86daccc4db08e477c01b1dc0cee08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bignon?= Date: Wed, 7 Feb 2018 10:53:04 +0100 Subject: [PATCH 4/4] Fixed build --- src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 1ab3be4dd8..2ece924fcc 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -261,7 +261,6 @@ internal UnitTestResult[] RunTestMethod() }; } - rowIndex++; watch.Stop(); foreach (var testResult in testResults) { @@ -269,6 +268,8 @@ internal UnitTestResult[] RunTestMethod() testResult.Duration = watch.Elapsed; } + rowIndex++; + results.AddRange(testResults); } }