From 82782081e4c871019d50762d71ef631a37ec7ade Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 2 Nov 2016 16:11:34 -0700 Subject: [PATCH 01/13] return and record test results incrementally --- .../Nodejs/TestFrameworks/mocha/mocha.js | 59 ++++++++++++-- .../Nodejs/TestFrameworks/run_tests.js | 6 +- Nodejs/Product/TestAdapter/TestExecutor.cs | 81 +++++++++++-------- 3 files changed, 103 insertions(+), 43 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index e15b0fbd8..071d4098a 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -70,7 +70,7 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testCases, callback) { +var run_tests = function (testCases, postResult) { function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -91,26 +91,64 @@ var run_tests = function (testCases, callback) { mocha.grep(new RegExp(testGrepString)); } - mocha.addFile(testCases[0].testFile); + var files = new Set(); + + for (var test in testCases) { + files.add(testCases[test].testFile); + } + + for (var file of files) { + mocha.addFile(file); + } // run tests - var runner = mocha.run(function (code) { }); + var runner = mocha.run(function (code) { process.exit(code); }); + runner.on('hook', function (hook) { + var event = { + type: 'hook start' + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('hook end', function (hook) { + var event = { + type: 'hook end', + title: hook.title + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); runner.on('start', function () { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('pending', function (test) { }); runner.on('test', function (test) { + var event = { + type: 'testStart', + title: test.fullTitle() + } + postResult(event); result.title = test.fullTitle(); result.time = Date.now(); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('end', function () { - callback(testResults); }); runner.on('pass', function (test) { result.passed = true; result.time = Date.now() - result.time; - testResults.push(result); + var event = { + type: 'result', + title: test.fullTitle(), + result: result + } + postResult(event); result = { 'title': '', 'passed': false, @@ -118,11 +156,18 @@ var run_tests = function (testCases, callback) { 'stdErr': '', 'time': '' } + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; result.time = Date.now() - result.time; - testResults.push(result); + var event = { + type: 'result', + title: test.fullTitle(), + result: result + } + postResult(event); result = { 'title': '', 'passed': false, @@ -130,6 +175,8 @@ var run_tests = function (testCases, callback) { 'stdErr': '', 'time': '' } + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index bc3a46f17..620d9d4a0 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -23,18 +23,16 @@ rl.on('line', (line) => { process.exit(1); } - function returnResult(result) { + function postResult(result) { // unhook stdout and stderr process.stdout.write = old_stdout; process.stderr.write = old_stderr; if (result) { console.log(JSON.stringify(result)); } - // end process, tests are done running. - process.exit(0); } // run the test - framework.run_tests(testCases, returnResult); + framework.run_tests(testCases, postResult); // close readline interface rl.close(); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index a825a9fe6..9ccbecf9e 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -34,6 +34,11 @@ namespace Microsoft.NodejsTools.TestAdapter { + class TestEvent { + public string type { get; set; } + public string title { get; set; } + public ResultObject result { get; set; } + } class ResultObject { public ResultObject() { title = String.Empty; @@ -46,7 +51,7 @@ public ResultObject() { public bool passed { get; set; } public string stdout { get; set; } public string stderr { get; set; } - public int time { get; set; } + public long time { get; set; } } [ExtensionUri(TestExecutor.ExecutorUriString)] @@ -62,6 +67,9 @@ class TestExecutor : ITestExecutor { private ProcessStartInfo _psi; private Process _nodeProcess; private object _syncObject = new object(); + private List _currentTests; + private IFrameworkHandle _frameworkHandle; + private TestResult _currentResult = null; public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -70,6 +78,26 @@ public void Cancel() { _cancelRequested.Set(); } + private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { + try { + TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) { + if (testEvent.type == "testStart") { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } + else if (testEvent.type == "result") { + if (_currentResult != null) { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + } + } + } + } catch (Exception) { } + } + /// /// This is the equivalent of "RunAll" functionality /// @@ -120,6 +148,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame args.AddRange(GetDebugArgs(settings, out port)); } + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); // launch node process @@ -149,6 +179,8 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame List args = new List(); args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + _currentTests = (List)tests; + _frameworkHandle = frameworkHandle; LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); // Run all test cases selected RunTestCases(tests, runContext, frameworkHandle, settings); @@ -169,7 +201,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I if (_cancelRequested.WaitOne(0)) { break; } - frameworkHandle.RecordStart(test); if (settings == null) { frameworkHandle.SendMessage( @@ -215,7 +246,8 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // } //} #if DEBUG - } catch (COMException ex) { + } + catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); @@ -228,16 +260,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I #endif } } - var results = GetTestResultFromProcess(_nodeProcess.StandardOutput); - - bool runCancelled = _cancelRequested.WaitOne(0); - - if (results != null) { - RecordEnd(frameworkHandle, tests, results); - } - else { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain results"); - } } private void KillNodeProcess() { @@ -273,7 +295,8 @@ private List ParseTestResult(string line) { List jsonResults = null; try { jsonResults = JsonConvert.DeserializeObject>(line); - } catch (Exception) { } + } + catch (Exception) { } return jsonResults; } @@ -301,6 +324,8 @@ private void LaunchNodeProcess(string workingDir, string nodeExePath, List tests, List results) { - if (tests.Count() == results.Count()) { - TestResult result; - foreach(var res in results) { - // If tests were run using "Run Selected Tests", the `tests` and `results` lists - // may not have the tests in the same order --so we query the test title from the `tests` list. - var test = tests.Where(n => n.DisplayName == res.title); - if(test.Count() == 1) { - result = new TestResult(test.First()); - result.Outcome = res.passed ? TestOutcome.Passed : TestOutcome.Failed; - result.Duration = new TimeSpan(0, 0, 0, 0, res.time); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, res.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, res.stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, res.stderr)); - frameworkHandle.RecordResult(result); - frameworkHandle.RecordEnd(test.First(), result.Outcome); - } - } - } + private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + result.EndTime = DateTimeOffset.Now; + result.Duration = result.EndTime - result.StartTime; + result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, resultObject.stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(test, result.Outcome); } } } From 49b050aece33efdb66674ed7cd908266176443f0 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 2 Nov 2016 16:14:00 -0700 Subject: [PATCH 02/13] fix formatting --- Nodejs/Product/TestAdapter/TestExecutor.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 9ccbecf9e..7e41fc324 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -246,8 +246,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // } //} #if DEBUG - } - catch (COMException ex) { + } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); @@ -295,8 +294,7 @@ private List ParseTestResult(string line) { List jsonResults = null; try { jsonResults = JsonConvert.DeserializeObject>(line); - } - catch (Exception) { } + } catch (Exception) { } return jsonResults; } From 5baa46de0f5fe1ba67f6201a90e93affa21a7229 Mon Sep 17 00:00:00 2001 From: ozyx Date: Thu, 3 Nov 2016 10:14:12 -0700 Subject: [PATCH 03/13] automatically fail tests that fail in before hooks --- Nodejs/Product/TestAdapter/TestExecutor.cs | 32 ++++++++-------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 7e41fc324..669fab3af 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -235,6 +235,15 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I _nodeProcess.WaitForExit(); } + // Automatically fail tests that haven't been run by this point (failures in before() hooks) + foreach(TestCase notRunTest in _currentTests) { + TestResult res = new TestResult(notRunTest); + res.Outcome = TestOutcome.Failed; + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, "Failure prior to test being run")); + frameworkHandle.RecordResult(res); + frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); + } + if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, @@ -290,26 +299,6 @@ private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, }; } - private List ParseTestResult(string line) { - List jsonResults = null; - try { - jsonResults = JsonConvert.DeserializeObject>(line); - } catch (Exception) { } - return jsonResults; - } - - private List GetTestResultFromProcess(StreamReader sr) { - List results = null; - while (sr.Peek() >= 0) { - results = ParseTestResult(sr.ReadLine()); - if (results == null) { - continue; - } - break; - } - return results; - } - private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { _psi = new ProcessStartInfo("cmd.exe") { Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", @@ -346,7 +335,7 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { return projSettings; } - private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + private void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime; result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; @@ -355,6 +344,7 @@ private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, T result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, result.Outcome); + _currentTests.Remove(test); } } } From 56ca7c5b2de8fefc21d204831c617ea0f3d017ad Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 7 Nov 2016 13:43:02 -0800 Subject: [PATCH 04/13] separate test suites by testFile instead of workingDir --- Nodejs/Product/TestAdapter/TestExecutor.cs | 90 ++++++++++++++-------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 669fab3af..73dc04db6 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -45,13 +45,11 @@ public ResultObject() { passed = false; stdout = String.Empty; stderr = String.Empty; - time = 0; } public string title { get; set; } public bool passed { get; set; } public string stdout { get; set; } public string stderr { get; set; } - public long time { get; set; } } [ExtensionUri(TestExecutor.ExecutorUriString)] @@ -70,6 +68,7 @@ class TestExecutor : ITestExecutor { private List _currentTests; private IFrameworkHandle _frameworkHandle; private TestResult _currentResult = null; + private ResultObject _currentResultObject = null; public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -90,10 +89,10 @@ private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { _frameworkHandle.RecordStart(test.First()); } else if (testEvent.type == "result") { - if (_currentResult != null) { - RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); - } + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); } + } else if (testEvent.type == "suite end") { + _currentResultObject = testEvent.result; } } catch (Exception) { } } @@ -122,26 +121,26 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // No VS instance just means no debugging, but everything else is // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { - // .njsproj file path -> project settings - var projectToTests = new Dictionary>(); + // .ts file path -> project settings + var fileToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; - // put tests into dictionary where key is their project working directory + // put tests into dictionary where key is their source file foreach (var test in receiver.Tests) { - if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - } - if (!projectToTests.ContainsKey(settings.WorkingDir)) { - projectToTests[settings.WorkingDir] = new List(); + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); } - projectToTests[settings.WorkingDir].Add(test); + fileToTests[test.CodeFilePath].Add(test); } - // where key is the workingDir and value is a list of tests - foreach (KeyValuePair> entry in projectToTests) { + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { List args = new List(); TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } int port = 0; if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); @@ -150,11 +149,12 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame _currentTests = entry.Value; _frameworkHandle = frameworkHandle; - args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); // launch node process LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases in a given project + // Run all test cases in a given file RunTestCases(entry.Value, runContext, frameworkHandle, settings); // dispose node process _nodeProcess.Dispose(); @@ -174,17 +174,46 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); - TestCase firstTest = tests.First(); - NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); - List args = new List(); - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); - - _currentTests = (List)tests; - _frameworkHandle = frameworkHandle; - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases selected - RunTestCases(tests, runContext, frameworkHandle, settings); - _nodeProcess.Dispose(); + using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + // .ts file path -> project settings + var fileToTests = new Dictionary>(); + var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + + // put tests into dictionary where key is their source file + foreach (var test in tests) { + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); + } + fileToTests[test.CodeFilePath].Add(test); + } + + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } + + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); + // Run all test cases in a given file + RunTestCases(entry.Value, runContext, frameworkHandle, settings); + // dispose node process + _nodeProcess.Dispose(); + } + } } private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, NodejsProjectSettings settings) { @@ -239,7 +268,8 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I foreach(TestCase notRunTest in _currentTests) { TestResult res = new TestResult(notRunTest); res.Outcome = TestOutcome.Failed; - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, "Failure prior to test being run")); + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); frameworkHandle.RecordResult(res); frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); } From dbd2605a5ba316f088f637a66db3b2aecac15b0e Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 7 Nov 2016 21:16:57 -0800 Subject: [PATCH 05/13] Fix newlines in TRX Test Result output --- Nodejs/Product/TestAdapter/TestExecutor.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 73dc04db6..0e92bb865 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -366,12 +366,14 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { } private void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + String[] standardOutputLines = resultObject.stdout.Split('\n'); + String[] standardErrorLines = resultObject.stderr.Split('\n'); result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime; result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultObject.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, resultObject.stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, String.Join(Environment.NewLine, standardOutputLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, String.Join(Environment.NewLine, standardErrorLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, String.Join(Environment.NewLine, standardErrorLines))); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, result.Outcome); _currentTests.Remove(test); From 16f0bc5ba96e1854cbf24fbf4120ad28052795e7 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 16 Nov 2016 16:10:38 -0800 Subject: [PATCH 06/13] modify exportrunner.js to return results incrementally --- .../ExportRunner/exportrunner.js | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index eec49a868..ff852eb63 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -5,8 +5,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { @@ -67,30 +66,32 @@ var find_tests = function (testFileList, discoverResultFile) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { - var test_results = []; - for (var test in testCases) { + var event = {}; + for (var test of testCases) { + event.type = 'testStart'; + event.title = test.testName; + callback(event); try { - var testCase = require(testCases[test].testFile); - result.title = testCases[test].testName; - result.time = Date.now(); - testCase[testCases[test].testName](); - result.time = Date.now() - result.time; + var testCase = require(test.testFile); + result.title = test.testName; + testCase[test.testName](); result.passed = true; } catch (err) { - result.time = Date.now() - result.time; result.passed = false; console.error(err.name); console.error(err.message); } - test_results.push(result) + event.type = 'result'; + event.result = result; + callback(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; } - callback(test_results); }; module.exports.run_tests = run_tests; \ No newline at end of file From 1efc9853cd48d7dac60f8589f7dc4461f602bcde Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 16 Nov 2016 16:12:10 -0800 Subject: [PATCH 07/13] modify mocha.js to return results incrementally --- .../Nodejs/TestFrameworks/mocha/mocha.js | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 071d4098a..8945f86f6 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -6,8 +6,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, @@ -91,22 +90,33 @@ var run_tests = function (testCases, postResult) { mocha.grep(new RegExp(testGrepString)); } - var files = new Set(); - - for (var test in testCases) { - files.add(testCases[test].testFile); - } - - for (var file of files) { - mocha.addFile(file); - } - + mocha.addFile(testCases[0].testFile); + // run tests var runner = mocha.run(function (code) { process.exit(code); }); - + runner.on('suite', function (suite) { + var event = { + type: 'suite start', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('suite end', function (suite) { + var event = { + type: 'suite end', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); runner.on('hook', function (hook) { var event = { - type: 'hook start' + type: 'hook start', + title: hook.title, + result: result } postResult(event); process.stdout.write = append_stdout; @@ -115,37 +125,48 @@ var run_tests = function (testCases, postResult) { runner.on('hook end', function (hook) { var event = { type: 'hook end', - title: hook.title + title: hook.title, + result: result } postResult(event); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('start', function () { + var event = { + type: 'start', + result: result + } + postResult(event); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('pending', function (test) { }); runner.on('test', function (test) { + result.title = test.fullTitle(); var event = { type: 'testStart', - title: test.fullTitle() + title: result.title } postResult(event); - result.title = test.fullTitle(); - result.time = Date.now(); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('end', function () { + var event = { + type: 'end', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); runner.on('pass', function (test) { result.passed = true; - result.time = Date.now() - result.time; var event = { type: 'result', - title: test.fullTitle(), + title: result.title, result: result } postResult(event); @@ -153,18 +174,16 @@ var run_tests = function (testCases, postResult) { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; - result.time = Date.now() - result.time; var event = { type: 'result', - title: test.fullTitle(), + title: result.title, result: result } postResult(event); @@ -172,8 +191,7 @@ var run_tests = function (testCases, postResult) { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } process.stdout.write = append_stdout; process.stderr.write = append_stderr; From 7349067cee1d2f654d955ebcf24ed25299c96179 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 2 Dec 2016 16:54:49 -0800 Subject: [PATCH 08/13] clean up code --- .../Nodejs/TestFrameworks/mocha/mocha.js | 84 +++++++------------ 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 8945f86f6..0c4813c2d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -18,8 +18,12 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); @@ -69,7 +73,11 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testCases, postResult) { +var run_tests = function (testCases, callback) { + function post(event) { + callback(event); + hook_outputs(); + } function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -91,110 +99,82 @@ var run_tests = function (testCases, postResult) { } mocha.addFile(testCases[0].testFile); - + // run tests var runner = mocha.run(function (code) { process.exit(code); }); runner.on('suite', function (suite) { - var event = { + post({ type: 'suite start', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('suite end', function (suite) { - var event = { + post({ type: 'suite end', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('hook', function (hook) { - var event = { + post({ type: 'hook start', title: hook.title, result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('hook end', function (hook) { - var event = { + post({ type: 'hook end', title: hook.title, result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('start', function () { - var event = { + post({ type: 'start', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; - }); - runner.on('pending', function (test) { + }); }); + runner.on('test', function (test) { result.title = test.fullTitle(); - var event = { - type: 'testStart', + post({ + type: 'test start', title: result.title - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('end', function () { - var event = { + post({ type: 'end', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('pass', function (test) { result.passed = true; - var event = { + post({ type: 'result', title: result.title, result: result - } - postResult(event); + }); result = { 'title': '', 'passed': false, 'stdOut': '', 'stdErr': '' } - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; - var event = { + post({ type: 'result', title: result.title, result: result - } - postResult(event); + }); result = { 'title': '', 'passed': false, 'stdOut': '', 'stdErr': '' } - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; }); }; From a0acb7ab57d7e97fbfd304cf7c7a4318e281020e Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 2 Dec 2016 16:56:11 -0800 Subject: [PATCH 09/13] display test stdout and stderr in exportRunner test results --- .../ExportRunner/exportrunner.js | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index ff852eb63..88dd747ed 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -14,8 +14,13 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile) { var debug; @@ -66,15 +71,21 @@ var find_tests = function (testFileList, discoverResultFile) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { - var event = {}; - for (var test of testCases) { - event.type = 'testStart'; - event.title = test.testName; + function post(event) { callback(event); + hook_outputs(); + } + + for (var test of testCases) { + event = { + type: 'test start', + title: test.testName + } + post(event); try { var testCase = require(test.testFile); - result.title = test.testName; testCase[test.testName](); + result.title = test.testName; result.passed = true; } catch (err) { result.passed = false; @@ -83,9 +94,7 @@ var run_tests = function (testCases, callback) { } event.type = 'result'; event.result = result; - callback(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + post(event); result = { 'title': '', 'passed': false, @@ -93,5 +102,9 @@ var run_tests = function (testCases, callback) { 'stdErr': '' }; } + callback({ + type: 'suite end', + result: result + }); }; module.exports.run_tests = run_tests; \ No newline at end of file From ef497da245154209689949cec663dd8c2f92deca Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 5 Dec 2016 13:20:29 -0800 Subject: [PATCH 10/13] code cleanup --- Nodejs/Product/TestAdapter/TestExecutor.cs | 78 +++++++++++----------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 0e92bb865..a5f08f611 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -33,25 +33,6 @@ using Newtonsoft.Json; namespace Microsoft.NodejsTools.TestAdapter { - - class TestEvent { - public string type { get; set; } - public string title { get; set; } - public ResultObject result { get; set; } - } - class ResultObject { - public ResultObject() { - title = String.Empty; - passed = false; - stdout = String.Empty; - stderr = String.Empty; - } - public string title { get; set; } - public bool passed { get; set; } - public string stdout { get; set; } - public string stderr { get; set; } - } - [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; @@ -79,20 +60,21 @@ public void Cancel() { private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { try { - TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); - // Extract test from list of tests - var test = _currentTests.Where(n => n.DisplayName == testEvent.title); - if (test.Count() > 0) { - if (testEvent.type == "testStart") { - _currentResult = new TestResult(test.First()); - _currentResult.StartTime = DateTimeOffset.Now; - _frameworkHandle.RecordStart(test.First()); - } - else if (testEvent.type == "result") { - RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + if (e.Data != null) { + TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) { + if (testEvent.type == "test start") { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } else if (testEvent.type == "result") { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + } + } else if (testEvent.type == "suite end") { + _currentResultObject = testEvent.result; } - } else if (testEvent.type == "suite end") { - _currentResultObject = testEvent.result; } } catch (Exception) { } } @@ -266,11 +248,13 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // Automatically fail tests that haven't been run by this point (failures in before() hooks) foreach(TestCase notRunTest in _currentTests) { - TestResult res = new TestResult(notRunTest); - res.Outcome = TestOutcome.Failed; - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); - frameworkHandle.RecordResult(res); + TestResult result = new TestResult(notRunTest); + result.Outcome = TestOutcome.Failed; + if(_currentResultObject != null) { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); + } + frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); } @@ -416,6 +400,25 @@ public NodejsProjectSettings() { public string ProjectRootDir { get; set; } } +class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } +} + +class TestEvent { + public string type { get; set; } + public string title { get; set; } + public ResultObject result { get; set; } +} + class TestCaseObject { public TestCaseObject() { framework = String.Empty; @@ -437,5 +440,4 @@ public TestCaseObject(string framework, string testName, string testFile, string public string testFile { get; set; } public string workingFolder { get; set; } public string projectFolder { get; set; } - } \ No newline at end of file From bf150b7f196631adbadee4a30441d7438c69de17 Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 5 Dec 2016 13:30:13 -0800 Subject: [PATCH 11/13] modify tape.run_tests to be compatible with new TestExecutor --- .../Nodejs/TestFrameworks/Tape/tape.js | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index bed15d75a..ee8329c9d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -6,13 +6,13 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { result.stdOut += string; } + function append_stderr(string, encoding, fd) { result.stdErr += string; } @@ -52,42 +52,63 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { module.exports.find_tests = find_tests; function run_tests(testInfo, callback) { - var testResults = []; - var testCases = loadTestCases(testInfo[0].testFile); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; - if (testCases === null) { - return; - } - var tape = findTape(testInfo[0].projectFolder); if (tape === null) { return; } - for (var test in testInfo) { - result.title = testInfo[test].testName; + var harness = tape.getHarness(); + + testInfo.forEach(function (info) { + runTest(info, harness, function (result) { + callback(result); + }); + }); + + tape.onFinish(function () { + // executes when all tests are done running + }); + + function runTest(testInfo, harness, done) { + var stream = harness.createStream({ objectMode: true }); + var title = testInfo.testName; + + stream.on(('data'), function (result) { + if (result.type === 'test') { + done({ + type: 'test start', + title: title + }); + } + }); + try { - result.time = Date.now(); - var harness = tape.getHarness(); - harness(testInfo[test].testName); - result.passed = true; + var htest = tape.test(title, {}, function (result) { + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': result._ok, + 'stdOut': '', + 'stdErr': '' + } + }); + }); } catch (e) { - result.passed = false; - logError('Error running test:', testInfo[test].testName, 'in', testInfo[test].testFile, e); + console.error('NTVS_ERROR:', e); + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': false, + 'stdOut': '', + 'stdErr': e.message + } + }); } - result.time = Date.now() - result.time; - testResults.push(result); - result = { - 'title': '', - 'passed': false, - 'stdOut': '', - 'stdErr': '', - 'time': 0 - }; } - - callback(testResults); } module.exports.run_tests = run_tests; From c5d1310dc6297b8423526a23f5cc7ca07ea40d80 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 6 Dec 2016 09:22:41 -0800 Subject: [PATCH 12/13] clean up code --- .../Nodejs/TestFrameworks/mocha/mocha.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 0c4813c2d..6d9109cc8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -78,6 +78,7 @@ var run_tests = function (testCases, callback) { callback(event); hook_outputs(); } + function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -93,27 +94,30 @@ var run_tests = function (testCases, callback) { var testGrepString = '^(' + testCases.map(function (testCase) { return testCase.testName }).join('|') + ')$'; - + if (testGrepString) { mocha.grep(new RegExp(testGrepString)); } - mocha.addFile(testCases[0].testFile); - // run tests - var runner = mocha.run(function (code) { process.exit(code); }); + var runner = mocha.run(function (code) { + process.exit(code); + }); + runner.on('suite', function (suite) { post({ type: 'suite start', result: result }); }); + runner.on('suite end', function (suite) { post({ type: 'suite end', result: result }); }); + runner.on('hook', function (hook) { post({ type: 'hook start', @@ -121,6 +125,7 @@ var run_tests = function (testCases, callback) { result: result }); }); + runner.on('hook end', function (hook) { post({ type: 'hook end', @@ -128,6 +133,7 @@ var run_tests = function (testCases, callback) { result: result }); }); + runner.on('start', function () { post({ type: 'start', @@ -142,12 +148,14 @@ var run_tests = function (testCases, callback) { title: result.title }); }); + runner.on('end', function () { post({ type: 'end', result: result }); }); + runner.on('pass', function (test) { result.passed = true; post({ @@ -162,6 +170,7 @@ var run_tests = function (testCases, callback) { 'stdErr': '' } }); + runner.on('fail', function (test, err) { result.passed = false; post({ From b189e21f38da2c9654f32a7d60a33f3f2556d319 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 6 Dec 2016 09:38:16 -0800 Subject: [PATCH 13/13] clean up code --- .../TestFrameworks/ExportRunner/exportrunner.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index 88dd747ed..4a0ded242 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -77,11 +77,10 @@ var run_tests = function (testCases, callback) { } for (var test of testCases) { - event = { + post({ type: 'test start', title: test.testName - } - post(event); + }); try { var testCase = require(test.testFile); testCase[test.testName](); @@ -92,9 +91,11 @@ var run_tests = function (testCases, callback) { console.error(err.name); console.error(err.message); } - event.type = 'result'; - event.result = result; - post(event); + post({ + type: 'result', + title: test.testName, + result: result + }); result = { 'title': '', 'passed': false,