diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index eec49a868..4a0ded242 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -5,8 +5,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { @@ -15,8 +14,13 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile) { var debug; @@ -67,30 +71,41 @@ var find_tests = function (testFileList, discoverResultFile) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { - var test_results = []; - for (var test in testCases) { + function post(event) { + callback(event); + hook_outputs(); + } + + for (var test of testCases) { + post({ + type: 'test start', + title: test.testName + }); try { - var testCase = require(testCases[test].testFile); - result.title = testCases[test].testName; - result.time = Date.now(); - testCase[testCases[test].testName](); - result.time = Date.now() - result.time; + var testCase = require(test.testFile); + testCase[test.testName](); + result.title = test.testName; result.passed = true; } catch (err) { - result.time = Date.now() - result.time; result.passed = false; console.error(err.name); console.error(err.message); } - test_results.push(result) + post({ + type: 'result', + title: test.testName, + result: result + }); result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; } - callback(test_results); + callback({ + type: 'suite end', + result: result + }); }; module.exports.run_tests = run_tests; \ No newline at end of file diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index bed15d75a..ee8329c9d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -6,13 +6,13 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { result.stdOut += string; } + function append_stderr(string, encoding, fd) { result.stdErr += string; } @@ -52,42 +52,63 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { module.exports.find_tests = find_tests; function run_tests(testInfo, callback) { - var testResults = []; - var testCases = loadTestCases(testInfo[0].testFile); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; - if (testCases === null) { - return; - } - var tape = findTape(testInfo[0].projectFolder); if (tape === null) { return; } - for (var test in testInfo) { - result.title = testInfo[test].testName; + var harness = tape.getHarness(); + + testInfo.forEach(function (info) { + runTest(info, harness, function (result) { + callback(result); + }); + }); + + tape.onFinish(function () { + // executes when all tests are done running + }); + + function runTest(testInfo, harness, done) { + var stream = harness.createStream({ objectMode: true }); + var title = testInfo.testName; + + stream.on(('data'), function (result) { + if (result.type === 'test') { + done({ + type: 'test start', + title: title + }); + } + }); + try { - result.time = Date.now(); - var harness = tape.getHarness(); - harness(testInfo[test].testName); - result.passed = true; + var htest = tape.test(title, {}, function (result) { + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': result._ok, + 'stdOut': '', + 'stdErr': '' + } + }); + }); } catch (e) { - result.passed = false; - logError('Error running test:', testInfo[test].testName, 'in', testInfo[test].testFile, e); + console.error('NTVS_ERROR:', e); + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': false, + 'stdOut': '', + 'stdErr': e.message + } + }); } - result.time = Date.now() - result.time; - testResults.push(result); - result = { - 'title': '', - 'passed': false, - 'stdOut': '', - 'stdErr': '', - 'time': 0 - }; } - - callback(testResults); } module.exports.run_tests = run_tests; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index e15b0fbd8..6d9109cc8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -6,8 +6,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, @@ -19,8 +18,12 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); @@ -71,6 +74,11 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { + function post(event) { + callback(event); + hook_outputs(); + } + function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -86,49 +94,95 @@ var run_tests = function (testCases, callback) { var testGrepString = '^(' + testCases.map(function (testCase) { return testCase.testName }).join('|') + ')$'; - + if (testGrepString) { mocha.grep(new RegExp(testGrepString)); } - mocha.addFile(testCases[0].testFile); - // run tests - var runner = mocha.run(function (code) { }); + var runner = mocha.run(function (code) { + process.exit(code); + }); + + runner.on('suite', function (suite) { + post({ + type: 'suite start', + result: result + }); + }); + + runner.on('suite end', function (suite) { + post({ + type: 'suite end', + result: result + }); + }); + + runner.on('hook', function (hook) { + post({ + type: 'hook start', + title: hook.title, + result: result + }); + }); + + runner.on('hook end', function (hook) { + post({ + type: 'hook end', + title: hook.title, + result: result + }); + }); runner.on('start', function () { + post({ + type: 'start', + result: result + }); }); + runner.on('test', function (test) { result.title = test.fullTitle(); - result.time = Date.now(); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + post({ + type: 'test start', + title: result.title + }); }); + runner.on('end', function () { - callback(testResults); + post({ + type: 'end', + result: result + }); }); + runner.on('pass', function (test) { result.passed = true; - result.time = Date.now() - result.time; - testResults.push(result); + post({ + type: 'result', + title: result.title, + result: result + }); result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } }); + runner.on('fail', function (test, err) { result.passed = false; - result.time = Date.now() - result.time; - testResults.push(result); + post({ + type: 'result', + title: result.title, + result: result + }); result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index bc3a46f17..620d9d4a0 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -23,18 +23,16 @@ rl.on('line', (line) => { process.exit(1); } - function returnResult(result) { + function postResult(result) { // unhook stdout and stderr process.stdout.write = old_stdout; process.stderr.write = old_stderr; if (result) { console.log(JSON.stringify(result)); } - // end process, tests are done running. - process.exit(0); } // run the test - framework.run_tests(testCases, returnResult); + framework.run_tests(testCases, postResult); // close readline interface rl.close(); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index a825a9fe6..a5f08f611 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -33,22 +33,6 @@ using Newtonsoft.Json; namespace Microsoft.NodejsTools.TestAdapter { - - class ResultObject { - public ResultObject() { - title = String.Empty; - passed = false; - stdout = String.Empty; - stderr = String.Empty; - time = 0; - } - public string title { get; set; } - public bool passed { get; set; } - public string stdout { get; set; } - public string stderr { get; set; } - public int time { get; set; } - } - [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; @@ -62,6 +46,10 @@ class TestExecutor : ITestExecutor { private ProcessStartInfo _psi; private Process _nodeProcess; private object _syncObject = new object(); + private List _currentTests; + private IFrameworkHandle _frameworkHandle; + private TestResult _currentResult = null; + private ResultObject _currentResultObject = null; public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -70,6 +58,27 @@ public void Cancel() { _cancelRequested.Set(); } + private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { + try { + if (e.Data != null) { + TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) { + if (testEvent.type == "test start") { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } else if (testEvent.type == "result") { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + } + } else if (testEvent.type == "suite end") { + _currentResultObject = testEvent.result; + } + } + } catch (Exception) { } + } + /// /// This is the equivalent of "RunAll" functionality /// @@ -94,37 +103,40 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // No VS instance just means no debugging, but everything else is // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { - // .njsproj file path -> project settings - var projectToTests = new Dictionary>(); + // .ts file path -> project settings + var fileToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; - // put tests into dictionary where key is their project working directory + // put tests into dictionary where key is their source file foreach (var test in receiver.Tests) { - if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - } - if (!projectToTests.ContainsKey(settings.WorkingDir)) { - projectToTests[settings.WorkingDir] = new List(); + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); } - projectToTests[settings.WorkingDir].Add(test); + fileToTests[test.CodeFilePath].Add(test); } - // where key is the workingDir and value is a list of tests - foreach (KeyValuePair> entry in projectToTests) { + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { List args = new List(); TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } int port = 0; if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); args.AddRange(GetDebugArgs(settings, out port)); } - args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); // launch node process LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases in a given project + // Run all test cases in a given file RunTestCases(entry.Value, runContext, frameworkHandle, settings); // dispose node process _nodeProcess.Dispose(); @@ -144,15 +156,46 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); - TestCase firstTest = tests.First(); - NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); - List args = new List(); - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + // .ts file path -> project settings + var fileToTests = new Dictionary>(); + var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + + // put tests into dictionary where key is their source file + foreach (var test in tests) { + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); + } + fileToTests[test.CodeFilePath].Add(test); + } + + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } + + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases selected - RunTestCases(tests, runContext, frameworkHandle, settings); - _nodeProcess.Dispose(); + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); + // Run all test cases in a given file + RunTestCases(entry.Value, runContext, frameworkHandle, settings); + // dispose node process + _nodeProcess.Dispose(); + } + } } private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, NodejsProjectSettings settings) { @@ -169,7 +212,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I if (_cancelRequested.WaitOne(0)) { break; } - frameworkHandle.RecordStart(test); if (settings == null) { frameworkHandle.SendMessage( @@ -204,6 +246,18 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I _nodeProcess.WaitForExit(); } + // Automatically fail tests that haven't been run by this point (failures in before() hooks) + foreach(TestCase notRunTest in _currentTests) { + TestResult result = new TestResult(notRunTest); + result.Outcome = TestOutcome.Failed; + if(_currentResultObject != null) { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); + } + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); + } + if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, @@ -228,16 +282,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I #endif } } - var results = GetTestResultFromProcess(_nodeProcess.StandardOutput); - - bool runCancelled = _cancelRequested.WaitOne(0); - - if (results != null) { - RecordEnd(frameworkHandle, tests, results); - } - else { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain results"); - } } private void KillNodeProcess() { @@ -269,26 +313,6 @@ private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, }; } - private List ParseTestResult(string line) { - List jsonResults = null; - try { - jsonResults = JsonConvert.DeserializeObject>(line); - } catch (Exception) { } - return jsonResults; - } - - private List GetTestResultFromProcess(StreamReader sr) { - List results = null; - while (sr.Peek() >= 0) { - results = ParseTestResult(sr.ReadLine()); - if (results == null) { - continue; - } - break; - } - return results; - } - private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { _psi = new ProcessStartInfo("cmd.exe") { Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", @@ -301,6 +325,8 @@ private void LaunchNodeProcess(string workingDir, string nodeExePath, List tests, List results) { - if (tests.Count() == results.Count()) { - TestResult result; - foreach(var res in results) { - // If tests were run using "Run Selected Tests", the `tests` and `results` lists - // may not have the tests in the same order --so we query the test title from the `tests` list. - var test = tests.Where(n => n.DisplayName == res.title); - if(test.Count() == 1) { - result = new TestResult(test.First()); - result.Outcome = res.passed ? TestOutcome.Passed : TestOutcome.Failed; - result.Duration = new TimeSpan(0, 0, 0, 0, res.time); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, res.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, res.stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, res.stderr)); - frameworkHandle.RecordResult(result); - frameworkHandle.RecordEnd(test.First(), result.Outcome); - } - } - } + private void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + String[] standardOutputLines = resultObject.stdout.Split('\n'); + String[] standardErrorLines = resultObject.stderr.Split('\n'); + result.EndTime = DateTimeOffset.Now; + result.Duration = result.EndTime - result.StartTime; + result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, String.Join(Environment.NewLine, standardOutputLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, String.Join(Environment.NewLine, standardErrorLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, String.Join(Environment.NewLine, standardErrorLines))); + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(test, result.Outcome); + _currentTests.Remove(test); } } } @@ -381,6 +400,25 @@ public NodejsProjectSettings() { public string ProjectRootDir { get; set; } } +class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } +} + +class TestEvent { + public string type { get; set; } + public string title { get; set; } + public ResultObject result { get; set; } +} + class TestCaseObject { public TestCaseObject() { framework = String.Empty; @@ -402,5 +440,4 @@ public TestCaseObject(string framework, string testName, string testFile, string public string testFile { get; set; } public string workingFolder { get; set; } public string projectFolder { get; set; } - } \ No newline at end of file