From 97a5d9e2e1b46837e7d8ad3b652cf3bf8e68a7f8 Mon Sep 17 00:00:00 2001 From: Jesse Mazzella Date: Fri, 14 Oct 2016 16:08:40 -0700 Subject: [PATCH 01/22] modifications to mocha.js and some TestExecutor logic (#1334) * have run_tests return structured result used by TestExecutor * have run_tests accept test data from stdin * run_tests receives test data over stdin * expose StandardOutput and disable async readline * TestExecutor receives test results over process StandardOutput * include testing framework in testInfo object sent to run_tests.js * comments * close readline interface after running test * revert ProcessOutput * use JsonConvert.DeserializeObject() instead * create GetTestResultFromProcess() method * use Process, copy over helper methods from ProcessOutput * close StandardInput after sending test info * include dependency for copied ProcessOutput method * launch node inside of RunTestCases instead * remove some debugging stuff * handle case if result is null * pass in streams as parameters to allow continual reading/writing * move setup logic to RunTests * only remove double quotes from test case info * move Process launch logic into function * mocha uses runner events to determine test pass/fail * get TestExecutor in working state, still launching one process per one test * get TextExecutor code into working state * fixes from PR feedback --- Common/Product/SharedProject/ProcessOutput.cs | 4 +- .../Nodejs/TestFrameworks/mocha/mocha.js | 39 ++- .../Nodejs/TestFrameworks/run_tests.js | 39 ++- Nodejs/Product/TestAdapter/TestExecutor.cs | 234 ++++++++++++++---- 4 files changed, 250 insertions(+), 66 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index 884b5c5e9..ee75aef72 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -304,7 +304,7 @@ public static ProcessOutput RunElevated( return result; } - private static string GetArguments(IEnumerable arguments, bool quoteArgs) { + public static string GetArguments(IEnumerable arguments, bool quoteArgs) { if (quoteArgs) { return string.Join(" ", arguments.Where(a => a != null).Select(QuoteSingleArgument)); } else { @@ -335,7 +335,7 @@ internal static IEnumerable SplitLines(string source) { } } - internal static string QuoteSingleArgument(string arg) { + public static string QuoteSingleArgument(string arg) { if (string.IsNullOrEmpty(arg)) { return "\"\""; } diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index d858fc0ef..1277b8b65 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -2,12 +2,25 @@ var EOL = require('os').EOL; var fs = require('fs'); var path = require('path'); - +var result = { + "title": "", + "passed": false, + "stdOut": "", + "stdErr": "" +}; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, // and 'xunit' does not print the stack trace from the test. var defaultMochaOptions = { ui: 'tdd', reporter: 'tap', timeout: 2000 }; +process.stdout.write = function (string, encoding, fd) { + result.stdOut += string; +} + +process.stderr.write = function (string, encoding, fd) { + result.stdErr += string; +} + var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); if (!Mocha) { @@ -56,7 +69,8 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testName, testFile, workingFolder, projectFolder) { +var run_tests = function (testName, testFile, workingFolder, projectFolder, callback) { + //var testResults = []; var Mocha = detectMocha(projectFolder); if (!Mocha) { return; @@ -70,10 +84,27 @@ var run_tests = function (testName, testFile, workingFolder, projectFolder) { else mocha.grep(testName); // prior Mocha 3.0.0 } + mocha.addFile(testFile); - mocha.run(function (code) { - process.exit(code); + // run tests + var runner = mocha.run(function (code) { }); + + runner.on('start', function () { + }); + runner.on('test', function (test) { + result.title = test.title; + }); + runner.on('end', function () { + callback(result); + }); + runner.on('pass', function (test) { + result.passed = true; + //testResults.push(result); + }); + runner.on('fail', function (test, err) { + result.passed = false; + //testResults.push(result); }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 3765fd252..4c95d3fae 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -1,10 +1,35 @@ var framework; -try { - framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); -} catch (exception) { - console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); - process.exit(1); -} +var readline = require('readline'); +var old_stdout = process.stdout.write; +var old_stderr = process.stderr.write; +var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); -framework.run_tests(process.argv[3], process.argv[4], process.argv[5], process.argv[6]); +rl.on('line', (line) => { + var testInfo = JSON.parse(line); + // get rid of leftover quotations from C# (necessary?) + for(var s in testInfo) { + testInfo[s] = testInfo[s].replace(/["]+/g, ''); + } + try { + framework = require('./' + testInfo.framework + '/' + testInfo.framework + '.js'); + } catch (exception) { + console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); + process.exit(1); + } + + function sendResult(result) { + process.stdout.write = old_stdout; + process.stderr.write = old_stderr; + console.log(JSON.stringify(result)); + process.exit(0); + } + // run the test + framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder, sendResult); + + // close readline interface + rl.close(); +}); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index dbabf6a8b..64c6c5cb7 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -29,9 +29,25 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudioTools; using Microsoft.VisualStudioTools.Project; +using Newtonsoft.Json.Linq; using MSBuild = Microsoft.Build.Evaluation; +using Newtonsoft.Json; namespace Microsoft.NodejsTools.TestAdapter { + + class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } + } + [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; @@ -41,7 +57,9 @@ class TestExecutor : ITestExecutor { private readonly ManualResetEvent _cancelRequested = new ManualResetEvent(false); - private ProcessOutput _nodeProcess; + private static readonly char[] _needToBeQuoted = new[] { ' ', '"' }; + private ProcessStartInfo _psi; + private Process _nodeProcess; private object _syncObject = new object(); public void Cancel() { @@ -71,8 +89,52 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame if (_cancelRequested.WaitOne(0)) { return; } + // May be null, but this is handled by RunTestCase if it matters. + // No VS instance just means no debugging, but everything else is + // okay. + using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + // .njsproj file path -> project settings + + var projectToTests = new Dictionary>(); + var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + + // put tests into dictionary where key is their project working directory + // NOTE: It seems to me that if we were to run all tests over multiple projects in a solution, + // we would have to separate the tests by their project in order to launch the node process + // correctly (to make sure we are using the correct working folder) and also to run + // groups of tests by test suite. + + foreach (var test in receiver.Tests) { + if (!sourceToSettings.TryGetValue(test.Source, out settings)) { + sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); + } + if (!projectToTests.ContainsKey(settings.WorkingDir)) { + projectToTests[settings.WorkingDir] = new List(); + } + projectToTests[settings.WorkingDir].Add(test); + } - RunTestCases(receiver.Tests, runContext, frameworkHandle); + // where key is the workingDir and value is a list of tests + foreach (KeyValuePair> entry in projectToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } + + //args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + + // eventually launch node process here + + // Run all test cases in a given project + RunTestCases(entry.Value, runContext, frameworkHandle); + + // dispose node process + } + } } public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { @@ -114,6 +176,7 @@ private void KillNodeProcess() { } } } + private static int GetFreePort() { return Enumerable.Range(new Random().Next(49152, 65536), 60000).Except( from connection in IPGlobalProperties.GetIPGlobalProperties().GetActiveTcpConnections() @@ -173,32 +236,32 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); return; } - lock (_syncObject) { - _nodeProcess = ProcessOutput.Run( - settings.NodeExePath, - args, - workingDir, - null, - false, - null, - false); + lock (_syncObject) { + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); - frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); + //frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); #endif - - _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); + // send test to run_tests.js + TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); + if (!_nodeProcess.HasExited) { + _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + _nodeProcess.StandardInput.Close(); + _nodeProcess.WaitForExit(5000); + } + //standardInput.Close(); if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); - while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { - if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { - break; - } - } + //while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { + // if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { + // break; + // } + //} #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); @@ -213,17 +276,58 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, #endif } } - - WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); + var result = GetTestResultFromProcess(_nodeProcess.StandardOutput); bool runCancelled = _cancelRequested.WaitOne(0); - RecordEnd(frameworkHandle, test, testResult, - string.Join(Environment.NewLine, _nodeProcess.StandardOutputLines), - string.Join(Environment.NewLine, _nodeProcess.StandardErrorLines), - (!runCancelled && _nodeProcess.ExitCode == 0) ? TestOutcome.Passed : TestOutcome.Failed); + + if (result != null) { + RecordEnd(frameworkHandle, test, testResult, + result.stdout, + result.stderr, + (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); + } else { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain result for " + test.DisplayName + " from TestRunner"); + } + + // dispose node process _nodeProcess.Dispose(); } + private ResultObject ParseTestResult(string line) { + ResultObject jsonResult = null; + try { + jsonResult = JsonConvert.DeserializeObject(line); + } catch (Exception) { } + return jsonResult; + } + + private ResultObject GetTestResultFromProcess(StreamReader sr) { + ResultObject result = null; + while (sr.Peek() >= 0) { + result = ParseTestResult(sr.ReadLine()); + if (result == null) { + continue; + } + break; + } + sr.DiscardBufferedData(); + return result; + } + + private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { + _psi = new ProcessStartInfo("cmd.exe") { + Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", + ProcessOutput.QuoteSingleArgument(workingDir), + ProcessOutput.QuoteSingleArgument(nodeExePath), + ProcessOutput.GetArguments(args, true)), + CreateNoWindow = true, + UseShellExecute = false + }; + _psi.RedirectStandardInput = true; + _psi.RedirectStandardOutput = true; + _nodeProcess = Process.Start(_psi); + } + private NodejsProjectSettings LoadProjectSettings(string projectFile) { var buildEngine = new MSBuild.ProjectCollection(); var proj = buildEngine.LoadProject(projectFile); @@ -255,40 +359,64 @@ private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, T frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, outcome); } + } +} - class DataReceiver { - public readonly StringBuilder Data = new StringBuilder(); +class DataReceiver { + public readonly StringBuilder Data = new StringBuilder(); - public void DataReceived(object sender, DataReceivedEventArgs e) { - if (e.Data != null) { - Data.AppendLine(e.Data); - } - } + public void DataReceived(object sender, DataReceivedEventArgs e) { + if (e.Data != null) { + Data.AppendLine(e.Data); } + } +} - class TestReceiver : ITestCaseDiscoverySink { - public List Tests { get; private set; } - - public TestReceiver() { - Tests = new List(); - } +class TestReceiver : ITestCaseDiscoverySink { + public List Tests { get; private set; } - public void SendTestCase(TestCase discoveredTest) { - Tests.Add(discoveredTest); - } - } + public TestReceiver() { + Tests = new List(); + } - class NodejsProjectSettings { - public NodejsProjectSettings() { - NodeExePath = String.Empty; - SearchPath = String.Empty; - WorkingDir = String.Empty; - } + public void SendTestCase(TestCase discoveredTest) { + Tests.Add(discoveredTest); + } +} - public string NodeExePath { get; set; } - public string SearchPath { get; set; } - public string WorkingDir { get; set; } - public string ProjectRootDir { get; set; } - } +class NodejsProjectSettings { + public NodejsProjectSettings() { + NodeExePath = String.Empty; + SearchPath = String.Empty; + WorkingDir = String.Empty; } + + public string NodeExePath { get; set; } + public string SearchPath { get; set; } + public string WorkingDir { get; set; } + public string ProjectRootDir { get; set; } } + +class TestCaseObject { + public TestCaseObject() { + framework = String.Empty; + testName = String.Empty; + testFile = String.Empty; + workingFolder = String.Empty; + projectFolder = String.Empty; + } + + public TestCaseObject(string framework, string testName, string testFile, string workingFolder, string projectFolder) { + this.framework = framework; + this.testName = testName; + this.testFile = testFile; + this.workingFolder = workingFolder; + this.projectFolder = projectFolder; + } + public string framework { get; set; } + public string testName { get; set; } + public string testFile { get; set; } + public string workingFolder { get; set; } + public string projectFolder { get; set; } + +} \ No newline at end of file From 332160249594c6dd92593731155590cb2c4aacfd Mon Sep 17 00:00:00 2001 From: Jesse Mazzella Date: Wed, 19 Oct 2016 14:59:57 -0700 Subject: [PATCH 02/22] prepare run_tests to run multiple tests (#1355) * prepare run_tests to run multiple tests * have tape.js run_tests use callback to return result * launch node if test run from 'run selected tests' --- .../Nodejs/TestFrameworks/Tape/tape.js | 22 +++++++++- .../Nodejs/TestFrameworks/mocha/mocha.js | 41 ++++++++++++------- .../Nodejs/TestFrameworks/run_tests.js | 4 +- Nodejs/Product/TestAdapter/TestExecutor.cs | 35 ++++++++++------ 4 files changed, 71 insertions(+), 31 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index 2bfec5671..00320a7cc 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -2,6 +2,20 @@ var EOL = require('os').EOL; var fs = require('fs'); var path = require('path'); +var result = { + "title": "", + "passed": false, + "stdOut": "", + "stdErr": "" +}; + +process.stdout.write = function (string, encoding, fd) { + result.stdOut += string; +} + +process.stderr.write = function (string, encoding, fd) { + result.stdErr += string; +} function find_tests(testFileList, discoverResultFile, projectFolder) { var test = findTape(projectFolder); @@ -37,8 +51,9 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -function run_tests(testName, testFile, workingFolder, projectFolder) { +function run_tests(testName, testFile, workingFolder, projectFolder, callback) { var testCases = loadTestCases(testFile); + result.title = testName; if (testCases === null) { return; } @@ -51,10 +66,13 @@ function run_tests(testName, testFile, workingFolder, projectFolder) { try { var harness = test.getHarness(); harness.only(testName); + result.passed = true; } catch (e) { logError("Error running test:", testName, "in", testFile, e); - return; + result.passed = false; } + + callback(result); } module.exports.run_tests = run_tests; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 1277b8b65..47168d429 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -12,14 +12,14 @@ var result = { // 'min' produces undisplayable text to stdout and stderr under piped/redirect, // and 'xunit' does not print the stack trace from the test. var defaultMochaOptions = { ui: 'tdd', reporter: 'tap', timeout: 2000 }; - -process.stdout.write = function (string, encoding, fd) { +function append_stdout(string, encoding, fd) { result.stdOut += string; } - -process.stderr.write = function (string, encoding, fd) { +function append_stderr(string, encoding, fd) { result.stdErr += string; } +process.stdout.write = append_stdout; +process.stderr.write = append_stderr; var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); @@ -78,33 +78,46 @@ var run_tests = function (testName, testFile, workingFolder, projectFolder, call var mocha = initializeMocha(Mocha, projectFolder); - if (testName) { - if (typeof mocha.fgrep === 'function') - mocha.fgrep(testName); // since Mocha 3.0.0 - else - mocha.grep(testName); // prior Mocha 3.0.0 - } + //if (testName) { + // if (typeof mocha.fgrep === 'function') + // mocha.fgrep(testName); // since Mocha 3.0.0 + // else + // mocha.grep(testName); // prior Mocha 3.0.0 + //} mocha.addFile(testFile); // run tests - var runner = mocha.run(function (code) { }); + var runner = mocha.run(function (code) { process.exit(code); }); runner.on('start', function () { }); runner.on('test', function (test) { result.title = test.title; + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); runner.on('end', function () { - callback(result); }); runner.on('pass', function (test) { result.passed = true; - //testResults.push(result); + callback(result); + result = { + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '' + } }); runner.on('fail', function (test, err) { result.passed = false; - //testResults.push(result); + callback(result); + result = { + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '' + } }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 4c95d3fae..6cd03870f 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -25,11 +25,11 @@ rl.on('line', (line) => { process.stdout.write = old_stdout; process.stderr.write = old_stderr; console.log(JSON.stringify(result)); - process.exit(0); + //process.exit(0); } // run the test framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder, sendResult); // close readline interface - rl.close(); + //rl.close(); }); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 64c6c5cb7..7660e9357 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -125,14 +125,14 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame args.AddRange(GetDebugArgs(settings, out port)); } - //args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); - - // eventually launch node process here + args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); // Run all test cases in a given project RunTestCases(entry.Value, runContext, frameworkHandle); - // dispose node process + _nodeProcess.Dispose(); } } } @@ -141,10 +141,26 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame ValidateArg.NotNull(tests, "tests"); ValidateArg.NotNull(runContext, "runContext"); ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); - _cancelRequested.Reset(); + bool hasExited = false; + bool isNull = _nodeProcess == null; + if (!isNull) { + hasExited = _nodeProcess.HasExited; + } + frameworkHandle.SendMessage(TestMessageLevel.Informational, isNull.ToString()); + frameworkHandle.SendMessage(TestMessageLevel.Informational, hasExited.ToString()); + if ( _nodeProcess == null || _nodeProcess.HasExited ) { + frameworkHandle.SendMessage(TestMessageLevel.Informational, "inside RunTests if statement"); + TestCase firstTest = tests.First(); + NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); + List args = new List(); + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); + } RunTestCases(tests, runContext, frameworkHandle); + + _nodeProcess.Dispose(); } private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { @@ -238,8 +254,6 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } lock (_syncObject) { - // launch node process - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); //frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); @@ -250,8 +264,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); _nodeProcess.StandardInput.Close(); _nodeProcess.WaitForExit(5000); - } - //standardInput.Close(); + } if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, @@ -288,9 +301,6 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } else { frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain result for " + test.DisplayName + " from TestRunner"); } - - // dispose node process - _nodeProcess.Dispose(); } private ResultObject ParseTestResult(string line) { @@ -310,7 +320,6 @@ private ResultObject GetTestResultFromProcess(StreamReader sr) { } break; } - sr.DiscardBufferedData(); return result; } From 4a188f68f2d9abe933d6d45c1a7ff753d9c41382 Mon Sep 17 00:00:00 2001 From: Jesse Mazzella Date: Mon, 24 Oct 2016 11:45:34 -0700 Subject: [PATCH 03/22] Have TestExecutor run multiple tests with a single instance of node (#1383) * run_tests accepts a list of tests * mocha.js runs a list of tests and records duration * use full title for tests and initialize time to 0 * run ExportRunner tests with single node instance * close readline interface after tests are done * have ExportRunner tests record test duration * tape framework runs multiple tests with one node instance * enable TestExecutor to run multiple tests with one instance of node * add comment --- .../ExportRunner/exportrunner.js | 45 +++- .../Nodejs/TestFrameworks/Tape/tape.js | 57 ++-- .../Nodejs/TestFrameworks/mocha/mocha.js | 54 ++-- .../Nodejs/TestFrameworks/run_tests.js | 26 +- Nodejs/Product/TestAdapter/TestExecutor.cs | 247 ++++++++---------- 5 files changed, 238 insertions(+), 191 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index 88dc6b431..eec49a868 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -1,6 +1,22 @@ var fs = require('fs'); var path = require('path'); var vm = require('vm'); +var result = { + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '', + 'time': 0 +}; + +function append_stdout(string, encoding, fd) { + result.stdOut += string; +} +function append_stderr(string, encoding, fd) { + result.stdErr += string; +} +process.stdout.write = append_stdout; +process.stderr.write = append_stderr; var find_tests = function (testFileList, discoverResultFile) { var debug; @@ -50,8 +66,31 @@ var find_tests = function (testFileList, discoverResultFile) { }; module.exports.find_tests = find_tests; -var run_tests = function (testName, testFile) { - var testCase = require(testFile); - testCase[testName](); +var run_tests = function (testCases, callback) { + var test_results = []; + for (var test in testCases) { + try { + var testCase = require(testCases[test].testFile); + result.title = testCases[test].testName; + result.time = Date.now(); + testCase[testCases[test].testName](); + result.time = Date.now() - result.time; + result.passed = true; + } catch (err) { + result.time = Date.now() - result.time; + result.passed = false; + console.error(err.name); + console.error(err.message); + } + test_results.push(result) + result = { + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '', + 'time': 0 + }; + } + callback(test_results); }; module.exports.run_tests = run_tests; \ No newline at end of file diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index 00320a7cc..bed15d75a 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -3,17 +3,17 @@ var EOL = require('os').EOL; var fs = require('fs'); var path = require('path'); var result = { - "title": "", - "passed": false, - "stdOut": "", - "stdErr": "" + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '', + 'time': 0 }; -process.stdout.write = function (string, encoding, fd) { +function append_stdout(string, encoding, fd) { result.stdOut += string; } - -process.stderr.write = function (string, encoding, fd) { +function append_stderr(string, encoding, fd) { result.stdErr += string; } @@ -24,7 +24,7 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { } var harness = test.getHarness({ exit: false }); - var tests = harness["_tests"]; + var tests = harness['_tests']; var count = 0; var testList = []; @@ -51,28 +51,43 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -function run_tests(testName, testFile, workingFolder, projectFolder, callback) { - var testCases = loadTestCases(testFile); - result.title = testName; +function run_tests(testInfo, callback) { + var testResults = []; + var testCases = loadTestCases(testInfo[0].testFile); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; if (testCases === null) { return; } - var test = findTape(projectFolder); - if (test === null) { + var tape = findTape(testInfo[0].projectFolder); + if (tape === null) { return; } - try { - var harness = test.getHarness(); - harness.only(testName); - result.passed = true; - } catch (e) { - logError("Error running test:", testName, "in", testFile, e); - result.passed = false; + for (var test in testInfo) { + result.title = testInfo[test].testName; + try { + result.time = Date.now(); + var harness = tape.getHarness(); + harness(testInfo[test].testName); + result.passed = true; + } catch (e) { + result.passed = false; + logError('Error running test:', testInfo[test].testName, 'in', testInfo[test].testFile, e); + } + result.time = Date.now() - result.time; + testResults.push(result); + result = { + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '', + 'time': 0 + }; } - callback(result); + callback(testResults); } module.exports.run_tests = run_tests; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 47168d429..e15b0fbd8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -3,10 +3,11 @@ var EOL = require('os').EOL; var fs = require('fs'); var path = require('path'); var result = { - "title": "", - "passed": false, - "stdOut": "", - "stdErr": "" + 'title': '', + 'passed': false, + 'stdOut': '', + 'stdErr': '', + 'time': 0 }; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, @@ -69,54 +70,65 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testName, testFile, workingFolder, projectFolder, callback) { - //var testResults = []; - var Mocha = detectMocha(projectFolder); +var run_tests = function (testCases, callback) { + function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string + } + + var testResults = []; + var Mocha = detectMocha(testCases[0].projectFolder); if (!Mocha) { return; } - var mocha = initializeMocha(Mocha, projectFolder); + var mocha = initializeMocha(Mocha, testCases[0].projectFolder); - //if (testName) { - // if (typeof mocha.fgrep === 'function') - // mocha.fgrep(testName); // since Mocha 3.0.0 - // else - // mocha.grep(testName); // prior Mocha 3.0.0 - //} + var testGrepString = '^(' + testCases.map(function (testCase) { + return testCase.testName + }).join('|') + ')$'; + + if (testGrepString) { + mocha.grep(new RegExp(testGrepString)); + } - mocha.addFile(testFile); + mocha.addFile(testCases[0].testFile); // run tests - var runner = mocha.run(function (code) { process.exit(code); }); + var runner = mocha.run(function (code) { }); runner.on('start', function () { }); runner.on('test', function (test) { - result.title = test.title; + result.title = test.fullTitle(); + result.time = Date.now(); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('end', function () { + callback(testResults); }); runner.on('pass', function (test) { result.passed = true; - callback(result); + result.time = Date.now() - result.time; + testResults.push(result); result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '' + 'stdErr': '', + 'time': '' } }); runner.on('fail', function (test, err) { result.passed = false; - callback(result); + result.time = Date.now() - result.time; + testResults.push(result); result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '' + 'stdErr': '', + 'time': '' } }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 6cd03870f..bc3a46f17 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -8,28 +8,34 @@ var rl = readline.createInterface({ }); rl.on('line', (line) => { - var testInfo = JSON.parse(line); + var testCases = JSON.parse(line); // get rid of leftover quotations from C# (necessary?) - for(var s in testInfo) { - testInfo[s] = testInfo[s].replace(/["]+/g, ''); + for (var test in testCases) { + for (var value in testCases[test]) { + testCases[test][value] = testCases[test][value].replace(/["]+/g, ''); + } } try { - framework = require('./' + testInfo.framework + '/' + testInfo.framework + '.js'); + framework = require('./' + testCases[0].framework + '/' + testCases[0].framework + '.js'); } catch (exception) { - console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); + console.log("NTVS_ERROR:Failed to load TestFramework (" + testCases[0].framework + "), " + exception); process.exit(1); } - function sendResult(result) { + function returnResult(result) { + // unhook stdout and stderr process.stdout.write = old_stdout; process.stderr.write = old_stderr; - console.log(JSON.stringify(result)); - //process.exit(0); + if (result) { + console.log(JSON.stringify(result)); + } + // end process, tests are done running. + process.exit(0); } // run the test - framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder, sendResult); + framework.run_tests(testCases, returnResult); // close readline interface - //rl.close(); + rl.close(); }); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 7660e9357..a825a9fe6 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -29,7 +29,6 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudioTools; using Microsoft.VisualStudioTools.Project; -using Newtonsoft.Json.Linq; using MSBuild = Microsoft.Build.Evaluation; using Newtonsoft.Json; @@ -41,11 +40,13 @@ public ResultObject() { passed = false; stdout = String.Empty; stderr = String.Empty; + time = 0; } public string title { get; set; } public bool passed { get; set; } public string stdout { get; set; } public string stderr { get; set; } + public int time { get; set; } } [ExtensionUri(TestExecutor.ExecutorUriString)] @@ -70,7 +71,7 @@ public void Cancel() { } /// - /// This is the equivallent of "RunAll" functionality + /// This is the equivalent of "RunAll" functionality /// /// Refers to the list of test sources passed to the test adapter from the client. (Client could be VS or command line) /// Defines the settings related to the current run @@ -94,17 +95,11 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { // .njsproj file path -> project settings - var projectToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; // put tests into dictionary where key is their project working directory - // NOTE: It seems to me that if we were to run all tests over multiple projects in a solution, - // we would have to separate the tests by their project in order to launch the node process - // correctly (to make sure we are using the correct working folder) and also to run - // groups of tests by test suite. - foreach (var test in receiver.Tests) { if (!sourceToSettings.TryGetValue(test.Source, out settings)) { sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); @@ -130,141 +125,85 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // launch node process LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); // Run all test cases in a given project - RunTestCases(entry.Value, runContext, frameworkHandle); + RunTestCases(entry.Value, runContext, frameworkHandle, settings); // dispose node process _nodeProcess.Dispose(); } } } + /// + /// This is the equivalent of "Run Selected Tests" functionality. + /// + /// The list of TestCases selected to run + /// Defines the settings related to the current run + /// Handle to framework. Used for recording results public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { ValidateArg.NotNull(tests, "tests"); ValidateArg.NotNull(runContext, "runContext"); ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); - bool hasExited = false; - bool isNull = _nodeProcess == null; - if (!isNull) { - hasExited = _nodeProcess.HasExited; - } - frameworkHandle.SendMessage(TestMessageLevel.Informational, isNull.ToString()); - frameworkHandle.SendMessage(TestMessageLevel.Informational, hasExited.ToString()); - if ( _nodeProcess == null || _nodeProcess.HasExited ) { - frameworkHandle.SendMessage(TestMessageLevel.Informational, "inside RunTests if statement"); - TestCase firstTest = tests.First(); - NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); - List args = new List(); - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - } - RunTestCases(tests, runContext, frameworkHandle); + TestCase firstTest = tests.First(); + NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); + List args = new List(); + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); + // Run all test cases selected + RunTestCases(tests, runContext, frameworkHandle, settings); _nodeProcess.Dispose(); } - private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { + private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, NodejsProjectSettings settings) { // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + int port = 0; // .njsproj file path -> project settings var sourceToSettings = new Dictionary(); - + TestCaseObject testObject; + List testObjects = new List(); foreach (var test in tests) { if (_cancelRequested.WaitOne(0)) { break; } + frameworkHandle.RecordStart(test); - try { - RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); - } catch (Exception ex) { - frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); + if (settings == null) { + frameworkHandle.SendMessage( + TestMessageLevel.Error, + "Unable to determine interpreter to use for " + test.Source); + frameworkHandle.RecordEnd(test, TestOutcome.Failed); } - } - } - } - - private void KillNodeProcess() { - lock (_syncObject) { - if (_nodeProcess != null) { - _nodeProcess.Kill(); - } - } - } - private static int GetFreePort() { - return Enumerable.Range(new Random().Next(49152, 65536), 60000).Except( - from connection in IPGlobalProperties.GetIPGlobalProperties().GetActiveTcpConnections() - select connection.LocalEndPoint.Port - ).First(); - } - - private IEnumerable GetInterpreterArgs(TestCase test, string workingDir, string projectRootDir) { - TestFrameworks.NodejsTestInfo testInfo = new TestFrameworks.NodejsTestInfo(test.FullyQualifiedName); - TestFrameworks.FrameworkDiscover discover = new TestFrameworks.FrameworkDiscover(); - return discover.Get(testInfo.TestFramework).ArgumentsToRunTests(testInfo.TestName, testInfo.ModulePath, workingDir, projectRootDir); - } - - private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, out int port) { - port = GetFreePort(); - - return new[] { - "--debug-brk=" + port.ToString() - }; - } - - private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary sourceToSettings) { - var testResult = new TestResult(test); - frameworkHandle.RecordStart(test); - testResult.StartTime = DateTimeOffset.Now; - NodejsProjectSettings settings; - if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - } - if (settings == null) { - frameworkHandle.SendMessage( - TestMessageLevel.Error, - "Unable to determine interpreter to use for " + test.Source); - RecordEnd( - frameworkHandle, - test, - testResult, - null, - "Unable to determine interpreter to use for " + test.Source, - TestOutcome.Failed); - return; - } - - NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); - List args = new List(); - int port = 0; - if (runContext.IsBeingDebugged && app != null) { - app.GetDTE().Debugger.DetachAll(); - args.AddRange(GetDebugArgs(settings, out port)); - } + NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); + List args = new List(); + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } - var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); - args.AddRange(GetInterpreterArgs(test, workingDir, settings.ProjectRootDir)); + var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); + args.AddRange(GetInterpreterArgs(test, workingDir, settings.ProjectRootDir)); - //Debug.Fail("attach debugger"); - if (!File.Exists(settings.NodeExePath)) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); - return; - } + //Debug.Fail("attach debugger"); + if (!File.Exists(settings.NodeExePath)) { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); + return; + } + testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); + testObjects.Add(testObject); + } - lock (_syncObject) { -#if DEBUG - frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); - //frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); -#endif - // send test to run_tests.js - TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); if (!_nodeProcess.HasExited) { - _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + // send testcases to run_tests.js + _nodeProcess.StandardInput.WriteLine(JsonConvert.SerializeObject(testObjects)); _nodeProcess.StandardInput.Close(); - _nodeProcess.WaitForExit(5000); - } + _nodeProcess.WaitForExit(); + } + if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, @@ -289,38 +228,65 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, #endif } } - var result = GetTestResultFromProcess(_nodeProcess.StandardOutput); + var results = GetTestResultFromProcess(_nodeProcess.StandardOutput); bool runCancelled = _cancelRequested.WaitOne(0); - if (result != null) { - RecordEnd(frameworkHandle, test, testResult, - result.stdout, - result.stderr, - (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); - } else { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain result for " + test.DisplayName + " from TestRunner"); + if (results != null) { + RecordEnd(frameworkHandle, tests, results); + } + else { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain results"); + } + } + + private void KillNodeProcess() { + lock (_syncObject) { + if (_nodeProcess != null) { + _nodeProcess.Kill(); + } } } - private ResultObject ParseTestResult(string line) { - ResultObject jsonResult = null; + private static int GetFreePort() { + return Enumerable.Range(new Random().Next(49152, 65536), 60000).Except( + from connection in IPGlobalProperties.GetIPGlobalProperties().GetActiveTcpConnections() + select connection.LocalEndPoint.Port + ).First(); + } + + private IEnumerable GetInterpreterArgs(TestCase test, string workingDir, string projectRootDir) { + TestFrameworks.NodejsTestInfo testInfo = new TestFrameworks.NodejsTestInfo(test.FullyQualifiedName); + TestFrameworks.FrameworkDiscover discover = new TestFrameworks.FrameworkDiscover(); + return discover.Get(testInfo.TestFramework).ArgumentsToRunTests(testInfo.TestName, testInfo.ModulePath, workingDir, projectRootDir); + } + + private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, out int port) { + port = GetFreePort(); + + return new[] { + "--debug-brk=" + port.ToString() + }; + } + + private List ParseTestResult(string line) { + List jsonResults = null; try { - jsonResult = JsonConvert.DeserializeObject(line); + jsonResults = JsonConvert.DeserializeObject>(line); } catch (Exception) { } - return jsonResult; + return jsonResults; } - private ResultObject GetTestResultFromProcess(StreamReader sr) { - ResultObject result = null; + private List GetTestResultFromProcess(StreamReader sr) { + List results = null; while (sr.Peek() >= 0) { - result = ParseTestResult(sr.ReadLine()); - if (result == null) { + results = ParseTestResult(sr.ReadLine()); + if (results == null) { continue; } break; } - return result; + return results; } private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { @@ -357,16 +323,25 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { return projSettings; } - private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, string stdout, string stderr, TestOutcome outcome) { - result.EndTime = DateTimeOffset.Now; - result.Duration = result.EndTime - result.StartTime; - result.Outcome = outcome; - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, stderr)); - - frameworkHandle.RecordResult(result); - frameworkHandle.RecordEnd(test, outcome); + private static void RecordEnd(IFrameworkHandle frameworkHandle, IEnumerable tests, List results) { + if (tests.Count() == results.Count()) { + TestResult result; + foreach(var res in results) { + // If tests were run using "Run Selected Tests", the `tests` and `results` lists + // may not have the tests in the same order --so we query the test title from the `tests` list. + var test = tests.Where(n => n.DisplayName == res.title); + if(test.Count() == 1) { + result = new TestResult(test.First()); + result.Outcome = res.passed ? TestOutcome.Passed : TestOutcome.Failed; + result.Duration = new TimeSpan(0, 0, 0, 0, res.time); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, res.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, res.stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, res.stderr)); + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(test.First(), result.Outcome); + } + } + } } } } From 988745cf74b88dfd0e8edb41b1fd76a144de1d0c Mon Sep 17 00:00:00 2001 From: Matt Bierner Date: Mon, 24 Oct 2016 16:32:59 -0700 Subject: [PATCH 04/22] Update appveyor.yml to pick up #1388 This will make the branch build on appveyor. --- appveyor.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1f99c1ad0..b9b5aaa64 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ - version: 1.0.{build} branches: - only: - - master + except: + - v1.1.x - v1.2.x skip_tags: true image: Visual Studio 2015 @@ -45,6 +45,25 @@ - path: NTVS_Out name: NtvsOut type: zip +- + version: 1.0.{build} + branches: + only: + - v1.2.x + skip_tags: true + image: Visual Studio 2015 + environment: + matrix: + - nodejs_version: 4 + vs_version: 14.0 + architecture: x64 + - nodejs_version: 5 + vs_version: 14.0 + architecture: x86 + install: *default_install_script + build_script: *default_build_script + test_script: *default_test_script + artifacts: *default_artifacts - version: 1.0.{build} branches: From 82782081e4c871019d50762d71ef631a37ec7ade Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 2 Nov 2016 16:11:34 -0700 Subject: [PATCH 05/22] return and record test results incrementally --- .../Nodejs/TestFrameworks/mocha/mocha.js | 59 ++++++++++++-- .../Nodejs/TestFrameworks/run_tests.js | 6 +- Nodejs/Product/TestAdapter/TestExecutor.cs | 81 +++++++++++-------- 3 files changed, 103 insertions(+), 43 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index e15b0fbd8..071d4098a 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -70,7 +70,7 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testCases, callback) { +var run_tests = function (testCases, postResult) { function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -91,26 +91,64 @@ var run_tests = function (testCases, callback) { mocha.grep(new RegExp(testGrepString)); } - mocha.addFile(testCases[0].testFile); + var files = new Set(); + + for (var test in testCases) { + files.add(testCases[test].testFile); + } + + for (var file of files) { + mocha.addFile(file); + } // run tests - var runner = mocha.run(function (code) { }); + var runner = mocha.run(function (code) { process.exit(code); }); + runner.on('hook', function (hook) { + var event = { + type: 'hook start' + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('hook end', function (hook) { + var event = { + type: 'hook end', + title: hook.title + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); runner.on('start', function () { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('pending', function (test) { }); runner.on('test', function (test) { + var event = { + type: 'testStart', + title: test.fullTitle() + } + postResult(event); result.title = test.fullTitle(); result.time = Date.now(); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('end', function () { - callback(testResults); }); runner.on('pass', function (test) { result.passed = true; result.time = Date.now() - result.time; - testResults.push(result); + var event = { + type: 'result', + title: test.fullTitle(), + result: result + } + postResult(event); result = { 'title': '', 'passed': false, @@ -118,11 +156,18 @@ var run_tests = function (testCases, callback) { 'stdErr': '', 'time': '' } + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; result.time = Date.now() - result.time; - testResults.push(result); + var event = { + type: 'result', + title: test.fullTitle(), + result: result + } + postResult(event); result = { 'title': '', 'passed': false, @@ -130,6 +175,8 @@ var run_tests = function (testCases, callback) { 'stdErr': '', 'time': '' } + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index bc3a46f17..620d9d4a0 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -23,18 +23,16 @@ rl.on('line', (line) => { process.exit(1); } - function returnResult(result) { + function postResult(result) { // unhook stdout and stderr process.stdout.write = old_stdout; process.stderr.write = old_stderr; if (result) { console.log(JSON.stringify(result)); } - // end process, tests are done running. - process.exit(0); } // run the test - framework.run_tests(testCases, returnResult); + framework.run_tests(testCases, postResult); // close readline interface rl.close(); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index a825a9fe6..9ccbecf9e 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -34,6 +34,11 @@ namespace Microsoft.NodejsTools.TestAdapter { + class TestEvent { + public string type { get; set; } + public string title { get; set; } + public ResultObject result { get; set; } + } class ResultObject { public ResultObject() { title = String.Empty; @@ -46,7 +51,7 @@ public ResultObject() { public bool passed { get; set; } public string stdout { get; set; } public string stderr { get; set; } - public int time { get; set; } + public long time { get; set; } } [ExtensionUri(TestExecutor.ExecutorUriString)] @@ -62,6 +67,9 @@ class TestExecutor : ITestExecutor { private ProcessStartInfo _psi; private Process _nodeProcess; private object _syncObject = new object(); + private List _currentTests; + private IFrameworkHandle _frameworkHandle; + private TestResult _currentResult = null; public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -70,6 +78,26 @@ public void Cancel() { _cancelRequested.Set(); } + private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { + try { + TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) { + if (testEvent.type == "testStart") { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } + else if (testEvent.type == "result") { + if (_currentResult != null) { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + } + } + } + } catch (Exception) { } + } + /// /// This is the equivalent of "RunAll" functionality /// @@ -120,6 +148,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame args.AddRange(GetDebugArgs(settings, out port)); } + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); // launch node process @@ -149,6 +179,8 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame List args = new List(); args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + _currentTests = (List)tests; + _frameworkHandle = frameworkHandle; LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); // Run all test cases selected RunTestCases(tests, runContext, frameworkHandle, settings); @@ -169,7 +201,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I if (_cancelRequested.WaitOne(0)) { break; } - frameworkHandle.RecordStart(test); if (settings == null) { frameworkHandle.SendMessage( @@ -215,7 +246,8 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // } //} #if DEBUG - } catch (COMException ex) { + } + catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); @@ -228,16 +260,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I #endif } } - var results = GetTestResultFromProcess(_nodeProcess.StandardOutput); - - bool runCancelled = _cancelRequested.WaitOne(0); - - if (results != null) { - RecordEnd(frameworkHandle, tests, results); - } - else { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain results"); - } } private void KillNodeProcess() { @@ -273,7 +295,8 @@ private List ParseTestResult(string line) { List jsonResults = null; try { jsonResults = JsonConvert.DeserializeObject>(line); - } catch (Exception) { } + } + catch (Exception) { } return jsonResults; } @@ -301,6 +324,8 @@ private void LaunchNodeProcess(string workingDir, string nodeExePath, List tests, List results) { - if (tests.Count() == results.Count()) { - TestResult result; - foreach(var res in results) { - // If tests were run using "Run Selected Tests", the `tests` and `results` lists - // may not have the tests in the same order --so we query the test title from the `tests` list. - var test = tests.Where(n => n.DisplayName == res.title); - if(test.Count() == 1) { - result = new TestResult(test.First()); - result.Outcome = res.passed ? TestOutcome.Passed : TestOutcome.Failed; - result.Duration = new TimeSpan(0, 0, 0, 0, res.time); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, res.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, res.stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, res.stderr)); - frameworkHandle.RecordResult(result); - frameworkHandle.RecordEnd(test.First(), result.Outcome); - } - } - } + private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + result.EndTime = DateTimeOffset.Now; + result.Duration = result.EndTime - result.StartTime; + result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, resultObject.stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(test, result.Outcome); } } } From 49b050aece33efdb66674ed7cd908266176443f0 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 2 Nov 2016 16:14:00 -0700 Subject: [PATCH 06/22] fix formatting --- Nodejs/Product/TestAdapter/TestExecutor.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 9ccbecf9e..7e41fc324 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -246,8 +246,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // } //} #if DEBUG - } - catch (COMException ex) { + } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); @@ -295,8 +294,7 @@ private List ParseTestResult(string line) { List jsonResults = null; try { jsonResults = JsonConvert.DeserializeObject>(line); - } - catch (Exception) { } + } catch (Exception) { } return jsonResults; } From 5baa46de0f5fe1ba67f6201a90e93affa21a7229 Mon Sep 17 00:00:00 2001 From: ozyx Date: Thu, 3 Nov 2016 10:14:12 -0700 Subject: [PATCH 07/22] automatically fail tests that fail in before hooks --- Nodejs/Product/TestAdapter/TestExecutor.cs | 32 ++++++++-------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 7e41fc324..669fab3af 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -235,6 +235,15 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I _nodeProcess.WaitForExit(); } + // Automatically fail tests that haven't been run by this point (failures in before() hooks) + foreach(TestCase notRunTest in _currentTests) { + TestResult res = new TestResult(notRunTest); + res.Outcome = TestOutcome.Failed; + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, "Failure prior to test being run")); + frameworkHandle.RecordResult(res); + frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); + } + if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, @@ -290,26 +299,6 @@ private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, }; } - private List ParseTestResult(string line) { - List jsonResults = null; - try { - jsonResults = JsonConvert.DeserializeObject>(line); - } catch (Exception) { } - return jsonResults; - } - - private List GetTestResultFromProcess(StreamReader sr) { - List results = null; - while (sr.Peek() >= 0) { - results = ParseTestResult(sr.ReadLine()); - if (results == null) { - continue; - } - break; - } - return results; - } - private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { _psi = new ProcessStartInfo("cmd.exe") { Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", @@ -346,7 +335,7 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { return projSettings; } - private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + private void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime; result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; @@ -355,6 +344,7 @@ private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, T result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, result.Outcome); + _currentTests.Remove(test); } } } From 56ca7c5b2de8fefc21d204831c617ea0f3d017ad Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 7 Nov 2016 13:43:02 -0800 Subject: [PATCH 08/22] separate test suites by testFile instead of workingDir --- Nodejs/Product/TestAdapter/TestExecutor.cs | 90 ++++++++++++++-------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 669fab3af..73dc04db6 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -45,13 +45,11 @@ public ResultObject() { passed = false; stdout = String.Empty; stderr = String.Empty; - time = 0; } public string title { get; set; } public bool passed { get; set; } public string stdout { get; set; } public string stderr { get; set; } - public long time { get; set; } } [ExtensionUri(TestExecutor.ExecutorUriString)] @@ -70,6 +68,7 @@ class TestExecutor : ITestExecutor { private List _currentTests; private IFrameworkHandle _frameworkHandle; private TestResult _currentResult = null; + private ResultObject _currentResultObject = null; public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -90,10 +89,10 @@ private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { _frameworkHandle.RecordStart(test.First()); } else if (testEvent.type == "result") { - if (_currentResult != null) { - RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); - } + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); } + } else if (testEvent.type == "suite end") { + _currentResultObject = testEvent.result; } } catch (Exception) { } } @@ -122,26 +121,26 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // No VS instance just means no debugging, but everything else is // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { - // .njsproj file path -> project settings - var projectToTests = new Dictionary>(); + // .ts file path -> project settings + var fileToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; - // put tests into dictionary where key is their project working directory + // put tests into dictionary where key is their source file foreach (var test in receiver.Tests) { - if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - } - if (!projectToTests.ContainsKey(settings.WorkingDir)) { - projectToTests[settings.WorkingDir] = new List(); + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); } - projectToTests[settings.WorkingDir].Add(test); + fileToTests[test.CodeFilePath].Add(test); } - // where key is the workingDir and value is a list of tests - foreach (KeyValuePair> entry in projectToTests) { + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { List args = new List(); TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } int port = 0; if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); @@ -150,11 +149,12 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame _currentTests = entry.Value; _frameworkHandle = frameworkHandle; - args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); // launch node process LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases in a given project + // Run all test cases in a given file RunTestCases(entry.Value, runContext, frameworkHandle, settings); // dispose node process _nodeProcess.Dispose(); @@ -174,17 +174,46 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); - TestCase firstTest = tests.First(); - NodejsProjectSettings settings = LoadProjectSettings(firstTest.Source); - List args = new List(); - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); - - _currentTests = (List)tests; - _frameworkHandle = frameworkHandle; - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases selected - RunTestCases(tests, runContext, frameworkHandle, settings); - _nodeProcess.Dispose(); + using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + // .ts file path -> project settings + var fileToTests = new Dictionary>(); + var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + + // put tests into dictionary where key is their source file + foreach (var test in tests) { + if (!fileToTests.ContainsKey(test.CodeFilePath)) { + fileToTests[test.CodeFilePath] = new List(); + } + fileToTests[test.CodeFilePath].Add(test); + } + + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } + + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; + + args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); + // Run all test cases in a given file + RunTestCases(entry.Value, runContext, frameworkHandle, settings); + // dispose node process + _nodeProcess.Dispose(); + } + } } private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle, NodejsProjectSettings settings) { @@ -239,7 +268,8 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I foreach(TestCase notRunTest in _currentTests) { TestResult res = new TestResult(notRunTest); res.Outcome = TestOutcome.Failed; - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, "Failure prior to test being run")); + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); frameworkHandle.RecordResult(res); frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); } From dbd2605a5ba316f088f637a66db3b2aecac15b0e Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 7 Nov 2016 21:16:57 -0800 Subject: [PATCH 09/22] Fix newlines in TRX Test Result output --- Nodejs/Product/TestAdapter/TestExecutor.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 73dc04db6..0e92bb865 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -366,12 +366,14 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { } private void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, ResultObject resultObject) { + String[] standardOutputLines = resultObject.stdout.Split('\n'); + String[] standardErrorLines = resultObject.stderr.Split('\n'); result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime; result.Outcome = resultObject.passed ? TestOutcome.Passed : TestOutcome.Failed; - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultObject.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, resultObject.stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, resultObject.stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, String.Join(Environment.NewLine, standardOutputLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, String.Join(Environment.NewLine, standardErrorLines))); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, String.Join(Environment.NewLine, standardErrorLines))); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, result.Outcome); _currentTests.Remove(test); From 16f0bc5ba96e1854cbf24fbf4120ad28052795e7 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 16 Nov 2016 16:10:38 -0800 Subject: [PATCH 10/22] modify exportrunner.js to return results incrementally --- .../ExportRunner/exportrunner.js | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index eec49a868..ff852eb63 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -5,8 +5,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { @@ -67,30 +66,32 @@ var find_tests = function (testFileList, discoverResultFile) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { - var test_results = []; - for (var test in testCases) { + var event = {}; + for (var test of testCases) { + event.type = 'testStart'; + event.title = test.testName; + callback(event); try { - var testCase = require(testCases[test].testFile); - result.title = testCases[test].testName; - result.time = Date.now(); - testCase[testCases[test].testName](); - result.time = Date.now() - result.time; + var testCase = require(test.testFile); + result.title = test.testName; + testCase[test.testName](); result.passed = true; } catch (err) { - result.time = Date.now() - result.time; result.passed = false; console.error(err.name); console.error(err.message); } - test_results.push(result) + event.type = 'result'; + event.result = result; + callback(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; } - callback(test_results); }; module.exports.run_tests = run_tests; \ No newline at end of file From 1efc9853cd48d7dac60f8589f7dc4461f602bcde Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 16 Nov 2016 16:12:10 -0800 Subject: [PATCH 11/22] modify mocha.js to return results incrementally --- .../Nodejs/TestFrameworks/mocha/mocha.js | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 071d4098a..8945f86f6 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -6,8 +6,7 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, @@ -91,22 +90,33 @@ var run_tests = function (testCases, postResult) { mocha.grep(new RegExp(testGrepString)); } - var files = new Set(); - - for (var test in testCases) { - files.add(testCases[test].testFile); - } - - for (var file of files) { - mocha.addFile(file); - } - + mocha.addFile(testCases[0].testFile); + // run tests var runner = mocha.run(function (code) { process.exit(code); }); - + runner.on('suite', function (suite) { + var event = { + type: 'suite start', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); + runner.on('suite end', function (suite) { + var event = { + type: 'suite end', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; + }); runner.on('hook', function (hook) { var event = { - type: 'hook start' + type: 'hook start', + title: hook.title, + result: result } postResult(event); process.stdout.write = append_stdout; @@ -115,37 +125,48 @@ var run_tests = function (testCases, postResult) { runner.on('hook end', function (hook) { var event = { type: 'hook end', - title: hook.title + title: hook.title, + result: result } postResult(event); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('start', function () { + var event = { + type: 'start', + result: result + } + postResult(event); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('pending', function (test) { }); runner.on('test', function (test) { + result.title = test.fullTitle(); var event = { type: 'testStart', - title: test.fullTitle() + title: result.title } postResult(event); - result.title = test.fullTitle(); - result.time = Date.now(); process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('end', function () { + var event = { + type: 'end', + result: result + } + postResult(event); + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; }); runner.on('pass', function (test) { result.passed = true; - result.time = Date.now() - result.time; var event = { type: 'result', - title: test.fullTitle(), + title: result.title, result: result } postResult(event); @@ -153,18 +174,16 @@ var run_tests = function (testCases, postResult) { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } process.stdout.write = append_stdout; process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; - result.time = Date.now() - result.time; var event = { type: 'result', - title: test.fullTitle(), + title: result.title, result: result } postResult(event); @@ -172,8 +191,7 @@ var run_tests = function (testCases, postResult) { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': '' + 'stdErr': '' } process.stdout.write = append_stdout; process.stderr.write = append_stderr; From 7349067cee1d2f654d955ebcf24ed25299c96179 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 2 Dec 2016 16:54:49 -0800 Subject: [PATCH 12/22] clean up code --- .../Nodejs/TestFrameworks/mocha/mocha.js | 84 +++++++------------ 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 8945f86f6..0c4813c2d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -18,8 +18,12 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); @@ -69,7 +73,11 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testCases, postResult) { +var run_tests = function (testCases, callback) { + function post(event) { + callback(event); + hook_outputs(); + } function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -91,110 +99,82 @@ var run_tests = function (testCases, postResult) { } mocha.addFile(testCases[0].testFile); - + // run tests var runner = mocha.run(function (code) { process.exit(code); }); runner.on('suite', function (suite) { - var event = { + post({ type: 'suite start', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('suite end', function (suite) { - var event = { + post({ type: 'suite end', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('hook', function (hook) { - var event = { + post({ type: 'hook start', title: hook.title, result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('hook end', function (hook) { - var event = { + post({ type: 'hook end', title: hook.title, result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('start', function () { - var event = { + post({ type: 'start', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; - }); - runner.on('pending', function (test) { + }); }); + runner.on('test', function (test) { result.title = test.fullTitle(); - var event = { - type: 'testStart', + post({ + type: 'test start', title: result.title - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('end', function () { - var event = { + post({ type: 'end', result: result - } - postResult(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + }); }); runner.on('pass', function (test) { result.passed = true; - var event = { + post({ type: 'result', title: result.title, result: result - } - postResult(event); + }); result = { 'title': '', 'passed': false, 'stdOut': '', 'stdErr': '' } - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; }); runner.on('fail', function (test, err) { result.passed = false; - var event = { + post({ type: 'result', title: result.title, result: result - } - postResult(event); + }); result = { 'title': '', 'passed': false, 'stdOut': '', 'stdErr': '' } - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; }); }; From a0acb7ab57d7e97fbfd304cf7c7a4318e281020e Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 2 Dec 2016 16:56:11 -0800 Subject: [PATCH 13/22] display test stdout and stderr in exportRunner test results --- .../ExportRunner/exportrunner.js | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index ff852eb63..88dd747ed 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -14,8 +14,13 @@ function append_stdout(string, encoding, fd) { function append_stderr(string, encoding, fd) { result.stdErr += string; } -process.stdout.write = append_stdout; -process.stderr.write = append_stderr; +function hook_outputs() { + process.stdout.write = append_stdout; + process.stderr.write = append_stderr; +} + + +hook_outputs(); var find_tests = function (testFileList, discoverResultFile) { var debug; @@ -66,15 +71,21 @@ var find_tests = function (testFileList, discoverResultFile) { module.exports.find_tests = find_tests; var run_tests = function (testCases, callback) { - var event = {}; - for (var test of testCases) { - event.type = 'testStart'; - event.title = test.testName; + function post(event) { callback(event); + hook_outputs(); + } + + for (var test of testCases) { + event = { + type: 'test start', + title: test.testName + } + post(event); try { var testCase = require(test.testFile); - result.title = test.testName; testCase[test.testName](); + result.title = test.testName; result.passed = true; } catch (err) { result.passed = false; @@ -83,9 +94,7 @@ var run_tests = function (testCases, callback) { } event.type = 'result'; event.result = result; - callback(event); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; + post(event); result = { 'title': '', 'passed': false, @@ -93,5 +102,9 @@ var run_tests = function (testCases, callback) { 'stdErr': '' }; } + callback({ + type: 'suite end', + result: result + }); }; module.exports.run_tests = run_tests; \ No newline at end of file From ef497da245154209689949cec663dd8c2f92deca Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 5 Dec 2016 13:20:29 -0800 Subject: [PATCH 14/22] code cleanup --- Nodejs/Product/TestAdapter/TestExecutor.cs | 78 +++++++++++----------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 0e92bb865..a5f08f611 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -33,25 +33,6 @@ using Newtonsoft.Json; namespace Microsoft.NodejsTools.TestAdapter { - - class TestEvent { - public string type { get; set; } - public string title { get; set; } - public ResultObject result { get; set; } - } - class ResultObject { - public ResultObject() { - title = String.Empty; - passed = false; - stdout = String.Empty; - stderr = String.Empty; - } - public string title { get; set; } - public bool passed { get; set; } - public string stdout { get; set; } - public string stderr { get; set; } - } - [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; @@ -79,20 +60,21 @@ public void Cancel() { private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { try { - TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); - // Extract test from list of tests - var test = _currentTests.Where(n => n.DisplayName == testEvent.title); - if (test.Count() > 0) { - if (testEvent.type == "testStart") { - _currentResult = new TestResult(test.First()); - _currentResult.StartTime = DateTimeOffset.Now; - _frameworkHandle.RecordStart(test.First()); - } - else if (testEvent.type == "result") { - RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + if (e.Data != null) { + TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) { + if (testEvent.type == "test start") { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } else if (testEvent.type == "result") { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); + } + } else if (testEvent.type == "suite end") { + _currentResultObject = testEvent.result; } - } else if (testEvent.type == "suite end") { - _currentResultObject = testEvent.result; } } catch (Exception) { } } @@ -266,11 +248,13 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // Automatically fail tests that haven't been run by this point (failures in before() hooks) foreach(TestCase notRunTest in _currentTests) { - TestResult res = new TestResult(notRunTest); - res.Outcome = TestOutcome.Failed; - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); - res.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); - frameworkHandle.RecordResult(res); + TestResult result = new TestResult(notRunTest); + result.Outcome = TestOutcome.Failed; + if(_currentResultObject != null) { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); + } + frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); } @@ -416,6 +400,25 @@ public NodejsProjectSettings() { public string ProjectRootDir { get; set; } } +class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } +} + +class TestEvent { + public string type { get; set; } + public string title { get; set; } + public ResultObject result { get; set; } +} + class TestCaseObject { public TestCaseObject() { framework = String.Empty; @@ -437,5 +440,4 @@ public TestCaseObject(string framework, string testName, string testFile, string public string testFile { get; set; } public string workingFolder { get; set; } public string projectFolder { get; set; } - } \ No newline at end of file From bf150b7f196631adbadee4a30441d7438c69de17 Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 5 Dec 2016 13:30:13 -0800 Subject: [PATCH 15/22] modify tape.run_tests to be compatible with new TestExecutor --- .../Nodejs/TestFrameworks/Tape/tape.js | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index bed15d75a..ee8329c9d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -6,13 +6,13 @@ var result = { 'title': '', 'passed': false, 'stdOut': '', - 'stdErr': '', - 'time': 0 + 'stdErr': '' }; function append_stdout(string, encoding, fd) { result.stdOut += string; } + function append_stderr(string, encoding, fd) { result.stdErr += string; } @@ -52,42 +52,63 @@ function find_tests(testFileList, discoverResultFile, projectFolder) { module.exports.find_tests = find_tests; function run_tests(testInfo, callback) { - var testResults = []; - var testCases = loadTestCases(testInfo[0].testFile); - process.stdout.write = append_stdout; - process.stderr.write = append_stderr; - if (testCases === null) { - return; - } - var tape = findTape(testInfo[0].projectFolder); if (tape === null) { return; } - for (var test in testInfo) { - result.title = testInfo[test].testName; + var harness = tape.getHarness(); + + testInfo.forEach(function (info) { + runTest(info, harness, function (result) { + callback(result); + }); + }); + + tape.onFinish(function () { + // executes when all tests are done running + }); + + function runTest(testInfo, harness, done) { + var stream = harness.createStream({ objectMode: true }); + var title = testInfo.testName; + + stream.on(('data'), function (result) { + if (result.type === 'test') { + done({ + type: 'test start', + title: title + }); + } + }); + try { - result.time = Date.now(); - var harness = tape.getHarness(); - harness(testInfo[test].testName); - result.passed = true; + var htest = tape.test(title, {}, function (result) { + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': result._ok, + 'stdOut': '', + 'stdErr': '' + } + }); + }); } catch (e) { - result.passed = false; - logError('Error running test:', testInfo[test].testName, 'in', testInfo[test].testFile, e); + console.error('NTVS_ERROR:', e); + done({ + type: 'result', + title: title, + result: { + 'title': title, + 'passed': false, + 'stdOut': '', + 'stdErr': e.message + } + }); } - result.time = Date.now() - result.time; - testResults.push(result); - result = { - 'title': '', - 'passed': false, - 'stdOut': '', - 'stdErr': '', - 'time': 0 - }; } - - callback(testResults); } module.exports.run_tests = run_tests; From c5d1310dc6297b8423526a23f5cc7ca07ea40d80 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 6 Dec 2016 09:22:41 -0800 Subject: [PATCH 16/22] clean up code --- .../Nodejs/TestFrameworks/mocha/mocha.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 0c4813c2d..6d9109cc8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -78,6 +78,7 @@ var run_tests = function (testCases, callback) { callback(event); hook_outputs(); } + function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } @@ -93,27 +94,30 @@ var run_tests = function (testCases, callback) { var testGrepString = '^(' + testCases.map(function (testCase) { return testCase.testName }).join('|') + ')$'; - + if (testGrepString) { mocha.grep(new RegExp(testGrepString)); } - mocha.addFile(testCases[0].testFile); - // run tests - var runner = mocha.run(function (code) { process.exit(code); }); + var runner = mocha.run(function (code) { + process.exit(code); + }); + runner.on('suite', function (suite) { post({ type: 'suite start', result: result }); }); + runner.on('suite end', function (suite) { post({ type: 'suite end', result: result }); }); + runner.on('hook', function (hook) { post({ type: 'hook start', @@ -121,6 +125,7 @@ var run_tests = function (testCases, callback) { result: result }); }); + runner.on('hook end', function (hook) { post({ type: 'hook end', @@ -128,6 +133,7 @@ var run_tests = function (testCases, callback) { result: result }); }); + runner.on('start', function () { post({ type: 'start', @@ -142,12 +148,14 @@ var run_tests = function (testCases, callback) { title: result.title }); }); + runner.on('end', function () { post({ type: 'end', result: result }); }); + runner.on('pass', function (test) { result.passed = true; post({ @@ -162,6 +170,7 @@ var run_tests = function (testCases, callback) { 'stdErr': '' } }); + runner.on('fail', function (test, err) { result.passed = false; post({ From b189e21f38da2c9654f32a7d60a33f3f2556d319 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 6 Dec 2016 09:38:16 -0800 Subject: [PATCH 17/22] clean up code --- .../TestFrameworks/ExportRunner/exportrunner.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index 88dd747ed..4a0ded242 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -77,11 +77,10 @@ var run_tests = function (testCases, callback) { } for (var test of testCases) { - event = { + post({ type: 'test start', title: test.testName - } - post(event); + }); try { var testCase = require(test.testFile); testCase[test.testName](); @@ -92,9 +91,11 @@ var run_tests = function (testCases, callback) { console.error(err.name); console.error(err.message); } - event.type = 'result'; - event.result = result; - post(event); + post({ + type: 'result', + title: test.testName, + result: result + }); result = { 'title': '', 'passed': false, From a931fc42ff8b48eb202aa776f2c5de9e6de7a3a0 Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Tue, 18 Apr 2017 23:21:30 -0700 Subject: [PATCH 18/22] Simplified test execution code --- Common/Product/SharedProject/ProcessOutput.cs | 37 ++- Common/Product/TestAdapter/VisualStudioApp.cs | 29 ++ .../ExportRunner/exportrunner.js | 2 +- .../Nodejs/TestFrameworks/mocha/mocha.js | 1 + Nodejs/Product/TestAdapter/TestExecutor.cs | 286 +++++++++--------- 5 files changed, 200 insertions(+), 155 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index ee75aef72..e70dd944a 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -58,6 +58,15 @@ public virtual void Show() { /// public virtual void ShowAndActivate() { } + + /// + /// Called to determine if stdin should be closed for a redirected process. + /// The default is true. + /// + public virtual bool CloseStandardInput() + { + return true; + } } sealed class TeeRedirector : Redirector, IDisposable { @@ -423,10 +432,16 @@ private ProcessOutput(Process process, Redirector redirector) { if (_process.StartInfo.RedirectStandardInput) { // Close standard input so that we don't get stuck trying to read input from the user. - try { - _process.StandardInput.Close(); - } catch (InvalidOperationException) { - // StandardInput not available + if (_redirector == null || (_redirector != null && _redirector.CloseStandardInput())) + { + try + { + _process.StandardInput.Close(); + } + catch (InvalidOperationException) + { + // StandardInput not available + } } } } @@ -557,6 +572,20 @@ public Redirector Redirector { get { return _redirector; } } + /// + /// Writes a line to stdin. A redirector must have been provided that indicates not + /// to close the StandardInput stream. + /// + /// + public void WriteInputLine(string line) + { + if (IsStarted && _redirector != null && !_redirector.CloseStandardInput()) + { + _process.StandardInput.WriteLine(line); + _process.StandardInput.Flush(); + } + } + private void FlushAndCloseOutput() { if (_process == null) { return; diff --git a/Common/Product/TestAdapter/VisualStudioApp.cs b/Common/Product/TestAdapter/VisualStudioApp.cs index 34f381b08..38c417071 100644 --- a/Common/Product/TestAdapter/VisualStudioApp.cs +++ b/Common/Product/TestAdapter/VisualStudioApp.cs @@ -15,7 +15,9 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using EnvDTE; using Microsoft.VisualStudio.OLE.Interop; @@ -75,7 +77,34 @@ public DTE GetDTE() { return dte; } +#if DEV15 + private static bool DTELoaded = false; +#endif + private static DTE GetDTE(int processId) { +#if DEV15 + // VS 2017 doesn't install some assemblies to the GAC that are needed to work with the + // debugger, and as the tests don't execute in the devenv.exe process, those assemblies + // fail to load - so load them manually from PublicAssemblies. + + // Use the executable name, as this is only needed for the out of proc test execution + // that may interact with the debugger (vstest.executionengine.x86.exe). + if (!DTELoaded) + { + string currentProc = Process.GetCurrentProcess().MainModule.FileName; + if (Path.GetFileName(currentProc).ToLowerInvariant().Equals("vstest.executionengine.x86.exe")) + { + string baseDir = Path.GetDirectoryName(currentProc); + string publicAssemblies = Path.Combine(baseDir, "..\\..\\..\\PublicAssemblies"); + + Assembly.LoadFrom(Path.Combine(publicAssemblies, "Microsoft.VisualStudio.OLE.Interop.dll")); + Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte90.dll")); + Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte80.dll")); + Assembly.LoadFrom(Path.Combine(publicAssemblies, "envdte.dll")); + } + DTELoaded = true; + } +#endif MessageFilter.Register(); var prefix = Process.GetProcessById(processId).ProcessName; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index 4a0ded242..dda49afcf 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -83,8 +83,8 @@ var run_tests = function (testCases, callback) { }); try { var testCase = require(test.testFile); - testCase[test.testName](); result.title = test.testName; + testCase[test.testName](); result.passed = true; } catch (err) { result.passed = false; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 6d9109cc8..16d89e45d 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -104,6 +104,7 @@ var run_tests = function (testCases, callback) { process.exit(code); }); + // See events available at https://github.com/mochajs/mocha/blob/8cae7a34f0b6eafeb16567beb8852b827cc5956b/lib/runner.js#L47-L57 runner.on('suite', function (suite) { post({ type: 'suite start', diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 96aab2a6f..73b0f1db8 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -20,6 +20,7 @@ using System.IO; using System.Linq; using System.Net.NetworkInformation; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading; @@ -31,9 +32,31 @@ using Microsoft.VisualStudioTools.Project; using MSBuild = Microsoft.Build.Evaluation; using Newtonsoft.Json; -using System.Globalization; namespace Microsoft.NodejsTools.TestAdapter { + class TestExecutionRedirector : Redirector + { + Action writer; + public TestExecutionRedirector(Action onWriteLine) + { + writer = onWriteLine; + } + public override void WriteErrorLine(string line) + { + writer(line); + } + + public override void WriteLine(string line) + { + writer(line); + } + + public override bool CloseStandardInput() + { + return false; + } + } + [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; @@ -44,8 +67,7 @@ class TestExecutor : ITestExecutor { private readonly ManualResetEvent _cancelRequested = new ManualResetEvent(false); private static readonly char[] _needToBeQuoted = new[] { ' ', '"' }; - private ProcessStartInfo _psi; - private Process _nodeProcess; + private ProcessOutput _nodeProcess; private object _syncObject = new object(); private List _currentTests; private IFrameworkHandle _frameworkHandle; @@ -59,25 +81,34 @@ public void Cancel() { _cancelRequested.Set(); } - private void ProcessTestEvent(object sender, DataReceivedEventArgs e) { - try { - if (e.Data != null) { - TestEvent testEvent = JsonConvert.DeserializeObject(e.Data); - // Extract test from list of tests - var test = _currentTests.Where(n => n.DisplayName == testEvent.title); - if (test.Count() > 0) { - if (testEvent.type == "test start") { - _currentResult = new TestResult(test.First()); - _currentResult.StartTime = DateTimeOffset.Now; - _frameworkHandle.RecordStart(test.First()); - } else if (testEvent.type == "result") { - RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); - } - } else if (testEvent.type == "suite end") { - _currentResultObject = testEvent.result; + private void ProcessTestRunnerEmit(string line) + { + try + { + TestEvent testEvent = JsonConvert.DeserializeObject(line); + // Extract test from list of tests + var test = _currentTests.Where(n => n.DisplayName == testEvent.title); + if (test.Count() > 0) + { + if (testEvent.type == "test start") + { + _currentResult = new TestResult(test.First()); + _currentResult.StartTime = DateTimeOffset.Now; + _frameworkHandle.RecordStart(test.First()); + } + else if (testEvent.type == "result") + { + RecordEnd(_frameworkHandle, test.First(), _currentResult, testEvent.result); } } - } catch (Exception) { } + else if (testEvent.type == "suite end") + { + _currentResultObject = testEvent.result; + } + } + catch (JsonReaderException) { + // Often lines emitted while running tests are not test results, and thus will fail to parse above + } } /// @@ -100,49 +131,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame if (_cancelRequested.WaitOne(0)) { return; } - // May be null, but this is handled by RunTestCase if it matters. - // No VS instance just means no debugging, but everything else is - // okay. - using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { - // .ts file path -> project settings - var fileToTests = new Dictionary>(); - var sourceToSettings = new Dictionary(); - NodejsProjectSettings settings = null; - // put tests into dictionary where key is their source file - foreach (var test in receiver.Tests) { - if (!fileToTests.ContainsKey(test.CodeFilePath)) { - fileToTests[test.CodeFilePath] = new List(); - } - fileToTests[test.CodeFilePath].Add(test); - } - - // where key is the file and value is a list of tests - foreach (KeyValuePair> entry in fileToTests) { - List args = new List(); - TestCase firstTest = entry.Value.ElementAt(0); - if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { - sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); - } - int port = 0; - if (runContext.IsBeingDebugged && app != null) { - app.GetDTE().Debugger.DetachAll(); - args.AddRange(GetDebugArgs(settings, out port)); - } - - _currentTests = entry.Value; - _frameworkHandle = frameworkHandle; - - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); - - // launch node process - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases in a given file - RunTestCases(entry.Value, runContext, frameworkHandle, settings); - // dispose node process - _nodeProcess.Dispose(); - } - } + RunTests(receiver.Tests, runContext, frameworkHandle); } /// @@ -151,51 +141,42 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame /// The list of TestCases selected to run /// Defines the settings related to the current run /// Handle to framework. Used for recording results - public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { + public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) + { ValidateArg.NotNull(tests, "tests"); ValidateArg.NotNull(runContext, "runContext"); ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); - using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { - // .ts file path -> project settings - var fileToTests = new Dictionary>(); - var sourceToSettings = new Dictionary(); - NodejsProjectSettings settings = null; - - // put tests into dictionary where key is their source file - foreach (var test in tests) { - if (!fileToTests.ContainsKey(test.CodeFilePath)) { - fileToTests[test.CodeFilePath] = new List(); - } - fileToTests[test.CodeFilePath].Add(test); + // .ts file path -> project settings + var fileToTests = new Dictionary>(); + var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + + // put tests into dictionary where key is their source file + foreach (var test in tests) + { + if (!fileToTests.ContainsKey(test.CodeFilePath)) + { + fileToTests[test.CodeFilePath] = new List(); } + fileToTests[test.CodeFilePath].Add(test); + } - // where key is the file and value is a list of tests - foreach (KeyValuePair> entry in fileToTests) { - List args = new List(); - TestCase firstTest = entry.Value.ElementAt(0); - if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) { - sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); - } - int port = 0; - if (runContext.IsBeingDebugged && app != null) { - app.GetDTE().Debugger.DetachAll(); - args.AddRange(GetDebugArgs(settings, out port)); - } - - _currentTests = entry.Value; - _frameworkHandle = frameworkHandle; + // where key is the file and value is a list of tests + foreach (KeyValuePair> entry in fileToTests) + { + TestCase firstTest = entry.Value.ElementAt(0); + if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) + { + sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + } - args.AddRange(GetInterpreterArgs(firstTest, settings.WorkingDir, settings.ProjectRootDir)); + _currentTests = entry.Value; + _frameworkHandle = frameworkHandle; - // launch node process - LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); - // Run all test cases in a given file - RunTestCases(entry.Value, runContext, frameworkHandle, settings); - // dispose node process - _nodeProcess.Dispose(); - } + // Run all test cases in a given file + RunTestCases(entry.Value, runContext, frameworkHandle, settings); } } @@ -203,12 +184,27 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. + if (tests.Count() == 0) + { + return; + } using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { int port = 0; + List nodeArgs = new List(); // .njsproj file path -> project settings var sourceToSettings = new Dictionary(); - TestCaseObject testObject; List testObjects = new List(); + + if (!File.Exists(settings.NodeExePath)) + { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); + return; + } + + // All tests being run are for the same test file, so just use the first test listed to get the working dir + NodejsTestInfo testInfo = new NodejsTestInfo(tests.First().FullyQualifiedName); + var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); + foreach (var test in tests) { if (_cancelRequested.WaitOne(0)) { break; @@ -221,54 +217,44 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I frameworkHandle.RecordEnd(test, TestOutcome.Failed); } - NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); List args = new List(); - if (runContext.IsBeingDebugged && app != null) { - app.GetDTE().Debugger.DetachAll(); - args.AddRange(GetDebugArgs(settings, out port)); - } - - var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); args.AddRange(GetInterpreterArgs(test, workingDir, settings.ProjectRootDir)); - //Debug.Fail("attach debugger"); - if (!File.Exists(settings.NodeExePath)) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); - return; + // Fetch the run_tests argument for starting node.exe if not specified yet + if(nodeArgs.Count == 0 && args.Count > 0) + { + nodeArgs.Add(args[0]); } - testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); - testObjects.Add(testObject); - } - if (!_nodeProcess.HasExited) { - // send testcases to run_tests.js - _nodeProcess.StandardInput.WriteLine(JsonConvert.SerializeObject(testObjects)); - _nodeProcess.StandardInput.Close(); - _nodeProcess.WaitForExit(); + testObjects.Add(new TestCaseObject(args[1], args[2], args[3], args[4], args[5])); } - // Automatically fail tests that haven't been run by this point (failures in before() hooks) - foreach(TestCase notRunTest in _currentTests) { - TestResult result = new TestResult(notRunTest); - result.Outcome = TestOutcome.Failed; - if(_currentResultObject != null) { - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); - } - frameworkHandle.RecordResult(result); - frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); + if (runContext.IsBeingDebugged && app != null) + { + app.GetDTE().Debugger.DetachAll(); + // Ensure that --debug-brk is the first argument + nodeArgs.InsertRange(0, GetDebugArgs(out port)); } + _nodeProcess = ProcessOutput.Run( + settings.NodeExePath, + nodeArgs, + settings.WorkingDir, + /* env */ null, + /* visible */ false, + /* redirector */ new TestExecutionRedirector(this.ProcessTestRunnerEmit), + /* quote args */ false); + if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); - //while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { - // if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { - // break; - // } - //} + while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { + if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { + break; + } + } #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); @@ -282,6 +268,21 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I } #endif } + // Send the process the list of tests to run and wait for it to complete + _nodeProcess.WriteInputLine(JsonConvert.SerializeObject(testObjects)); + _nodeProcess.Wait(); + + // Automatically fail tests that haven't been run by this point (failures in before() hooks) + foreach(TestCase notRunTest in _currentTests) { + TestResult result = new TestResult(notRunTest); + result.Outcome = TestOutcome.Failed; + if(_currentResultObject != null) { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, _currentResultObject.stdout)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, _currentResultObject.stderr)); + } + frameworkHandle.RecordResult(result); + frameworkHandle.RecordEnd(notRunTest, TestOutcome.Failed); + } } } @@ -306,30 +307,15 @@ private IEnumerable GetInterpreterArgs(TestCase test, string workingDir, return discover.Get(testInfo.TestFramework).ArgumentsToRunTests(testInfo.TestName, testInfo.ModulePath, workingDir, projectRootDir); } - private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, out int port) { + private static IEnumerable GetDebugArgs(out int port) { port = GetFreePort(); + // TODO: Need to use --inspect-brk on Node.js 8 or later return new[] { "--debug-brk=" + port.ToString() }; } - private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { - _psi = new ProcessStartInfo("cmd.exe") { - Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", - ProcessOutput.QuoteSingleArgument(workingDir), - ProcessOutput.QuoteSingleArgument(nodeExePath), - ProcessOutput.GetArguments(args, true)), - CreateNoWindow = true, - UseShellExecute = false - }; - _psi.RedirectStandardInput = true; - _psi.RedirectStandardOutput = true; - _nodeProcess = Process.Start(_psi); - _nodeProcess.BeginOutputReadLine(); - _nodeProcess.OutputDataReceived += ProcessTestEvent; - } - private NodejsProjectSettings LoadProjectSettings(string projectFile) { var env = new Dictionary(); #if DEV15 From a7fbc3eebd5232e42f55c5f324ae8c3a86836abe Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Wed, 19 Apr 2017 14:04:54 -0700 Subject: [PATCH 19/22] Review feedback --- Common/Product/TestAdapter/VisualStudioApp.cs | 3 ++- Nodejs/Product/TestAdapter/TestExecutor.cs | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Common/Product/TestAdapter/VisualStudioApp.cs b/Common/Product/TestAdapter/VisualStudioApp.cs index 38c417071..c932162a1 100644 --- a/Common/Product/TestAdapter/VisualStudioApp.cs +++ b/Common/Product/TestAdapter/VisualStudioApp.cs @@ -92,7 +92,8 @@ private static DTE GetDTE(int processId) { if (!DTELoaded) { string currentProc = Process.GetCurrentProcess().MainModule.FileName; - if (Path.GetFileName(currentProc).ToLowerInvariant().Equals("vstest.executionengine.x86.exe")) + if (StringComparer.OrdinalIgnoreCase.Equals( + Path.GetFileName(currentProc), "vstest.executionengine.x86.exe")) { string baseDir = Path.GetDirectoryName(currentProc); string publicAssemblies = Path.Combine(baseDir, "..\\..\\..\\PublicAssemblies"); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index f5a76a718..2e6a2070d 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -152,7 +152,7 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame // .ts file path -> project settings var fileToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); - NodejsProjectSettings settings = null; + NodejsProjectSettings projectSettings = null; // put tests into dictionary where key is their source file foreach (var test in tests) @@ -168,16 +168,16 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame foreach (KeyValuePair> entry in fileToTests) { TestCase firstTest = entry.Value.ElementAt(0); - if (!sourceToSettings.TryGetValue(firstTest.Source, out settings)) + if (!sourceToSettings.TryGetValue(firstTest.Source, out projectSettings)) { - sourceToSettings[firstTest.Source] = settings = LoadProjectSettings(firstTest.Source); + sourceToSettings[firstTest.Source] = projectSettings = LoadProjectSettings(firstTest.Source); } _currentTests = entry.Value; _frameworkHandle = frameworkHandle; // Run all test cases in a given file - RunTestCases(entry.Value, runContext, frameworkHandle, settings); + RunTestCases(entry.Value, runContext, frameworkHandle, projectSettings); } } From 5cfb3e2e1e5075b52f0b87350ad046e6a7524ff2 Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Thu, 20 Apr 2017 01:35:45 -0700 Subject: [PATCH 20/22] Fixed tape runner --- .../Nodejs/TestFrameworks/Tape/tape.js | 107 ++++++++++-------- 1 file changed, 61 insertions(+), 46 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index ee8329c9d..d87e95ea8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -57,58 +57,73 @@ function run_tests(testInfo, callback) { return; } - var harness = tape.getHarness(); - - testInfo.forEach(function (info) { - runTest(info, harness, function (result) { - callback(result); - }); - }); - - tape.onFinish(function () { - // executes when all tests are done running + var harness = tape.getHarness({objectMode: true}); + var capture = false; // Only capture between 'test' and 'end' events to avoid skipped test events. + harness.createStream({ objectMode: true }).on('data', function (evt){ + switch (evt.type) { + case 'test': + capture = true; + // Test is starting. Reset the result object. Send a "test start" event. + result = { + 'title': evt.name, + 'passed': true, + 'stdOut': '', + 'stdErr': '' + }; + callback({ + 'type': 'test start', + 'title': result.title, + 'result': result + }); + break; + case 'assert': + if (!capture) break; + // Correlate the success/failure asserts for this test. There may be multiple per test + var msg = "Operator: " + evt.operator + ". Expected: " + evt.expected + ". Actual: " + evt.actual + "\n"; + if (evt.ok) { + result.stdOut += msg; + } else { + result.stdErr += msg + (evt.error.stack || evt.error.message) + "\n"; + result.passed = false; + } + break; + case 'end': + if (!capture) break; + // Test is done. Send a "result" event. + callback({ + 'type': 'result', + 'title': result.title, + 'result': result + }); + capture = false; + break; + default: + break; + } }); - function runTest(testInfo, harness, done) { - var stream = harness.createStream({ objectMode: true }); - var title = testInfo.testName; + loadTestCases(testInfo[0].testFile); - stream.on(('data'), function (result) { - if (result.type === 'test') { - done({ - type: 'test start', - title: title - }); - } - }); + // Skip those not selected to run. The rest will start running on the next tick. + harness['_tests'].forEach(function(test){ + if( !testInfo.some( function(ti){ return ti.testName == test.name; }) ) { + test._skip = true; + } + }); - try { - var htest = tape.test(title, {}, function (result) { - done({ - type: 'result', - title: title, - result: { - 'title': title, - 'passed': result._ok, - 'stdOut': '', - 'stdErr': '' - } - }); - }); - } catch (e) { - console.error('NTVS_ERROR:', e); - done({ - type: 'result', - title: title, - result: { - 'title': title, - 'passed': false, - 'stdOut': '', - 'stdErr': e.message - } + harness.onFinish(function () { + // TODO: This still doesn't seem to handle async tests with plan issues. + if (capture) { + // Something didn't finish. Finish it now. + result.passed = false; + callback({ + 'type': 'result', + 'title': result.title, + 'result': result }); } - } + process.exit(0); + }); } module.exports.run_tests = run_tests; From b40d16eb8ca8ee3745cada58954803f7be0d3bb6 Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Sat, 29 Apr 2017 00:56:48 -0700 Subject: [PATCH 21/22] Ensure things close down correctly --- .../Nodejs/TestFrameworks/ExportRunner/exportrunner.js | 1 + Nodejs/Product/Nodejs/TestFrameworks/run_tests.js | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js index dda49afcf..42cbaa368 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/ExportRunner/exportrunner.js @@ -107,5 +107,6 @@ var run_tests = function (testCases, callback) { type: 'suite end', result: result }); + process.exit(); }; module.exports.run_tests = run_tests; \ No newline at end of file diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 620d9d4a0..3cae0bbc5 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -8,6 +8,7 @@ var rl = readline.createInterface({ }); rl.on('line', (line) => { + rl.close(); var testCases = JSON.parse(line); // get rid of leftover quotations from C# (necessary?) for (var test in testCases) { @@ -33,7 +34,4 @@ rl.on('line', (line) => { } // run the test framework.run_tests(testCases, postResult); - - // close readline interface - rl.close(); }); From 8ff40680a5914ce72bce4de7799347e4fec981aa Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Mon, 1 May 2017 17:00:09 -0700 Subject: [PATCH 22/22] Minor fixes to test runners --- Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js | 1 - Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js index d87e95ea8..f60f9c79c 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/Tape/tape.js @@ -112,7 +112,6 @@ function run_tests(testInfo, callback) { }); harness.onFinish(function () { - // TODO: This still doesn't seem to handle async tests with plan issues. if (capture) { // Something didn't finish. Finish it now. result.passed = false; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 16d89e45d..9a6c4cf7f 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -92,9 +92,9 @@ var run_tests = function (testCases, callback) { var mocha = initializeMocha(Mocha, testCases[0].projectFolder); var testGrepString = '^(' + testCases.map(function (testCase) { - return testCase.testName + return escapeRegExp(testCase.testName); }).join('|') + ')$'; - + if (testGrepString) { mocha.grep(new RegExp(testGrepString)); }