From 53d9f1aa2312a424a23c6229e18c3bbbdb3c61dc Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 3 Oct 2016 11:15:25 -0700 Subject: [PATCH 01/25] have run_tests return structured result used by TestExecutor --- .../Nodejs/TestFrameworks/run_tests.js | 29 ++++++++++++++++++- Nodejs/Product/TestAdapter/TestExecutor.cs | 28 ++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 3765fd252..c302326a8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -1,4 +1,25 @@ var framework; +var result = { + "title": "", + "passed": false, + "stdOut": "", + "stdErr": "" +}; + +process.stdout.write = (function (write) { + return function (string, encoding, fileDescriptor) { + result.stdOut += string; + write.apply(process.stdout, arguments); + }; +})(process.stdout.write); + +process.stderr.write = (function (write) { + return function (string, encoding, fileDescriptor) { + result.stdErr += string; + write.apply(process.stderr, arguments); + }; +})(process.stderr.write); + try { framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); } catch (exception) { @@ -6,5 +27,11 @@ try { process.exit(1); } -framework.run_tests(process.argv[3], process.argv[4], process.argv[5], process.argv[6]); +process.on('exit', (code) => { + result.passed = code === 0 ? true : false; + console.log(JSON.stringify(result)); +}); + +result.title = process.argv[3].replace(' ', '::'); +framework.run_tests(process.argv[3], process.argv[4], process.argv[5], process.argv[6]); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index dbabf6a8b..451b9c2f1 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -29,6 +29,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudioTools; using Microsoft.VisualStudioTools.Project; +using Newtonsoft.Json.Linq; using MSBuild = Microsoft.Build.Evaluation; namespace Microsoft.NodejsTools.TestAdapter { @@ -216,14 +217,22 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); + var result = ParseTestResult(_nodeProcess.StandardOutputLines); + bool runCancelled = _cancelRequested.WaitOne(0); RecordEnd(frameworkHandle, test, testResult, - string.Join(Environment.NewLine, _nodeProcess.StandardOutputLines), - string.Join(Environment.NewLine, _nodeProcess.StandardErrorLines), - (!runCancelled && _nodeProcess.ExitCode == 0) ? TestOutcome.Passed : TestOutcome.Failed); + result.stdout, + result.stderr, + (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); _nodeProcess.Dispose(); } + private ResultObject ParseTestResult(IEnumerable standardOutputLines) { + JObject jsonResult = JObject.Parse(standardOutputLines.ElementAt(standardOutputLines.Count() - 1)); + ResultObject result = jsonResult.ToObject(); + return result; + } + private NodejsProjectSettings LoadProjectSettings(string projectFile) { var buildEngine = new MSBuild.ProjectCollection(); var proj = buildEngine.LoadProject(projectFile); @@ -290,5 +299,18 @@ public NodejsProjectSettings() { public string WorkingDir { get; set; } public string ProjectRootDir { get; set; } } + class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } + } } + } From b0472246686b3867dbee5a558f0183c0ed026260 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 4 Oct 2016 13:12:16 -0700 Subject: [PATCH 02/25] have run_tests accept test data from stdin --- .../Nodejs/TestFrameworks/run_tests.js | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index c302326a8..731746b34 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -1,10 +1,29 @@ var framework; +var readline = require('readline'); var result = { "title": "", "passed": false, "stdOut": "", "stdErr": "" }; +var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +try { + framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); +} catch (exception) { + console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); + process.exit(1); +} + +rl.on('line', (line) => { + var data = JSON.parse(line); + console.log(JSON.stringify(data)); + result.title = data.testName.replace(' ', '::'); + framework.run_tests(data.testName, data.testFile, data.workingFolder, data.projectFolder); +}); process.stdout.write = (function (write) { return function (string, encoding, fileDescriptor) { @@ -20,18 +39,10 @@ process.stderr.write = (function (write) { }; })(process.stderr.write); -try { - framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); -} catch (exception) { - console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); - process.exit(1); -} - process.on('exit', (code) => { result.passed = code === 0 ? true : false; console.log(JSON.stringify(result)); + // clear stdOut and stdErr + result.stdErr = ""; + result.stdOut = ""; }); - -result.title = process.argv[3].replace(' ', '::'); - -framework.run_tests(process.argv[3], process.argv[4], process.argv[5], process.argv[6]); From 63d75a249ab0c8b5250909698f714d90d392425f Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 4 Oct 2016 16:03:46 -0700 Subject: [PATCH 03/25] run_tests receives test data over stdin --- Common/Product/SharedProject/ProcessOutput.cs | 15 ++- .../Nodejs/TestFrameworks/run_tests.js | 43 +++++---- Nodejs/Product/TestAdapter/TestExecutor.cs | 92 ++++++++++++++++--- 3 files changed, 116 insertions(+), 34 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index 884b5c5e9..8b3fa9fee 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -424,7 +424,7 @@ private ProcessOutput(Process process, Redirector redirector) { if (_process.StartInfo.RedirectStandardInput) { // Close standard input so that we don't get stuck trying to read input from the user. try { - _process.StandardInput.Close(); + //_process.StandardInput.Close(); } catch (InvalidOperationException) { // StandardInput not available } @@ -499,6 +499,19 @@ public string Arguments { } } + /// + /// TODO + /// + public StreamWriter StandardInput { + get + { + if (_process == null) { + return null; + } + return _process.StandardInput; + } + } + /// /// True if the process started. False if an error occurred. /// diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 731746b34..b2de49396 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -6,24 +6,6 @@ var result = { "stdOut": "", "stdErr": "" }; -var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}); - -try { - framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); -} catch (exception) { - console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); - process.exit(1); -} - -rl.on('line', (line) => { - var data = JSON.parse(line); - console.log(JSON.stringify(data)); - result.title = data.testName.replace(' ', '::'); - framework.run_tests(data.testName, data.testFile, data.workingFolder, data.projectFolder); -}); process.stdout.write = (function (write) { return function (string, encoding, fileDescriptor) { @@ -39,10 +21,33 @@ process.stderr.write = (function (write) { }; })(process.stderr.write); +var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +rl.on('line', (line) => { + var data = JSON.parse(line); + result.title = data.testName.replace(' ', '::'); + // get rid of leftover quotations from C# + for(var s in data) { + data[s] = data[s].replace(/['"]+/g, ''); + } + // run the test + framework.run_tests(data.testName, data.testFile, data.workingFolder, data.projectFolder); +}); + +try { + framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); +} catch (exception) { + console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); + process.exit(1); +} + process.on('exit', (code) => { result.passed = code === 0 ? true : false; console.log(JSON.stringify(result)); // clear stdOut and stdErr result.stdErr = ""; result.stdOut = ""; -}); +}); \ No newline at end of file diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 451b9c2f1..ac15d4256 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -33,8 +33,56 @@ using MSBuild = Microsoft.Build.Evaluation; namespace Microsoft.NodejsTools.TestAdapter { + + class ResultObject { + public ResultObject() { + title = String.Empty; + passed = false; + stdout = String.Empty; + stderr = String.Empty; + } + public string title { get; set; } + public bool passed { get; set; } + public string stdout { get; set; } + public string stderr { get; set; } + } + [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { + class TestRedirector : Redirector { + public override void WriteErrorLine(string line) { + throw new NotImplementedException(); + } + + public override void WriteLine(string line) { + var result = ParseTestResult(line); + if (result == null) { + return; + } + + Console.WriteLine("TEST! " + result.ToString()); + } + + public override void Show() { + base.Show(); + } + + public override void ShowAndActivate() { + base.ShowAndActivate(); + } + + private ResultObject ParseTestResult(string line) { + JObject jsonResult = null; + try { + jsonResult = JObject.Parse(line); + } + catch (Exception ex) { + Console.WriteLine(ex.Message); + } + return jsonResult != null ? jsonResult.ToObject() : null; + } + } + public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; public static readonly Uri ExecutorUri = new Uri(ExecutorUriString); //get from NodeRemoteDebugPortSupplier::PortSupplierId @@ -44,6 +92,7 @@ class TestExecutor : ITestExecutor { private ProcessOutput _nodeProcess; private object _syncObject = new object(); + private TestRedirector _testRedirector = new TestRedirector(); public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -72,8 +121,9 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame if (_cancelRequested.WaitOne(0)) { return; } - + // launch node process here? RunTestCases(receiver.Tests, runContext, frameworkHandle); + // dispose node process } public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { @@ -188,6 +238,10 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); #endif + // send test to run_tests.js + TestCaseObject testObject = new TestCaseObject(args[2], args[3], args[4], args[5]); + string o = Newtonsoft.Json.JsonConvert.SerializeObject(testObject); + _nodeProcess.StandardInput.WriteLine(o); _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); if (runContext.IsBeingDebugged && app != null) { @@ -215,8 +269,9 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } } - WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); + + WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); var result = ParseTestResult(_nodeProcess.StandardOutputLines); bool runCancelled = _cancelRequested.WaitOne(0); @@ -229,8 +284,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, private ResultObject ParseTestResult(IEnumerable standardOutputLines) { JObject jsonResult = JObject.Parse(standardOutputLines.ElementAt(standardOutputLines.Count() - 1)); - ResultObject result = jsonResult.ToObject(); - return result; + return jsonResult.ToObject(); } private NodejsProjectSettings LoadProjectSettings(string projectFile) { @@ -299,18 +353,28 @@ public NodejsProjectSettings() { public string WorkingDir { get; set; } public string ProjectRootDir { get; set; } } - class ResultObject { - public ResultObject() { - title = String.Empty; - passed = false; - stdout = String.Empty; - stderr = String.Empty; + + class TestCaseObject { + public TestCaseObject() { + testName = String.Empty; + testFile = String.Empty; + workingFolder = String.Empty; + projectFolder = String.Empty; } - public string title { get; set; } - public bool passed { get; set; } - public string stdout { get; set; } - public string stderr { get; set; } + + public TestCaseObject(string testName, string testFile, string workingFolder, string projectFolder) { + this.testName = testName; + this.testFile = testFile; + this.workingFolder = workingFolder; + this.projectFolder = projectFolder; + } + public string testName { get; set; } + public string testFile { get; set; } + public string workingFolder { get; set; } + public string projectFolder { get; set; } + } + } } From f86fa0c66972c73ed1474c5c42df2f05881f5267 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 5 Oct 2016 14:19:10 -0700 Subject: [PATCH 04/25] expose StandardOutput and disable async readline --- Common/Product/SharedProject/ProcessOutput.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index 8b3fa9fee..b85c5240e 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -415,10 +415,10 @@ private ProcessOutput(Process process, Redirector redirector) { if (_process != null) { if (_process.StartInfo.RedirectStandardOutput) { - _process.BeginOutputReadLine(); + //_process.BeginOutputReadLine(); } if (_process.StartInfo.RedirectStandardError) { - _process.BeginErrorReadLine(); + //_process.BeginErrorReadLine(); } if (_process.StartInfo.RedirectStandardInput) { @@ -512,6 +512,20 @@ public StreamWriter StandardInput { } } + /// + /// TODO + /// + public StreamReader StandardOutput { + get + { + if(_process == null || !_process.StartInfo.RedirectStandardOutput || _process.StartInfo.UseShellExecute) { + return null; + } + + return _process.StandardOutput; + } + } + /// /// True if the process started. False if an error occurred. /// From 20a9973b98a7ef67e0c323297bdb7eb58630433a Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 5 Oct 2016 14:21:24 -0700 Subject: [PATCH 05/25] TestExecutor receives test results over process StandardOutput --- Nodejs/Product/TestAdapter/TestExecutor.cs | 67 ++++++++-------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index ac15d4256..6843ba4fc 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -49,40 +49,6 @@ public ResultObject() { [ExtensionUri(TestExecutor.ExecutorUriString)] class TestExecutor : ITestExecutor { - class TestRedirector : Redirector { - public override void WriteErrorLine(string line) { - throw new NotImplementedException(); - } - - public override void WriteLine(string line) { - var result = ParseTestResult(line); - if (result == null) { - return; - } - - Console.WriteLine("TEST! " + result.ToString()); - } - - public override void Show() { - base.Show(); - } - - public override void ShowAndActivate() { - base.ShowAndActivate(); - } - - private ResultObject ParseTestResult(string line) { - JObject jsonResult = null; - try { - jsonResult = JObject.Parse(line); - } - catch (Exception ex) { - Console.WriteLine(ex.Message); - } - return jsonResult != null ? jsonResult.ToObject() : null; - } - } - public const string ExecutorUriString = "executor://NodejsTestExecutor/v1"; public static readonly Uri ExecutorUri = new Uri(ExecutorUriString); //get from NodeRemoteDebugPortSupplier::PortSupplierId @@ -92,7 +58,6 @@ private ResultObject ParseTestResult(string line) { private ProcessOutput _nodeProcess; private object _syncObject = new object(); - private TestRedirector _testRedirector = new TestRedirector(); public void Cancel() { //let us just kill the node process there, rather do it late, because VS engine process @@ -165,6 +130,7 @@ private void KillNodeProcess() { } } } + private static int GetFreePort() { return Enumerable.Range(new Random().Next(49152, 65536), 60000).Except( from connection in IPGlobalProperties.GetIPGlobalProperties().GetActiveTcpConnections() @@ -224,6 +190,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); return; } + lock (_syncObject) { _nodeProcess = ProcessOutput.Run( settings.NodeExePath, @@ -240,8 +207,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, #endif // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[2], args[3], args[4], args[5]); - string o = Newtonsoft.Json.JsonConvert.SerializeObject(testObject); - _nodeProcess.StandardInput.WriteLine(o); + _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); if (runContext.IsBeingDebugged && app != null) { @@ -268,11 +234,19 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, #endif } } - - - WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); - var result = ParseTestResult(_nodeProcess.StandardOutputLines); + // at this point the test is definitely done executing + ResultObject result = null; + + using (StreamReader sr = _nodeProcess.StandardOutput) { + while(sr.Peek() >= 0) { + result = ParseTestResult(sr.ReadLine()); + if(result == null) { + continue; + } + break; + } + } bool runCancelled = _cancelRequested.WaitOne(0); RecordEnd(frameworkHandle, test, testResult, @@ -282,9 +256,14 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, _nodeProcess.Dispose(); } - private ResultObject ParseTestResult(IEnumerable standardOutputLines) { - JObject jsonResult = JObject.Parse(standardOutputLines.ElementAt(standardOutputLines.Count() - 1)); - return jsonResult.ToObject(); + private ResultObject ParseTestResult(string line) { + JObject jsonResult = null; + try { + jsonResult = JObject.Parse(line); + } + catch (Exception) { + } + return jsonResult != null ? jsonResult.ToObject() : null; } private NodejsProjectSettings LoadProjectSettings(string projectFile) { From eb19e28d48264ce8adf664b88fa1d8152166aec0 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 5 Oct 2016 15:33:08 -0700 Subject: [PATCH 06/25] include testing framework in testInfo object sent to run_tests.js --- .../Nodejs/TestFrameworks/run_tests.js | 24 +++++++++---------- Nodejs/Product/TestAdapter/TestExecutor.cs | 7 ++++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index b2de49396..6df9dab86 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -27,23 +27,23 @@ var rl = readline.createInterface({ }); rl.on('line', (line) => { - var data = JSON.parse(line); - result.title = data.testName.replace(' ', '::'); + var testInfo = JSON.parse(line); + result.title = testInfo.testName.replace(' ', '::'); // get rid of leftover quotations from C# - for(var s in data) { - data[s] = data[s].replace(/['"]+/g, ''); + for(var s in testInfo) { + testInfo[s] = testInfo[s].replace(/['"]+/g, ''); + } + + try { + framework = require('./' + testInfo.framework + '/' + testInfo.framework + '.js'); + } catch (exception) { + console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); + process.exit(1); } // run the test - framework.run_tests(data.testName, data.testFile, data.workingFolder, data.projectFolder); + framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder); }); -try { - framework = require('./' + process.argv[2] + '/' + process.argv[2] + '.js'); -} catch (exception) { - console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); - process.exit(1); -} - process.on('exit', (code) => { result.passed = code === 0 ? true : false; console.log(JSON.stringify(result)); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 6843ba4fc..110651366 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -206,7 +206,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); #endif // send test to run_tests.js - TestCaseObject testObject = new TestCaseObject(args[2], args[3], args[4], args[5]); + TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); @@ -335,18 +335,21 @@ public NodejsProjectSettings() { class TestCaseObject { public TestCaseObject() { + framework = String.Empty; testName = String.Empty; testFile = String.Empty; workingFolder = String.Empty; projectFolder = String.Empty; } - public TestCaseObject(string testName, string testFile, string workingFolder, string projectFolder) { + public TestCaseObject(string framework, string testName, string testFile, string workingFolder, string projectFolder) { + this.framework = framework; this.testName = testName; this.testFile = testFile; this.workingFolder = workingFolder; this.projectFolder = projectFolder; } + public string framework { get; set; } public string testName { get; set; } public string testFile { get; set; } public string workingFolder { get; set; } From 831f60a5fec0144e113ff7e723cdd5fb78093a2b Mon Sep 17 00:00:00 2001 From: ozyx Date: Thu, 6 Oct 2016 08:47:00 -0700 Subject: [PATCH 07/25] comments --- Common/Product/SharedProject/ProcessOutput.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index b85c5240e..f310cd626 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -500,7 +500,8 @@ public string Arguments { } /// - /// TODO + /// The StandardInput stream of the process or null if the process hasn't + /// started yet. /// public StreamWriter StandardInput { get @@ -513,7 +514,8 @@ public StreamWriter StandardInput { } /// - /// TODO + /// The StandardOutput stream of the process or null if process hasn't started + /// or if conditions are not correct to use the stream. /// public StreamReader StandardOutput { get From 22a5d5d0f5f9df0b93c545b6c965855cd1a100f7 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 13:10:48 -0700 Subject: [PATCH 08/25] close readline interface after running test --- Nodejs/Product/Nodejs/TestFrameworks/run_tests.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 6df9dab86..65c2a1ea8 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -42,6 +42,9 @@ rl.on('line', (line) => { } // run the test framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder); + + // close readline interface + rl.close(); }); process.on('exit', (code) => { From 566851c4dd914974b05c249f37b4db6f6b3b9eb6 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 13:12:08 -0700 Subject: [PATCH 09/25] revert ProcessOutput --- Common/Product/SharedProject/ProcessOutput.cs | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index f310cd626..884b5c5e9 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -415,16 +415,16 @@ private ProcessOutput(Process process, Redirector redirector) { if (_process != null) { if (_process.StartInfo.RedirectStandardOutput) { - //_process.BeginOutputReadLine(); + _process.BeginOutputReadLine(); } if (_process.StartInfo.RedirectStandardError) { - //_process.BeginErrorReadLine(); + _process.BeginErrorReadLine(); } if (_process.StartInfo.RedirectStandardInput) { // Close standard input so that we don't get stuck trying to read input from the user. try { - //_process.StandardInput.Close(); + _process.StandardInput.Close(); } catch (InvalidOperationException) { // StandardInput not available } @@ -499,35 +499,6 @@ public string Arguments { } } - /// - /// The StandardInput stream of the process or null if the process hasn't - /// started yet. - /// - public StreamWriter StandardInput { - get - { - if (_process == null) { - return null; - } - return _process.StandardInput; - } - } - - /// - /// The StandardOutput stream of the process or null if process hasn't started - /// or if conditions are not correct to use the stream. - /// - public StreamReader StandardOutput { - get - { - if(_process == null || !_process.StartInfo.RedirectStandardOutput || _process.StartInfo.UseShellExecute) { - return null; - } - - return _process.StandardOutput; - } - } - /// /// True if the process started. False if an error occurred. /// From 1faba483be543d10e29777fecd3946b749376604 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:05:24 -0700 Subject: [PATCH 10/25] use JsonConvert.DeserializeObject() instead --- Nodejs/Product/TestAdapter/TestExecutor.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 110651366..6a3eeb1a4 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -31,6 +31,7 @@ using Microsoft.VisualStudioTools.Project; using Newtonsoft.Json.Linq; using MSBuild = Microsoft.Build.Evaluation; +using Newtonsoft.Json; namespace Microsoft.NodejsTools.TestAdapter { @@ -257,13 +258,13 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } private ResultObject ParseTestResult(string line) { - JObject jsonResult = null; + ResultObject jsonResult = null; try { - jsonResult = JObject.Parse(line); + jsonResult = JsonConvert.DeserializeObject(line); } catch (Exception) { } - return jsonResult != null ? jsonResult.ToObject() : null; + return jsonResult; } private NodejsProjectSettings LoadProjectSettings(string projectFile) { From b82e88d4fc668c0be8633147b85217db78d0b49b Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:08:22 -0700 Subject: [PATCH 11/25] create GetTestResultFromProcess() method --- Nodejs/Product/TestAdapter/TestExecutor.cs | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 6a3eeb1a4..5ff33664c 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -236,18 +236,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } } WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); - // at this point the test is definitely done executing - ResultObject result = null; - - using (StreamReader sr = _nodeProcess.StandardOutput) { - while(sr.Peek() >= 0) { - result = ParseTestResult(sr.ReadLine()); - if(result == null) { - continue; - } - break; - } - } + var result = GetTestResultFromProcess(); bool runCancelled = _cancelRequested.WaitOne(0); RecordEnd(frameworkHandle, test, testResult, @@ -267,6 +256,20 @@ private ResultObject ParseTestResult(string line) { return jsonResult; } + private ResultObject GetTestResultFromProcess() { + ResultObject result = null; + using (StreamReader sr = _nodeProcess.StandardOutput) { + while (sr.Peek() >= 0) { + result = ParseTestResult(sr.ReadLine()); + if (result == null) { + continue; + } + break; + } + } + return result; + } + private NodejsProjectSettings LoadProjectSettings(string projectFile) { var buildEngine = new MSBuild.ProjectCollection(); var proj = buildEngine.LoadProject(projectFile); From 2113caf1fc31f471a73f0e3ed0e240446e2bd6fd Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:16:56 -0700 Subject: [PATCH 12/25] use Process, copy over helper methods from ProcessOutput --- Nodejs/Product/TestAdapter/TestExecutor.cs | 49 +++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 5ff33664c..339506f15 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -57,7 +57,8 @@ class TestExecutor : ITestExecutor { private readonly ManualResetEvent _cancelRequested = new ManualResetEvent(false); - private ProcessOutput _nodeProcess; + private ProcessStartInfo _psi; + private Process _nodeProcess; private object _syncObject = new object(); public void Cancel() { @@ -290,6 +291,52 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { return projSettings; } + private static string GetArguments(IEnumerable arguments, bool quoteArgs) { + if (quoteArgs) { + return string.Join(" ", arguments.Where(a => a != null).Select(QuoteSingleArgument)); + } + else { + return string.Join(" ", arguments.Where(a => a != null)); + } + } + + internal static string QuoteSingleArgument(string arg) { + if (string.IsNullOrEmpty(arg)) { + return "\"\""; + } + if (arg.IndexOfAny(_needToBeQuoted) < 0) { + return arg; + } + + if (arg.StartsWith("\"") && arg.EndsWith("\"")) { + bool inQuote = false; + int consecutiveBackslashes = 0; + foreach (var c in arg) { + if (c == '"') { + if (consecutiveBackslashes % 2 == 0) { + inQuote = !inQuote; + } + } + + if (c == '\\') { + consecutiveBackslashes += 1; + } + else { + consecutiveBackslashes = 0; + } + } + if (!inQuote) { + return arg; + } + } + + var newArg = arg.Replace("\"", "\\\""); + if (newArg.EndsWith("\\")) { + newArg += "\\"; + } + return "\"" + newArg + "\""; + } + private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, string stdout, string stderr, TestOutcome outcome) { result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime; From a512751e1f0ff4fb3dc3c390bfebec34ab271d87 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:24:41 -0700 Subject: [PATCH 13/25] close StandardInput after sending test info --- Nodejs/Product/TestAdapter/TestExecutor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 339506f15..224e543ab 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -210,6 +210,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + _nodeProcess.StandardInput.Close(); _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); if (runContext.IsBeingDebugged && app != null) { From 3e8e08deb19c3ef661c12f4bd06357f10ce30c36 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:25:50 -0700 Subject: [PATCH 14/25] include dependency for copied ProcessOutput method --- Nodejs/Product/TestAdapter/TestExecutor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 224e543ab..bbbbe5dc8 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -57,6 +57,7 @@ class TestExecutor : ITestExecutor { private readonly ManualResetEvent _cancelRequested = new ManualResetEvent(false); + private static readonly char[] _needToBeQuoted = new[] { ' ', '"' }; private ProcessStartInfo _psi; private Process _nodeProcess; private object _syncObject = new object(); From ca7f182bf61612d914cbd5837569118171be86a5 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:30:11 -0700 Subject: [PATCH 15/25] launch node inside of RunTestCases instead --- Nodejs/Product/TestAdapter/TestExecutor.cs | 143 ++++++++++++++------- 1 file changed, 97 insertions(+), 46 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index bbbbe5dc8..eaa085fb8 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -35,7 +35,7 @@ namespace Microsoft.NodejsTools.TestAdapter { - class ResultObject { + class ResultObject { public ResultObject() { title = String.Empty; passed = false; @@ -110,18 +110,80 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { // .njsproj file path -> project settings + + var projectToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); + NodejsProjectSettings settings = null; + // put tests into dictionary where key is their project working directory + // NOTE: Not sure if this is necessary, but it seems to me that if we were + // to run all tests over multiple projects in a solution, we would have to + // separate the tests by their project in order to launch the node process + // correctly (to make sure we are using the correct working folder). foreach (var test in tests) { - if (_cancelRequested.WaitOne(0)) { - break; + if (!sourceToSettings.TryGetValue(test.Source, out settings)) { + sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); + } + if (!projectToTests.ContainsKey(settings.WorkingDir)) { + projectToTests[settings.WorkingDir] = new List(); + } + projectToTests[settings.WorkingDir].Add(test); + } + + foreach (KeyValuePair> entry in projectToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + + NodejsTestInfo testInfo = new NodejsTestInfo(firstTest.FullyQualifiedName); + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); } - try { - RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); - } catch (Exception ex) { - frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); + var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); + args.AddRange(GetInterpreterArgs(firstTest, workingDir, settings.ProjectRootDir)); + + foreach (string s in args) { + Console.WriteLine(s); + } + + // launch node process + _psi = new ProcessStartInfo("cmd.exe") { + Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", + QuoteSingleArgument(entry.Key), + QuoteSingleArgument(settings.NodeExePath), + GetArguments(args, true)), + CreateNoWindow = true, + UseShellExecute = false + }; + _psi.RedirectStandardInput = true; + _psi.RedirectStandardOutput = true; + _nodeProcess = Process.Start(_psi); + StreamWriter sw = _nodeProcess.StandardInput; + + foreach (TestCase test in entry.Value) { + if (_cancelRequested.WaitOne(0)) { + break; + } + + //Debug.Fail("attach debugger"); + if (!File.Exists(settings.NodeExePath)) { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); + return; + } + + // call RunTestCase to run each test + try { + RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); + } + catch (Exception ex) { + frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); + } + } + // dispose node process + _nodeProcess.Dispose(); } } } @@ -195,50 +257,40 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } lock (_syncObject) { - _nodeProcess = ProcessOutput.Run( - settings.NodeExePath, - args, - workingDir, - null, - false, - null, - false); - -#if DEBUG - frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); - frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); -#endif + //#if DEBUG + // frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); + // frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); + //#endif // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); _nodeProcess.StandardInput.Close(); - _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); - if (runContext.IsBeingDebugged && app != null) { - try { - //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, - //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. - string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); - while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { - if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { - break; - } - } -#if DEBUG - } catch (COMException ex) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); - frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); - KillNodeProcess(); - } -#else - } catch (COMException) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); - KillNodeProcess(); - } -#endif - } + //_nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); + // if (runContext.IsBeingDebugged && app != null) { + // try { + // //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, + // //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. + // string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); + // while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { + // if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { + // break; + // } + // } + //#if DEBUG + // } catch (COMException ex) { + // frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); + // frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); + // KillNodeProcess(); + // } + //#else + // } catch (COMException) { + // frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); + // KillNodeProcess(); + // } + //#endif + // } } - WaitHandle.WaitAll(new WaitHandle[] { _nodeProcess.WaitHandle }); var result = GetTestResultFromProcess(); bool runCancelled = _cancelRequested.WaitOne(0); @@ -246,7 +298,6 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, result.stdout, result.stderr, (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); - _nodeProcess.Dispose(); } private ResultObject ParseTestResult(string line) { From bc92e13aa7fa75c4d7cb9579638298b719062bee Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 14:43:20 -0700 Subject: [PATCH 16/25] remove some debugging stuff --- Nodejs/Product/TestAdapter/TestExecutor.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index eaa085fb8..bec43944a 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -144,10 +144,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); args.AddRange(GetInterpreterArgs(firstTest, workingDir, settings.ProjectRootDir)); - foreach (string s in args) { - Console.WriteLine(s); - } - // launch node process _psi = new ProcessStartInfo("cmd.exe") { Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", From 0b78611ab087814a346d21869a310f1cf1b39f4f Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 16:09:24 -0700 Subject: [PATCH 17/25] handle case if result is null --- Nodejs/Product/TestAdapter/TestExecutor.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index bec43944a..3f16ba8c1 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -290,10 +290,15 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, var result = GetTestResultFromProcess(); bool runCancelled = _cancelRequested.WaitOne(0); - RecordEnd(frameworkHandle, test, testResult, - result.stdout, - result.stderr, - (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); + result = null; + if (result != null) { + RecordEnd(frameworkHandle, test, testResult, + result.stdout, + result.stderr, + (!runCancelled && result.passed) ? TestOutcome.Passed : TestOutcome.Failed); + } else { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain result for " + test.DisplayName + " from TestRunner"); + } } private ResultObject ParseTestResult(string line) { From 5338a2725ab24d7280b8f39aa35a156518cc71d2 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 7 Oct 2016 16:38:42 -0700 Subject: [PATCH 18/25] pass in streams as parameters to allow continual reading/writing --- Nodejs/Product/TestAdapter/TestExecutor.cs | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 3f16ba8c1..12597b11b 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -156,7 +156,6 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I _psi.RedirectStandardInput = true; _psi.RedirectStandardOutput = true; _nodeProcess = Process.Start(_psi); - StreamWriter sw = _nodeProcess.StandardInput; foreach (TestCase test in entry.Value) { if (_cancelRequested.WaitOne(0)) { @@ -171,7 +170,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // call RunTestCase to run each test try { - RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); + RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings, _nodeProcess.StandardInput, _nodeProcess.StandardOutput); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); @@ -213,7 +212,7 @@ private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, }; } - private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary sourceToSettings) { + private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary sourceToSettings, StreamWriter standardInput, StreamReader standardOutput) { var testResult = new TestResult(test); frameworkHandle.RecordStart(test); testResult.StartTime = DateTimeOffset.Now; @@ -259,8 +258,8 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, //#endif // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); - _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); - _nodeProcess.StandardInput.Close(); + standardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + standardInput.Close(); //_nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); // if (runContext.IsBeingDebugged && app != null) { @@ -287,10 +286,10 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, //#endif // } } - var result = GetTestResultFromProcess(); + var result = GetTestResultFromProcess(standardOutput); bool runCancelled = _cancelRequested.WaitOne(0); - result = null; + if (result != null) { RecordEnd(frameworkHandle, test, testResult, result.stdout, @@ -311,16 +310,14 @@ private ResultObject ParseTestResult(string line) { return jsonResult; } - private ResultObject GetTestResultFromProcess() { + private ResultObject GetTestResultFromProcess(StreamReader sr) { ResultObject result = null; - using (StreamReader sr = _nodeProcess.StandardOutput) { - while (sr.Peek() >= 0) { - result = ParseTestResult(sr.ReadLine()); - if (result == null) { - continue; - } - break; + while (sr.Peek() >= 0) { + result = ParseTestResult(sr.ReadLine()); + if (result == null) { + continue; } + break; } return result; } From 7a93d78712119c0b2f0b7844ab0d7ea3fac739d0 Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 10 Oct 2016 11:59:36 -0700 Subject: [PATCH 19/25] move setup logic to RunTests --- Nodejs/Product/TestAdapter/TestExecutor.cs | 73 +++++++++++----------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 12597b11b..f57bed681 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -90,21 +90,6 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame return; } // launch node process here? - RunTestCases(receiver.Tests, runContext, frameworkHandle); - // dispose node process - } - - public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { - ValidateArg.NotNull(tests, "tests"); - ValidateArg.NotNull(runContext, "runContext"); - ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); - - _cancelRequested.Reset(); - - RunTestCases(tests, runContext, frameworkHandle); - } - - private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. @@ -120,7 +105,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I // to run all tests over multiple projects in a solution, we would have to // separate the tests by their project in order to launch the node process // correctly (to make sure we are using the correct working folder). - foreach (var test in tests) { + foreach (var test in receiver.Tests) { if (!sourceToSettings.TryGetValue(test.Source, out settings)) { sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); } @@ -133,16 +118,13 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I foreach (KeyValuePair> entry in projectToTests) { List args = new List(); TestCase firstTest = entry.Value.ElementAt(0); - - NodejsTestInfo testInfo = new NodejsTestInfo(firstTest.FullyQualifiedName); int port = 0; if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); args.AddRange(GetDebugArgs(settings, out port)); } - var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); - args.AddRange(GetInterpreterArgs(firstTest, workingDir, settings.ProjectRootDir)); + args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); // launch node process _psi = new ProcessStartInfo("cmd.exe") { @@ -157,28 +139,43 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I _psi.RedirectStandardOutput = true; _nodeProcess = Process.Start(_psi); - foreach (TestCase test in entry.Value) { - if (_cancelRequested.WaitOne(0)) { - break; - } + RunTestCases(entry.Value, runContext, frameworkHandle); - //Debug.Fail("attach debugger"); - if (!File.Exists(settings.NodeExePath)) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); - return; - } + // dispose node process + _nodeProcess.Dispose(); + } + } + } - // call RunTestCase to run each test - try { - RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings, _nodeProcess.StandardInput, _nodeProcess.StandardOutput); - } - catch (Exception ex) { - frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); - } + public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { + ValidateArg.NotNull(tests, "tests"); + ValidateArg.NotNull(runContext, "runContext"); + ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); + + _cancelRequested.Reset(); + RunTestCases(tests, runContext, frameworkHandle); + } + + private void RunTestCases(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { + // May be null, but this is handled by RunTestCase if it matters. + // No VS instance just means no debugging, but everything else is + // okay. + using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { + // .njsproj file path -> project settings + var sourceToSettings = new Dictionary(); + + foreach (var test in tests) { + if (_cancelRequested.WaitOne(0)) { + break; + } + + try { + RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings, _nodeProcess.StandardInput, _nodeProcess.StandardOutput); + } + catch (Exception ex) { + frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); } - // dispose node process - _nodeProcess.Dispose(); } } } From 7eb9142dae31981b09a824faaf6a014397233d5b Mon Sep 17 00:00:00 2001 From: ozyx Date: Mon, 10 Oct 2016 12:58:56 -0700 Subject: [PATCH 20/25] only remove double quotes from test case info --- Nodejs/Product/Nodejs/TestFrameworks/run_tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 65c2a1ea8..2a693dd08 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -31,7 +31,7 @@ rl.on('line', (line) => { result.title = testInfo.testName.replace(' ', '::'); // get rid of leftover quotations from C# for(var s in testInfo) { - testInfo[s] = testInfo[s].replace(/['"]+/g, ''); + testInfo[s] = testInfo[s].replace(/["]+/g, ''); } try { From f0dc92d0960ac07d9c0d5f64e80d4e56720aa249 Mon Sep 17 00:00:00 2001 From: ozyx Date: Tue, 11 Oct 2016 14:37:19 -0700 Subject: [PATCH 21/25] move Process launch logic into function --- Nodejs/Product/TestAdapter/TestExecutor.cs | 103 +++++++++++---------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index f57bed681..236d989db 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -89,7 +89,6 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame if (_cancelRequested.WaitOne(0)) { return; } - // launch node process here? // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. @@ -101,9 +100,8 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame NodejsProjectSettings settings = null; // put tests into dictionary where key is their project working directory - // NOTE: Not sure if this is necessary, but it seems to me that if we were - // to run all tests over multiple projects in a solution, we would have to - // separate the tests by their project in order to launch the node process + // NOTE: It seems to me that if we were to run all tests over multiple projects in a solution, + // we would have to separate the tests by their project in order to launch the node process // correctly (to make sure we are using the correct working folder). foreach (var test in receiver.Tests) { if (!sourceToSettings.TryGetValue(test.Source, out settings)) { @@ -115,6 +113,7 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame projectToTests[settings.WorkingDir].Add(test); } + // where key is the workingDir and value is a list of tests foreach (KeyValuePair> entry in projectToTests) { List args = new List(); TestCase firstTest = entry.Value.ElementAt(0); @@ -127,18 +126,9 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); // launch node process - _psi = new ProcessStartInfo("cmd.exe") { - Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", - QuoteSingleArgument(entry.Key), - QuoteSingleArgument(settings.NodeExePath), - GetArguments(args, true)), - CreateNoWindow = true, - UseShellExecute = false - }; - _psi.RedirectStandardInput = true; - _psi.RedirectStandardOutput = true; - _nodeProcess = Process.Start(_psi); + LaunchNodeProcess(entry.Key, settings.NodeExePath, args); + // Run all test cases in a given project RunTestCases(entry.Value, runContext, frameworkHandle); // dispose node process @@ -249,39 +239,41 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } lock (_syncObject) { - //#if DEBUG - // frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); - // frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); - //#endif +#if DEBUG + frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); + //frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); +#endif // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); - standardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); - standardInput.Close(); - - //_nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); - // if (runContext.IsBeingDebugged && app != null) { - // try { - // //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, - // //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. - // string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); - // while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { - // if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { - // break; - // } - // } - //#if DEBUG - // } catch (COMException ex) { - // frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); - // frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); - // KillNodeProcess(); - // } - //#else - // } catch (COMException) { - // frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); - // KillNodeProcess(); - // } - //#endif - // } + if (!_nodeProcess.HasExited) { + standardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + _nodeProcess.WaitForExit(5000); + } + //standardInput.Close(); + if (runContext.IsBeingDebugged && app != null) { + try { + //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, + //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. + string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); + //while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { + // if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { + // break; + // } + //} +#if DEBUG + } + catch (COMException ex) { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); + frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); + KillNodeProcess(); + } +#else + } catch (COMException) { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); + KillNodeProcess(); + } +#endif + } } var result = GetTestResultFromProcess(standardOutput); @@ -316,9 +308,24 @@ private ResultObject GetTestResultFromProcess(StreamReader sr) { } break; } + sr.DiscardBufferedData(); return result; } + private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { + _psi = new ProcessStartInfo("cmd.exe") { + Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", + QuoteSingleArgument(workingDir), + QuoteSingleArgument(nodeExePath), + GetArguments(args, true)), + CreateNoWindow = true, + UseShellExecute = false + }; + _psi.RedirectStandardInput = true; + _psi.RedirectStandardOutput = true; + _nodeProcess = Process.Start(_psi); + } + private NodejsProjectSettings LoadProjectSettings(string projectFile) { var buildEngine = new MSBuild.ProjectCollection(); var proj = buildEngine.LoadProject(projectFile); @@ -390,8 +397,8 @@ private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, T result.Duration = result.EndTime - result.StartTime; result.Outcome = outcome; result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, stderr)); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, outcome); From f48b8c95927f24c99cb3b94ba369b627a0c6561f Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 12 Oct 2016 13:00:11 -0700 Subject: [PATCH 22/25] mocha uses runner events to determine test pass/fail --- .../Nodejs/TestFrameworks/mocha/mocha.js | 50 +++++++++++++++++-- .../Nodejs/TestFrameworks/run_tests.js | 31 +----------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index d858fc0ef..6488a493a 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -2,12 +2,32 @@ var EOL = require('os').EOL; var fs = require('fs'); var path = require('path'); - +var result = { + "title": "", + "passed": false, + "stdOut": "", + "stdErr": "" +}; // Choose 'tap' rather than 'min' or 'xunit'. The reason is that // 'min' produces undisplayable text to stdout and stderr under piped/redirect, // and 'xunit' does not print the stack trace from the test. var defaultMochaOptions = { ui: 'tdd', reporter: 'tap', timeout: 2000 }; +function hook_stdout(callback) { + var old_write = process.stdout.write; + + process.stdout.write = (function (write) { + return function (string, encoding, fd) { + callback(string, encoding, fd) + //write.apply(process.stdout, arguments) + } + })(process.stdout.write) + + return function () { + process.stdout.write = old_write + } +} + var find_tests = function (testFileList, discoverResultFile, projectFolder) { var Mocha = detectMocha(projectFolder); if (!Mocha) { @@ -57,11 +77,16 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { module.exports.find_tests = find_tests; var run_tests = function (testName, testFile, workingFolder, projectFolder) { + //var testResults = []; var Mocha = detectMocha(projectFolder); if (!Mocha) { return; } + var unhook = hook_stdout(function (string, encoding, fd) { + result.stdOut += string; + }); + var mocha = initializeMocha(Mocha, projectFolder); if (testName) { @@ -70,10 +95,29 @@ var run_tests = function (testName, testFile, workingFolder, projectFolder) { else mocha.grep(testName); // prior Mocha 3.0.0 } + mocha.addFile(testFile); - mocha.run(function (code) { - process.exit(code); + // run tests + var runner = mocha.run(function (code) { }); + + runner.on('start', function () { + }); + runner.on('test', function (test) { + result.title = test.title; + }); + runner.on('end', function () { + unhook(); + console.log(JSON.stringify(result)); + process.exit(0); + }); + runner.on('pass', function (test) { + result.passed = true; + //testResults.push(result); + }); + runner.on('fail', function (test, err) { + result.passed = false; + //testResults.push(result); }); }; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 2a693dd08..06932719e 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -1,25 +1,5 @@ var framework; var readline = require('readline'); -var result = { - "title": "", - "passed": false, - "stdOut": "", - "stdErr": "" -}; - -process.stdout.write = (function (write) { - return function (string, encoding, fileDescriptor) { - result.stdOut += string; - write.apply(process.stdout, arguments); - }; -})(process.stdout.write); - -process.stderr.write = (function (write) { - return function (string, encoding, fileDescriptor) { - result.stdErr += string; - write.apply(process.stderr, arguments); - }; -})(process.stderr.write); var rl = readline.createInterface({ input: process.stdin, @@ -28,8 +8,7 @@ var rl = readline.createInterface({ rl.on('line', (line) => { var testInfo = JSON.parse(line); - result.title = testInfo.testName.replace(' ', '::'); - // get rid of leftover quotations from C# + // get rid of leftover quotations from C# (necessary?) for(var s in testInfo) { testInfo[s] = testInfo[s].replace(/["]+/g, ''); } @@ -46,11 +25,3 @@ rl.on('line', (line) => { // close readline interface rl.close(); }); - -process.on('exit', (code) => { - result.passed = code === 0 ? true : false; - console.log(JSON.stringify(result)); - // clear stdOut and stdErr - result.stdErr = ""; - result.stdOut = ""; -}); \ No newline at end of file From 89837b616632f2c0172e5aa48865cc6fcefed190 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 12 Oct 2016 13:00:52 -0700 Subject: [PATCH 23/25] get TestExecutor in working state, still launching one process per one test --- Nodejs/Product/TestAdapter/TestExecutor.cs | 60 ++++++++++++---------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 236d989db..4a7ac343d 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -95,7 +95,7 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { // .njsproj file path -> project settings - var projectToTests = new Dictionary>(); + //var projectToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; @@ -103,36 +103,36 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame // NOTE: It seems to me that if we were to run all tests over multiple projects in a solution, // we would have to separate the tests by their project in order to launch the node process // correctly (to make sure we are using the correct working folder). - foreach (var test in receiver.Tests) { - if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - } - if (!projectToTests.ContainsKey(settings.WorkingDir)) { - projectToTests[settings.WorkingDir] = new List(); - } - projectToTests[settings.WorkingDir].Add(test); - } + + //foreach (var test in receiver.Tests) { + // if (!sourceToSettings.TryGetValue(test.Source, out settings)) { + // sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); + // } + // if (!projectToTests.ContainsKey(settings.WorkingDir)) { + // projectToTests[settings.WorkingDir] = new List(); + // } + // projectToTests[settings.WorkingDir].Add(test); + //} // where key is the workingDir and value is a list of tests - foreach (KeyValuePair> entry in projectToTests) { - List args = new List(); - TestCase firstTest = entry.Value.ElementAt(0); - int port = 0; - if (runContext.IsBeingDebugged && app != null) { - app.GetDTE().Debugger.DetachAll(); - args.AddRange(GetDebugArgs(settings, out port)); - } - args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + //foreach (KeyValuePair> entry in projectToTests) { + // List args = new List(); + // TestCase firstTest = entry.Value.ElementAt(0); + // int port = 0; + // if (runContext.IsBeingDebugged && app != null) { + // app.GetDTE().Debugger.DetachAll(); + // args.AddRange(GetDebugArgs(settings, out port)); + // } + + // args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); - // launch node process - LaunchNodeProcess(entry.Key, settings.NodeExePath, args); + // eventually launch node process here // Run all test cases in a given project RunTestCases(entry.Value, runContext, frameworkHandle); // dispose node process - _nodeProcess.Dispose(); } } } @@ -161,7 +161,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I } try { - RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings, _nodeProcess.StandardInput, _nodeProcess.StandardOutput); + RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); @@ -199,7 +199,7 @@ private static IEnumerable GetDebugArgs(NodejsProjectSettings settings, }; } - private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary sourceToSettings, StreamWriter standardInput, StreamReader standardOutput) { + private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary sourceToSettings) { var testResult = new TestResult(test); frameworkHandle.RecordStart(test); testResult.StartTime = DateTimeOffset.Now; @@ -221,6 +221,8 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, return; } + + NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); List args = new List(); int port = 0; @@ -239,6 +241,8 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } lock (_syncObject) { + // launch node process + LaunchNodeProcess(settings.WorkingDir, settings.NodeExePath, args); #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); //frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); @@ -246,7 +250,8 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, // send test to run_tests.js TestCaseObject testObject = new TestCaseObject(args[1], args[2], args[3], args[4], args[5]); if (!_nodeProcess.HasExited) { - standardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + _nodeProcess.StandardInput.WriteLine(Newtonsoft.Json.JsonConvert.SerializeObject(testObject)); + _nodeProcess.StandardInput.Close(); _nodeProcess.WaitForExit(5000); } //standardInput.Close(); @@ -275,7 +280,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, #endif } } - var result = GetTestResultFromProcess(standardOutput); + var result = GetTestResultFromProcess(_nodeProcess.StandardOutput); bool runCancelled = _cancelRequested.WaitOne(0); @@ -287,6 +292,9 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, } else { frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to obtain result for " + test.DisplayName + " from TestRunner"); } + + // dispose node process + _nodeProcess.Dispose(); } private ResultObject ParseTestResult(string line) { From dc5df19a558c427dadee67ca12d2df6fd936c772 Mon Sep 17 00:00:00 2001 From: ozyx Date: Wed, 12 Oct 2016 13:14:14 -0700 Subject: [PATCH 24/25] get TextExecutor code into working state --- Nodejs/Product/TestAdapter/TestExecutor.cs | 150 ++++++++++----------- 1 file changed, 74 insertions(+), 76 deletions(-) diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 4a7ac343d..94bb95672 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -95,37 +95,37 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { // .njsproj file path -> project settings - //var projectToTests = new Dictionary>(); + var projectToTests = new Dictionary>(); var sourceToSettings = new Dictionary(); NodejsProjectSettings settings = null; // put tests into dictionary where key is their project working directory // NOTE: It seems to me that if we were to run all tests over multiple projects in a solution, // we would have to separate the tests by their project in order to launch the node process - // correctly (to make sure we are using the correct working folder). - - //foreach (var test in receiver.Tests) { - // if (!sourceToSettings.TryGetValue(test.Source, out settings)) { - // sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); - // } - // if (!projectToTests.ContainsKey(settings.WorkingDir)) { - // projectToTests[settings.WorkingDir] = new List(); - // } - // projectToTests[settings.WorkingDir].Add(test); - //} + // correctly (to make sure we are using the correct working folder) and also to run + // groups of tests by test suite. - // where key is the workingDir and value is a list of tests + foreach (var test in receiver.Tests) { + if (!sourceToSettings.TryGetValue(test.Source, out settings)) { + sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); + } + if (!projectToTests.ContainsKey(settings.WorkingDir)) { + projectToTests[settings.WorkingDir] = new List(); + } + projectToTests[settings.WorkingDir].Add(test); + } - //foreach (KeyValuePair> entry in projectToTests) { - // List args = new List(); - // TestCase firstTest = entry.Value.ElementAt(0); - // int port = 0; - // if (runContext.IsBeingDebugged && app != null) { - // app.GetDTE().Debugger.DetachAll(); - // args.AddRange(GetDebugArgs(settings, out port)); - // } + // where key is the workingDir and value is a list of tests + foreach (KeyValuePair> entry in projectToTests) { + List args = new List(); + TestCase firstTest = entry.Value.ElementAt(0); + int port = 0; + if (runContext.IsBeingDebugged && app != null) { + app.GetDTE().Debugger.DetachAll(); + args.AddRange(GetDebugArgs(settings, out port)); + } - // args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); + //args.AddRange(GetInterpreterArgs(firstTest, entry.Key, settings.ProjectRootDir)); // eventually launch node process here @@ -221,7 +221,7 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, return; } - + NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); List args = new List(); @@ -405,72 +405,70 @@ private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, T result.Duration = result.EndTime - result.StartTime; result.Outcome = outcome; result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, stdout)); - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, stderr)); - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, stderr)); + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, stderr)); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(test, outcome); } + } +} - class DataReceiver { - public readonly StringBuilder Data = new StringBuilder(); +class DataReceiver { + public readonly StringBuilder Data = new StringBuilder(); - public void DataReceived(object sender, DataReceivedEventArgs e) { - if (e.Data != null) { - Data.AppendLine(e.Data); - } - } + public void DataReceived(object sender, DataReceivedEventArgs e) { + if (e.Data != null) { + Data.AppendLine(e.Data); } + } +} - class TestReceiver : ITestCaseDiscoverySink { - public List Tests { get; private set; } - - public TestReceiver() { - Tests = new List(); - } - - public void SendTestCase(TestCase discoveredTest) { - Tests.Add(discoveredTest); - } - } +class TestReceiver : ITestCaseDiscoverySink { + public List Tests { get; private set; } - class NodejsProjectSettings { - public NodejsProjectSettings() { - NodeExePath = String.Empty; - SearchPath = String.Empty; - WorkingDir = String.Empty; - } + public TestReceiver() { + Tests = new List(); + } - public string NodeExePath { get; set; } - public string SearchPath { get; set; } - public string WorkingDir { get; set; } - public string ProjectRootDir { get; set; } - } + public void SendTestCase(TestCase discoveredTest) { + Tests.Add(discoveredTest); + } +} - class TestCaseObject { - public TestCaseObject() { - framework = String.Empty; - testName = String.Empty; - testFile = String.Empty; - workingFolder = String.Empty; - projectFolder = String.Empty; - } +class NodejsProjectSettings { + public NodejsProjectSettings() { + NodeExePath = String.Empty; + SearchPath = String.Empty; + WorkingDir = String.Empty; + } - public TestCaseObject(string framework, string testName, string testFile, string workingFolder, string projectFolder) { - this.framework = framework; - this.testName = testName; - this.testFile = testFile; - this.workingFolder = workingFolder; - this.projectFolder = projectFolder; - } - public string framework { get; set; } - public string testName { get; set; } - public string testFile { get; set; } - public string workingFolder { get; set; } - public string projectFolder { get; set; } + public string NodeExePath { get; set; } + public string SearchPath { get; set; } + public string WorkingDir { get; set; } + public string ProjectRootDir { get; set; } +} - } +class TestCaseObject { + public TestCaseObject() { + framework = String.Empty; + testName = String.Empty; + testFile = String.Empty; + workingFolder = String.Empty; + projectFolder = String.Empty; + } + public TestCaseObject(string framework, string testName, string testFile, string workingFolder, string projectFolder) { + this.framework = framework; + this.testName = testName; + this.testFile = testFile; + this.workingFolder = workingFolder; + this.projectFolder = projectFolder; } + public string framework { get; set; } + public string testName { get; set; } + public string testFile { get; set; } + public string workingFolder { get; set; } + public string projectFolder { get; set; } -} +} \ No newline at end of file From 36f018a31fc4ecdf80eea8347344a7e0690d7429 Mon Sep 17 00:00:00 2001 From: ozyx Date: Fri, 14 Oct 2016 13:50:14 -0700 Subject: [PATCH 25/25] fixes from PR feedback --- Common/Product/SharedProject/ProcessOutput.cs | 4 +- .../Nodejs/TestFrameworks/mocha/mocha.js | 27 ++----- .../Nodejs/TestFrameworks/run_tests.js | 14 +++- Nodejs/Product/TestAdapter/TestExecutor.cs | 72 +++---------------- 4 files changed, 30 insertions(+), 87 deletions(-) diff --git a/Common/Product/SharedProject/ProcessOutput.cs b/Common/Product/SharedProject/ProcessOutput.cs index 884b5c5e9..ee75aef72 100644 --- a/Common/Product/SharedProject/ProcessOutput.cs +++ b/Common/Product/SharedProject/ProcessOutput.cs @@ -304,7 +304,7 @@ public static ProcessOutput RunElevated( return result; } - private static string GetArguments(IEnumerable arguments, bool quoteArgs) { + public static string GetArguments(IEnumerable arguments, bool quoteArgs) { if (quoteArgs) { return string.Join(" ", arguments.Where(a => a != null).Select(QuoteSingleArgument)); } else { @@ -335,7 +335,7 @@ internal static IEnumerable SplitLines(string source) { } } - internal static string QuoteSingleArgument(string arg) { + public static string QuoteSingleArgument(string arg) { if (string.IsNullOrEmpty(arg)) { return "\"\""; } diff --git a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js index 6488a493a..1277b8b65 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/mocha/mocha.js @@ -13,19 +13,12 @@ var result = { // and 'xunit' does not print the stack trace from the test. var defaultMochaOptions = { ui: 'tdd', reporter: 'tap', timeout: 2000 }; -function hook_stdout(callback) { - var old_write = process.stdout.write; - - process.stdout.write = (function (write) { - return function (string, encoding, fd) { - callback(string, encoding, fd) - //write.apply(process.stdout, arguments) - } - })(process.stdout.write) +process.stdout.write = function (string, encoding, fd) { + result.stdOut += string; +} - return function () { - process.stdout.write = old_write - } +process.stderr.write = function (string, encoding, fd) { + result.stdErr += string; } var find_tests = function (testFileList, discoverResultFile, projectFolder) { @@ -76,17 +69,13 @@ var find_tests = function (testFileList, discoverResultFile, projectFolder) { }; module.exports.find_tests = find_tests; -var run_tests = function (testName, testFile, workingFolder, projectFolder) { +var run_tests = function (testName, testFile, workingFolder, projectFolder, callback) { //var testResults = []; var Mocha = detectMocha(projectFolder); if (!Mocha) { return; } - var unhook = hook_stdout(function (string, encoding, fd) { - result.stdOut += string; - }); - var mocha = initializeMocha(Mocha, projectFolder); if (testName) { @@ -107,9 +96,7 @@ var run_tests = function (testName, testFile, workingFolder, projectFolder) { result.title = test.title; }); runner.on('end', function () { - unhook(); - console.log(JSON.stringify(result)); - process.exit(0); + callback(result); }); runner.on('pass', function (test) { result.passed = true; diff --git a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js index 06932719e..4c95d3fae 100644 --- a/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js +++ b/Nodejs/Product/Nodejs/TestFrameworks/run_tests.js @@ -1,6 +1,7 @@ var framework; var readline = require('readline'); - +var old_stdout = process.stdout.write; +var old_stderr = process.stderr.write; var rl = readline.createInterface({ input: process.stdin, output: process.stdout @@ -19,9 +20,16 @@ rl.on('line', (line) => { console.log("NTVS_ERROR:Failed to load TestFramework (" + process.argv[2] + "), " + exception); process.exit(1); } + + function sendResult(result) { + process.stdout.write = old_stdout; + process.stderr.write = old_stderr; + console.log(JSON.stringify(result)); + process.exit(0); + } // run the test - framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder); - + framework.run_tests(testInfo.testName, testInfo.testFile, testInfo.workingFolder, testInfo.projectFolder, sendResult); + // close readline interface rl.close(); }); diff --git a/Nodejs/Product/TestAdapter/TestExecutor.cs b/Nodejs/Product/TestAdapter/TestExecutor.cs index 94bb95672..64c6c5cb7 100644 --- a/Nodejs/Product/TestAdapter/TestExecutor.cs +++ b/Nodejs/Product/TestAdapter/TestExecutor.cs @@ -162,8 +162,7 @@ private void RunTestCases(IEnumerable tests, IRunContext runContext, I try { RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); - } - catch (Exception ex) { + } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); } } @@ -221,8 +220,6 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, return; } - - NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); List args = new List(); int port = 0; @@ -266,17 +263,16 @@ private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, // } //} #if DEBUG - } - catch (COMException ex) { + } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); } #else - } catch (COMException) { - frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); - KillNodeProcess(); - } + } catch (COMException) { + frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); + KillNodeProcess(); + } #endif } } @@ -301,9 +297,7 @@ private ResultObject ParseTestResult(string line) { ResultObject jsonResult = null; try { jsonResult = JsonConvert.DeserializeObject(line); - } - catch (Exception) { - } + } catch (Exception) { } return jsonResult; } @@ -323,9 +317,9 @@ private ResultObject GetTestResultFromProcess(StreamReader sr) { private void LaunchNodeProcess(string workingDir, string nodeExePath, List args) { _psi = new ProcessStartInfo("cmd.exe") { Arguments = string.Format(@"/S /C pushd {0} & {1} {2}", - QuoteSingleArgument(workingDir), - QuoteSingleArgument(nodeExePath), - GetArguments(args, true)), + ProcessOutput.QuoteSingleArgument(workingDir), + ProcessOutput.QuoteSingleArgument(nodeExePath), + ProcessOutput.GetArguments(args, true)), CreateNoWindow = true, UseShellExecute = false }; @@ -354,52 +348,6 @@ private NodejsProjectSettings LoadProjectSettings(string projectFile) { return projSettings; } - private static string GetArguments(IEnumerable arguments, bool quoteArgs) { - if (quoteArgs) { - return string.Join(" ", arguments.Where(a => a != null).Select(QuoteSingleArgument)); - } - else { - return string.Join(" ", arguments.Where(a => a != null)); - } - } - - internal static string QuoteSingleArgument(string arg) { - if (string.IsNullOrEmpty(arg)) { - return "\"\""; - } - if (arg.IndexOfAny(_needToBeQuoted) < 0) { - return arg; - } - - if (arg.StartsWith("\"") && arg.EndsWith("\"")) { - bool inQuote = false; - int consecutiveBackslashes = 0; - foreach (var c in arg) { - if (c == '"') { - if (consecutiveBackslashes % 2 == 0) { - inQuote = !inQuote; - } - } - - if (c == '\\') { - consecutiveBackslashes += 1; - } - else { - consecutiveBackslashes = 0; - } - } - if (!inQuote) { - return arg; - } - } - - var newArg = arg.Replace("\"", "\\\""); - if (newArg.EndsWith("\\")) { - newArg += "\\"; - } - return "\"" + newArg + "\""; - } - private static void RecordEnd(IFrameworkHandle frameworkHandle, TestCase test, TestResult result, string stdout, string stderr, TestOutcome outcome) { result.EndTime = DateTimeOffset.Now; result.Duration = result.EndTime - result.StartTime;