diff --git a/external/Touch.Unit b/external/Touch.Unit
index ea4480690288..71c2c44f926e 160000
--- a/external/Touch.Unit
+++ b/external/Touch.Unit
@@ -1 +1 @@
-Subproject commit ea44806902885d33925463f8f47a67278385a559
+Subproject commit 71c2c44f926e8f6bcf37c8f5823ed7e841aabc59
diff --git a/jenkins/nunit-summary.xslt b/jenkins/nunit-summary.xslt
new file mode 100644
index 000000000000..88304a43e537
--- /dev/null
+++ b/jenkins/nunit-summary.xslt
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ [PASS]
+
+
+ [FAIL]
+
+
+ [IGNORED]
+
+
+ [INCONCLUSIVE]
+
+
+
+ .
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh
index fe7e67f9d738..c3f39a463b61 100755
--- a/jenkins/run-tests.sh
+++ b/jenkins/run-tests.sh
@@ -1,5 +1,6 @@
#!/bin/bash -e
+export BUILD_REVISION=jenkins
cd $WORKSPACE
# Unlock
security default-keychain -s builder.keychain
@@ -11,6 +12,3 @@ security set-keychain-settings -lut 7200
# Run tests
make -C tests jenkins
-
-# Lock
-security lock-keychain
diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs
index 3ed2b9c29d3e..c8ba102b992d 100644
--- a/tests/xharness/AppRunner.cs
+++ b/tests/xharness/AppRunner.cs
@@ -1,13 +1,15 @@
-using System;
+using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
+using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
+using System.Xml.Xsl;
namespace xharness
{
@@ -309,6 +311,125 @@ public bool EnsureCleanSimulatorState {
}
}
+ void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument doc)
+ {
+ // load the resource that contains the xslt and apply it to the doc and write the logs
+ if (File.Exists (finalPath)) {
+ // if the file does exist, remove it
+ File.Delete (finalPath);
+ }
+
+ using (var strm = Assembly.GetExecutingAssembly ().GetManifestResourceStream ("xharness.nunit-summary.xslt"))
+ using (var xsltReader = XmlReader.Create (strm))
+ using (var xmlReader = new XmlNodeReader (doc))
+ using (var writer = new StreamWriter (finalPath)) {
+ writer.Write (logHeader);
+ var xslt = new XslCompiledTransform ();
+ xslt.Load (xsltReader);
+ xslt.Transform (xmlReader, null, writer);
+ writer.Flush ();
+ }
+ }
+
+ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed)
+ {
+ string log;
+ using (var reader = listener_log.GetReader ())
+ log = reader.ReadToEnd ();
+ // parsing the result is different if we are in jenkins or nor.
+ if (Harness.InJenkins) {
+ // we have to parse the xml result
+ crashed = false;
+ if (log.Contains ("test-results")) {
+ // remove any possible extra info
+ var index = log.IndexOf ("{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}
");
+ main_log.WriteLine ("Test run failed");
+ return false;
+ } else {
+ Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} succeeded: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: 0 Ignored: {ignored}
");
+ main_log.WriteLine ("Test run succeeded");
+ return true;
+ }
+ } else if (timed_out) {
+ Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
");
+ return false;
+ } else {
+ Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} crashed
");
+ main_log.WriteLine ("Test run crashed");
+ crashed = true;
+ return false;
+ }
+ } else {
+ // parsing the human readable results
+ if (log.Contains ("Tests run")) {
+ var tests_run = string.Empty;
+ var log_lines = log.Split ('\n');
+ var failed = false;
+ foreach (var line in log_lines) {
+ if (line.Contains ("Tests run:")) {
+ Console.WriteLine (line);
+ tests_run = line.Replace ("Tests run: ", "");
+ break;
+ } else if (line.Contains ("FAIL")) {
+ Console.WriteLine (line);
+ failed = true;
+ }
+ }
+
+ if (failed) {
+ Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run);
+ main_log.WriteLine ("Test run failed");
+ return false;
+ } else {
+ Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run);
+ main_log.WriteLine ("Test run succeeded");
+ return true;
+ }
+ } else if (timed_out) {
+ Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode);
+ return false;
+ } else {
+ Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode);
+ main_log.WriteLine ("Test run crashed");
+ crashed = true;
+ return false;
+ }
+ }
+ }
+
[DllImport ("/usr/lib/libc.dylib")]
extern static IntPtr ttyname (int filedes);
@@ -346,6 +467,10 @@ public async Task RunAsync ()
args.Append (" -setenv=NUNIT_AUTOEXIT=true");
args.Append (" -argument=-app-arg:-enablenetwork");
args.Append (" -setenv=NUNIT_ENABLE_NETWORK=true");
+ // detect if we are using a jenkins bot.
+ if (Harness.InJenkins)
+ args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true");
+
if (isSimulator) {
args.Append (" -argument=-app-arg:-hostname:127.0.0.1");
args.Append (" -setenv=NUNIT_HOSTNAME=127.0.0.1");
@@ -555,50 +680,15 @@ public async Task RunAsync ()
var crashed = false;
if (File.Exists (listener_log.FullPath)) {
Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", listener_log.FullPath);
- string log;
- using (var reader = listener_log.GetReader ())
- log = reader.ReadToEnd ();
- if (log.Contains ("Tests run")) {
- var tests_run = string.Empty;
- var log_lines = log.Split ('\n');
- var failed = false;
- foreach (var line in log_lines) {
- if (line.Contains ("Tests run:")) {
- Console.WriteLine (line);
- tests_run = line.Replace ("Tests run: ", "");
- break;
- } else if (line.Contains ("FAIL")) {
- Console.WriteLine (line);
- failed = true;
- }
- }
-
- if (failed) {
- Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run);
- main_log.WriteLine ("Test run failed");
- success = false;
- } else {
- Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run);
- main_log.WriteLine ("Test run succeeded");
- success = true;
- }
- } else if (timed_out) {
- Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode);
- success = false;
- } else if (launch_failure) {
- Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode);
- main_log.WriteLine ("Test run failed to launch");
- success = false;
- } else {
- Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode);
- main_log.WriteLine ("Test run crashed");
- crashed = true;
- success = false;
- }
+ success = TestsSucceeded (listener_log, timed_out, crashed);
} else if (timed_out) {
Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode);
main_log.WriteLine ("Test run never launched");
success = false;
+ } else if (launch_failure) {
+ Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode);
+ main_log.WriteLine ("Test run failed to launch");
+ success = false;
} else {
Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed at startup (no log)
", mode);
main_log.WriteLine ("Test run crashed before it started (no log file produced)");
@@ -620,7 +710,6 @@ public async Task RunAsync ()
} else {
Result = TestExecutingResult.Failed;
}
-
return success.Value ? 0 : 1;
}
@@ -657,4 +746,3 @@ async Task KillPidAsync (Log log, int pid, TimeSpan kill_separation, TimeSpan ti
}
}
}
-
diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs
index c28c8de08462..34425dad6a9f 100644
--- a/tests/xharness/Harness.cs
+++ b/tests/xharness/Harness.cs
@@ -498,7 +498,15 @@ public void LogWrench (string message)
public bool InWrench {
get {
- return !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION"));
+ var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION");
+ return !string.IsNullOrEmpty (buildRev) && buildRev != "jenkins";
+ }
+ }
+
+ public bool InJenkins {
+ get {
+ var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION");
+ return !string.IsNullOrEmpty (buildRev) && buildRev == "jenkins";
}
}
diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj
index dc435bdbb780..ffc3b90ac95e 100644
--- a/tests/xharness/xharness.csproj
+++ b/tests/xharness/xharness.csproj
@@ -70,5 +70,10 @@
+
+
+ nunit-summary.xslt
+
+