diff --git a/external/Touch.Unit b/external/Touch.Unit index ea4480690288..71c2c44f926e 160000 --- a/external/Touch.Unit +++ b/external/Touch.Unit @@ -1 +1 @@ -Subproject commit ea44806902885d33925463f8f47a67278385a559 +Subproject commit 71c2c44f926e8f6bcf37c8f5823ed7e841aabc59 diff --git a/jenkins/nunit-summary.xslt b/jenkins/nunit-summary.xslt new file mode 100644 index 000000000000..88304a43e537 --- /dev/null +++ b/jenkins/nunit-summary.xslt @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + [PASS] + + + [FAIL] + + + [IGNORED] + + + [INCONCLUSIVE] + + + + . + + + + + + + + + + + + + + diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index fe7e67f9d738..c3f39a463b61 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash -e +export BUILD_REVISION=jenkins cd $WORKSPACE # Unlock security default-keychain -s builder.keychain @@ -11,6 +12,3 @@ security set-keychain-settings -lut 7200 # Run tests make -C tests jenkins - -# Lock -security lock-keychain diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 3ed2b9c29d3e..c8ba102b992d 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -1,13 +1,15 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; +using System.Xml.Xsl; namespace xharness { @@ -309,6 +311,125 @@ public bool EnsureCleanSimulatorState { } } + void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument doc) + { + // load the resource that contains the xslt and apply it to the doc and write the logs + if (File.Exists (finalPath)) { + // if the file does exist, remove it + File.Delete (finalPath); + } + + using (var strm = Assembly.GetExecutingAssembly ().GetManifestResourceStream ("xharness.nunit-summary.xslt")) + using (var xsltReader = XmlReader.Create (strm)) + using (var xmlReader = new XmlNodeReader (doc)) + using (var writer = new StreamWriter (finalPath)) { + writer.Write (logHeader); + var xslt = new XslCompiledTransform (); + xslt.Load (xsltReader); + xslt.Transform (xmlReader, null, writer); + writer.Flush (); + } + } + + public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed) + { + string log; + using (var reader = listener_log.GetReader ()) + log = reader.ReadToEnd (); + // parsing the result is different if we are in jenkins or nor. + if (Harness.InJenkins) { + // we have to parse the xml result + crashed = false; + if (log.Contains ("test-results")) { + // remove any possible extra info + var index = log.IndexOf ("{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}
"); + main_log.WriteLine ("Test run failed"); + return false; + } else { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} succeeded: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: 0 Ignored: {ignored}
"); + main_log.WriteLine ("Test run succeeded"); + return true; + } + } else if (timed_out) { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); + return false; + } else { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} crashed
"); + main_log.WriteLine ("Test run crashed"); + crashed = true; + return false; + } + } else { + // parsing the human readable results + if (log.Contains ("Tests run")) { + var tests_run = string.Empty; + var log_lines = log.Split ('\n'); + var failed = false; + foreach (var line in log_lines) { + if (line.Contains ("Tests run:")) { + Console.WriteLine (line); + tests_run = line.Replace ("Tests run: ", ""); + break; + } else if (line.Contains ("FAIL")) { + Console.WriteLine (line); + failed = true; + } + } + + if (failed) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run); + main_log.WriteLine ("Test run failed"); + return false; + } else { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run); + main_log.WriteLine ("Test run succeeded"); + return true; + } + } else if (timed_out) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode); + return false; + } else { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode); + main_log.WriteLine ("Test run crashed"); + crashed = true; + return false; + } + } + } + [DllImport ("/usr/lib/libc.dylib")] extern static IntPtr ttyname (int filedes); @@ -346,6 +467,10 @@ public async Task RunAsync () args.Append (" -setenv=NUNIT_AUTOEXIT=true"); args.Append (" -argument=-app-arg:-enablenetwork"); args.Append (" -setenv=NUNIT_ENABLE_NETWORK=true"); + // detect if we are using a jenkins bot. + if (Harness.InJenkins) + args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true"); + if (isSimulator) { args.Append (" -argument=-app-arg:-hostname:127.0.0.1"); args.Append (" -setenv=NUNIT_HOSTNAME=127.0.0.1"); @@ -555,50 +680,15 @@ public async Task RunAsync () var crashed = false; if (File.Exists (listener_log.FullPath)) { Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); - string log; - using (var reader = listener_log.GetReader ()) - log = reader.ReadToEnd (); - if (log.Contains ("Tests run")) { - var tests_run = string.Empty; - var log_lines = log.Split ('\n'); - var failed = false; - foreach (var line in log_lines) { - if (line.Contains ("Tests run:")) { - Console.WriteLine (line); - tests_run = line.Replace ("Tests run: ", ""); - break; - } else if (line.Contains ("FAIL")) { - Console.WriteLine (line); - failed = true; - } - } - - if (failed) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run); - main_log.WriteLine ("Test run failed"); - success = false; - } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run); - main_log.WriteLine ("Test run succeeded"); - success = true; - } - } else if (timed_out) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode); - success = false; - } else if (launch_failure) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); - main_log.WriteLine ("Test run failed to launch"); - success = false; - } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode); - main_log.WriteLine ("Test run crashed"); - crashed = true; - success = false; - } + success = TestsSucceeded (listener_log, timed_out, crashed); } else if (timed_out) { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode); main_log.WriteLine ("Test run never launched"); success = false; + } else if (launch_failure) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); + main_log.WriteLine ("Test run failed to launch"); + success = false; } else { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed at startup (no log)
", mode); main_log.WriteLine ("Test run crashed before it started (no log file produced)"); @@ -620,7 +710,6 @@ public async Task RunAsync () } else { Result = TestExecutingResult.Failed; } - return success.Value ? 0 : 1; } @@ -657,4 +746,3 @@ async Task KillPidAsync (Log log, int pid, TimeSpan kill_separation, TimeSpan ti } } } - diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index c28c8de08462..34425dad6a9f 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -498,7 +498,15 @@ public void LogWrench (string message) public bool InWrench { get { - return !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")); + var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); + return !string.IsNullOrEmpty (buildRev) && buildRev != "jenkins"; + } + } + + public bool InJenkins { + get { + var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); + return !string.IsNullOrEmpty (buildRev) && buildRev == "jenkins"; } } diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index dc435bdbb780..ffc3b90ac95e 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -70,5 +70,10 @@ + + + nunit-summary.xslt + +