From fae27b4f455ffa982e4211eca01241ea442aae7b Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 21 Sep 2016 18:44:01 +0200 Subject: [PATCH 01/17] [Jenkins] Make test to write output as an xml file so that it can be parsed by the jenkins bot. --- external/Touch.Unit | 2 +- tests/xharness/AppRunner.cs | 138 +++++++++++++++++++++++++----------- 2 files changed, 97 insertions(+), 43 deletions(-) diff --git a/external/Touch.Unit b/external/Touch.Unit index ea4480690288..8163298ba756 160000 --- a/external/Touch.Unit +++ b/external/Touch.Unit @@ -1 +1 @@ -Subproject commit ea44806902885d33925463f8f47a67278385a559 +Subproject commit 8163298ba756fc0f5e564d7ce998ef441231ba54 diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 3ed2b9c29d3e..da089e51f9f3 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -309,6 +309,95 @@ public bool EnsureCleanSimulatorState { } } + public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed) { + string log; + using (var reader = listener_log.GetReader ()) + log = reader.ReadToEnd (); + // parsing the result is different if we are in jenkins or nor. + if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")) + && Environment.GetEnvironmentVariable ("BUILD_REVISION") == "jenkins") { + // we have to parse the xml result + crashed = false; + if (log.Contains ("test-results")) { + // remove any possible extra info + var index = log.IndexOf ("{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}
"); + main_log.WriteLine ("Test run failed"); + return false; + } else { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} succeeded: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: 0 Ignored: {ignored}
"); + main_log.WriteLine ("Test run succeeded"); + return true; + } + } else if (timed_out) { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); + return false; + } else { + Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} crashed
"); + main_log.WriteLine ("Test run crashed"); + crashed = true; + return false; + } + } else { + // parsing the human readable results + if (log.Contains ("Tests run")) { + var tests_run = string.Empty; + var log_lines = log.Split ('\n'); + var failed = false; + foreach (var line in log_lines) { + if (line.Contains ("Tests run:")) { + Console.WriteLine (line); + tests_run = line.Replace ("Tests run: ", ""); + break; + } else if (line.Contains ("FAIL")) { + Console.WriteLine (line); + failed = true; + } + } + + if (failed) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run); + main_log.WriteLine ("Test run failed"); + return false; + } else { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run); + main_log.WriteLine ("Test run succeeded"); + return true; + } + } else if (timed_out) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode); + return false; + } else { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode); + main_log.WriteLine ("Test run crashed"); + crashed = true; + return false; + } + } + } + [DllImport ("/usr/lib/libc.dylib")] extern static IntPtr ttyname (int filedes); @@ -346,6 +435,11 @@ public async Task RunAsync () args.Append (" -setenv=NUNIT_AUTOEXIT=true"); args.Append (" -argument=-app-arg:-enablenetwork"); args.Append (" -setenv=NUNIT_ENABLE_NETWORK=true"); + // detect if we are using a jenkins bot. + if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")) + && Environment.GetEnvironmentVariable ("BUILD_REVISION") == "jenkins") + args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true"); + if (isSimulator) { args.Append (" -argument=-app-arg:-hostname:127.0.0.1"); args.Append (" -setenv=NUNIT_HOSTNAME=127.0.0.1"); @@ -555,46 +649,7 @@ public async Task RunAsync () var crashed = false; if (File.Exists (listener_log.FullPath)) { Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); - string log; - using (var reader = listener_log.GetReader ()) - log = reader.ReadToEnd (); - if (log.Contains ("Tests run")) { - var tests_run = string.Empty; - var log_lines = log.Split ('\n'); - var failed = false; - foreach (var line in log_lines) { - if (line.Contains ("Tests run:")) { - Console.WriteLine (line); - tests_run = line.Replace ("Tests run: ", ""); - break; - } else if (line.Contains ("FAIL")) { - Console.WriteLine (line); - failed = true; - } - } - - if (failed) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run); - main_log.WriteLine ("Test run failed"); - success = false; - } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run); - main_log.WriteLine ("Test run succeeded"); - success = true; - } - } else if (timed_out) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode); - success = false; - } else if (launch_failure) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); - main_log.WriteLine ("Test run failed to launch"); - success = false; - } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode); - main_log.WriteLine ("Test run crashed"); - crashed = true; - success = false; - } + success = TestsSucceeded (listener_log, timed_out, crashed); } else if (timed_out) { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode); main_log.WriteLine ("Test run never launched"); @@ -657,4 +712,3 @@ async Task KillPidAsync (Log log, int pid, TimeSpan kill_separation, TimeSpan ti } } } - From 7e0410c2fdc67e486361a0cc76dfa3fdbe412aac Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 09:43:47 +0200 Subject: [PATCH 02/17] Point to the correct Touvh.Unit repo. --- tests/xharness/AppRunner.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index da089e51f9f3..71db07c13d00 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -675,7 +675,6 @@ public async Task RunAsync () } else { Result = TestExecutingResult.Failed; } - return success.Value ? 0 : 1; } From 238eb297122b9d519a49e21b0c718fef6bfda4e4 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 10:47:06 +0200 Subject: [PATCH 03/17] Use the available property to determine if we are being ran in Jenkins. --- tests/xharness/AppRunner.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 71db07c13d00..fbd060045abb 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -314,8 +314,7 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed using (var reader = listener_log.GetReader ()) log = reader.ReadToEnd (); // parsing the result is different if we are in jenkins or nor. - if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")) - && Environment.GetEnvironmentVariable ("BUILD_REVISION") == "jenkins") { + if (Harness.InWrench) { // we have to parse the xml result crashed = false; if (log.Contains ("test-results")) { @@ -436,8 +435,7 @@ public async Task RunAsync () args.Append (" -argument=-app-arg:-enablenetwork"); args.Append (" -setenv=NUNIT_ENABLE_NETWORK=true"); // detect if we are using a jenkins bot. - if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")) - && Environment.GetEnvironmentVariable ("BUILD_REVISION") == "jenkins") + if (Harness.InWrench) args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true"); if (isSimulator) { From eb07a27264d4c451bfa3c1b477822b9171685b01 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 12:05:17 +0200 Subject: [PATCH 04/17] Log where are test results stored. --- tests/xharness/AppRunner.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index fbd060045abb..a508316865c7 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -327,6 +327,7 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed // store a clean version of the logs, later this will be used by the bots to show results in github/web var path = listener_log.FullPath; path = path.Replace (".log", ".xml"); + Harness.LogWrench ($"Added tests results from {mode} to {path}"); testsResults.Save (path); var mainResultNode = testsResults.SelectSingleNode("test-results"); From 7b3c3d2c235eeb5419f8bdd27981c59f06e2e8f5 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 15:12:20 +0200 Subject: [PATCH 05/17] Add @MonkeyWrench: prefix. --- tests/xharness/AppRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index a508316865c7..ec0963249b02 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -327,7 +327,7 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed // store a clean version of the logs, later this will be used by the bots to show results in github/web var path = listener_log.FullPath; path = path.Replace (".log", ".xml"); - Harness.LogWrench ($"Added tests results from {mode} to {path}"); + Harness.LogWrench ($"@MonkeyWrench: Added tests results from {mode} to {path}"); testsResults.Save (path); var mainResultNode = testsResults.SelectSingleNode("test-results"); From c806ab9f1da22e30d74b5c0bf355b1071c8e63ce Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 16:30:06 +0200 Subject: [PATCH 06/17] Ensure that we do set the build env in jenkins/run-tests.sh --- jenkins/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index fe7e67f9d738..0ade3d32069f 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -1,6 +1,7 @@ #!/bin/bash -e cd $WORKSPACE +export BUILD_REVISION=jenkins # Unlock security default-keychain -s builder.keychain security list-keychains -s builder.keychain From 2d75b5393799ae6b4accfe075462d13922f90832 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 23 Sep 2016 18:39:51 +0200 Subject: [PATCH 07/17] Do not mix Wrench with Jenkins. The reports in jenkins can be Xml, in Wrench we prefer the old style. --- tests/xharness/AppRunner.cs | 23 +++++++++++++++-------- tests/xharness/Harness.cs | 10 +++++++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index ec0963249b02..25d5fbe8d274 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -314,7 +314,8 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed using (var reader = listener_log.GetReader ()) log = reader.ReadToEnd (); // parsing the result is different if we are in jenkins or nor. - if (Harness.InWrench) { + if (Harness.InJenkins) { + Harness.LogWrench ($"We are in jenkins."); // we have to parse the xml result crashed = false; if (log.Contains ("test-results")) { @@ -323,18 +324,24 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed log = log.Remove (0, index - 1); var testsResults = new XmlDocument (); testsResults.LoadXml (log); - - // store a clean version of the logs, later this will be used by the bots to show results in github/web - var path = listener_log.FullPath; - path = path.Replace (".log", ".xml"); - Harness.LogWrench ($"@MonkeyWrench: Added tests results from {mode} to {path}"); - testsResults.Save (path); + Harness.LogWrench ($"Loaded xml."); var mainResultNode = testsResults.SelectSingleNode("test-results"); + Harness.LogWrench ($"Got main node."); if (mainResultNode == null) { + Harness.LogWrench ($"Node is null."); crashed = true; return false; } + // update the information of the main node to add information about the mode and the test that is excuted. This will later create + // nicer reports in jenkins + Harness.LogWrench ($"Setting name node to {Target}."); + mainResultNode.Attributes["name"].Value = Target; + // store a clean version of the logs, later this will be used by the bots to show results in github/web + var path = listener_log.FullPath; + path = path.Replace (".log", ".xml"); + testsResults.Save (path); + int ignored = Convert.ToInt16(mainResultNode.Attributes["ignored"].Value); int invalid = Convert.ToInt16(mainResultNode.Attributes["invalid"].Value); int inconclusive = Convert.ToInt16(mainResultNode.Attributes["inconclusive"].Value); @@ -436,7 +443,7 @@ public async Task RunAsync () args.Append (" -argument=-app-arg:-enablenetwork"); args.Append (" -setenv=NUNIT_ENABLE_NETWORK=true"); // detect if we are using a jenkins bot. - if (Harness.InWrench) + if (Harness.InJenkins) args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true"); if (isSimulator) { diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index c28c8de08462..34425dad6a9f 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -498,7 +498,15 @@ public void LogWrench (string message) public bool InWrench { get { - return !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("BUILD_REVISION")); + var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); + return !string.IsNullOrEmpty (buildRev) && buildRev != "jenkins"; + } + } + + public bool InJenkins { + get { + var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); + return !string.IsNullOrEmpty (buildRev) && buildRev == "jenkins"; } } From e28220630b3773de34a51bc95548c6795044d262 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 26 Sep 2016 16:05:41 +0200 Subject: [PATCH 08/17] Ensure that the main node of the unit tests does contain the target, that will improve the tests results reporting. --- tests/xharness/AppRunner.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 25d5fbe8d274..6695bb2e5499 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -315,7 +315,6 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed log = reader.ReadToEnd (); // parsing the result is different if we are in jenkins or nor. if (Harness.InJenkins) { - Harness.LogWrench ($"We are in jenkins."); // we have to parse the xml result crashed = false; if (log.Contains ("test-results")) { @@ -324,10 +323,8 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed log = log.Remove (0, index - 1); var testsResults = new XmlDocument (); testsResults.LoadXml (log); - Harness.LogWrench ($"Loaded xml."); var mainResultNode = testsResults.SelectSingleNode("test-results"); - Harness.LogWrench ($"Got main node."); if (mainResultNode == null) { Harness.LogWrench ($"Node is null."); crashed = true; @@ -335,7 +332,6 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed } // update the information of the main node to add information about the mode and the test that is excuted. This will later create // nicer reports in jenkins - Harness.LogWrench ($"Setting name node to {Target}."); mainResultNode.Attributes["name"].Value = Target; // store a clean version of the logs, later this will be used by the bots to show results in github/web var path = listener_log.FullPath; From 70d89bd0a9e6e564912f2797b7263207db890e08 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 24 Oct 2016 07:13:51 +0200 Subject: [PATCH 09/17] Revert "Fix binding project LinkWithAttributes generation to prevent rebuilds" (#1018) --- jenkins/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index 0ade3d32069f..4d7788fa3328 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash -e +export BUILD_REVISION=jenkins cd $WORKSPACE export BUILD_REVISION=jenkins # Unlock From 63f0894bf44890f5644578c2988564e836ff8852 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 25 Oct 2016 12:49:02 +0200 Subject: [PATCH 10/17] Added xslt to be used to keep the old Test Reports until we move to only Jenkins reports. --- jenkins/nunit-summary.xslt | 45 ++++++++++++++++++++++++++++++++++++++ jenkins/run-tests.sh | 8 ++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 jenkins/nunit-summary.xslt diff --git a/jenkins/nunit-summary.xslt b/jenkins/nunit-summary.xslt new file mode 100644 index 000000000000..88304a43e537 --- /dev/null +++ b/jenkins/nunit-summary.xslt @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + [PASS] + + + [FAIL] + + + [IGNORED] + + + [INCONCLUSIVE] + + + + . + + + + + + + + + + + + + + diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index 4d7788fa3328..76f84685eb65 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -2,7 +2,6 @@ export BUILD_REVISION=jenkins cd $WORKSPACE -export BUILD_REVISION=jenkins # Unlock security default-keychain -s builder.keychain security list-keychains -s builder.keychain @@ -14,5 +13,12 @@ security set-keychain-settings -lut 7200 # Run tests make -C tests jenkins +# go through the diff xml results and do an xslt transformation for the old Test Report +for i in tests/logs/*/*.xml; do + logname=$(basename "${i%.*}"); + dirname=$(dirname "$i"); + full_path="$dirname/$logname.log"; + xsltproc jenkins/nunit-summary.xslt $i > $full_path; +done # Lock security lock-keychain From 64d7b81af51a28a039a59726a28fb64e9bd4a375 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 28 Oct 2016 16:31:35 +0200 Subject: [PATCH 11/17] Add an extra log for the xslt transformation. --- external/Touch.Unit | 2 +- jenkins/run-tests.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/external/Touch.Unit b/external/Touch.Unit index 8163298ba756..71c2c44f926e 160000 --- a/external/Touch.Unit +++ b/external/Touch.Unit @@ -1 +1 @@ -Subproject commit 8163298ba756fc0f5e564d7ce998ef441231ba54 +Subproject commit 71c2c44f926e8f6bcf37c8f5823ed7e841aabc59 diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index 76f84685eb65..dd1c63a37c71 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -18,6 +18,7 @@ for i in tests/logs/*/*.xml; do logname=$(basename "${i%.*}"); dirname=$(dirname "$i"); full_path="$dirname/$logname.log"; + echo "Cleaning xml log on $full_path"; xsltproc jenkins/nunit-summary.xslt $i > $full_path; done # Lock From 68701672686e1be5647a16790a26c3aea921b13f Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 7 Nov 2016 10:11:15 +0100 Subject: [PATCH 12/17] Point to the correcto dir in jenkins. --- jenkins/run-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index dd1c63a37c71..a13d9ef6f990 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -14,12 +14,12 @@ security set-keychain-settings -lut 7200 make -C tests jenkins # go through the diff xml results and do an xslt transformation for the old Test Report -for i in tests/logs/*/*.xml; do +for i in jenkins-results/tests/*/*.xml; do logname=$(basename "${i%.*}"); dirname=$(dirname "$i"); full_path="$dirname/$logname.log"; echo "Cleaning xml log on $full_path"; - xsltproc jenkins/nunit-summary.xslt $i > $full_path; + xsltproc jenkins/nunit-summary.xslt $i > "$full_path"; done # Lock security lock-keychain From 4dfdec7a751e6d29a89d3d58884875b874da8b41 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 10 Nov 2016 12:20:17 +0100 Subject: [PATCH 13/17] Deal with the xslt once we have finished rather than in a batch. --- jenkins/run-tests.sh | 8 -------- tests/xharness/AppRunner.cs | 27 +++++++++++++++++++++++++++ tests/xharness/xharness.csproj | 5 +++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index a13d9ef6f990..fddb1ebe5979 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -13,13 +13,5 @@ security set-keychain-settings -lut 7200 # Run tests make -C tests jenkins -# go through the diff xml results and do an xslt transformation for the old Test Report -for i in jenkins-results/tests/*/*.xml; do - logname=$(basename "${i%.*}"); - dirname=$(dirname "$i"); - full_path="$dirname/$logname.log"; - echo "Cleaning xml log on $full_path"; - xsltproc jenkins/nunit-summary.xslt $i > "$full_path"; -done # Lock security lock-keychain diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 6695bb2e5499..294cf9d5dd10 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -3,11 +3,13 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; +using System.Xml.Xsl; namespace xharness { @@ -309,6 +311,25 @@ public bool EnsureCleanSimulatorState { } } + void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument doc){ + // load the resource that contains the xslt and apply it to the doc and write the logs + if (File.Exists (finalPath)) { + // if the file does exist, remove it + File.Delete (finalPath); + } + + using (var strm = Assembly.GetExecutingAssembly ().GetManifestResourceStream ("xharness.nunit-summary.xslt")) + using (var xsltReader = XmlReader.Create (strm)) + using (var xmlReader = new XmlNodeReader (doc)) + using (var writer = new StreamWriter (finalPath)) { + writer.Write (logHeader); + var xslt = new XslCompiledTransform (); + xslt.Load (xsltReader); + xslt.Transform (xmlReader, null, writer); + writer.Flush (); + } + } + public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed) { string log; using (var reader = listener_log.GetReader ()) @@ -320,6 +341,7 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed if (log.Contains ("test-results")) { // remove any possible extra info var index = log.IndexOf ("{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}
"); diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index dc435bdbb780..ffc3b90ac95e 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -70,5 +70,10 @@ + + + nunit-summary.xslt + + From 6f1308c8dae9404d6c73e3b5bf53748998b1a961 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 14 Nov 2016 23:23:32 +0100 Subject: [PATCH 14/17] Remove noise. --- tests/xharness/AppRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 294cf9d5dd10..1c633871bf75 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -341,7 +341,7 @@ public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed if (log.Contains ("test-results")) { // remove any possible extra info var index = log.IndexOf (" Date: Mon, 14 Nov 2016 23:34:55 +0100 Subject: [PATCH 15/17] Readd case removed in rebase. --- tests/xharness/AppRunner.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 1c633871bf75..9b60d7b1dde2 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -311,7 +311,8 @@ public bool EnsureCleanSimulatorState { } } - void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument doc){ + void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument doc) + { // load the resource that contains the xslt and apply it to the doc and write the logs if (File.Exists (finalPath)) { // if the file does exist, remove it @@ -330,7 +331,8 @@ void GenerateHumanReadableLogs (string finalPath, string logHeader, XmlDocument } } - public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed) { + public bool TestsSucceeded (LogStream listener_log, bool timed_out, bool crashed) + { string log; using (var reader = listener_log.GetReader ()) log = reader.ReadToEnd (); @@ -683,6 +685,10 @@ public async Task RunAsync () Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode); main_log.WriteLine ("Test run never launched"); success = false; + } else if (launch_failure) { + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); + main_log.WriteLine ("Test run failed to launch"); + success = false; } else { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed at startup (no log)
", mode); main_log.WriteLine ("Test run crashed before it started (no log file produced)"); From 0001e3aed4ecf5190bc5d899f9a02d5c0715140e Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 15 Nov 2016 10:17:33 +0100 Subject: [PATCH 16/17] Fix indentation. --- tests/xharness/AppRunner.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 9b60d7b1dde2..c8ba102b992d 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -686,9 +686,9 @@ public async Task RunAsync () main_log.WriteLine ("Test run never launched"); success = false; } else if (launch_failure) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); - main_log.WriteLine ("Test run failed to launch"); - success = false; + Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); + main_log.WriteLine ("Test run failed to launch"); + success = false; } else { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed at startup (no log)
", mode); main_log.WriteLine ("Test run crashed before it started (no log file produced)"); From 20a98f928588881f9e2fbece9edadcbfbe459616 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 15 Nov 2016 17:55:17 +0100 Subject: [PATCH 17/17] Skip lock keychain. --- jenkins/run-tests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/jenkins/run-tests.sh b/jenkins/run-tests.sh index fddb1ebe5979..c3f39a463b61 100755 --- a/jenkins/run-tests.sh +++ b/jenkins/run-tests.sh @@ -12,6 +12,3 @@ security set-keychain-settings -lut 7200 # Run tests make -C tests jenkins - -# Lock -security lock-keychain