From e642e1a3631fcd8bd1bb8c302f76dcbf8e180430 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 31 Jul 2020 15:01:29 +0200 Subject: [PATCH] [xharness] Improve parsing of NUnitV2 xml reports to fix an issue with failures in parameterized tests. We weren't properly reporting all failures when there were multiple failures in parameterized tests, because we'd incorrectly skip too many xml nodes when we were looking for the next test failure. --- ...ft.DotNet.XHarness.iOS.Shared.Tests.csproj | 2 + .../Samples/TestCaseFailures.xml | 62 +++++++++++++++++++ .../XmlResultParserTests.cs | 26 +++++++- .../XmlResultParser.cs | 27 +++++--- 4 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj index 50aab4f24b53..364f4b1d55ad 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj @@ -18,6 +18,7 @@ + @@ -52,6 +53,7 @@ PreserveNewest + diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml new file mode 100644 index 000000000000..5a2d4e5193b9 --- /dev/null +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + message + stacktrace + + + + + + + message + stacktrace + + + + + + + + + + + + diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs index f0f6d5eb5073..876061161a4e 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs @@ -361,9 +361,31 @@ public void NUnitV2GenerateTestReport () message
  • -ErrorTest2: SingleLineErrorMessage
    • -
  • NUnit.Tests.Assemblies.MockTestFixture.FailingTest: Intentional failure
    • +
  • +NUnit.Tests.Assemblies.MockTestFixture.TestWithException: System.ApplicationException : Intentional Exception
    • + + +"; + Assert.AreEqual (expectedOutput, writer.ToString (), "Output"); + } + + + [Test] + public void NUnitV2GenerateTestReportWithTestCaseFailures () + { + using var writer = new StringWriter (); + using var stream = GetType ().Assembly.GetManifestResourceStream ("Microsoft.DotNet.XHarness.iOS.Shared.Tests.Samples.TestCaseFailures.xml"); + using var reader = new StreamReader (stream); + resultParser.GenerateTestReport (writer, reader, XmlResultJargon.NUnitV2); + Console.WriteLine (writer.ToString ()); + var expectedOutput = +@"
    +
      +
    • +Xamarin.MTouch.FastDev_LinkAll(iOS): message
      • +
    • +Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode: message
    "; diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs index 0e2f448484ba..8e787e676242 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs @@ -518,7 +518,14 @@ static void GenerateNUnitV2TestReport (TextWriter writer, XmlReader reader) void write_failure () { var name = reader ["name"]; - var message = reader.ReadToDescendant ("message") ? reader.ReadElementContentAsString () : null; + string? message = null; + var depth = reader.Depth; + if (reader.ReadToDescendant ("message")) { + message = reader.ReadElementContentAsString (); + // ReadToParent + while (depth > reader.Depth && reader.Read ()) { + } + } var message_block = message?.IndexOf ('\n') >= 0; writer.WriteLine ("
  • "); writer.Write (name.AsHtml ()); @@ -544,16 +551,18 @@ void write_failure () if (result == "Error") write_failure (); - if (!reader.ReadToDescendant ("test-case")) - continue; + var depth = reader.Depth; - do { - result = reader ["result"]; - if (result != "Error" && result != "Failure") - continue; + while (reader.Read ()) { + if (reader.NodeType != XmlNodeType.Element || reader.Name != "test-case") + continue; - write_failure (); - } while (reader.ReadToNextSibling ("test-case")); + result = reader ["result"]; + if (result == "Error" || result == "Failure") + write_failure (); + if (reader.Depth < depth) + break; + } } writer.WriteLine ("");