From 8e1413eb76bb1fd2af54acfea4540a393bf8ac0e Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sun, 19 Nov 2023 13:17:16 +0000 Subject: [PATCH 01/19] Adds handling of It's marked as inconclusive --- src/Pester.RSpec.ps1 | 7 +++++++ src/Pester.Runtime.ps1 | 11 ++++++++++- src/csharp/Pester/Container.cs | 2 ++ src/csharp/Pester/Run.cs | 2 ++ src/csharp/Pester/Test.cs | 1 + src/csharp/Pester/ToStringConverter.cs | 1 + src/en-US/about_Pester.help.txt | 2 +- src/functions/It.ps1 | 2 +- src/functions/Output.ps1 | 17 +++++++++-------- src/functions/Set-ItResult.ps1 | 14 ++++++++++---- src/functions/TestResults.NUnit25.ps1 | 2 +- src/functions/TestResults.NUnit3.ps1 | 5 +++-- 12 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/Pester.RSpec.ps1 b/src/Pester.RSpec.ps1 index ee7729459..c9f8d5973 100644 --- a/src/Pester.RSpec.ps1 +++ b/src/Pester.RSpec.ps1 @@ -107,6 +107,9 @@ function Add-RSpecTestObjectProperties { $TestObject.Result = if ($TestObject.Skipped) { "Skipped" } + elseif ($TestObject.Inconclusive) { + "Inconclusive" + } elseif ($TestObject.Passed) { "Passed" } @@ -160,6 +163,9 @@ function PostProcess-RspecTestRun ($TestRun) { "Skipped" { $null = $TestRun.Skipped.Add($t) } + "Inconclusive" { + $null = $TestRun.Inconclusive.Add($t) + } default { throw "Result $($t.Result) is not supported." } } @@ -236,6 +242,7 @@ function PostProcess-RspecTestRun ($TestRun) { $TestRun.PassedCount = $TestRun.Passed.Count $TestRun.FailedCount = $TestRun.Failed.Count $TestRun.SkippedCount = $TestRun.Skipped.Count + $TestRun.InconclusiveCount = $TestRun.Inconclusive.Count $TestRun.NotRunCount = $TestRun.NotRun.Count $TestRun.TotalCount = $TestRun.Tests.Count diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index 72b4914d6..68fa87ba0 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -687,7 +687,12 @@ function Invoke-TestItem { Write-PesterDebugMessage -Scope Skip "($path) Test is skipped." } $Test.Passed = $true - $Test.Skipped = $true + if ('PesterTestInconclusive' -eq $Result.ErrorRecord.FullyQualifiedErrorId) { + $Test.Inconclusive = $true + } + else { + $Test.Skipped = $true + } } else { $Test.Passed = $result.Success @@ -2194,6 +2199,7 @@ function PostProcess-ExecutedBlock { $b.OwnFailedCount = 0 $b.OwnPassedCount = 0 $b.OwnSkippedCount = 0 + $b.OwnInconclusiveCount = 0 $b.OwnNotRunCount = 0 $testDuration = [TimeSpan]::Zero @@ -2245,6 +2251,7 @@ function PostProcess-ExecutedBlock { $b.FailedCount = $b.OwnFailedCount $b.PassedCount = $b.OwnPassedCount $b.SkippedCount = $b.OwnSkippedCount + $b.InconclusiveCount = $b.OwnInconclusiveCount $b.NotRunCount = $b.OwnNotRunCount } else { @@ -2266,6 +2273,7 @@ function PostProcess-ExecutedBlock { $b.PassedCount += $child.PassedCount $b.FailedCount += $child.FailedCount $b.SkippedCount += $child.SkippedCount + $b.InconclusiveCount += $child.OwnInconclusiveCount $b.NotRunCount += $child.NotRunCount } @@ -2274,6 +2282,7 @@ function PostProcess-ExecutedBlock { $b.PassedCount += $b.OwnPassedCount $b.FailedCount += $b.OwnFailedCount $b.SkippedCount += $b.OwnSkippedCount + $b.InconclusiveCount += $b.OwnInconclusiveCount $b.NotRunCount += $b.OwnNotRunCount $b.Passed = -not ($thisBlockFailed -or $anyTestFailed -or $anyChildBlockFailed) diff --git a/src/csharp/Pester/Container.cs b/src/csharp/Pester/Container.cs index 7c15e2c9b..76bd80c69 100644 --- a/src/csharp/Pester/Container.cs +++ b/src/csharp/Pester/Container.cs @@ -18,6 +18,7 @@ public static Container CreateFromBlock (Block block) { FailedCount = block.FailedCount, PassedCount = block.PassedCount, SkippedCount = block.SkippedCount, + InconclusiveCount = block.InconclusiveCount, NotRunCount = block.NotRunCount, TotalCount = block.TotalCount, ErrorRecord = block.ErrorRecord ?? new List(), @@ -56,6 +57,7 @@ public static Container CreateFromFile(FileInfo file) public int FailedCount { get; set; } public int PassedCount { get; set; } public int SkippedCount { get; set; } + public int InconclusiveCount { get; set; } public int NotRunCount { get; set; } public int TotalCount { get; set; } public List ErrorRecord { get; set; } = new List(); diff --git a/src/csharp/Pester/Run.cs b/src/csharp/Pester/Run.cs index 66b7d8212..40c1cda3e 100644 --- a/src/csharp/Pester/Run.cs +++ b/src/csharp/Pester/Run.cs @@ -20,6 +20,7 @@ public static Run Create() public int PassedCount { get; set; } public int SkippedCount { get; set; } + public int InconclusiveCount { get; set; } public int NotRunCount { get; set; } public int TotalCount { get; set; } @@ -47,6 +48,7 @@ public static Run Create() public List Passed { get; set; } = new List(); public List Skipped { get; set; } = new List(); + public List Inconclusive { get; set; } = new List(); public List NotRun { get; set; } = new List(); public List Tests { get; set; } = new List(); diff --git a/src/csharp/Pester/Test.cs b/src/csharp/Pester/Test.cs index 1b52fe94c..a44031ba8 100644 --- a/src/csharp/Pester/Test.cs +++ b/src/csharp/Pester/Test.cs @@ -63,6 +63,7 @@ public Test() public DateTime? ExecutedAt { get; set; } public bool Passed { get; set; } public bool Skipped { get; set; } + public bool Inconclusive { get; set; } public TimeSpan UserDuration { get; set; } public TimeSpan FrameworkDuration { get; set; } diff --git a/src/csharp/Pester/ToStringConverter.cs b/src/csharp/Pester/ToStringConverter.cs index 4fe3fd62b..cc6f77f45 100644 --- a/src/csharp/Pester/ToStringConverter.cs +++ b/src/csharp/Pester/ToStringConverter.cs @@ -12,6 +12,7 @@ static string ResultToString(string result) "Passed" => "[+]", "Failed" => "[-]", "Skipped" => "[!]", + "Inconclusive" => "[?]", "NotRun" => "[ ]", _ => "[ERR]", }; diff --git a/src/en-US/about_Pester.help.txt b/src/en-US/about_Pester.help.txt index a7f0ec7a9..a32ecf78b 100644 --- a/src/en-US/about_Pester.help.txt +++ b/src/en-US/about_Pester.help.txt @@ -255,7 +255,7 @@ LONG DESCRIPTION The output ends with a summary of the test results. Tests completed in 3.47s - Passed: 20 Failed: 1 Skipped: 0 Pending: 0 Inconclusive: 0 + Tests Passed: 20, Failed: 1, Skipped: 0, Inconclusive: 0, NotRun: 0 However, because Pester uses Write-Host, it does not write to the output stream (stdout), so there are no output objects to save in a variable or diff --git a/src/functions/It.ps1 b/src/functions/It.ps1 index a8d1012b4..227fc7c3a 100644 --- a/src/functions/It.ps1 +++ b/src/functions/It.ps1 @@ -15,7 +15,7 @@ In addition to using your own logic to test expectations and throw exceptions, you may also use Pester's Should command to perform assertions in plain language. - You can intentionally mark It block result as inconclusive by using Set-TestInconclusive + You can intentionally mark It block result as inconclusive by using Set-ItResult -Inconclusive command as the first tested statement in the It block. .PARAMETER Name diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 264d5ea0b..2e519aad3 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -26,7 +26,7 @@ $script:ReportStrings = DATA { TestsPassed = 'Tests Passed: {0}, ' TestsFailed = 'Failed: {0}, ' - TestsSkipped = 'Skipped: {0} ' + TestsSkipped = 'Skipped: {0}, ' TestsPending = 'Pending: {0}, ' TestsInconclusive = 'Inconclusive: {0}, ' TestsNotRun = 'NotRun: {0}' @@ -345,12 +345,12 @@ function Write-PesterReport { # else { # $ReportTheme.Information # } - # $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) { - # $ReportTheme.Inconclusive - # } - # else { - # $ReportTheme.Information - # } + $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) { + $ReportTheme.Inconclusive + } + else { + $ReportTheme.Information + } # Try { # $PesterStatePassedScenariosCount = $PesterState.PassedScenarios.Count @@ -374,6 +374,7 @@ function Write-PesterReport { Write-PesterHostMessage ($ReportStrings.TestsPassed -f $RunResult.PassedCount) -Foreground $Success -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsFailed -f $RunResult.FailedCount) -Foreground $Failure -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsSkipped -f $RunResult.SkippedCount) -Foreground $Skipped -NoNewLine + Write-PesterHostMessage ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsTotal -f $RunResult.TotalCount) -Foreground $Total -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsNotRun -f $RunResult.NotRunCount) -Foreground $NotRun @@ -843,7 +844,7 @@ function Get-WriteScreenPlugin ($Verbosity) { if ($PesterPreference.Output.Verbosity.Value -in 'Detailed', 'Diagnostic') { $because = if ($_test.FailureMessage) { ", because $($_test.FailureMessage)" } else { $null } Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$margin[?] $out" -NoNewLine - Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive ", is inconclusive$because" -NoNewLine + Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$because" -NoNewLine Write-PesterHostMessage -ForegroundColor $ReportTheme.InconclusiveTime " $humanTime" } diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index 8698e1eac..0fc3099fc 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -58,13 +58,9 @@ $result = $PSCmdlet.ParameterSetName - [String]$Message = "is skipped" if ($Result -ne 'Skipped') { [String]$Because = if ($Because) { $Result.ToUpper(), $Because -join ': ' } else { $Result.ToUpper() } } - if ($Because) { - [String]$Message += ", because $Because" - } switch ($null) { $File { @@ -81,15 +77,25 @@ switch ($result) { 'Inconclusive' { [String]$errorId = 'PesterTestInconclusive' + [String]$message = "is inconclusive" + break } 'Pending' { [String]$errorId = 'PesterTestPending' + [String]$message = "is pending" + break } 'Skipped' { [String]$errorId = 'PesterTestSkipped' + [String]$message = "is skipped" + break } } + if ($Because) { + [String]$message += ", because $Because" + } + throw [Pester.Factory]::CreateErrorRecord( $errorId, #string errorId $Message, #string message diff --git a/src/functions/TestResults.NUnit25.ps1 b/src/functions/TestResults.NUnit25.ps1 index 64cd6cffb..04593e9e7 100644 --- a/src/functions/TestResults.NUnit25.ps1 +++ b/src/functions/TestResults.NUnit25.ps1 @@ -23,7 +23,7 @@ function Write-NUnitTestResultAttributes { $XmlWriter.WriteAttributeString('errors', '0') $XmlWriter.WriteAttributeString('failures', $Result.FailedCount) $XmlWriter.WriteAttributeString('not-run', $Result.NotRunCount) - $XmlWriter.WriteAttributeString('inconclusive', '0') # $Result.PendingCount + $Result.InconclusiveCount) #TODO: reflect inconclusive count once it is added + $XmlWriter.WriteAttributeString('inconclusive', $Result.InconclusiveCount) $XmlWriter.WriteAttributeString('ignored', '0') $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount) $XmlWriter.WriteAttributeString('invalid', '0') diff --git a/src/functions/TestResults.NUnit3.ps1 b/src/functions/TestResults.NUnit3.ps1 index 2a47fbffe..f2c62a8d1 100644 --- a/src/functions/TestResults.NUnit3.ps1 +++ b/src/functions/TestResults.NUnit3.ps1 @@ -30,7 +30,7 @@ function Write-NUnit3TestRunAttributes { $XmlWriter.WriteAttributeString('total', ($Result.TotalCount - $Result.NotRunCount)) # testcasecount - filtered $XmlWriter.WriteAttributeString('passed', $Result.PassedCount) $XmlWriter.WriteAttributeString('failed', $Result.FailedCount) - $XmlWriter.WriteAttributeString('inconclusive', '0') # required attr. $Result.PendingCount + $Result.InconclusiveCount when/if implemented? + $XmlWriter.WriteAttributeString('inconclusive', $Resul.InconclusiveCount) $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount) $XmlWriter.WriteAttributeString('warnings', '0') # required attr. $XmlWriter.WriteAttributeString('start-time', (Get-UTCTimeString $Result.ExecutedAt)) @@ -297,6 +297,7 @@ function Get-NUnit3TestSuiteInfo { passed = $TestSuite.PassedCount failed = $TestSuite.FailedCount skipped = $TestSuite.SkippedCount + inconclusive = $TestSuite.InconclusiveCount site = $site shouldrun = $TestSuite.ShouldRun } @@ -327,7 +328,7 @@ function Write-NUnit3TestSuiteAttributes { $XmlWriter.WriteAttributeString('total', $TestSuiteInfo.total) $XmlWriter.WriteAttributeString('passed', $TestSuiteInfo.passed) $XmlWriter.WriteAttributeString('failed', $TestSuiteInfo.failed) - $XmlWriter.WriteAttributeString('inconclusive', '0') # required attribute + $XmlWriter.WriteAttributeString('inconclusive', $TestSuiteInfo.inconclusive) # required attribute $XmlWriter.WriteAttributeString('warnings', '0') # required attribute $XmlWriter.WriteAttributeString('skipped', $TestSuiteInfo.skipped) $XmlWriter.WriteAttributeString('asserts', $TestSuiteInfo.testcasecount) # required attr. hardcode 1:1 per testcase From 5ec791c836a015bfda15fa8f19ccfcc1ac7f8b3c Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sun, 19 Nov 2023 14:27:19 +0000 Subject: [PATCH 02/19] Removes **DEPRECATED** from Inconclusive parameter help text --- src/functions/Set-ItResult.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index 0fc3099fc..d9112d0e8 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -14,7 +14,7 @@ backwards compatible .PARAMETER Inconclusive - **DEPRECATED** Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped + Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped .PARAMETER Pending **DEPRECATED** Sets the test result to pending. Cannot be used at the same time as -Inconclusive or -Skipped From 7eb17cd5840a5d910228e1503bab235330bba2df Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sun, 19 Nov 2023 14:30:35 +0000 Subject: [PATCH 03/19] Adds deprecation notice to pester report when Set-ItResult -Pending is used --- src/functions/Output.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 2e519aad3..8d83f9700 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -61,6 +61,7 @@ $script:ReportTheme = DATA { Discovery = 'Magenta' Container = 'Magenta' BlockFail = 'Red' + Warning = 'Yellow' } } @@ -309,6 +310,10 @@ function Write-PesterReport { ) # if(-not ($PesterState.Show | Has-Flag Summary)) { return } + if ($RunResult.Tests.ErrorRecord.FullyQualifiedErrorID -contains 'PesterTestPending') { + Write-PesterHostMessage '**DEPRECATED**: The Set-ItResult -Pending parameter is deprecated. It will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + } + Write-PesterHostMessage ($ReportStrings.Timing -f (Get-HumanTime ($RunResult.Duration))) -Foreground $ReportTheme.Foreground $Success, $Failure = if ($RunResult.FailedCount -gt 0) { From f5626bfcd470e677ca53c397fa2fdc1d0070d49e Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Mon, 20 Nov 2023 16:50:26 +0000 Subject: [PATCH 04/19] Fix typo and add changes to fix failing P tests --- src/functions/TestResults.NUnit3.ps1 | 72 +++++++++++++++------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/functions/TestResults.NUnit3.ps1 b/src/functions/TestResults.NUnit3.ps1 index f2c62a8d1..5a80318b2 100644 --- a/src/functions/TestResults.NUnit3.ps1 +++ b/src/functions/TestResults.NUnit3.ps1 @@ -19,7 +19,7 @@ function Write-NUnit3Report($Result, [System.Xml.XmlWriter] $XmlWriter) { } function Write-NUnit3TestRunAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('id', '0') @@ -30,7 +30,7 @@ function Write-NUnit3TestRunAttributes { $XmlWriter.WriteAttributeString('total', ($Result.TotalCount - $Result.NotRunCount)) # testcasecount - filtered $XmlWriter.WriteAttributeString('passed', $Result.PassedCount) $XmlWriter.WriteAttributeString('failed', $Result.FailedCount) - $XmlWriter.WriteAttributeString('inconclusive', $Resul.InconclusiveCount) + $XmlWriter.WriteAttributeString('inconclusive', $Result.InconclusiveCount) $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount) $XmlWriter.WriteAttributeString('warnings', '0') # required attr. $XmlWriter.WriteAttributeString('start-time', (Get-UTCTimeString $Result.ExecutedAt)) @@ -306,14 +306,14 @@ function Get-NUnit3TestSuiteInfo { } function Write-NUnit3TestSuiteAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestSuiteInfo, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('type', $TestSuiteInfo.type) $XmlWriter.WriteAttributeString('id', (Get-NUnit3NodeId)) $XmlWriter.WriteAttributeString('name', $TestSuiteInfo.name) $XmlWriter.WriteAttributeString('fullname', $TestSuiteInfo.fullname) - if ($TestSuiteInfo.type -in 'TestFixture','ParameterizedMethod') { + if ($TestSuiteInfo.type -in 'TestFixture', 'ParameterizedMethod') { $XmlWriter.WriteAttributeString('classname', $TestSuiteInfo.classname) } $XmlWriter.WriteAttributeString('runstate', $TestSuiteInfo.runstate) @@ -392,20 +392,21 @@ function Get-NUnit3ParameterizedMethodSuiteInfo { $sampleTest = $TestSuiteGroup.Group[0] $node = [PSCustomObject] @{ - Name = $sampleTest.Name - ExpandedName = $sampleTest.Name - Path = $sampleTest.Block.Path # used for classname -> block path - Data = $null - TotalCount = 0 - Duration = [timespan]0 - ExecutedAt = [datetime]::MinValue - PassedCount = 0 - FailedCount = 0 - SkippedCount = 0 - NotRunCount = 0 - OwnTotalCount = 0 - ShouldRun = $true - Skip = $sampleTest.Skip + Name = $sampleTest.Name + ExpandedName = $sampleTest.Name + Path = $sampleTest.Block.Path # used for classname -> block path + Data = $null + TotalCount = 0 + Duration = [timespan]0 + ExecutedAt = [datetime]::MinValue + PassedCount = 0 + FailedCount = 0 + SkippedCount = 0 + InconclusiveCount = 0 + NotRunCount = 0 + OwnTotalCount = 0 + ShouldRun = $true + Skip = $sampleTest.Skip } foreach ($testCase in $TestSuiteGroup.Group) { @@ -418,6 +419,7 @@ function Get-NUnit3ParameterizedMethodSuiteInfo { Passed { $node.PassedCount++; break; } Failed { $node.FailedCount++; break; } Skipped { $node.SkippedCount++; break; } + Inconclusive { $node.InconclusiveCount++; break; } NotRun { $node.NotRunCount++; break; } } @@ -439,20 +441,21 @@ function Get-NUnit3ParameterizedFixtureSuiteInfo { $sampleBlock = $TestSuiteGroup.Group[0] $node = [PSCustomObject] @{ - Name = $sampleBlock.Name - ExpandedName = $sampleBlock.Name - Path = $sampleBlock.Path - Data = $null - TotalCount = 0 - Duration = [timespan]0 - ExecutedAt = [datetime]::MinValue - PassedCount = 0 - FailedCount = 0 - SkippedCount = 0 - NotRunCount = 0 - OwnTotalCount = 0 - ShouldRun = $true - Skip = $false # ParameterizedFixture are always Runnable, even with -Skip + Name = $sampleBlock.Name + ExpandedName = $sampleBlock.Name + Path = $sampleBlock.Path + Data = $null + TotalCount = 0 + Duration = [timespan]0 + ExecutedAt = [datetime]::MinValue + PassedCount = 0 + FailedCount = 0 + SkippedCount = 0 + InconclusiveCount = 0 + NotRunCount = 0 + OwnTotalCount = 0 + ShouldRun = $true + Skip = $false # ParameterizedFixture are always Runnable, even with -Skip } foreach ($block in $TestSuiteGroup.Group) { @@ -464,6 +467,7 @@ function Get-NUnit3ParameterizedFixtureSuiteInfo { $node.PassedCount += $block.PassedCount $node.FailedCount += $block.FailedCount $node.SkippedCount += $block.SkippedCount + $node.InconclusiveCount += $block.InconclusiveCount $node.NotRunCount += $block.NotRunCount $node.TotalCount += $block.TotalCount @@ -506,7 +510,7 @@ function Write-NUnit3TestCaseElement { } function Write-NUnit3TestCaseAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestResult, [string] $ParentPath, [System.Xml.XmlWriter] $XmlWriter) # add parameters to name for testcase with data when not using variables in name @@ -578,7 +582,7 @@ function Write-NUnit3FailureElement ($TestResult, [System.Xml.XmlWriter] $XmlWri $XmlWriter.WriteEndElement() # Close failure } -function Write-NUnitReasonElement ($TestResult,[System.Xml.XmlWriter] $XmlWriter) { +function Write-NUnitReasonElement ($TestResult, [System.Xml.XmlWriter] $XmlWriter) { # TODO: do not format the errors here, instead format them in the core using some unified function so we get the same thing on the screen and in nunit $result = Get-ErrorForXmlReport -TestResult $TestResult From 53acb19c8879031726143aa6a58c60125724d728 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 19 Jan 2024 19:53:17 +0000 Subject: [PATCH 05/19] Moves deprecation notice to end of Pester report --- src/functions/Output.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 8d83f9700..6e09d0357 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -310,10 +310,6 @@ function Write-PesterReport { ) # if(-not ($PesterState.Show | Has-Flag Summary)) { return } - if ($RunResult.Tests.ErrorRecord.FullyQualifiedErrorID -contains 'PesterTestPending') { - Write-PesterHostMessage '**DEPRECATED**: The Set-ItResult -Pending parameter is deprecated. It will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning - } - Write-PesterHostMessage ($ReportStrings.Timing -f (Get-HumanTime ($RunResult.Duration))) -Foreground $ReportTheme.Foreground $Success, $Failure = if ($RunResult.FailedCount -gt 0) { @@ -408,6 +404,11 @@ function Write-PesterReport { # & $SafeCommands['Write-Host'] ($ReportStrings.TestsPending -f $RunResult.PendingCount) -Foreground $Pending -NoNewLine # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive # } + + if ($RunResult.Tests.ErrorRecord.FullyQualifiedErrorID -contains 'PesterTestPending') { + Write-PesterHostMessage '' + Write-PesterHostMessage '**DEPRECATED**: The Set-ItResult -Pending parameter is deprecated. It will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + } } function Write-CoverageReport { From 79d8327d39129838e8335e373d3ad83480368232 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 19 Jan 2024 20:07:47 +0000 Subject: [PATCH 06/19] Fix InconclusiveCount typo --- src/Pester.Runtime.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index 68fa87ba0..88e1ae217 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -2273,7 +2273,7 @@ function PostProcess-ExecutedBlock { $b.PassedCount += $child.PassedCount $b.FailedCount += $child.FailedCount $b.SkippedCount += $child.SkippedCount - $b.InconclusiveCount += $child.OwnInconclusiveCount + $b.InconclusiveCount += $child.InconclusiveCount $b.NotRunCount += $child.NotRunCount } From 6710d33d0297ff13a4265e79f8b6aa15dfd17f48 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 19 Jan 2024 21:16:07 +0000 Subject: [PATCH 07/19] Adds missing increment of OwnInconclusiveCount --- src/Pester.Runtime.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index 88e1ae217..d682e9d32 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -2211,6 +2211,9 @@ function PostProcess-ExecutedBlock { if (-not $t.ShouldRun) { $b.OwnNotRunCount++ } + elseif ($t.ShouldRun -and $t.Inconclusive) { + $b.OwnInconclusiveCount++ + } elseif ($t.ShouldRun -and $t.Skipped) { $b.OwnSkippedCount++ } From b9237e816fafde3578845b38eaf9f94b9f106ead Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 19 Jan 2024 21:26:34 +0000 Subject: [PATCH 08/19] Increment InconclusiveCount in Pester4Result object --- src/Main.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Main.ps1 b/src/Main.ps1 index 8bfb3568c..3b923fc66 100644 --- a/src/Main.ps1 +++ b/src/Main.ps1 @@ -747,12 +747,12 @@ function Invoke-Pester { } $plugins.Add(( - # decorator plugin needs to be added after output - # because on teardown they will run in opposite order - # and that way output can consume the fixed object that decorator - # decorated, not nice but works - Get-RSpecObjectDecoratorPlugin - )) + # decorator plugin needs to be added after output + # because on teardown they will run in opposite order + # and that way output can consume the fixed object that decorator + # decorated, not nice but works + Get-RSpecObjectDecoratorPlugin + )) if ($PesterPreference.TestDrive.Enabled.Value) { $plugins.Add((Get-TestDrivePlugin)) @@ -1458,11 +1458,13 @@ function ConvertTo-Pester4Result { "Skipped" { $legacyResult.SkippedCount++ } + "Inconclusive" { + $legacyResult.InconclusiveCount++ + } } } $legacyResult.TotalCount = $legacyResult.TestResult.Count $legacyResult.PendingCount = 0 - $legacyResult.InconclusiveCount = 0 $legacyResult.Time = $PesterResult.Duration $legacyResult From 7c219c25784afcad3927e81e530fcdfb9bc173fe Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sat, 3 Feb 2024 16:14:53 +0000 Subject: [PATCH 09/19] Update example in Set-ItResult comment based help --- src/functions/Set-ItResult.ps1 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index d9112d0e8..06c025673 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -29,6 +29,9 @@ .EXAMPLE ```powershell Describe "Example" { + It "Inconclusive test" { + Set-ItResult -Inconclusive -Because "we want it to be inconclusice" + } It "Skipped test" { Set-ItResult -Skipped -Because "we want it to be skipped" } @@ -38,9 +41,11 @@ the output should be ``` - [!] Skipped test is skipped, because we want it to be skipped - Tests completed in 0ms - Tests Passed: 0, Failed: 0, Skipped: 0, Pending: 0, Inconclusive 1 + Describing Example + [?] Inconclusive test is inconclusive, because INCONCLUSIVE: we want it to be inconclusice 35ms (32ms|3ms) + [!] Skipped test is skipped, because we want it to be skipped 3ms (2ms|1ms) + Tests completed in 78ms + Tests Passed: 0, Failed: 0, Skipped: 1, Inconclusive: 1, NotRun: 0 ``` .LINK From 9ba12d50b5251d8ccb1f7bb72d30a70711c4407b Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sat, 3 Feb 2024 16:16:10 +0000 Subject: [PATCH 10/19] Test for Inconclusive tests and count in ResultObject --- tst/Pester.RSpec.ResultObject.ts.ps1 | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tst/Pester.RSpec.ResultObject.ts.ps1 b/tst/Pester.RSpec.ResultObject.ts.ps1 index cd518bbc7..69439ebd0 100644 --- a/tst/Pester.RSpec.ResultObject.ts.ps1 +++ b/tst/Pester.RSpec.ResultObject.ts.ps1 @@ -100,6 +100,14 @@ i -PassThru:$PassThru { It "not run" -Tag "Slow" { 1 | Should -Be 1 } + + It "Set-ItResult -Inconclusive" { + Set-ItResult -Inconclusive + } + + It "Set-ItResult -Skipped" { + Set-ItResult -Skipped + } } } $result = Invoke-Pester -Configuration @{ @@ -119,7 +127,7 @@ i -PassThru:$PassThru { $result | Verify-Property "Containers" $result.Containers.Count | Verify-Equal 2 - $result.TotalCount | Verify-Equal 4 + $result.TotalCount | Verify-Equal 6 $result.Tests | Verify-NotNull $result.PassedCount | Verify-Equal 1 @@ -128,12 +136,15 @@ i -PassThru:$PassThru { $result.FailedCount | Verify-Equal 1 $result.Failed | Verify-NotNull - $result.SkippedCount | Verify-Equal 1 + $result.SkippedCount | Verify-Equal 2 $result.Skipped | Verify-NotNull $result.NotRunCount | Verify-Equal 1 $result.NotRun | Verify-NotNull + $result.InconclusiveCount | Verify-Equal 1 + $result.Inconclusive | Verify-NotNull + $result.Duration | Verify-Equal ($result.Containers[0].Duration + $result.Containers[1].Duration) $result.UserDuration | Verify-Equal ($result.Containers[0].UserDuration + $result.Containers[1].UserDuration) $result.FrameworkDuration | Verify-Equal ($result.Containers[0].FrameworkDuration + $result.Containers[1].FrameworkDuration) From a21a5506487797f59651a6ef675deee5a4162f65 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sat, 3 Feb 2024 21:14:09 +0000 Subject: [PATCH 11/19] Adds inconclusive and skipped tests to nunit report 'ts' files --- tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 | 98 +++++++++++++++++ tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 113 +++++++++++++++++++- 2 files changed, 208 insertions(+), 3 deletions(-) diff --git a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 index 3712bc6ed..8b81d3d7f 100644 --- a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 @@ -144,6 +144,51 @@ i -PassThru:$PassThru { } + t "should write a skipped test result" { + $sb = { + Describe "Mocked Describe 1" { + It "Skipped testcase" -Skip { + } + } + Describe "Mocked Describe 2" { + It "Skipped testcase" { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport + $xmlTestSuite = $xmlResult.'test-results'.'test-suite'.'results'.'test-suite'.'results'.'test-suite' + $xmlTestCase1 = $xmlTestSuite.results.'test-case'[0] + $xmlTestCase2 = $xmlTestSuite.results.'test-case'[1] + + $xmlTestCase1.name | Verify-Equal "Mocked Describe 1.Skipped testcase" + $xmlTestCase1.result | Verify-Equal "Ignored" + $xmlTestCase1.time | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + + $xmlTestCase2.name | Verify-Equal "Mocked Describe 2.Skipped testcase" + $xmlTestCase2.result | Verify-Equal "Ignored" + $xmlTestCase2.time | Verify-XmlTime $r.Containers[0].Blocks[1].Tests[0].Duration + } + + t "should write an inconclusive test result" { + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport + $xmlTestCase = $xmlResult.'test-results'.'test-suite'.'results'.'test-suite'.'results'.'test-suite'.'results'.'test-case' + $xmlTestCase.name | Verify-Equal "Mocked Describe.Inconclusive testcase" + $xmlTestCase.result | Verify-Equal "Inconclusive" + $xmlTestCase.time | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + } + t "should write the test summary" { $sb = { Describe "Mocked Describe" { @@ -162,6 +207,59 @@ i -PassThru:$PassThru { $xmlTestResult.time | Verify-Equal (Get-Date -Format "HH:mm:ss" $r.ExecutedAt) } + t "should write inconclusive count" { + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.inconclusive | Verify-Equal 1 + + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase 1" { + Set-ItResult -Inconclusive + } + It "Inconclusive testcase 2" { + Set-ItResult -Inconclusive + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.inconclusive | Verify-Equal 2 + } + + t "should write skipped count" { + $sb = { + Describe "Mocked Describe" { + It "Skipped testcase" { + Set-ItResult -Skipped + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.skipped | Verify-Equal 1 + + $sb = { + Describe "Mocked Describe" { + It "Skipped testcase 1" -Skip { + } + It "Skippde testcase 2" { + Set-ItResult -Skipped + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.skipped | Verify-Equal 2 + } + t "should write the test-suite information" { $sb = { Describe "Mocked Describe" { diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index e81f147f9..e8715db26 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -147,6 +147,60 @@ i -PassThru:$PassThru { } + t 'should write a skipped test result' { + $sb = { + Describe 'Describe 1' { + It 'Skipped testcase' -Skip { + } + } + Describe 'Describe 2' { + It 'Skipped testcase' { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlTestSuite = $xmlResult.'test-run'.'test-suite'.'test-suite' + $xmlTestCase1 = $xmlTestSuite.'test-case'[0] + $xmlTestCase2 = $xmlTestSuite.'test-case'[1] + + $xmlTestCase1.name | Verify-Equal 'Describe 1.Skipped testcase' + $xmlTestCase1.methodname | Verify-Equal 'Skipped testcase' + $xmlTestCase1.classname | Verify-Equal 'Describe 1' + $xmlTestCase1.fullname | Verify-Equal 'Describe 1.Skipped testcase' + $xmlTestCase1.result | Verify-Equal 'Skipped' + $xmlTestCase1.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + + $xmlTestCase2.name | Verify-Equal 'Describe 2.Skipped testcase' + $xmlTestCase2.methodname | Verify-Equal 'Skipped testcase' + $xmlTestCase2.classname | Verify-Equal 'Describe 2' + $xmlTestCase2.fullname | Verify-Equal 'Describe 2.Skipped testcase' + $xmlTestCase2.result | Verify-Equal 'Skipped' + $xmlTestCase2.duration | Verify-XmlTime $r.Containers[0].Blocks[1].Tests[0].Duration + } + + t 'should write an inconclusive test result' { + $sb = { + Describe 'Describe' { + It 'Inconclusive testcase' { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlTestCase = $xmlResult.'test-run'.'test-suite'.'test-suite'.'test-case' + $xmlTestCase.name | Verify-Equal 'Describe.Inconclusive testcase' + $xmlTestCase.methodname | Verify-Equal 'Inconclusive testcase' + $xmlTestCase.classname | Verify-Equal 'Describe' + $xmlTestCase.fullname | Verify-Equal 'Describe.Inconclusive testcase' + $xmlTestCase.result | Verify-Equal 'Inconclusive' + $xmlTestCase.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + } + t 'should write the test summary' { $sb = { Describe 'Describe' { @@ -167,6 +221,59 @@ i -PassThru:$PassThru { $xmlTestResult.'end-time' | Verify-Equal (($r.ExecutedAt + $r.Duration).ToUniversalTime().ToString('o')) } + t "should write inconclusive count" { + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.inconclusive | Verify-Equal 1 + + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase 1" { + Set-ItResult -Inconclusive + } + It "Inconclusive testcase 2" { + Set-ItResult -Inconclusive + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.inconclusive | Verify-Equal 2 + } + + t "should write skipped count" { + $sb = { + Describe "Mocked Describe" { + It "Skipped testcase" { + Set-ItResult -Skipped + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.skipped | Verify-Equal 1 + + $sb = { + Describe "Mocked Describe" { + It "Skipped testcase 1" -Skip { + } + It "Skippde testcase 2" { + Set-ItResult -Skipped + } + } + } + $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.skipped | Verify-Equal 2 + } + t 'should write the test-suite information' { $sb = { Describe 'Describe' { @@ -317,7 +424,7 @@ i -PassThru:$PassThru { t 'should add tags as Category-properties to blocks and tests' { $sb = { Describe 'Describe' -Tag 'abc' { - It 'Successful testcase' -Tag 'hello','world' { + It 'Successful testcase' -Tag 'hello', 'world' { $true | Should -Be $true } } @@ -524,7 +631,7 @@ i -PassThru:$PassThru { # behavior based on NUnit3 runner $sb = { Describe 'Describe' { - It 'Testcase <_>' -Tag 'hello','world' -ForEach @(1,2) { + It 'Testcase <_>' -Tag 'hello', 'world' -ForEach @(1, 2) { $true | Should -Be $true } } @@ -681,7 +788,7 @@ i -PassThru:$PassThru { t 'should add tags as Category-properties on child test-suites only' { # behavior based on NUnit3 runner $sb = { - Describe 'Describe <_>' -Tag 'abc' -ForEach @(1,2) { + Describe 'Describe <_>' -Tag 'abc' -ForEach @(1, 2) { It 'Testcase' { $true | Should -Be $true } From ce3da790e7470e6e8072a74ee3683123587c8051 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 5 Apr 2024 12:11:01 +0000 Subject: [PATCH 12/19] Suppress output and use consistent configuration style --- tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 | 8 ++++---- tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 index 8b81d3d7f..ffba530c1 100644 --- a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 @@ -215,7 +215,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport $xmlResult.'test-results'.inconclusive | Verify-Equal 1 @@ -229,7 +229,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport $xmlResult.'test-results'.inconclusive | Verify-Equal 2 } @@ -242,7 +242,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport $xmlResult.'test-results'.skipped | Verify-Equal 1 @@ -255,7 +255,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport $xmlResult.'test-results'.skipped | Verify-Equal 2 } diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index e8715db26..0456edb04 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -229,7 +229,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.inconclusive | Verify-Equal 1 @@ -243,7 +243,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.inconclusive | Verify-Equal 2 } @@ -256,7 +256,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.skipped | Verify-Equal 1 @@ -269,7 +269,7 @@ i -PassThru:$PassThru { } } } - $r = invoke-pester -container ( new-pestercontainer -ScriptBlock $sb) -passThru -Output Detailed + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.skipped | Verify-Equal 2 } From b77ce8a224e2a94f1f24e29e6e59270c3d903cc5 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 5 Apr 2024 12:18:26 +0000 Subject: [PATCH 13/19] Adds skipped and inconclusive testcases to schema validation test --- tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 | 16 ++++++++++++++++ tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 index ffba530c1..0e4a1f405 100644 --- a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 @@ -344,6 +344,22 @@ i -PassThru:$PassThru { $false | Should -Be $true 5 } } + + Describe "Describe #3" { + It "Skipped testcase #1" -Skip {} + } + + Describe "Describe #4" { + It "Skipped testcase #2" { + Set-ItResult -Skipped + } + } + + Describe "Describe #5" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } } $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index 0456edb04..897752d92 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -365,6 +365,22 @@ i -PassThru:$PassThru { $false | Should -Be $true } } + + Describe "Describe #3" { + It "Skipped testcase #1" -Skip {} + } + + Describe "Describe #4" { + It "Skipped testcase #2" { + Set-ItResult -Skipped + } + } + + Describe "Describe #5" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } } $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) From 82e93f3e6b03d3f935bde75c4f62c0a22e683c1e Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 5 Apr 2024 14:57:08 +0000 Subject: [PATCH 14/19] Refactors nunit reports inconclusive count and skipped count tests --- tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 | 30 ++++-------------- tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 34 ++++++--------------- 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 index 0e4a1f405..996e1197b 100644 --- a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 @@ -209,21 +209,12 @@ i -PassThru:$PassThru { t "should write inconclusive count" { $sb = { - Describe "Mocked Describe" { - It "Inconclusive testcase" { - Set-ItResult -Inconclusive - } - } - } - $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) - $xmlResult = $r | ConvertTo-NUnitReport - $xmlResult.'test-results'.inconclusive | Verify-Equal 1 - - $sb = { - Describe "Mocked Describe" { + Describe "Mocked Describe 1" { It "Inconclusive testcase 1" { Set-ItResult -Inconclusive } + } + Describe "Mocked Describe 2" { It "Inconclusive testcase 2" { Set-ItResult -Inconclusive } @@ -236,20 +227,11 @@ i -PassThru:$PassThru { t "should write skipped count" { $sb = { - Describe "Mocked Describe" { - It "Skipped testcase" { - Set-ItResult -Skipped - } - } - } - $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) - $xmlResult = $r | ConvertTo-NUnitReport - $xmlResult.'test-results'.skipped | Verify-Equal 1 - - $sb = { - Describe "Mocked Describe" { + Describe "Mocked Describe 1" { It "Skipped testcase 1" -Skip { } + } + Describe "Mocked Describe 2" { It "Skippde testcase 2" { Set-ItResult -Skipped } diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index 897752d92..6aa07f673 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -223,21 +223,12 @@ i -PassThru:$PassThru { t "should write inconclusive count" { $sb = { - Describe "Mocked Describe" { - It "Inconclusive testcase" { - Set-ItResult -Inconclusive - } - } - } - $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) - $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 - $xmlResult.'test-run'.inconclusive | Verify-Equal 1 - - $sb = { - Describe "Mocked Describe" { + Describe "Mocked Describe 1" { It "Inconclusive testcase 1" { Set-ItResult -Inconclusive } + } + Describe "Mocked Describe 2" { It "Inconclusive testcase 2" { Set-ItResult -Inconclusive } @@ -246,24 +237,17 @@ i -PassThru:$PassThru { $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.inconclusive | Verify-Equal 2 + $xmlResult.'test-run'.'test-suite'.'test-suite'[0].inconclusive | Verify-Equal 1 + $xmlResult.'test-run'.'test-suite'.'test-suite'[1].inconclusive | Verify-Equal 1 } t "should write skipped count" { $sb = { - Describe "Mocked Describe" { - It "Skipped testcase" { - Set-ItResult -Skipped - } - } - } - $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) - $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 - $xmlResult.'test-run'.skipped | Verify-Equal 1 - - $sb = { - Describe "Mocked Describe" { + Describe "Mocked Describe 1" { It "Skipped testcase 1" -Skip { } + } + Describe "Mocked Describe 2" { It "Skippde testcase 2" { Set-ItResult -Skipped } @@ -272,6 +256,8 @@ i -PassThru:$PassThru { $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlResult.'test-run'.skipped | Verify-Equal 2 + $xmlResult.'test-run'.'test-suite'.'test-suite'[0].skipped | Verify-Equal 1 + $xmlResult.'test-run'.'test-suite'.'test-suite'[1].skipped | Verify-Equal 1 } t 'should write the test-suite information' { From 6cf60c6c3980b7efe6612321a311dd4e281e0caf Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Fri, 5 Apr 2024 15:01:36 +0000 Subject: [PATCH 15/19] Removes unnecessary assertions --- tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index 6aa07f673..bc593ff59 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -167,16 +167,10 @@ i -PassThru:$PassThru { $xmlTestCase2 = $xmlTestSuite.'test-case'[1] $xmlTestCase1.name | Verify-Equal 'Describe 1.Skipped testcase' - $xmlTestCase1.methodname | Verify-Equal 'Skipped testcase' - $xmlTestCase1.classname | Verify-Equal 'Describe 1' - $xmlTestCase1.fullname | Verify-Equal 'Describe 1.Skipped testcase' $xmlTestCase1.result | Verify-Equal 'Skipped' $xmlTestCase1.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration $xmlTestCase2.name | Verify-Equal 'Describe 2.Skipped testcase' - $xmlTestCase2.methodname | Verify-Equal 'Skipped testcase' - $xmlTestCase2.classname | Verify-Equal 'Describe 2' - $xmlTestCase2.fullname | Verify-Equal 'Describe 2.Skipped testcase' $xmlTestCase2.result | Verify-Equal 'Skipped' $xmlTestCase2.duration | Verify-XmlTime $r.Containers[0].Blocks[1].Tests[0].Duration } @@ -194,9 +188,6 @@ i -PassThru:$PassThru { $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 $xmlTestCase = $xmlResult.'test-run'.'test-suite'.'test-suite'.'test-case' $xmlTestCase.name | Verify-Equal 'Describe.Inconclusive testcase' - $xmlTestCase.methodname | Verify-Equal 'Inconclusive testcase' - $xmlTestCase.classname | Verify-Equal 'Describe' - $xmlTestCase.fullname | Verify-Equal 'Describe.Inconclusive testcase' $xmlTestCase.result | Verify-Equal 'Inconclusive' $xmlTestCase.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration } From 066ca5dae44cc6e651100342cbcb51c311830273 Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Sun, 7 Apr 2024 17:56:00 +0000 Subject: [PATCH 16/19] Set Inconclusive result in NUnit reports --- src/functions/TestResults.NUnit25.ps1 | 13 ++++++++----- src/functions/TestResults.NUnit3.ps1 | 5 ++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/functions/TestResults.NUnit25.ps1 b/src/functions/TestResults.NUnit25.ps1 index 04593e9e7..3d3cf6a41 100644 --- a/src/functions/TestResults.NUnit25.ps1 +++ b/src/functions/TestResults.NUnit25.ps1 @@ -13,7 +13,7 @@ } function Write-NUnitTestResultAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('xmlns', 'xsi', $null, 'http://www.w3.org/2001/XMLSchema-instance') @@ -32,7 +32,7 @@ function Write-NUnitTestResultAttributes { } function Write-NUnitTestResultChildNodes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) Write-NUnitEnvironmentInformation -Result $Result -XmlWriter $XmlWriter @@ -98,7 +98,7 @@ function Write-NUnitCultureInformation { } function Write-NUnitTestSuiteElements { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Node, [System.Xml.XmlWriter] $XmlWriter, [string] $Path) $suiteInfo = Get-TestSuiteInfo -TestSuite $Node -Path $Path @@ -249,7 +249,7 @@ function Get-TestSuiteInfo { } function Write-NUnitTestSuiteAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestSuiteInfo, [string] $TestSuiteType = 'TestFixture', [System.Xml.XmlWriter] $XmlWriter, [string] $Path) $name = $TestSuiteInfo.Name @@ -279,7 +279,7 @@ function Write-NUnitTestCaseElement { } function Write-NUnitTestCaseAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $ParameterizedSuiteName) $testName = $TestResult.ExpandedPath @@ -399,5 +399,8 @@ function Get-GroupResult ($InputObject) { if ($InputObject.PendingCount -gt 0) { return 'Inconclusive' } + if ($InputObject.InconclusiveCount -gt 0) { + return 'Inconclusive' + } return 'Success' } diff --git a/src/functions/TestResults.NUnit3.ps1 b/src/functions/TestResults.NUnit3.ps1 index 5a80318b2..5ac45d5be 100644 --- a/src/functions/TestResults.NUnit3.ps1 +++ b/src/functions/TestResults.NUnit3.ps1 @@ -345,9 +345,12 @@ function Get-NUnit3Result ($InputObject) { elseif ($InputObject.SkippedCount -gt 0) { 'Skipped' } - else { + elseif ($InputObject.PassedCount -gt 0) { 'Passed' } + else { + 'Inconclusive' + } } function Get-NUnit3Site ($Node) { From 9d80eb5f9e1faec7cbba6cffb921b273b229304f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 10 Apr 2024 23:09:12 +0200 Subject: [PATCH 17/19] Add cheaper check for deprecation, and a test, remove INCONCLUSIVE from message --- src/Pester.Runtime.ps1 | 6 ++++++ src/functions/Output.ps1 | 5 ++--- src/functions/Set-ItResult.ps1 | 8 ++------ tst/Pester.RSpec.Output.ts.ps1 | 21 +++++++++++++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index d682e9d32..07c23d57b 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -692,6 +692,12 @@ function Invoke-TestItem { } else { $Test.Skipped = $true + + # Pending test is still considered a skipped, we don't have a special category for it. + # Mark the run to show deprecation message. + if ('PesterTestPending' -eq $Result.ErrorRecord.FullyQualifiedErrorId) { + $test.Block.Root.FrameworkData['ShowPendingDeprecation'] = $true + } } } else { diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 6e09d0357..027e4abb2 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -405,9 +405,8 @@ function Write-PesterReport { # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive # } - if ($RunResult.Tests.ErrorRecord.FullyQualifiedErrorID -contains 'PesterTestPending') { - Write-PesterHostMessage '' - Write-PesterHostMessage '**DEPRECATED**: The Set-ItResult -Pending parameter is deprecated. It will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + if ($RunResult.Containers.Blocks.Root.FrameworkData['ShowPendingDeprecation'] -eq $true) { + Write-PesterHostMessage '**DEPRECATED**: The -Pending parameter of Set-ItResult is deprecated. The parameter will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning } } diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index 06c025673..6371eebc1 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -30,7 +30,7 @@ ```powershell Describe "Example" { It "Inconclusive test" { - Set-ItResult -Inconclusive -Because "we want it to be inconclusice" + Set-ItResult -Inconclusive -Because "we want it to be inconclusive" } It "Skipped test" { Set-ItResult -Skipped -Because "we want it to be skipped" @@ -42,7 +42,7 @@ ``` Describing Example - [?] Inconclusive test is inconclusive, because INCONCLUSIVE: we want it to be inconclusice 35ms (32ms|3ms) + [?] Inconclusive test is inconclusive, because we want it to be inconclusive 35ms (32ms|3ms) [!] Skipped test is skipped, because we want it to be skipped 3ms (2ms|1ms) Tests completed in 78ms Tests Passed: 0, Failed: 0, Skipped: 1, Inconclusive: 1, NotRun: 0 @@ -63,10 +63,6 @@ $result = $PSCmdlet.ParameterSetName - if ($Result -ne 'Skipped') { - [String]$Because = if ($Because) { $Result.ToUpper(), $Because -join ': ' } else { $Result.ToUpper() } - } - switch ($null) { $File { [String]$File = $MyInvocation.ScriptName diff --git a/tst/Pester.RSpec.Output.ts.ps1 b/tst/Pester.RSpec.Output.ts.ps1 index f900f8d3a..92a72c106 100644 --- a/tst/Pester.RSpec.Output.ts.ps1 +++ b/tst/Pester.RSpec.Output.ts.ps1 @@ -295,4 +295,25 @@ i -PassThru:$PassThru { $normalOutput | Verify-Equal $writehostOutput } } + + b 'Pending is deprecated' { + t 'Shows deprecated message when -pending is used' { + $sb = { + $container = New-PesterContainer -ScriptBlock { + Describe 'd' { + It 'i' { + Set-ItResult -Pending + } + } + } + + Invoke-Pester -Container $container + } + + $output = Invoke-InNewProcess -ScriptBlock $sb + + $deprecated = $output | Select-String -Pattern '\*DEPRECATED\*' + @($deprecated).Count | Verify-Equal 1 + } + } } From c156209238ddc623938bbd0bee8d9295539d8cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 10 Apr 2024 23:13:04 +0200 Subject: [PATCH 18/19] Format because --- src/functions/Set-ItResult.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index 6371eebc1..1d3e97a04 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -94,7 +94,7 @@ } if ($Because) { - [String]$message += ", because $Because" + [String]$message += ", because $(Format-Because $Because)" } throw [Pester.Factory]::CreateErrorRecord( From d5a5e1a370afcb064a52b5601b41e7f71a6a8508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 10 Apr 2024 23:30:37 +0200 Subject: [PATCH 19/19] Fix pending detection on failing run --- src/functions/Output.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 027e4abb2..53712bb26 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -405,8 +405,13 @@ function Write-PesterReport { # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive # } - if ($RunResult.Containers.Blocks.Root.FrameworkData['ShowPendingDeprecation'] -eq $true) { - Write-PesterHostMessage '**DEPRECATED**: The -Pending parameter of Set-ItResult is deprecated. The parameter will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + $rootFrameworkData = @($RunResult.Containers.Blocks.Root.FrameworkData) + foreach ($frameworkData in $rootFrameworkData) { + if ($null -ne $frameworkData -and $frameworkData['ShowPendingDeprecation']) { + Write-PesterHostMessage '**DEPRECATED**: The -Pending parameter of Set-ItResult is deprecated. The parameter will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + # Show it only once. + break + } } }