diff --git a/Examples/Gherkin/Gherkin-ScenarioData.feature b/Examples/Gherkin/Gherkin-ScenarioData.feature index 6841ca678..7caef73fa 100644 --- a/Examples/Gherkin/Gherkin-ScenarioData.feature +++ b/Examples/Gherkin/Gherkin-ScenarioData.feature @@ -23,7 +23,7 @@ Feature: Pester displays scenario data in the console | 1 | 1 | 2 | | 2 | 2 | 4 | And a single column data table: - | PropNames | + | PropertyNames | | ModuleVersion | | GUID | | Author | @@ -36,3 +36,24 @@ Feature: Pester displays scenario data in the console | PrivateData.PSData.Tags | When this scenario is run Then the tables are displayed correctly in the console + + Scenario: Can classify steps as undefined + Given this step definition does not have an implementation + When this scenario is run + Then all of these steps are classified as undefined + +Scenario Outline: Pester can display scenario example tables to the console + + Given a number '' and a numbxer '' + When I add them together + Then I should get '' + + Examples: Elementary, my dear Watson + | x | y | result | + | 1 | 1 | 2 | + | 2 | 2 | 4 | + + Scenario: Now for something a little bit different + | x | y | result | + | 1 | 2 | 3 | + | 2 | 3 | 5 | diff --git a/Examples/Gherkin/JustForReporting1.feature b/Examples/Gherkin/JustForReporting1.feature index 489613648..35c0814ec 100644 --- a/Examples/Gherkin/JustForReporting1.feature +++ b/Examples/Gherkin/JustForReporting1.feature @@ -10,9 +10,9 @@ Feature: A test feature for reporting 1 Given step_ And and_ - When step_ + When step_ And and_ - Then step_ + Then step_ And and_ Examples: Examples 1 diff --git a/Examples/Gherkin/ScenarioData.Steps.ps1 b/Examples/Gherkin/ScenarioData.Steps.ps1 index c887fdc48..ba00bc58c 100644 --- a/Examples/Gherkin/ScenarioData.Steps.ps1 +++ b/Examples/Gherkin/ScenarioData.Steps.ps1 @@ -1,12 +1,18 @@ Given "the following DocString:?" { param([string]$DocString) - + #throw + Set-StepPending $DocString | Should -Not -BeNull } -When "this scenario is run" { } +When "this scenario is run" { #Set-StepPending +} Then "the DocString is displayed in the console" { } GherkinStep "a (square|rectangular|single column) data table:?" { param($table) } Then "the tables are displayed correctly in the console" { } + +Given "a number '(\d+)' and a number '(\d+)" { param([int]$x, [int]$y) } +When "I add them together" { } +Then "I should get '(\d+)" { param([int]$result) } diff --git a/Functions/Assertions/Set-StepPending.ps1 b/Functions/Assertions/Set-StepPending.ps1 new file mode 100644 index 000000000..6e25c0fc5 --- /dev/null +++ b/Functions/Assertions/Set-StepPending.ps1 @@ -0,0 +1,70 @@ +function New-UndefinedStepErrorRecord ([string] $File, [string] $Line, [string] $LineText) { + $exception = New-Object Exception "No matching step definition found." + $errorID = 'PesterUndefinedGherkinStep' + $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult + # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system. + $targetObject = @{Message = $Message; File = $File; Line = $Line; LineText = $LineText} + $errorRecord = New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject + return $errorRecord +} + +function New-PendingStepErrorRecord ([string] $File, [string] $Line, [string] $LineText) { + $exception = New-Object Exception "# TODO: (Pester::Pending)" + $errorID = 'PesterPendingGherkinStep' + $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult + # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system. + $targetObject = @{Message = $Message; File = $File; Line = $Line; LineText = $LineText} + $errorRecord = New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject + return $errorRecord +} + +function Set-StepPending { +<# + + .SYNOPSIS + Set-StepPending used inside Step Definition blocks will cause those steps to be + considered as pending. + + .DESCRIPTION + If Set-StepPending is used inside a step definition block, the test will be + considered as Pending. It's not a passed result, nor a failed result, + but something in between. It indicates that the results of the test could not + be verified. A step definition with a Pending result will be considered as + Inconclusive when output to the NUnitXml report, unless overridden by + specifying pending and inconclusive tests as failed. + + .EXAMPLE + + Invoke-Gherkin + + Given "this step is not yet implemented" { + + Set-StepPending + + } + + The test result. + + Scenario: Tests with steps using Set-StepPending are pending + [?] Given this step is not yet implemented 96ms + # TODO: (Pester::Pending) + at C:\Users\\features\Example.Steps.ps1: line 10 + at C:\Users\\features\MyNewFeature.feature: line 10 + [!] When something else 0ms + [!] Then this should happen 0ms + + 1 scenario (1 pending) + 3 steps (2 skipped, 1 pending) + Tests completed in 408ms + +#> + [CmdletBinding()] + param ( ) + + Assert-DescribeInProgress -CommandName Set-StepPending + $lineText = $MyInvocation.Line.TrimEnd($([System.Environment]::NewLine)) + $line = $MyInvocation.ScriptLineNumber + $file = $MyInvocation.ScriptName + + throw ( New-PendingStepErrorRecord -File $file -Line $line -LineText $lineText) +} diff --git a/Functions/Gherkin.Tests.ps1 b/Functions/Gherkin.Tests.ps1 index 3c3741e69..ac283c088 100644 --- a/Functions/Gherkin.Tests.ps1 +++ b/Functions/Gherkin.Tests.ps1 @@ -277,7 +277,7 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None #-HideStepData } } @@ -302,7 +302,6 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin { 'The Pester test report shows scenario names with examples [Failing Scenario (inconclusive) 1]' ) } - } Describe "Check test results of steps" -Tag Gherkin { @@ -313,7 +312,7 @@ Describe "Check test results of steps" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None -HideStepData } } @@ -369,11 +368,11 @@ Describe "Check test results of steps" -Tag Gherkin { } It "Test result 11 is correct" { - $testResults[10] | Should -Be 'Inconclusive' + $testResults[10] | Should -Be 'Skipped' } It "Test result 12 is correct" { - $testResults[11] | Should -Be 'Inconclusive' + $testResults[11] | Should -Be 'Skipped' } It "Test result 13 is correct" { @@ -381,7 +380,7 @@ Describe "Check test results of steps" -Tag Gherkin { } It "Test result 14 is correct" { - $testResults[13] | Should -Be 'Inconclusive' + $testResults[13] | Should -Be 'Passed' } It "Test result 15 is correct" { @@ -402,7 +401,7 @@ Describe "A generated NUnit report" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\JustForReporting*.feature) -PassThru -Show None -OutputFile $reportFile + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\JustForReporting*.feature) -PassThru -Show None -OutputFile $reportFile -HideStepData } } @@ -522,7 +521,7 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario2Examples1StepsXPath/@result)[3]" | Should -Be "Success" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[4]" | Should -Be "Success" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[5]" | Should -Be "Failure" - Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Ignored" Get-XmlInnerText "$scenario2Examples1StepsXPath[5]/failure/message" | Should -Be "An example error in the then clause" if ($expectFeatureFileNameInStackTrace) { diff --git a/Functions/Gherkin.ps1 b/Functions/Gherkin.ps1 index ffd746336..28065fbf4 100644 --- a/Functions/Gherkin.ps1 +++ b/Functions/Gherkin.ps1 @@ -227,7 +227,11 @@ function Invoke-Gherkin { [switch]$PassThru ) begin { - & $SafeCommands["Import-LocalizedData"] -BindingVariable Script:ReportStrings -BaseDirectory $PesterRoot -FileName Gherkin.psd1 -ErrorAction SilentlyContinue + # & $SafeCommands["Import-LocalizedData"] -BindingVariable Script:ReportStrings -BaseDirectory $PesterRoot -FileName Gherkin.psd1 -ErrorAction SilentlyContinue + & $SafeCommands["Import-LocalizedData"] -BindingVariable GherkinReportData -BaseDirectory $PesterRoot -Filename Gherkin.psd1 -ErrorAction SilentlyContinue + + $Script:ReportStrings = $GherkinReportData.ReportStrings + $Script:ReportTheme = $GherkinReportData.ReportTheme #Fallback to en-US culture strings If ([String]::IsNullOrEmpty($ReportStrings)) { @@ -265,20 +269,39 @@ function Invoke-Gherkin { throw "There are no existing failed tests to re-run." } } + $sessionState = Set-SessionStateHint -PassThru -Hint "Caller - Captured in Invoke-Gherkin" -SessionState $PSCmdlet.SessionState - $pester = New-PesterState -TagFilter $Tag -ExcludeTagFilter $ExcludeTag -TestNameFilter $ScenarioName -SessionState $sessionState -Strict $Strict -Show $Show -PesterOption $PesterOption | + $pester = New-PesterState -TagFilter $Tag -ExcludeTagFilter $ExcludeTag -TestNameFilter $ScenarioName -SessionState $sessionState -Strict:$Strict -Show $Show -PesterOption $PesterOption | & $SafeCommands["Add-Member"] -MemberType NoteProperty -Name Features -Value (& $SafeCommands["New-Object"] System.Collections.Generic.List[PSObject] ) -PassThru | & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name FailedScenarios -PassThru -Value { $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | - & $SafeCommands["Where-Object"] { $_.Group | - & $SafeCommands["Where-Object"] { -not $_.Passed } } | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Failed' }) } | & $SafeCommands["Select-Object"] -ExpandProperty Name $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } } | & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name PassedScenarios -PassThru -Value { $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | - & $SafeCommands["Where-Object"] { -not ($_.Group | - & $SafeCommands["Where-Object"] { -not $_.Passed }) } | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -ne 'Passed' }) } | + & $SafeCommands["Select-Object"] -ExpandProperty Name + $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { -not ($Names -contains $_.Name) } + } | + & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name PendingScenarios -PassThru -Value { + $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Pending' }) } | + & $SafeCommands["Select-Object"] -ExpandProperty Name + $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } + } | + & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name UndefinedScenarios -PassThru -Value { + $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Inconclusive' }) } | & $SafeCommands["Select-Object"] -ExpandProperty Name $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } } @@ -297,7 +320,7 @@ function Invoke-Gherkin { $Location | & $SafeCommands["Set-Location"] [Environment]::CurrentDirectory = $CWD - $pester | Write-PesterReport + $pester | Write-GherkinReport $coverageReport = Get-CoverageReport -PesterState $pester Write-CoverageReport -CoverageReport $coverageReport Exit-CoverageAnalysis -PesterState $pester @@ -309,7 +332,7 @@ function Invoke-Gherkin { if ($PassThru) { # Remove all runtime properties like current* and Scope $properties = @( - "Path", "Features", "TagFilter", "TestNameFilter", "TotalCount", "PassedCount", "FailedCount", "Time", "TestResult", "PassedScenarios", "FailedScenarios" + "Path", "Features", "TagFilter", "TestNameFilter", "TotalCount", "PassedCount", "FailedCount", "Time", "TestResult", "PassedScenarios", "FailedScenarios", "PendingScenarios", "UndefinedScenarios" if ($CodeCoverage) { @{ Name = 'CodeCoverage'; Expression = { $coverageReport } } @@ -399,6 +422,7 @@ function Import-GherkinFeature { $Scenarios = $( :scenarios foreach ($Child in $Feature.Children) { $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Child.Location -Name "Path" -Value $Path + $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Child -Name "Result" -Value 'Undefined' -Force foreach ($Step in $Child.Steps) { $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Step.Location -Name "Path" -Value $Path } @@ -416,17 +440,66 @@ function Import-GherkinFeature { } } - if( $Scenario -is [Gherkin.Ast.ScenarioOutline] ) { + if ($Scenario -is [Gherkin.Ast.ScenarioOutline]) { + $ScenarioName = $Scenario.Name + $exampleTableName = "" + $formattedExampleTable = "" + # If there is no example set name, the following index will be included in the scenario name - $ScenarioIndex = 0 - foreach ($ExampleSet in $Scenario.Examples) { - ${Column Names} = @($ExampleSet.TableHeader.Cells | & $SafeCommands["Select-Object"] -ExpandProperty Value) + $ExampleTableIndex = 0 + + foreach ($ExampleTable in $Scenario.Examples) { + # TODO: Rename 'HideStepData' to something more appropriate, such as, 'PrintTablesAndDocStrings' + if (!$HideStepData) { + #region Get formatted representation of examples table to be printed to the console + # TODO: Pull some of this logic out into its own function. I duplicated a lot of this from below where I formatted step DataTable arguments + + $exampleTableRowValues = $ExampleTable.TableBody | + & $SafeCommands['ForEach-Object'] { ,@($_ | + & $SafeCommands['Select-Object'] -ExpandProperty Cells | + & $SafeCommands['Select-Object'] -ExpandProperty Value) } + + if ($ExampleTable.TableBody[0].Cells.Length -gt 1) { + $transposedExampleTableRowValues = for ($i = $exampleTableRowValues[0].Length - 1; $i -ge 0; $i--) { + ,@(for ($j = 0; $j -lt $exampleTableRowValues.Count; $j++) { + $exampleTableRowValues[$j][$i] + }) + } + + [Array]::Reverse($transposedExampleTableRowValues) + $cellWidths = $transposedExampleTableRowValues | + & $SafeCommands['ForEach-Object'] { $_ | + & $SafeCommands['Measure-Object'] -Property Length -Maximum | + & $SafeCommands['Select-Object'] -ExpandProperty Maximum } + } else { + $cellWidths = @($exampleTableRowValues | + & $SafeCommands['ForEach-Object'] { $_ } | + & $SafeCommands['Measure-Object'] -Property Length -Maximum | + & $SafeCommands['Select-Object'] -ExpandProperty Maximum) + } + + #$formattedExampleTable = "`n {0}:{1}" -f $ExampleTable.Keyword," $($ExampleTable.Name)".Trim() + + foreach ($row in $ExampleTable.TableBody) { + $rowText = " |" + for ($j = 0; $j -lt $row.Cells.Count; $j++) { + $rowText += " {0,$(-$cellWidths[$j])} |" -f $row.Cells[$j].Value + } + + $formattedExampleTable += "$([Environment]::NewLine)$rowText" + } + + #endregion + } + + ${Column Names} = @($ExampleTable.TableHeader.Cells | & $SafeCommands["Select-Object"] -ExpandProperty Value) $NamesPattern = "<(?:" + (${Column Names} -join "|") + ")>" + # If there is an example set name, the following index will be included in the scenario name - $ExampleSetIndex = 0 - foreach ($Example in $ExampleSet.TableBody) { + $ExampleIndex = 0 + foreach ($Example in $ExampleTable.TableBody) { $ScenarioIndex++ - $ExampleSetIndex++ + $ExampleIndex++ $Steps = foreach ($Step in $Scenario.Steps) { [string]$StepText = $Step.Text if ($StepText -match $NamesPattern) { @@ -437,21 +510,47 @@ function Import-GherkinFeature { } } } + if ($StepText -ne $Step.Text) { - & $SafeCommands["New-Object"] Gherkin.Ast.Step $Step.Location, $Step.Keyword.Trim(), $StepText, $Step.Argument + & $SafeCommands["New-Object"] Gherkin.Ast.Step $Step.Location, $Step.Keyword.Trim(), $StepText, $Step.Argument } else { $Step } } - $ScenarioName = $Scenario.Name - if ($ExampleSet.Name) { - # Include example set name and index of example - $ScenarioName = $ScenarioName + " [$($ExampleSet.Name.Trim()) $ExampleSetIndex]" + + $exampleTablename = if ($null -ne $ExampleTable.Name) { + "{0}: {1}" -f $ExampleTable.Keyword,$ExampleTable.Name.Trim() } else { - # Only include index of scenario - $ScenarioName = $ScenarioName + " [$ScenarioIndex]" + "{0}: Table {1}" -f $ExampleTable.Keyword,$ExampleTableIndex + } + + # If we're not showing DataTables or DocScrings in the console, and the DataTable + # has a description (name), add the example DataTable name to the scenario name, + # else, create a generic DataTable name to be added to the scenario name, and also + # add a 1-based index of the associated example row to the scenario name (since + # each row in the DataTable reperesents a scenario). + # TODO: Don't change the scenario name. Instead, we should do this where we write the NUnitXml, + # TODO: because this function is only responsible for parsing the feature file and creating + # TODO: features, scenarios, and steps for execution. + + # TODO: Investigate NUnitXML and how TestCases are output -- because example table scenarios are really TestCases + # TODO: So, having a separate scenario for each table row is going to cause problems with pretty-printing tables + # Might need to introduce the concept of "scenario sets"--which could be facilitated just like Scenario Outlines + # in gherkin. Essentially, a Scenario is a Scenario outline with a single example, the secnario itself. + # The real problem here, is that we're trying to create "printable" data along with test execution data. + # These separate concerns need to be, well, separated. + # Since in Gherkin tests, the unit of "printability" is the feature and its "scenario outlines" (again, where + # a single scenario is a scenario outline with a single example), this "printable" data should be stored + # separate from the test execution data. + if ($HideStepData) { + $ScenarioName += ", $exampleTableName (Example $ExampleIndex)" } - & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $ExampleSet.Tags, $Scenario.Location, $Scenario.Keyword.Trim(), $ScenarioName, $Scenario.Description, $Steps | Convert-Tags $Scenario.Tags + + & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $ExampleSet.Tags, $Scenario.Location, $Scenario.Keyword.Trim(), $ScenarioName, $Scenario.Description, $Steps | + & $SafeCommands['Add-Member'] -MemberType "NoteProperty" -Name "ExampleTableName" -Value $exampleTableName -PassThru | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "ExampleTable" -Value $formattedExampleTable -PassThru | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "Result" -Value 'Inconclusive' -PassThru | + Convert-Tags $Scenario.Tags } } } elseif ($HideStepData) { @@ -505,7 +604,9 @@ function Import-GherkinFeature { $Step } } - & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $null, $Scenario.Location, $Scenario.Keyword.Trim(), $Scenario.Name, $Scenario.Description, $Steps | Convert-Tags $Scenario.Tags + & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $null, $Scenario.Location, $Scenario.Keyword.Trim(), $Scenario.Name, $Scenario.Description, $Steps | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "Result" -Value 'Inconclusive' -PassThru | + Convert-Tags $Scenario.Tags } } ) @@ -613,7 +714,8 @@ function Invoke-GherkinScenario { $script:mockTable = @{} # Create a clean variable scope in each scenario - $script:GherkinScenarioScope = New-Module Scenario { $a = 4 + $script:GherkinScenarioScope = New-Module Scenario { + $a = 4 } $script:GherkinSessionState = Set-SessionStateHint -PassThru -Hint Scenario -SessionState $Script:GherkinScenarioScope.SessionState @@ -623,17 +725,38 @@ function Invoke-GherkinScenario { Invoke-GherkinHook BeforeEachScenario $Scenario.Name $Scenario.Tags $testResultIndexStart = $Pester.TestResult.Count + $previousStepResult = 'Passed'; # If there's a background, run that before the test, but after hooks if ($Background) { foreach ($Step in $Background.Steps) { - # Run Background steps -Background so they don't output in each scenario - Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($previousStepResult -ne 'Passed' -and $previousStepResult -ne 'Inconclusive' <# i.e. Pending #>) { + $Pester.AddTestResult(("{0} {1}" -f $Step.Keyword.Trim(), $Step.Text), "Skipped", $null, $null, $null, $null, $null) + $Pester.TestResult[-1] | Write-PesterResult + } else { + # Run Background steps -Background so they don't output in each scenario + Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($Pester.TestResult[-1].Result -ne 'Passed' -or $Pester.TestResult[-1].Result -ne 'Inconclusive') { + $previousStepResult = $Pester.TestResult[-1].Result + } + } } } foreach ($Step in $Scenario.Steps) { - Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($previousStepResult -ne 'Passed' -and $previousStepResult -ne 'Inconclusive') { + $Pester.AddTestResult(("{0} {1}" -f $Step.Keyword.Trim(), $Step.Text), "Skipped", $null, $null, $null, $null, $null) + $Pester.TestResult[-1] | Write-PesterResult + } else { + Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($Pester.TestResult[-1].Result -ne 'Passed') { + $previousStepResult = $Pester.TestResult[-1].Result + } + } + } + + if ($previousStepResult -ne 'Passed') { + $Scenario.Result = $previousStepResult } Invoke-GherkinHook AfterEachScenario $Scenario.Name $Scenario.Tags @@ -782,18 +905,16 @@ function Invoke-GherkinStep { # Iterate over the test results of the previous steps for ($i = $TestResultIndexStart; $i -lt ($Pester.TestResult.Count); $i++) { $previousTestResult = $Pester.TestResult[$i].Result - if ($previousTestResult -eq "Failed" -or $previousTestResult -eq "Inconclusive") { + if ($previousTestResult -eq "Failed") { $previousStepsNotSuccessful = $true break } } - if (!$StepCommand -or $previousStepsNotSuccessful) { - $skipMessage = if (!$StepCommand) { - "Could not find implementation for step!" - } else { - "Step skipped (previous step did not pass)" - } - $PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message $skipMessage -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText + + if (!$StepCommand) { + $PesterErrorRecord = New-UndefinedStepErrorRecord -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText + } elseif ($previousStepsNotSuccessful) { + $PesterErrorRecord = Set-ItResult -Skipped } else { $NamedArguments, $Parameters = Get-StepParameters $Step $StepCommand $watch = & $SafeCommands["New-Object"] System.Diagnostics.Stopwatch @@ -834,7 +955,10 @@ function Invoke-GherkinStep { # For Gherkin, we want to show the step, but not pretend to be a StackTrace if (${Pester Result}.Result -eq 'Inconclusive') { - ${Pester Result}.StackTrace = "At " + $Step.Keyword.Trim() + ', ' + $Step.Location.Path + ': line ' + $Step.Location.Line + # TODO: Attempt to show skeleton implentation of step?? + ${Pester Result}.StackTrace = "at " + $Step.Keyword.Trim() + ', ' + $Step.Location.Path + ': line ' + $Step.Location.Line + } elseif (${Pester Result}.Result -eq 'Pending') { + ${Pester Result}.StackTrace += "$([Environment]::NewLine)From $($Step.Location.Path): line $($Step.Location.Line)" } else { # Unless we really are a StackTrace... ${Pester Result}.StackTrace += "`nFrom " + $Step.Location.Path + ': line ' + $Step.Location.Line diff --git a/Functions/Output.ps1 b/Functions/Output.ps1 index 1a8c456c0..864c8a2f8 100644 --- a/Functions/Output.ps1 +++ b/Functions/Output.ps1 @@ -185,7 +185,7 @@ function ConvertTo-PesterResult { return $testResult } - if (@('PesterAssertionFailed', 'PesterTestSkipped', 'PesterTestInconclusive', 'PesterTestPending') -contains $ErrorRecord.FullyQualifiedErrorID) { + if (@('PesterAssertionFailed', 'PesterTestSkipped', 'PesterTestInconclusive', 'PesterTestPending', 'PesterUndefinedGherkinStep', 'PesterPendingGherkinStep') -contains $ErrorRecord.FullyQualifiedErrorID) { # we use TargetObject to pass structured information about the error. $details = $ErrorRecord.TargetObject @@ -198,6 +198,8 @@ function ConvertTo-PesterResult { PesterTestInconclusive { $testResult.Result = "Inconclusive"; break; } PesterTestPending { $testResult.Result = "Pending"; break; } PesterTestSkipped { $testResult.Result = "Skipped"; break; } + PesterUndefinedGherkinStep { $testResult.Result = "Inconclusive"; break; } + PesterPendingGherkinStep { $testResult.Result = "Pending"; break; } } } else { $failureMessage = $ErrorRecord.ToString() @@ -308,6 +310,18 @@ function Write-PesterResult { & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.PendingTime $humanTime } } + + if ($TestResult.ErrorRecord.FullyQualifiedErrorId -eq 'PesterPendingGherkinStep') { + if($pester.IncludeVSCodeMarker) { + & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending $($TestResult.failureMessage -replace '(?m)^',($error_margin*2)) + & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending $($TestResult.stackTrace -replace '(?m)^',($error_margin*2)) + } else { + $TestResult | + foreach { ,($_.ErrorRecord.Exception.Message -replace "Exception: ","") + $_.StackTrace -split "\r?\n" } | + & $SafeCommands['Select-Object'] -Index 0,1,3 | + foreach { & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending ($_ -replace '(?m)^',($error_margin*2))} + } + } break } @@ -343,6 +357,153 @@ function Write-PesterResult { } } +function Write-GherkinReport { + param ( + [Parameter(Mandatory = $True, ValueFromPipeline = $True)] + $PesterState + ) + + if (-not ($PesterState.Show | Has-Flag Summary)) { return } + + $Success, $Failure = if($PesterState.FailedCount -gt 0) { + $ReportTheme.Foreground, $ReportTheme.Fail + } else { + $ReportTheme.Pass, $ReportTheme.Information + } + $Skipped = if($PesterState.SkippedCount -gt 0) { $ReportTheme.Skipped } else { $ReportTheme.Information } + $Pending = if($PesterState.PendingCount -gt 0) { $ReportTheme.Pending } else { $ReportTheme.Information } + $Inconclusive = if($PesterState.InconclusiveCount -gt 0) { $ReportTheme.Inconclusive } else { $ReportTheme.Information } + + Try { + $PesterStatePassedScenariosCount = $PesterState.PassedScenarios.Count + } + Catch { + $PesterStatePassedScenariosCount = 0 + } + + Try { + $PesterStateFailedScenariosCount = $PesterState.FailedScenarios.Count + } + Catch { + $PesterStateFailedScenariosCount = 0 + } + + Try { + $PesterStatePendingScenariosCount = $PesterState.PendingScenarios.Count + } + Catch { + $PesterStatePendingScenariosCount = 0 + } + + Try { + $PesterStateUndefinedScenariosCount = $PesterState.UndefinedScenarios.Count + } + Catch { + $PesterStateUndefinedScenariosCount = 0 + } + + $PesterStateTotalScenariosCount = $PesterStateFailedScenariosCount + $PesterStateUndefinedScenariosCount + $PesterStatePendingScenariosCount + $PesterStatePassedScenariosCount + + if($ReportStrings.ContextsPassed) { + [string[]]$PesterStateScenarioSummaryCounts = ,($ReportStrings.ContextSummary -f $PesterStateTotalScenariosCount) + if ($PesterStateTotalScenariosCount -eq 1) { + $PesterStateScenarioSummaryCounts[0] = $PesterStateScenarioSummaryCounts[0] -replace "scenarios","scenario" + } + + $PesterStateScenarioSummaryCounts += @(($ReportStrings.ContextsFailed -f $PesterStateFailedScenariosCount), + ($ReportStrings.ContextsUndefined -f $PesterStateUndefinedScenariosCount), + ($ReportStrings.ContextsPending -f $PesterStatePendingScenariosCount), + ($ReportStrings.ContextsPassed -f $PesterStatePassedScenariosCount)) + + $PesterStateScenarioSummaryData = $PesterStateScenarioSummaryCounts | & $SafeCommands["ForEach-Object"] { + $_ -match "^(?\d+) (?failed|undefined|pending|passed|scenarios \()" | Out-Null + switch ($Matches['Result']) { + failed { $Foreground = $Failure } + undefined { $Foreground = $Inconclusive } + pending { $Foreground = $Pending } + passed { $Foreground = $Success } + default { $Foreground = $ReportTheme.Foreground } + } + + if ($Matches['ScenarioCount'] -gt 0) { + [PSCustomObject]@{ Foreground = $Foreground; Text = $_ } + } + } + + & $SafeCommands['Write-Host'] '' + for ($i = 0; $i -lt $PesterStateScenarioSummaryData.Length; $i++) { + $summaryData = $PesterStateScenarioSummaryData[$i] + if ($i -eq $PesterStateScenarioSummaryData.Length - 1) { + & $SafeCommands['Write-Host'] ($summaryData.Text -replace ", ",'') -Foreground $summaryData.Foreground -NoNewLine + & $SafeCommands['Write-Host'] ")" -Foreground $ReportTheme.Foreground + } else { + & $SafeCommands['Write-Host'] $summaryData.Text -Foreground $summaryData.Foreground -NoNewLine + if ($i) { + &$SafeCommands['Write-Host'] ", " -Foreground $ReportTheme.Foreground -NoNewLine + } + } + } + } + if($ReportStrings.TestsPassed) { + $PesterStateTotalStepsCount = $PesterState.FailedCount + $PesterState.Inconclusive + $PesterState.SkippedCount + $PesterState.PendingCount + $PesterState.PassedCount + [string[]]$PesterStateStepSummaryCounts = ,($ReportStrings.TestsSummary -f $PesterStateTotalStepsCount) + if ($PesterStateTotalStepCount -eq 1) { + $PesterStateStepSummaryCounts[0] = $PesterStateStepSummaryCounts[0] -replace "steps","step" + } + + $PesterStateStepSummaryCounts += @(($ReportStrings.TestsFailed -f $PesterState.FailedCount), + ($ReportStrings.TestsInconclusive -f $PesterState.InconclusiveCount), + ($ReportStrings.TestsSkipped -f $PesterState.SkippedCount), + ($ReportStrings.TestsPending -f $PesterState.PendingCount), + ($ReportStrings.TestsPassed -f $PesterState.PassedCount)) + + $PesterStateStepSummaryData = $PesterStateStepSummaryCounts | & $SafeCommands["ForEach-Object"] { + $_ -match "^(?\d+) (?failed|undefined|skipped|pending|passed|steps \()" | Out-Null + switch ($Matches['Result']) { + failed { $Foreground = $Failure } + undefined { $Foreground = $Inconclusive } + skipped { $Foreground = $Skipped } + pending { $Foreground = $Pending } + passed { $Foreground = $Success } + default { $Foreground = $ReportTheme.Foreground } + } + + if ($Matches['StepCount'] -gt 0) { + [PSCustomObject]@{ Foreground = $Foreground; Text = $_ } + } + } + + for ($i = 0; $i -lt $PesterStateStepSummaryData.Length; $i++) { + $summaryData = $PesterStateStepSummaryData[$i] + if ($i -eq $PesterStateStepSummaryData.Length - 1) { + & $SafeCommands['Write-Host'] ($summaryData.Text -replace ", ",'') -Foreground $summaryData.Foreground -NoNewLine + & $SafeCommands['Write-Host'] ")" -Foreground $ReportTheme.Foreground + } else { + & $SafeCommands['Write-Host'] $summaryData.Text -Foreground $summaryData.Foreground -NoNewLine + if ($i) { + &$SafeCommands['Write-Host'] ", " -Foreground $ReportTheme.Foreground -NoNewLine + } + } + } + } + + & $SafeCommands['Write-Host'] ($ReportStrings.Timing -f (Get-HumanTime $PesterState.Time.TotalSeconds)) -Foreground $ReportTheme.Foreground + & $SafeCommands['Write-Host'] '' + + # TODO: Can we create a method that would auto-generate the Step Definition script blocks to the console for undefined steps? + + # {Count} Scenario(s?) ({FailedCount} failed, {UndefinedCount} undefined, {PendingCount} pending, {PassedCount} passed) + # {Count} Step(s?) ({FailedCount} failed, {UndefinedCount} undefined, {SkippedCount} skipped, {PendingCount} pending, {PassedCount} passed) + # 0m0.002s + # + # You can implement step definitions for undefined steps with these snippets: + # + # Given "the input '(.*?)'" { + # param($arg1) + # Set-TestPending + # } +} + function Write-PesterReport { param ( [Parameter(mandatory=$true, valueFromPipeline=$true)] diff --git a/Pester.psd1 b/Pester.psd1 index 3f44feee0..9fec0d12b 100644 --- a/Pester.psd1 +++ b/Pester.psd1 @@ -49,6 +49,7 @@ FunctionsToExport = @( 'AfterAll' 'Get-MockDynamicParameter' 'Set-DynamicParameterVariable' + 'Set-StepPending' 'Set-TestInconclusive' 'Set-ItResult' 'SafeGetCommand' diff --git a/Pester.psm1 b/Pester.psm1 index 2fd929b18..a2468af12 100644 --- a/Pester.psm1 +++ b/Pester.psm1 @@ -1555,7 +1555,7 @@ Set-SessionStateHint -Hint Pester -SessionState $ExecutionContext.SessionState # in the future rename the function to Add-ShouldOperator Set-Alias -Name Add-ShouldOperator -Value Add-AssertionOperator -& $script:SafeCommands['Export-ModuleMember'] Describe, Context, It, In, Mock, Assert-VerifiableMock, Assert-VerifiableMocks, Assert-MockCalled, Set-TestInconclusive, Set-ItResult +& $script:SafeCommands['Export-ModuleMember'] Describe, Context, It, In, Mock, Assert-VerifiableMock, Assert-VerifiableMocks, Assert-MockCalled, Set-TestInconclusive, Set-ItResult, Set-StepPending & $script:SafeCommands['Export-ModuleMember'] New-Fixture, Get-TestDriveItem, Should, Invoke-Pester, Setup, InModuleScope, Invoke-Mock & $script:SafeCommands['Export-ModuleMember'] BeforeEach, AfterEach, BeforeAll, AfterAll & $script:SafeCommands['Export-ModuleMember'] Get-MockDynamicParameter, Set-DynamicParameterVariable diff --git a/en-US/Gherkin.psd1 b/en-US/Gherkin.psd1 index d4c04ff99..48588cb24 100644 --- a/en-US/Gherkin.psd1 +++ b/en-US/Gherkin.psd1 @@ -1,29 +1,58 @@ @{ - StartMessage = "Testing all features in '{0}'" - FilterMessage = " for scenarios matching '{0}'" - TagMessage = " with tags: '{0}'" - MessageOfs = "', '" + ReportStrings = @{ + StartMessage = "Testing all features in '{0}'" + FilterMessage = " for scenarios matching '{0}'" + TagMessage = " with tags: '{0}'" + MessageOfs = "', '" - CoverageTitle = "Code coverage report:" - CoverageMessage = "Covered {2:P2} of {3:N0} analyzed {0} in {4:N0} {1}." - MissedSingular = 'Missed command:' - MissedPlural = 'Missed commands:' - CommandSingular = 'Command' - CommandPlural = 'Commands' - FileSingular = 'File' - FilePlural = 'Files' + CoverageTitle = "Code coverage report:" + CoverageMessage = "Covered {2:P2} of {3:N0} analyzed {0} in {4:N0} {1}." + MissedSingular = 'Missed command:' + MissedPlural = 'Missed commands:' + CommandSingular = 'Command' + CommandPlural = 'Commands' + FileSingular = 'File' + FilePlural = 'Files' - Describe = "Feature: {0}" - Context = "Scenario: {0}" - Margin = " " - Timing = "Testing completed in {0}" + Describe = "Feature: {0}" + Context = "Scenario: {0}" + Margin = " " + Timing = "Testing completed in {0}" - # If this is set to an empty string, the count won't be printed - ContextsPassed = "Scenarios Passed: {0} " - ContextsFailed = "Failed: {0}" - TestsPassed = "Steps Passed: {0} " - TestsFailed = "Failed: {0} " - TestsSkipped = 'Skipped: {0} ' - TestsPending = 'Pending: {0} ' - TestsInconclusive = 'Inconclusive: {0} ' + # If this is set to an empty string, the count won't be printed + ContextSummary = '{0} scenarios (' + ContextsFailed = '{0} failed' + ContextsUndefined = '{0} undefined' + ContextsPending = '{0} pending' + ContextsPassed = '{0} passed' + TestsSummary = '{0} steps (' + TestsFailed = '{0} failed' + TestsInconclusive = '{0} undefined' + TestsSkipped = '{0} skipped' + TestsPending = '{0} pending' + TestsPassed = '{0} passed' + } + + ReportTheme = @{ + Describe = 'Green' + DescribeDetail = 'DarkYellow' + Context = 'Cyan' + ContextDetail = 'DarkCyan' + Pass = 'DarkGreen' + PassTime = 'DarkGray' + Fail = 'Red' + FailTime = 'DarkGray' + Skipped = 'Yellow' + SkippedTime = 'DarkGray' + Pending = 'Yellow' + PendingTime = 'DarkGray' + Inconclusive = 'Yellow' + InconclusiveTime = 'DarkGray' + Incomplete = 'Yellow' + IncompleteTime = 'DarkGray' + Foreground = 'White' + Information = 'DarkGray' + Coverage = 'White' + CoverageWarn = 'DarkRed' + } }