From 5af19ce0a646e2b93975a1af804a8bca53dc7765 Mon Sep 17 00:00:00 2001 From: fourpastmidnight Date: Tue, 6 Nov 2018 18:29:55 -0500 Subject: [PATCH 1/2] Add a Set-StepPending function to Assertions In Cucumber, when you write a Feature file and then run cucmuber on it without implementing any steps whatsoever, the result is that the scenario (and steps) are considered 'undefined'. For Pester, we currently treat these as Inconclusive (which is fine--we can synonomize Inconclusive with Undefined). However, there's no explicit mechanism to mark a step as pending. This commit adds a Set-StepPending "assertion" which is similar in nature to the Set-TestInconclusive "assertion". This will mark that step as 'Pending'. This commit only adds the Set-StepPending function. The next series of commits will update Pester's Gherkin implementation to use it. --- Functions/Assertions/Set-StepPending.ps1 | 60 ++++++++++++++++++++++++ Pester.psd1 | 1 + Pester.psm1 | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 Functions/Assertions/Set-StepPending.ps1 diff --git a/Functions/Assertions/Set-StepPending.ps1 b/Functions/Assertions/Set-StepPending.ps1 new file mode 100644 index 000000000..db7f09938 --- /dev/null +++ b/Functions/Assertions/Set-StepPending.ps1 @@ -0,0 +1,60 @@ +function New-PendingStepErrorRecord ([string] $File, [string] $Line, [string] $LineText) { + $exception = New-Object Exception "# TODO: (Pester::Pending)" + $errorID = 'PesterPendingGherkinStep' + $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult + # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system. + $targetObject = @{Message = $Message; File = $File; Line = $Line; LineText = $LineText} + $errorRecord = New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject + return $errorRecord +} + +function Set-StepPending { +<# + + .SYNOPSIS + Set-StepPending used inside Step Definition blocks will cause those steps to be + considered as pending. + + .DESCRIPTION + If Set-StepPending is used inside a step definition block, the test will be + considered as Pending. It's not a passed result, nor a failed result, + but something in between. It indicates that the results of the test could not + be verified. A step definition with a Pending result will be considered as + Inconclusive when output to the NUnitXml report, unless overridden by + specifying pending and inconclusive tests as failed. + + .EXAMPLE + + Invoke-Gherkin + + Given "this step is not yet implemented" { + + Set-StepPending + + } + + The test result. + + Scenario: Tests with steps using Set-StepPending are pending + [?] Given this step is not yet implemented 96ms + # TODO: (Pester::Pending) + at C:\Users\\features\Example.Steps.ps1: line 10 + at C:\Users\\features\MyNewFeature.feature: line 10 + [!] When something else 0ms + [!] Then this should happen 0ms + + 1 scenario (1 pending) + 3 steps (2 skipped, 1 pending) + Tests completed in 408ms + +#> + [CmdletBinding()] + param ( ) + + Assert-DescribeInProgress -CommandName Set-StepPending + $lineText = $MyInvocation.Line.TrimEnd($([System.Environment]::NewLine)) + $line = $MyInvocation.ScriptLineNumber + $file = $MyInvocation.ScriptName + + throw ( New-PendingStepErrorRecord -File $file -Line $line -LineText $lineText) +} diff --git a/Pester.psd1 b/Pester.psd1 index 3f44feee0..9fec0d12b 100644 --- a/Pester.psd1 +++ b/Pester.psd1 @@ -49,6 +49,7 @@ FunctionsToExport = @( 'AfterAll' 'Get-MockDynamicParameter' 'Set-DynamicParameterVariable' + 'Set-StepPending' 'Set-TestInconclusive' 'Set-ItResult' 'SafeGetCommand' diff --git a/Pester.psm1 b/Pester.psm1 index 2fd929b18..a2468af12 100644 --- a/Pester.psm1 +++ b/Pester.psm1 @@ -1555,7 +1555,7 @@ Set-SessionStateHint -Hint Pester -SessionState $ExecutionContext.SessionState # in the future rename the function to Add-ShouldOperator Set-Alias -Name Add-ShouldOperator -Value Add-AssertionOperator -& $script:SafeCommands['Export-ModuleMember'] Describe, Context, It, In, Mock, Assert-VerifiableMock, Assert-VerifiableMocks, Assert-MockCalled, Set-TestInconclusive, Set-ItResult +& $script:SafeCommands['Export-ModuleMember'] Describe, Context, It, In, Mock, Assert-VerifiableMock, Assert-VerifiableMocks, Assert-MockCalled, Set-TestInconclusive, Set-ItResult, Set-StepPending & $script:SafeCommands['Export-ModuleMember'] New-Fixture, Get-TestDriveItem, Should, Invoke-Pester, Setup, InModuleScope, Invoke-Mock & $script:SafeCommands['Export-ModuleMember'] BeforeEach, AfterEach, BeforeAll, AfterAll & $script:SafeCommands['Export-ModuleMember'] Get-MockDynamicParameter, Set-DynamicParameterVariable From 26fb0a80e91152da4eefd2ec1c3428c2242a9db0 Mon Sep 17 00:00:00 2001 From: fourpastmidnight Date: Mon, 19 Nov 2018 00:30:21 -0500 Subject: [PATCH 2/2] @wip - add undefined/pending scenarios/steps This commit contains work-in-progress towards the following: * Adding the ability for Gherkin-style Pester tests to recognize and report undefined steps and scenarios * Undefined steps are steps in a feature file for which no PowerShell script block has been written. * TODO: Still need to add output that shows a "suggested" implementation * TODO: If any preceding step definitions are undefined, then any defined steps in the scenario should be skipped. * If at least one scenario step is undefined, then the scenario as a whole is undefined. * Added the ability for Gherkin-style Pester tests to recognize pending steps and scenarios * Pending steps are steps whose implementation explicitly sets the step to pending (via the new Set-StepPending function), which is similar to cucumber's pending function. * Once a pending step has been encountered, all other steps in the scenario are skipped -- afterall, even if other steps are implemented, how can the test possibly pass if there's a pending step? * If at least one scenario step is pending, then the scenario is also Pending. * If a scenario step fails, all other steps are skipped. Afterall, what's the point in running the rest of the scenario if a previous step has failed? The scenario has still failed. * As alluded, if at least one step has failed, then the scenario has failed. * Customized the output of Gherkin style tests to match that of cucumber. * Mostly a stylistic change, which is optional * HOWEVER, the 'Inconclusive' state is used for 'Undefined'. Gherkin doesn't actually make use of an Inconclusive state, and since Pester expects certain states, all of which are used in Gherkin _except_ Inconclusive, it made sense to usurp this and use it for Undefined. * NOTE: I'm actually not happy about having to do this. The mental mapping that needs to take place when working on the Pester code, while simple now, will become much more complex as time goes on. Something more extensible needs to be implemented. * For the stylistic changes, I'm setting up the codebase to allow "output theming". But, this is in its very infancy and is not well thought out yet. --- Examples/Gherkin/Gherkin-ScenarioData.feature | 23 +- Examples/Gherkin/JustForReporting1.feature | 4 +- Examples/Gherkin/ScenarioData.Steps.ps1 | 10 +- Functions/Assertions/Set-StepPending.ps1 | 10 + Functions/Gherkin.Tests.ps1 | 15 +- Functions/Gherkin.ps1 | 198 ++++++++++++++---- Functions/Output.ps1 | 163 +++++++++++++- en-US/Gherkin.psd1 | 77 ++++--- 8 files changed, 425 insertions(+), 75 deletions(-) diff --git a/Examples/Gherkin/Gherkin-ScenarioData.feature b/Examples/Gherkin/Gherkin-ScenarioData.feature index 6841ca678..7caef73fa 100644 --- a/Examples/Gherkin/Gherkin-ScenarioData.feature +++ b/Examples/Gherkin/Gherkin-ScenarioData.feature @@ -23,7 +23,7 @@ Feature: Pester displays scenario data in the console | 1 | 1 | 2 | | 2 | 2 | 4 | And a single column data table: - | PropNames | + | PropertyNames | | ModuleVersion | | GUID | | Author | @@ -36,3 +36,24 @@ Feature: Pester displays scenario data in the console | PrivateData.PSData.Tags | When this scenario is run Then the tables are displayed correctly in the console + + Scenario: Can classify steps as undefined + Given this step definition does not have an implementation + When this scenario is run + Then all of these steps are classified as undefined + +Scenario Outline: Pester can display scenario example tables to the console + + Given a number '' and a numbxer '' + When I add them together + Then I should get '' + + Examples: Elementary, my dear Watson + | x | y | result | + | 1 | 1 | 2 | + | 2 | 2 | 4 | + + Scenario: Now for something a little bit different + | x | y | result | + | 1 | 2 | 3 | + | 2 | 3 | 5 | diff --git a/Examples/Gherkin/JustForReporting1.feature b/Examples/Gherkin/JustForReporting1.feature index 489613648..35c0814ec 100644 --- a/Examples/Gherkin/JustForReporting1.feature +++ b/Examples/Gherkin/JustForReporting1.feature @@ -10,9 +10,9 @@ Feature: A test feature for reporting 1 Given step_ And and_ - When step_ + When step_ And and_ - Then step_ + Then step_ And and_ Examples: Examples 1 diff --git a/Examples/Gherkin/ScenarioData.Steps.ps1 b/Examples/Gherkin/ScenarioData.Steps.ps1 index c887fdc48..ba00bc58c 100644 --- a/Examples/Gherkin/ScenarioData.Steps.ps1 +++ b/Examples/Gherkin/ScenarioData.Steps.ps1 @@ -1,12 +1,18 @@ Given "the following DocString:?" { param([string]$DocString) - + #throw + Set-StepPending $DocString | Should -Not -BeNull } -When "this scenario is run" { } +When "this scenario is run" { #Set-StepPending +} Then "the DocString is displayed in the console" { } GherkinStep "a (square|rectangular|single column) data table:?" { param($table) } Then "the tables are displayed correctly in the console" { } + +Given "a number '(\d+)' and a number '(\d+)" { param([int]$x, [int]$y) } +When "I add them together" { } +Then "I should get '(\d+)" { param([int]$result) } diff --git a/Functions/Assertions/Set-StepPending.ps1 b/Functions/Assertions/Set-StepPending.ps1 index db7f09938..6e25c0fc5 100644 --- a/Functions/Assertions/Set-StepPending.ps1 +++ b/Functions/Assertions/Set-StepPending.ps1 @@ -1,3 +1,13 @@ +function New-UndefinedStepErrorRecord ([string] $File, [string] $Line, [string] $LineText) { + $exception = New-Object Exception "No matching step definition found." + $errorID = 'PesterUndefinedGherkinStep' + $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult + # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system. + $targetObject = @{Message = $Message; File = $File; Line = $Line; LineText = $LineText} + $errorRecord = New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject + return $errorRecord +} + function New-PendingStepErrorRecord ([string] $File, [string] $Line, [string] $LineText) { $exception = New-Object Exception "# TODO: (Pester::Pending)" $errorID = 'PesterPendingGherkinStep' diff --git a/Functions/Gherkin.Tests.ps1 b/Functions/Gherkin.Tests.ps1 index 3c3741e69..ac283c088 100644 --- a/Functions/Gherkin.Tests.ps1 +++ b/Functions/Gherkin.Tests.ps1 @@ -277,7 +277,7 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None #-HideStepData } } @@ -302,7 +302,6 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin { 'The Pester test report shows scenario names with examples [Failing Scenario (inconclusive) 1]' ) } - } Describe "Check test results of steps" -Tag Gherkin { @@ -313,7 +312,7 @@ Describe "Check test results of steps" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None -HideStepData } } @@ -369,11 +368,11 @@ Describe "Check test results of steps" -Tag Gherkin { } It "Test result 11 is correct" { - $testResults[10] | Should -Be 'Inconclusive' + $testResults[10] | Should -Be 'Skipped' } It "Test result 12 is correct" { - $testResults[11] | Should -Be 'Inconclusive' + $testResults[11] | Should -Be 'Skipped' } It "Test result 13 is correct" { @@ -381,7 +380,7 @@ Describe "Check test results of steps" -Tag Gherkin { } It "Test result 14 is correct" { - $testResults[13] | Should -Be 'Inconclusive' + $testResults[13] | Should -Be 'Passed' } It "Test result 15 is correct" { @@ -402,7 +401,7 @@ Describe "A generated NUnit report" -Tag Gherkin { Import-Module $scriptRoot\Pester.psd1 -Force New-Object psobject -Property @{ - Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\JustForReporting*.feature) -PassThru -Show None -OutputFile $reportFile + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\JustForReporting*.feature) -PassThru -Show None -OutputFile $reportFile -HideStepData } } @@ -522,7 +521,7 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario2Examples1StepsXPath/@result)[3]" | Should -Be "Success" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[4]" | Should -Be "Success" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[5]" | Should -Be "Failure" - Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Ignored" Get-XmlInnerText "$scenario2Examples1StepsXPath[5]/failure/message" | Should -Be "An example error in the then clause" if ($expectFeatureFileNameInStackTrace) { diff --git a/Functions/Gherkin.ps1 b/Functions/Gherkin.ps1 index ffd746336..28065fbf4 100644 --- a/Functions/Gherkin.ps1 +++ b/Functions/Gherkin.ps1 @@ -227,7 +227,11 @@ function Invoke-Gherkin { [switch]$PassThru ) begin { - & $SafeCommands["Import-LocalizedData"] -BindingVariable Script:ReportStrings -BaseDirectory $PesterRoot -FileName Gherkin.psd1 -ErrorAction SilentlyContinue + # & $SafeCommands["Import-LocalizedData"] -BindingVariable Script:ReportStrings -BaseDirectory $PesterRoot -FileName Gherkin.psd1 -ErrorAction SilentlyContinue + & $SafeCommands["Import-LocalizedData"] -BindingVariable GherkinReportData -BaseDirectory $PesterRoot -Filename Gherkin.psd1 -ErrorAction SilentlyContinue + + $Script:ReportStrings = $GherkinReportData.ReportStrings + $Script:ReportTheme = $GherkinReportData.ReportTheme #Fallback to en-US culture strings If ([String]::IsNullOrEmpty($ReportStrings)) { @@ -265,20 +269,39 @@ function Invoke-Gherkin { throw "There are no existing failed tests to re-run." } } + $sessionState = Set-SessionStateHint -PassThru -Hint "Caller - Captured in Invoke-Gherkin" -SessionState $PSCmdlet.SessionState - $pester = New-PesterState -TagFilter $Tag -ExcludeTagFilter $ExcludeTag -TestNameFilter $ScenarioName -SessionState $sessionState -Strict $Strict -Show $Show -PesterOption $PesterOption | + $pester = New-PesterState -TagFilter $Tag -ExcludeTagFilter $ExcludeTag -TestNameFilter $ScenarioName -SessionState $sessionState -Strict:$Strict -Show $Show -PesterOption $PesterOption | & $SafeCommands["Add-Member"] -MemberType NoteProperty -Name Features -Value (& $SafeCommands["New-Object"] System.Collections.Generic.List[PSObject] ) -PassThru | & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name FailedScenarios -PassThru -Value { $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | - & $SafeCommands["Where-Object"] { $_.Group | - & $SafeCommands["Where-Object"] { -not $_.Passed } } | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Failed' }) } | & $SafeCommands["Select-Object"] -ExpandProperty Name $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } } | & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name PassedScenarios -PassThru -Value { $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | - & $SafeCommands["Where-Object"] { -not ($_.Group | - & $SafeCommands["Where-Object"] { -not $_.Passed }) } | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -ne 'Passed' }) } | + & $SafeCommands["Select-Object"] -ExpandProperty Name + $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { -not ($Names -contains $_.Name) } + } | + & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name PendingScenarios -PassThru -Value { + $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Pending' }) } | + & $SafeCommands["Select-Object"] -ExpandProperty Name + $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } + } | + & $SafeCommands["Add-Member"] -MemberType ScriptProperty -Name UndefinedScenarios -PassThru -Value { + $Names = $this.TestResult | & $SafeCommands["Group-Object"] Describe | + & $SafeCommands["Where-Object"] { ($_.Group | + & $SafeCommands["Select-Object"] -ExpandProperty Result | + & $SafeCommands["Where-Object"] { $_ -eq 'Inconclusive' }) } | & $SafeCommands["Select-Object"] -ExpandProperty Name $this.Features | Select-Object -ExpandProperty Scenarios | & $SafeCommands["Where-Object"] { $Names -contains $_.Name } } @@ -297,7 +320,7 @@ function Invoke-Gherkin { $Location | & $SafeCommands["Set-Location"] [Environment]::CurrentDirectory = $CWD - $pester | Write-PesterReport + $pester | Write-GherkinReport $coverageReport = Get-CoverageReport -PesterState $pester Write-CoverageReport -CoverageReport $coverageReport Exit-CoverageAnalysis -PesterState $pester @@ -309,7 +332,7 @@ function Invoke-Gherkin { if ($PassThru) { # Remove all runtime properties like current* and Scope $properties = @( - "Path", "Features", "TagFilter", "TestNameFilter", "TotalCount", "PassedCount", "FailedCount", "Time", "TestResult", "PassedScenarios", "FailedScenarios" + "Path", "Features", "TagFilter", "TestNameFilter", "TotalCount", "PassedCount", "FailedCount", "Time", "TestResult", "PassedScenarios", "FailedScenarios", "PendingScenarios", "UndefinedScenarios" if ($CodeCoverage) { @{ Name = 'CodeCoverage'; Expression = { $coverageReport } } @@ -399,6 +422,7 @@ function Import-GherkinFeature { $Scenarios = $( :scenarios foreach ($Child in $Feature.Children) { $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Child.Location -Name "Path" -Value $Path + $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Child -Name "Result" -Value 'Undefined' -Force foreach ($Step in $Child.Steps) { $null = & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -InputObject $Step.Location -Name "Path" -Value $Path } @@ -416,17 +440,66 @@ function Import-GherkinFeature { } } - if( $Scenario -is [Gherkin.Ast.ScenarioOutline] ) { + if ($Scenario -is [Gherkin.Ast.ScenarioOutline]) { + $ScenarioName = $Scenario.Name + $exampleTableName = "" + $formattedExampleTable = "" + # If there is no example set name, the following index will be included in the scenario name - $ScenarioIndex = 0 - foreach ($ExampleSet in $Scenario.Examples) { - ${Column Names} = @($ExampleSet.TableHeader.Cells | & $SafeCommands["Select-Object"] -ExpandProperty Value) + $ExampleTableIndex = 0 + + foreach ($ExampleTable in $Scenario.Examples) { + # TODO: Rename 'HideStepData' to something more appropriate, such as, 'PrintTablesAndDocStrings' + if (!$HideStepData) { + #region Get formatted representation of examples table to be printed to the console + # TODO: Pull some of this logic out into its own function. I duplicated a lot of this from below where I formatted step DataTable arguments + + $exampleTableRowValues = $ExampleTable.TableBody | + & $SafeCommands['ForEach-Object'] { ,@($_ | + & $SafeCommands['Select-Object'] -ExpandProperty Cells | + & $SafeCommands['Select-Object'] -ExpandProperty Value) } + + if ($ExampleTable.TableBody[0].Cells.Length -gt 1) { + $transposedExampleTableRowValues = for ($i = $exampleTableRowValues[0].Length - 1; $i -ge 0; $i--) { + ,@(for ($j = 0; $j -lt $exampleTableRowValues.Count; $j++) { + $exampleTableRowValues[$j][$i] + }) + } + + [Array]::Reverse($transposedExampleTableRowValues) + $cellWidths = $transposedExampleTableRowValues | + & $SafeCommands['ForEach-Object'] { $_ | + & $SafeCommands['Measure-Object'] -Property Length -Maximum | + & $SafeCommands['Select-Object'] -ExpandProperty Maximum } + } else { + $cellWidths = @($exampleTableRowValues | + & $SafeCommands['ForEach-Object'] { $_ } | + & $SafeCommands['Measure-Object'] -Property Length -Maximum | + & $SafeCommands['Select-Object'] -ExpandProperty Maximum) + } + + #$formattedExampleTable = "`n {0}:{1}" -f $ExampleTable.Keyword," $($ExampleTable.Name)".Trim() + + foreach ($row in $ExampleTable.TableBody) { + $rowText = " |" + for ($j = 0; $j -lt $row.Cells.Count; $j++) { + $rowText += " {0,$(-$cellWidths[$j])} |" -f $row.Cells[$j].Value + } + + $formattedExampleTable += "$([Environment]::NewLine)$rowText" + } + + #endregion + } + + ${Column Names} = @($ExampleTable.TableHeader.Cells | & $SafeCommands["Select-Object"] -ExpandProperty Value) $NamesPattern = "<(?:" + (${Column Names} -join "|") + ")>" + # If there is an example set name, the following index will be included in the scenario name - $ExampleSetIndex = 0 - foreach ($Example in $ExampleSet.TableBody) { + $ExampleIndex = 0 + foreach ($Example in $ExampleTable.TableBody) { $ScenarioIndex++ - $ExampleSetIndex++ + $ExampleIndex++ $Steps = foreach ($Step in $Scenario.Steps) { [string]$StepText = $Step.Text if ($StepText -match $NamesPattern) { @@ -437,21 +510,47 @@ function Import-GherkinFeature { } } } + if ($StepText -ne $Step.Text) { - & $SafeCommands["New-Object"] Gherkin.Ast.Step $Step.Location, $Step.Keyword.Trim(), $StepText, $Step.Argument + & $SafeCommands["New-Object"] Gherkin.Ast.Step $Step.Location, $Step.Keyword.Trim(), $StepText, $Step.Argument } else { $Step } } - $ScenarioName = $Scenario.Name - if ($ExampleSet.Name) { - # Include example set name and index of example - $ScenarioName = $ScenarioName + " [$($ExampleSet.Name.Trim()) $ExampleSetIndex]" + + $exampleTablename = if ($null -ne $ExampleTable.Name) { + "{0}: {1}" -f $ExampleTable.Keyword,$ExampleTable.Name.Trim() } else { - # Only include index of scenario - $ScenarioName = $ScenarioName + " [$ScenarioIndex]" + "{0}: Table {1}" -f $ExampleTable.Keyword,$ExampleTableIndex + } + + # If we're not showing DataTables or DocScrings in the console, and the DataTable + # has a description (name), add the example DataTable name to the scenario name, + # else, create a generic DataTable name to be added to the scenario name, and also + # add a 1-based index of the associated example row to the scenario name (since + # each row in the DataTable reperesents a scenario). + # TODO: Don't change the scenario name. Instead, we should do this where we write the NUnitXml, + # TODO: because this function is only responsible for parsing the feature file and creating + # TODO: features, scenarios, and steps for execution. + + # TODO: Investigate NUnitXML and how TestCases are output -- because example table scenarios are really TestCases + # TODO: So, having a separate scenario for each table row is going to cause problems with pretty-printing tables + # Might need to introduce the concept of "scenario sets"--which could be facilitated just like Scenario Outlines + # in gherkin. Essentially, a Scenario is a Scenario outline with a single example, the secnario itself. + # The real problem here, is that we're trying to create "printable" data along with test execution data. + # These separate concerns need to be, well, separated. + # Since in Gherkin tests, the unit of "printability" is the feature and its "scenario outlines" (again, where + # a single scenario is a scenario outline with a single example), this "printable" data should be stored + # separate from the test execution data. + if ($HideStepData) { + $ScenarioName += ", $exampleTableName (Example $ExampleIndex)" } - & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $ExampleSet.Tags, $Scenario.Location, $Scenario.Keyword.Trim(), $ScenarioName, $Scenario.Description, $Steps | Convert-Tags $Scenario.Tags + + & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $ExampleSet.Tags, $Scenario.Location, $Scenario.Keyword.Trim(), $ScenarioName, $Scenario.Description, $Steps | + & $SafeCommands['Add-Member'] -MemberType "NoteProperty" -Name "ExampleTableName" -Value $exampleTableName -PassThru | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "ExampleTable" -Value $formattedExampleTable -PassThru | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "Result" -Value 'Inconclusive' -PassThru | + Convert-Tags $Scenario.Tags } } } elseif ($HideStepData) { @@ -505,7 +604,9 @@ function Import-GherkinFeature { $Step } } - & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $null, $Scenario.Location, $Scenario.Keyword.Trim(), $Scenario.Name, $Scenario.Description, $Steps | Convert-Tags $Scenario.Tags + & $SafeCommands["New-Object"] Gherkin.Ast.Scenario $null, $Scenario.Location, $Scenario.Keyword.Trim(), $Scenario.Name, $Scenario.Description, $Steps | + & $SafeCommands["Add-Member"] -MemberType "NoteProperty" -Name "Result" -Value 'Inconclusive' -PassThru | + Convert-Tags $Scenario.Tags } } ) @@ -613,7 +714,8 @@ function Invoke-GherkinScenario { $script:mockTable = @{} # Create a clean variable scope in each scenario - $script:GherkinScenarioScope = New-Module Scenario { $a = 4 + $script:GherkinScenarioScope = New-Module Scenario { + $a = 4 } $script:GherkinSessionState = Set-SessionStateHint -PassThru -Hint Scenario -SessionState $Script:GherkinScenarioScope.SessionState @@ -623,17 +725,38 @@ function Invoke-GherkinScenario { Invoke-GherkinHook BeforeEachScenario $Scenario.Name $Scenario.Tags $testResultIndexStart = $Pester.TestResult.Count + $previousStepResult = 'Passed'; # If there's a background, run that before the test, but after hooks if ($Background) { foreach ($Step in $Background.Steps) { - # Run Background steps -Background so they don't output in each scenario - Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($previousStepResult -ne 'Passed' -and $previousStepResult -ne 'Inconclusive' <# i.e. Pending #>) { + $Pester.AddTestResult(("{0} {1}" -f $Step.Keyword.Trim(), $Step.Text), "Skipped", $null, $null, $null, $null, $null) + $Pester.TestResult[-1] | Write-PesterResult + } else { + # Run Background steps -Background so they don't output in each scenario + Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($Pester.TestResult[-1].Result -ne 'Passed' -or $Pester.TestResult[-1].Result -ne 'Inconclusive') { + $previousStepResult = $Pester.TestResult[-1].Result + } + } } } foreach ($Step in $Scenario.Steps) { - Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($previousStepResult -ne 'Passed' -and $previousStepResult -ne 'Inconclusive') { + $Pester.AddTestResult(("{0} {1}" -f $Step.Keyword.Trim(), $Step.Text), "Skipped", $null, $null, $null, $null, $null) + $Pester.TestResult[-1] | Write-PesterResult + } else { + Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart + if ($Pester.TestResult[-1].Result -ne 'Passed') { + $previousStepResult = $Pester.TestResult[-1].Result + } + } + } + + if ($previousStepResult -ne 'Passed') { + $Scenario.Result = $previousStepResult } Invoke-GherkinHook AfterEachScenario $Scenario.Name $Scenario.Tags @@ -782,18 +905,16 @@ function Invoke-GherkinStep { # Iterate over the test results of the previous steps for ($i = $TestResultIndexStart; $i -lt ($Pester.TestResult.Count); $i++) { $previousTestResult = $Pester.TestResult[$i].Result - if ($previousTestResult -eq "Failed" -or $previousTestResult -eq "Inconclusive") { + if ($previousTestResult -eq "Failed") { $previousStepsNotSuccessful = $true break } } - if (!$StepCommand -or $previousStepsNotSuccessful) { - $skipMessage = if (!$StepCommand) { - "Could not find implementation for step!" - } else { - "Step skipped (previous step did not pass)" - } - $PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message $skipMessage -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText + + if (!$StepCommand) { + $PesterErrorRecord = New-UndefinedStepErrorRecord -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText + } elseif ($previousStepsNotSuccessful) { + $PesterErrorRecord = Set-ItResult -Skipped } else { $NamedArguments, $Parameters = Get-StepParameters $Step $StepCommand $watch = & $SafeCommands["New-Object"] System.Diagnostics.Stopwatch @@ -834,7 +955,10 @@ function Invoke-GherkinStep { # For Gherkin, we want to show the step, but not pretend to be a StackTrace if (${Pester Result}.Result -eq 'Inconclusive') { - ${Pester Result}.StackTrace = "At " + $Step.Keyword.Trim() + ', ' + $Step.Location.Path + ': line ' + $Step.Location.Line + # TODO: Attempt to show skeleton implentation of step?? + ${Pester Result}.StackTrace = "at " + $Step.Keyword.Trim() + ', ' + $Step.Location.Path + ': line ' + $Step.Location.Line + } elseif (${Pester Result}.Result -eq 'Pending') { + ${Pester Result}.StackTrace += "$([Environment]::NewLine)From $($Step.Location.Path): line $($Step.Location.Line)" } else { # Unless we really are a StackTrace... ${Pester Result}.StackTrace += "`nFrom " + $Step.Location.Path + ': line ' + $Step.Location.Line diff --git a/Functions/Output.ps1 b/Functions/Output.ps1 index 1a8c456c0..864c8a2f8 100644 --- a/Functions/Output.ps1 +++ b/Functions/Output.ps1 @@ -185,7 +185,7 @@ function ConvertTo-PesterResult { return $testResult } - if (@('PesterAssertionFailed', 'PesterTestSkipped', 'PesterTestInconclusive', 'PesterTestPending') -contains $ErrorRecord.FullyQualifiedErrorID) { + if (@('PesterAssertionFailed', 'PesterTestSkipped', 'PesterTestInconclusive', 'PesterTestPending', 'PesterUndefinedGherkinStep', 'PesterPendingGherkinStep') -contains $ErrorRecord.FullyQualifiedErrorID) { # we use TargetObject to pass structured information about the error. $details = $ErrorRecord.TargetObject @@ -198,6 +198,8 @@ function ConvertTo-PesterResult { PesterTestInconclusive { $testResult.Result = "Inconclusive"; break; } PesterTestPending { $testResult.Result = "Pending"; break; } PesterTestSkipped { $testResult.Result = "Skipped"; break; } + PesterUndefinedGherkinStep { $testResult.Result = "Inconclusive"; break; } + PesterPendingGherkinStep { $testResult.Result = "Pending"; break; } } } else { $failureMessage = $ErrorRecord.ToString() @@ -308,6 +310,18 @@ function Write-PesterResult { & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.PendingTime $humanTime } } + + if ($TestResult.ErrorRecord.FullyQualifiedErrorId -eq 'PesterPendingGherkinStep') { + if($pester.IncludeVSCodeMarker) { + & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending $($TestResult.failureMessage -replace '(?m)^',($error_margin*2)) + & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending $($TestResult.stackTrace -replace '(?m)^',($error_margin*2)) + } else { + $TestResult | + foreach { ,($_.ErrorRecord.Exception.Message -replace "Exception: ","") + $_.StackTrace -split "\r?\n" } | + & $SafeCommands['Select-Object'] -Index 0,1,3 | + foreach { & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending ($_ -replace '(?m)^',($error_margin*2))} + } + } break } @@ -343,6 +357,153 @@ function Write-PesterResult { } } +function Write-GherkinReport { + param ( + [Parameter(Mandatory = $True, ValueFromPipeline = $True)] + $PesterState + ) + + if (-not ($PesterState.Show | Has-Flag Summary)) { return } + + $Success, $Failure = if($PesterState.FailedCount -gt 0) { + $ReportTheme.Foreground, $ReportTheme.Fail + } else { + $ReportTheme.Pass, $ReportTheme.Information + } + $Skipped = if($PesterState.SkippedCount -gt 0) { $ReportTheme.Skipped } else { $ReportTheme.Information } + $Pending = if($PesterState.PendingCount -gt 0) { $ReportTheme.Pending } else { $ReportTheme.Information } + $Inconclusive = if($PesterState.InconclusiveCount -gt 0) { $ReportTheme.Inconclusive } else { $ReportTheme.Information } + + Try { + $PesterStatePassedScenariosCount = $PesterState.PassedScenarios.Count + } + Catch { + $PesterStatePassedScenariosCount = 0 + } + + Try { + $PesterStateFailedScenariosCount = $PesterState.FailedScenarios.Count + } + Catch { + $PesterStateFailedScenariosCount = 0 + } + + Try { + $PesterStatePendingScenariosCount = $PesterState.PendingScenarios.Count + } + Catch { + $PesterStatePendingScenariosCount = 0 + } + + Try { + $PesterStateUndefinedScenariosCount = $PesterState.UndefinedScenarios.Count + } + Catch { + $PesterStateUndefinedScenariosCount = 0 + } + + $PesterStateTotalScenariosCount = $PesterStateFailedScenariosCount + $PesterStateUndefinedScenariosCount + $PesterStatePendingScenariosCount + $PesterStatePassedScenariosCount + + if($ReportStrings.ContextsPassed) { + [string[]]$PesterStateScenarioSummaryCounts = ,($ReportStrings.ContextSummary -f $PesterStateTotalScenariosCount) + if ($PesterStateTotalScenariosCount -eq 1) { + $PesterStateScenarioSummaryCounts[0] = $PesterStateScenarioSummaryCounts[0] -replace "scenarios","scenario" + } + + $PesterStateScenarioSummaryCounts += @(($ReportStrings.ContextsFailed -f $PesterStateFailedScenariosCount), + ($ReportStrings.ContextsUndefined -f $PesterStateUndefinedScenariosCount), + ($ReportStrings.ContextsPending -f $PesterStatePendingScenariosCount), + ($ReportStrings.ContextsPassed -f $PesterStatePassedScenariosCount)) + + $PesterStateScenarioSummaryData = $PesterStateScenarioSummaryCounts | & $SafeCommands["ForEach-Object"] { + $_ -match "^(?\d+) (?failed|undefined|pending|passed|scenarios \()" | Out-Null + switch ($Matches['Result']) { + failed { $Foreground = $Failure } + undefined { $Foreground = $Inconclusive } + pending { $Foreground = $Pending } + passed { $Foreground = $Success } + default { $Foreground = $ReportTheme.Foreground } + } + + if ($Matches['ScenarioCount'] -gt 0) { + [PSCustomObject]@{ Foreground = $Foreground; Text = $_ } + } + } + + & $SafeCommands['Write-Host'] '' + for ($i = 0; $i -lt $PesterStateScenarioSummaryData.Length; $i++) { + $summaryData = $PesterStateScenarioSummaryData[$i] + if ($i -eq $PesterStateScenarioSummaryData.Length - 1) { + & $SafeCommands['Write-Host'] ($summaryData.Text -replace ", ",'') -Foreground $summaryData.Foreground -NoNewLine + & $SafeCommands['Write-Host'] ")" -Foreground $ReportTheme.Foreground + } else { + & $SafeCommands['Write-Host'] $summaryData.Text -Foreground $summaryData.Foreground -NoNewLine + if ($i) { + &$SafeCommands['Write-Host'] ", " -Foreground $ReportTheme.Foreground -NoNewLine + } + } + } + } + if($ReportStrings.TestsPassed) { + $PesterStateTotalStepsCount = $PesterState.FailedCount + $PesterState.Inconclusive + $PesterState.SkippedCount + $PesterState.PendingCount + $PesterState.PassedCount + [string[]]$PesterStateStepSummaryCounts = ,($ReportStrings.TestsSummary -f $PesterStateTotalStepsCount) + if ($PesterStateTotalStepCount -eq 1) { + $PesterStateStepSummaryCounts[0] = $PesterStateStepSummaryCounts[0] -replace "steps","step" + } + + $PesterStateStepSummaryCounts += @(($ReportStrings.TestsFailed -f $PesterState.FailedCount), + ($ReportStrings.TestsInconclusive -f $PesterState.InconclusiveCount), + ($ReportStrings.TestsSkipped -f $PesterState.SkippedCount), + ($ReportStrings.TestsPending -f $PesterState.PendingCount), + ($ReportStrings.TestsPassed -f $PesterState.PassedCount)) + + $PesterStateStepSummaryData = $PesterStateStepSummaryCounts | & $SafeCommands["ForEach-Object"] { + $_ -match "^(?\d+) (?failed|undefined|skipped|pending|passed|steps \()" | Out-Null + switch ($Matches['Result']) { + failed { $Foreground = $Failure } + undefined { $Foreground = $Inconclusive } + skipped { $Foreground = $Skipped } + pending { $Foreground = $Pending } + passed { $Foreground = $Success } + default { $Foreground = $ReportTheme.Foreground } + } + + if ($Matches['StepCount'] -gt 0) { + [PSCustomObject]@{ Foreground = $Foreground; Text = $_ } + } + } + + for ($i = 0; $i -lt $PesterStateStepSummaryData.Length; $i++) { + $summaryData = $PesterStateStepSummaryData[$i] + if ($i -eq $PesterStateStepSummaryData.Length - 1) { + & $SafeCommands['Write-Host'] ($summaryData.Text -replace ", ",'') -Foreground $summaryData.Foreground -NoNewLine + & $SafeCommands['Write-Host'] ")" -Foreground $ReportTheme.Foreground + } else { + & $SafeCommands['Write-Host'] $summaryData.Text -Foreground $summaryData.Foreground -NoNewLine + if ($i) { + &$SafeCommands['Write-Host'] ", " -Foreground $ReportTheme.Foreground -NoNewLine + } + } + } + } + + & $SafeCommands['Write-Host'] ($ReportStrings.Timing -f (Get-HumanTime $PesterState.Time.TotalSeconds)) -Foreground $ReportTheme.Foreground + & $SafeCommands['Write-Host'] '' + + # TODO: Can we create a method that would auto-generate the Step Definition script blocks to the console for undefined steps? + + # {Count} Scenario(s?) ({FailedCount} failed, {UndefinedCount} undefined, {PendingCount} pending, {PassedCount} passed) + # {Count} Step(s?) ({FailedCount} failed, {UndefinedCount} undefined, {SkippedCount} skipped, {PendingCount} pending, {PassedCount} passed) + # 0m0.002s + # + # You can implement step definitions for undefined steps with these snippets: + # + # Given "the input '(.*?)'" { + # param($arg1) + # Set-TestPending + # } +} + function Write-PesterReport { param ( [Parameter(mandatory=$true, valueFromPipeline=$true)] diff --git a/en-US/Gherkin.psd1 b/en-US/Gherkin.psd1 index d4c04ff99..48588cb24 100644 --- a/en-US/Gherkin.psd1 +++ b/en-US/Gherkin.psd1 @@ -1,29 +1,58 @@ @{ - StartMessage = "Testing all features in '{0}'" - FilterMessage = " for scenarios matching '{0}'" - TagMessage = " with tags: '{0}'" - MessageOfs = "', '" + ReportStrings = @{ + StartMessage = "Testing all features in '{0}'" + FilterMessage = " for scenarios matching '{0}'" + TagMessage = " with tags: '{0}'" + MessageOfs = "', '" - CoverageTitle = "Code coverage report:" - CoverageMessage = "Covered {2:P2} of {3:N0} analyzed {0} in {4:N0} {1}." - MissedSingular = 'Missed command:' - MissedPlural = 'Missed commands:' - CommandSingular = 'Command' - CommandPlural = 'Commands' - FileSingular = 'File' - FilePlural = 'Files' + CoverageTitle = "Code coverage report:" + CoverageMessage = "Covered {2:P2} of {3:N0} analyzed {0} in {4:N0} {1}." + MissedSingular = 'Missed command:' + MissedPlural = 'Missed commands:' + CommandSingular = 'Command' + CommandPlural = 'Commands' + FileSingular = 'File' + FilePlural = 'Files' - Describe = "Feature: {0}" - Context = "Scenario: {0}" - Margin = " " - Timing = "Testing completed in {0}" + Describe = "Feature: {0}" + Context = "Scenario: {0}" + Margin = " " + Timing = "Testing completed in {0}" - # If this is set to an empty string, the count won't be printed - ContextsPassed = "Scenarios Passed: {0} " - ContextsFailed = "Failed: {0}" - TestsPassed = "Steps Passed: {0} " - TestsFailed = "Failed: {0} " - TestsSkipped = 'Skipped: {0} ' - TestsPending = 'Pending: {0} ' - TestsInconclusive = 'Inconclusive: {0} ' + # If this is set to an empty string, the count won't be printed + ContextSummary = '{0} scenarios (' + ContextsFailed = '{0} failed' + ContextsUndefined = '{0} undefined' + ContextsPending = '{0} pending' + ContextsPassed = '{0} passed' + TestsSummary = '{0} steps (' + TestsFailed = '{0} failed' + TestsInconclusive = '{0} undefined' + TestsSkipped = '{0} skipped' + TestsPending = '{0} pending' + TestsPassed = '{0} passed' + } + + ReportTheme = @{ + Describe = 'Green' + DescribeDetail = 'DarkYellow' + Context = 'Cyan' + ContextDetail = 'DarkCyan' + Pass = 'DarkGreen' + PassTime = 'DarkGray' + Fail = 'Red' + FailTime = 'DarkGray' + Skipped = 'Yellow' + SkippedTime = 'DarkGray' + Pending = 'Yellow' + PendingTime = 'DarkGray' + Inconclusive = 'Yellow' + InconclusiveTime = 'DarkGray' + Incomplete = 'Yellow' + IncompleteTime = 'DarkGray' + Foreground = 'White' + Information = 'DarkGray' + Coverage = 'White' + CoverageWarn = 'DarkRed' + } }