diff --git a/FaultCustomRunbookScripts/Zonal/Canary/Fault-AksZones.ps1 b/FaultCustomRunbookScripts/Zonal/Canary/Fault-AksZones.ps1 index 2409d50..13bfbcd 100644 --- a/FaultCustomRunbookScripts/Zonal/Canary/Fault-AksZones.ps1 +++ b/FaultCustomRunbookScripts/Zonal/Canary/Fault-AksZones.ps1 @@ -500,9 +500,18 @@ if ($operationObjects.Count -eq 0 -and $unexpectedOutputs.Count -gt 0) { $scriptEnd = Get-Date $successCount = ($operationObjects | Where-Object { $_.IsSuccess }).Count -$failureCount = ($operationObjects | Where-Object { -not $_.IsSuccess }).Count +$skippedCount = ($operationObjects | Where-Object { $_.Status -eq 'Skipped' }).Count +$failureCount = ($operationObjects | Where-Object { -not $_.IsSuccess -and $_.Status -ne 'Skipped' }).Count $failureCount += $unexpectedOutputs.Count -$overallStatus = if ($failureCount -eq 0) { 'Success' } elseif ($successCount -gt 0) { 'PartialSuccess' } else { 'Failed' } +# A skipped resource (no eligible nodes to fault in the target zone) is surfaced as a dedicated +# user error (RHDSUserErrorAKSNoNodesToFaultInTargetZone) and downgrades the run to PartialSuccess. +$overallStatus = if ($failureCount -gt 0 -and $successCount -eq 0 -and $skippedCount -eq 0) { + 'Failed' +} elseif ($failureCount -gt 0 -or $skippedCount -gt 0) { + 'PartialSuccess' +} else { + 'Success' +} $resourceResults = @() foreach ($op in $operationObjects) { @@ -522,11 +531,16 @@ foreach ($op in $operationObjects) { } $durationMs = [int]([Math]::Round((($endTime) - $startTime).TotalMilliseconds)) $err = $null - if (-not $op.IsSuccess) { + $metadata = @{ Status = $op.Status } + if ($op.Status -eq 'Skipped') { + # No eligible nodes to fault in the target zone - report as a dedicated user error. + $err = @{ ErrorCode='RHDSUserErrorAKSNoNodesToFaultInTargetZone'; Message=$op.ErrorMessage; Details=$op.ErrorMessage; Category='Skipped'; IsRetryable=$false } + } + elseif (-not $op.IsSuccess) { $err = @{ ErrorCode='FailedToFaultResource'; Message=$op.ErrorMessage; Details=$op.ErrorMessage; Category=$op.Status; IsRetryable=$false } } $processedAtUtc = $endTime.ToUniversalTime() - $resourceResults += @{ ResourceId=$op.ResourceId; IsSuccess=$op.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=@{ Status=$op.Status } } + $resourceResults += @{ ResourceId=$op.ResourceId; IsSuccess=$op.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=$metadata } } foreach ($unexpected in $unexpectedOutputs) { @@ -546,13 +560,18 @@ $executionResult = [ordered]@{ $executionJson = $executionResult | ConvertTo-Json -Depth 6 Write-Output $executionJson -# Fail the runbook if any resource could not be faulted +# Fail the runbook only on genuine faults. Skipped resources (no eligible nodes to fault in the +# target zone) are reported as a dedicated user error with PartialSuccess and do not fail the run. if ($failureCount -gt 0) { $errorMsg = "Runbook failed: $failureCount out of $($AksTargetList.Count) AKS cluster(s) could not be faulted. Status: $overallStatus" Write-Error $errorMsg -ErrorAction Stop throw $errorMsg } -Write-Verbose "All AKS zone fault operations completed successfully." +if ($skippedCount -gt 0) { + Write-Verbose "AKS zone fault completed with PartialSuccess. $skippedCount of $($AksTargetList.Count) cluster(s) had no eligible nodes to fault in the target zone (RHDSUserErrorAKSNoNodesToFaultInTargetZone)." +} else { + Write-Verbose "All AKS zone fault operations completed successfully." +} #endregion \ No newline at end of file diff --git a/FaultCustomRunbookScripts/Zonal/Canary/Fault-LoadBalancerZones.ps1 b/FaultCustomRunbookScripts/Zonal/Canary/Fault-LoadBalancerZones.ps1 index 3747217..fac1675 100644 --- a/FaultCustomRunbookScripts/Zonal/Canary/Fault-LoadBalancerZones.ps1 +++ b/FaultCustomRunbookScripts/Zonal/Canary/Fault-LoadBalancerZones.ps1 @@ -330,7 +330,7 @@ $functions = { $uniqueProbesToOverride = $probesToOverride | Select-Object -Unique if (-not $uniqueProbesToOverride) { Write-Log "No health probes found for zoned backends on LB '$LBName'. Nothing to override." "WARNING" - return [pscustomobject]@{ IsSuccess = $true; Status = 'Skipped'; Message = 'No health probes found for zoned backends.' } + return [pscustomobject]@{ IsSuccess = $false; Status = 'Skipped'; Message = 'No health probes found for zoned backends in the target zone.' } } # Capture original probe settings before override @@ -484,9 +484,18 @@ if ($results.Count -eq 0 -and $unexpectedOutputs.Count -gt 0) { $scriptEnd = Get-Date $successCount = ($results | Where-Object { $_.IsSuccess }).Count -$failureCount = ($results | Where-Object { -not $_.IsSuccess }).Count +$skippedCount = ($results | Where-Object { $_.Status -eq 'Skipped' }).Count +$failureCount = ($results | Where-Object { -not $_.IsSuccess -and $_.Status -ne 'Skipped' }).Count $failureCount += $unexpectedOutputs.Count -$overallStatus = if ($failureCount -eq 0) { 'Success' } elseif ($successCount -gt 0) { 'PartialSuccess' } else { 'Failed' } +# A skipped resource (no probes for zoned backends to fault in the target zone) is surfaced as a +# dedicated user error (RHDSUserErrorLBNoProbesToFaultInTargetZone) and downgrades the run to PartialSuccess. +$overallStatus = if ($failureCount -gt 0 -and $successCount -eq 0 -and $skippedCount -eq 0) { + 'Failed' +} elseif ($failureCount -gt 0 -or $skippedCount -gt 0) { + 'PartialSuccess' +} else { + 'Success' +} $resourceResults = @() foreach ($r in $results) { @@ -506,9 +515,16 @@ foreach ($r in $results) { } $durationMs = [int]([Math]::Round((($endTime) - $startTime).TotalMilliseconds)) $err = $null - if (-not $r.IsSuccess) { $err = @{ ErrorCode='FailedToFaultResource'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category=$r.Status; IsRetryable=$false } } + $metadata = @{ Status = $r.Status; DurationMinutes = $Duration } + if ($r.Status -eq 'Skipped') { + # No probes for zoned backends to fault in the target zone - report as a dedicated user error. + $err = @{ ErrorCode='RHDSUserErrorLBNoProbesToFaultInTargetZone'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category='Skipped'; IsRetryable=$false } + } + elseif (-not $r.IsSuccess) { + $err = @{ ErrorCode='FailedToFaultResource'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category=$r.Status; IsRetryable=$false } + } $processedAtUtc = $endTime.ToUniversalTime() - $resourceResults += @{ ResourceId=$r.ResourceId; IsSuccess=$r.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=@{ Status=$r.Status; DurationMinutes=$Duration } } + $resourceResults += @{ ResourceId=$r.ResourceId; IsSuccess=$r.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=$metadata } } foreach ($unexpected in $unexpectedOutputs) { @@ -528,13 +544,18 @@ $executionResult = [ordered]@{ $executionJson = $executionResult | ConvertTo-Json -Depth 6 Write-Output $executionJson -# Fail the runbook if any resource could not be faulted +# Fail the runbook only on genuine faults. Skipped resources (no probes for zoned backends to fault +# in the target zone) are reported as a dedicated user error with PartialSuccess and do not fail the run. if ($failureCount -gt 0) { $errorMsg = "Runbook failed: $failureCount out of $($lbTargets.Count) Load Balancer(s) could not be faulted. Status: $overallStatus" Write-Error $errorMsg -ErrorAction Stop throw $errorMsg } -Write-Verbose "All Load Balancer health probe override operations completed successfully." +if ($skippedCount -gt 0) { + Write-Verbose "Load Balancer probe override completed with PartialSuccess. $skippedCount of $($lbTargets.Count) load balancer(s) had no probes for zoned backends to fault in the target zone (RHDSUserErrorLBNoProbesToFaultInTargetZone)." +} else { + Write-Verbose "All Load Balancer health probe override operations completed successfully." +} #endregion \ No newline at end of file