Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions FaultCustomRunbookScripts/Zonal/Canary/Fault-AksZones.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -500,9 +500,18 @@ if ($operationObjects.Count -eq 0 -and $unexpectedOutputs.Count -gt 0) {

$scriptEnd = Get-Date
$successCount = ($operationObjects | Where-Object { $_.IsSuccess }).Count
$failureCount = ($operationObjects | Where-Object { -not $_.IsSuccess }).Count
$skippedCount = ($operationObjects | Where-Object { $_.Status -eq 'Skipped' }).Count
$failureCount = ($operationObjects | Where-Object { -not $_.IsSuccess -and $_.Status -ne 'Skipped' }).Count
$failureCount += $unexpectedOutputs.Count
$overallStatus = if ($failureCount -eq 0) { 'Success' } elseif ($successCount -gt 0) { 'PartialSuccess' } else { 'Failed' }
# A skipped resource (no eligible nodes to fault in the target zone) is surfaced as a dedicated
# user error (RHDSUserErrorAKSNoNodesToFaultInTargetZone) and downgrades the run to PartialSuccess.
$overallStatus = if ($failureCount -gt 0 -and $successCount -eq 0 -and $skippedCount -eq 0) {
'Failed'
} elseif ($failureCount -gt 0 -or $skippedCount -gt 0) {
'PartialSuccess'
} else {
'Success'
}

$resourceResults = @()
foreach ($op in $operationObjects) {
Expand All @@ -522,11 +531,16 @@ foreach ($op in $operationObjects) {
}
$durationMs = [int]([Math]::Round((($endTime) - $startTime).TotalMilliseconds))
$err = $null
if (-not $op.IsSuccess) {
$metadata = @{ Status = $op.Status }
if ($op.Status -eq 'Skipped') {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hope the skipped status is returned by the $op. Please check or confirm once.

# No eligible nodes to fault in the target zone - report as a dedicated user error.
$err = @{ ErrorCode='RHDSUserErrorAKSNoNodesToFaultInTargetZone'; Message=$op.ErrorMessage; Details=$op.ErrorMessage; Category='Skipped'; IsRetryable=$false }
}
elseif (-not $op.IsSuccess) {
$err = @{ ErrorCode='FailedToFaultResource'; Message=$op.ErrorMessage; Details=$op.ErrorMessage; Category=$op.Status; IsRetryable=$false }
}
$processedAtUtc = $endTime.ToUniversalTime()
$resourceResults += @{ ResourceId=$op.ResourceId; IsSuccess=$op.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=@{ Status=$op.Status } }
$resourceResults += @{ ResourceId=$op.ResourceId; IsSuccess=$op.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=$metadata }
}

foreach ($unexpected in $unexpectedOutputs) {
Expand All @@ -546,13 +560,18 @@ $executionResult = [ordered]@{
$executionJson = $executionResult | ConvertTo-Json -Depth 6
Write-Output $executionJson

# Fail the runbook if any resource could not be faulted
# Fail the runbook only on genuine faults. Skipped resources (no eligible nodes to fault in the
# target zone) are reported as a dedicated user error with PartialSuccess and do not fail the run.
if ($failureCount -gt 0) {
$errorMsg = "Runbook failed: $failureCount out of $($AksTargetList.Count) AKS cluster(s) could not be faulted. Status: $overallStatus"
Write-Error $errorMsg -ErrorAction Stop
throw $errorMsg
}

Write-Verbose "All AKS zone fault operations completed successfully."
if ($skippedCount -gt 0) {
Write-Verbose "AKS zone fault completed with PartialSuccess. $skippedCount of $($AksTargetList.Count) cluster(s) had no eligible nodes to fault in the target zone (RHDSUserErrorAKSNoNodesToFaultInTargetZone)."
} else {
Write-Verbose "All AKS zone fault operations completed successfully."
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How are we handling the this skipped return at the Drills code?


#endregion
35 changes: 28 additions & 7 deletions FaultCustomRunbookScripts/Zonal/Canary/Fault-LoadBalancerZones.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ $functions = {
$uniqueProbesToOverride = $probesToOverride | Select-Object -Unique
if (-not $uniqueProbesToOverride) {
Write-Log "No health probes found for zoned backends on LB '$LBName'. Nothing to override." "WARNING"
return [pscustomobject]@{ IsSuccess = $true; Status = 'Skipped'; Message = 'No health probes found for zoned backends.' }
return [pscustomobject]@{ IsSuccess = $false; Status = 'Skipped'; Message = 'No health probes found for zoned backends in the target zone.' }
}

# Capture original probe settings before override
Expand Down Expand Up @@ -484,9 +484,18 @@ if ($results.Count -eq 0 -and $unexpectedOutputs.Count -gt 0) {

$scriptEnd = Get-Date
$successCount = ($results | Where-Object { $_.IsSuccess }).Count
$failureCount = ($results | Where-Object { -not $_.IsSuccess }).Count
$skippedCount = ($results | Where-Object { $_.Status -eq 'Skipped' }).Count
$failureCount = ($results | Where-Object { -not $_.IsSuccess -and $_.Status -ne 'Skipped' }).Count
$failureCount += $unexpectedOutputs.Count
$overallStatus = if ($failureCount -eq 0) { 'Success' } elseif ($successCount -gt 0) { 'PartialSuccess' } else { 'Failed' }
# A skipped resource (no probes for zoned backends to fault in the target zone) is surfaced as a
# dedicated user error (RHDSUserErrorLBNoProbesToFaultInTargetZone) and downgrades the run to PartialSuccess.
$overallStatus = if ($failureCount -gt 0 -and $successCount -eq 0 -and $skippedCount -eq 0) {
'Failed'
} elseif ($failureCount -gt 0 -or $skippedCount -gt 0) {
'PartialSuccess'
} else {
'Success'
}

$resourceResults = @()
foreach ($r in $results) {
Expand All @@ -506,9 +515,16 @@ foreach ($r in $results) {
}
$durationMs = [int]([Math]::Round((($endTime) - $startTime).TotalMilliseconds))
$err = $null
if (-not $r.IsSuccess) { $err = @{ ErrorCode='FailedToFaultResource'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category=$r.Status; IsRetryable=$false } }
$metadata = @{ Status = $r.Status; DurationMinutes = $Duration }
if ($r.Status -eq 'Skipped') {
# No probes for zoned backends to fault in the target zone - report as a dedicated user error.
$err = @{ ErrorCode='RHDSUserErrorLBNoProbesToFaultInTargetZone'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category='Skipped'; IsRetryable=$false }
}
elseif (-not $r.IsSuccess) {
$err = @{ ErrorCode='FailedToFaultResource'; Message=$r.ErrorMessage; Details=$r.ErrorMessage; Category=$r.Status; IsRetryable=$false }
}
$processedAtUtc = $endTime.ToUniversalTime()
$resourceResults += @{ ResourceId=$r.ResourceId; IsSuccess=$r.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=@{ Status=$r.Status; DurationMinutes=$Duration } }
$resourceResults += @{ ResourceId=$r.ResourceId; IsSuccess=$r.IsSuccess; Error=$err; ProcessedAt=$processedAtUtc; ProcessingDurationMs=$durationMs; Metadata=$metadata }
}

foreach ($unexpected in $unexpectedOutputs) {
Expand All @@ -528,13 +544,18 @@ $executionResult = [ordered]@{
$executionJson = $executionResult | ConvertTo-Json -Depth 6
Write-Output $executionJson

# Fail the runbook if any resource could not be faulted
# Fail the runbook only on genuine faults. Skipped resources (no probes for zoned backends to fault
# in the target zone) are reported as a dedicated user error with PartialSuccess and do not fail the run.
if ($failureCount -gt 0) {
$errorMsg = "Runbook failed: $failureCount out of $($lbTargets.Count) Load Balancer(s) could not be faulted. Status: $overallStatus"
Write-Error $errorMsg -ErrorAction Stop
throw $errorMsg
}

Write-Verbose "All Load Balancer health probe override operations completed successfully."
if ($skippedCount -gt 0) {
Write-Verbose "Load Balancer probe override completed with PartialSuccess. $skippedCount of $($lbTargets.Count) load balancer(s) had no probes for zoned backends to fault in the target zone (RHDSUserErrorLBNoProbesToFaultInTargetZone)."
} else {
Write-Verbose "All Load Balancer health probe override operations completed successfully."
}

#endregion