From 257f169308d4ea131facca9ae202d985015764d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:07:45 +0000 Subject: [PATCH 1/4] Initial plan From ff307c4e4d1f336710841a48c9f1df6bc8a14ff5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:22:39 +0000 Subject: [PATCH 2/4] feat: classify driver-exit (0-turn) failures separately from agent-logic failures Add isDriverExitFailure() helper to logs_models.go that detects failed runs with zero agent turns (CLI wrapper or infra step exited before the agent ran). WorkflowHealth gains DriverExitCount and AgentLogicFailureCount so per-workflow health dashboards can distinguish infra flakiness from real agent regressions. LogsSummary gains TotalDriverExitFailures and TotalAgentLogicFailures rollup counters so operators can see fleet-wide breakdown at a glance. RunData gains FailureKind ("driver_exit" | "agent_logic" | "") per-run so individual runs in the JSON output carry the classification. Tests added for the new helper and all three surfaces. Closes #47103 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/health_metrics.go | 63 +++++++++++++---------- pkg/cli/health_metrics_test.go | 77 ++++++++++++++++++++++++++++ pkg/cli/logs_models.go | 8 +++ pkg/cli/logs_report.go | 93 ++++++++++++++++++++++------------ pkg/cli/logs_report_test.go | 59 +++++++++++++++++++++ 5 files changed, 241 insertions(+), 59 deletions(-) diff --git a/pkg/cli/health_metrics.go b/pkg/cli/health_metrics.go index c3c15d8125c..ec8bfd545d4 100644 --- a/pkg/cli/health_metrics.go +++ b/pkg/cli/health_metrics.go @@ -14,19 +14,21 @@ var healthMetricsLog = logger.New("cli:health_metrics") // WorkflowHealth represents health metrics for a single workflow type WorkflowHealth struct { - WorkflowName string `json:"workflow_name" console:"header:Workflow"` - TotalRuns int `json:"total_runs" console:"-"` - SuccessCount int `json:"success_count" console:"-"` - FailureCount int `json:"failure_count" console:"-"` - SuccessRate float64 `json:"success_rate" console:"-"` - DisplayRate string `json:"-" console:"header:Success Rate"` - Trend string `json:"trend" console:"header:Trend"` - AvgDuration time.Duration `json:"avg_duration" console:"-"` - DisplayDur string `json:"-" console:"header:Avg Duration"` - TotalTokens int `json:"total_tokens" console:"-"` - AvgTokens int `json:"avg_tokens" console:"-"` - DisplayTokens string `json:"-" console:"header:Avg Tokens"` - BelowThresh bool `json:"below_threshold" console:"-"` + WorkflowName string `json:"workflow_name" console:"header:Workflow"` + TotalRuns int `json:"total_runs" console:"-"` + SuccessCount int `json:"success_count" console:"-"` + FailureCount int `json:"failure_count" console:"-"` + DriverExitCount int `json:"driver_exit_count" console:"-"` + AgentLogicFailureCount int `json:"agent_logic_failure_count" console:"-"` + SuccessRate float64 `json:"success_rate" console:"-"` + DisplayRate string `json:"-" console:"header:Success Rate"` + Trend string `json:"trend" console:"header:Trend"` + AvgDuration time.Duration `json:"avg_duration" console:"-"` + DisplayDur string `json:"-" console:"header:Avg Duration"` + TotalTokens int `json:"total_tokens" console:"-"` + AvgTokens int `json:"avg_tokens" console:"-"` + DisplayTokens string `json:"-" console:"header:Avg Tokens"` + BelowThresh bool `json:"below_threshold" console:"-"` } // HealthSummary represents aggregated health metrics across all workflows @@ -78,6 +80,8 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold // Accumulate success/failure counts and numerical metrics. successCount := 0 failureCount := 0 + driverExitCount := 0 + agentLogicFailureCount := 0 var durationStats, tokenStats stats.StatVar var totalTokens int @@ -86,6 +90,11 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold successCount++ } else if isFailureConclusion(run.Conclusion) { failureCount++ + if isDriverExitFailure(run) { + driverExitCount++ + } else { + agentLogicFailureCount++ + } } totalTokens += run.TokenUsage durationStats.Add(float64(run.Duration)) @@ -109,19 +118,21 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold belowThreshold := successRate < threshold health := WorkflowHealth{ - WorkflowName: workflowName, - TotalRuns: totalRuns, - SuccessCount: successCount, - FailureCount: failureCount, - SuccessRate: successRate, - DisplayRate: displayRate, - Trend: trend.String(), - AvgDuration: avgDuration, - DisplayDur: displayDur, - TotalTokens: totalTokens, - AvgTokens: avgTokens, - DisplayTokens: displayTokens, - BelowThresh: belowThreshold, + WorkflowName: workflowName, + TotalRuns: totalRuns, + SuccessCount: successCount, + FailureCount: failureCount, + DriverExitCount: driverExitCount, + AgentLogicFailureCount: agentLogicFailureCount, + SuccessRate: successRate, + DisplayRate: displayRate, + Trend: trend.String(), + AvgDuration: avgDuration, + DisplayDur: displayDur, + TotalTokens: totalTokens, + AvgTokens: avgTokens, + DisplayTokens: displayTokens, + BelowThresh: belowThreshold, } healthMetricsLog.Printf("Health calculated: workflow=%s, successRate=%.2f%%, trend=%s", workflowName, successRate, trend.String()) diff --git a/pkg/cli/health_metrics_test.go b/pkg/cli/health_metrics_test.go index 870a93f5b3c..caabf9bd381 100644 --- a/pkg/cli/health_metrics_test.go +++ b/pkg/cli/health_metrics_test.go @@ -257,3 +257,80 @@ func TestFormatTokens(t *testing.T) { }) } } + +func TestCalculateWorkflowHealthDriverExitClassification(t *testing.T) { + runs := []WorkflowRun{ + {Conclusion: "success", Duration: 2 * time.Minute, Turns: 3}, + // driver-exit: failed with zero turns + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0}, + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0}, + // agent-logic: failed but agent ran (turns > 0) + {Conclusion: "failure", Duration: 2 * time.Minute, Turns: 5}, + } + + health := CalculateWorkflowHealth("test-workflow", runs, 80.0) + + assert.Equal(t, 4, health.TotalRuns, "total runs should be 4") + assert.Equal(t, 1, health.SuccessCount, "success count should be 1") + assert.Equal(t, 3, health.FailureCount, "failure count should be 3") + assert.Equal(t, 2, health.DriverExitCount, "driver-exit count should be 2 (zero-turn failures)") + assert.Equal(t, 1, health.AgentLogicFailureCount, "agent-logic count should be 1 (non-zero-turn failure)") +} + +func TestCalculateWorkflowHealthDriverExitCountZeroWhenNoFailures(t *testing.T) { + runs := []WorkflowRun{ + {Conclusion: "success", Duration: 2 * time.Minute, Turns: 3}, + {Conclusion: "success", Duration: 3 * time.Minute, Turns: 4}, + } + + health := CalculateWorkflowHealth("test-workflow", runs, 80.0) + + assert.Equal(t, 0, health.DriverExitCount, "driver-exit count should be zero when all runs succeed") + assert.Equal(t, 0, health.AgentLogicFailureCount, "agent-logic count should be zero when all runs succeed") +} + +func TestIsDriverExitFailure(t *testing.T) { + tests := []struct { + name string + run WorkflowRun + expected bool + }{ + { + name: "failure with zero turns is driver-exit", + run: WorkflowRun{Conclusion: "failure", Turns: 0}, + expected: true, + }, + { + name: "timed_out with zero turns is driver-exit", + run: WorkflowRun{Conclusion: "timed_out", Turns: 0}, + expected: true, + }, + { + name: "cancelled with zero turns is driver-exit", + run: WorkflowRun{Conclusion: "cancelled", Turns: 0}, + expected: true, + }, + { + name: "failure with non-zero turns is agent-logic (not driver-exit)", + run: WorkflowRun{Conclusion: "failure", Turns: 3}, + expected: false, + }, + { + name: "success with zero turns is not driver-exit", + run: WorkflowRun{Conclusion: "success", Turns: 0}, + expected: false, + }, + { + name: "success with non-zero turns is not driver-exit", + run: WorkflowRun{Conclusion: "success", Turns: 5}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isDriverExitFailure(tt.run) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/cli/logs_models.go b/pkg/cli/logs_models.go index 1b43bd99ead..d460c2c8638 100644 --- a/pkg/cli/logs_models.go +++ b/pkg/cli/logs_models.go @@ -349,3 +349,11 @@ func isFailureConclusion(conclusion string) bool { } return isFailure } + +// isDriverExitFailure returns true when a failed run shows no agent turns, which +// indicates the CLI wrapper or a pre/post-agent infrastructure step exited non-zero +// before the agent had a chance to run. Runs with Turns > 0 are classified as +// agent-logic failures instead because the agent did execute. +func isDriverExitFailure(run WorkflowRun) bool { + return isFailureConclusion(run.Conclusion) && run.Turns == 0 +} diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index cd35d40b5c0..619d9cb8284 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -54,29 +54,36 @@ type ContinuationData struct { // LogsSummary contains aggregate metrics across all runs type LogsSummary struct { - TotalRuns int `json:"total_runs" console:"header:Total Runs"` - TotalDuration string `json:"total_duration" console:"header:Total Duration"` - TotalAIC float64 `json:"total_aic,omitempty"` - TotalTokens int `json:"total_tokens,omitempty" console:"header:Total Tokens,format:number,omitempty"` - TotalActionMinutes float64 `json:"total_action_minutes" console:"header:Total Action Minutes"` - TotalTurns int `json:"total_turns" console:"header:Total Turns"` - TotalSteeringEvents int `json:"total_steering_events,omitempty" console:"header:Total Steering Events,format:number,omitempty"` - TotalErrors int `json:"total_errors" console:"header:Total Errors"` - TotalWarnings int `json:"total_warnings" console:"header:Total Warnings"` - TotalMissingTools int `json:"total_missing_tools" console:"header:Total Missing Tools"` - TotalMissingData int `json:"total_missing_data" console:"header:Total Missing Data"` - TotalSafeItems int `json:"total_safe_items" console:"header:Total Safe Items"` - RunsWithTemporaryIDChains int `json:"runs_with_temporary_id_chains,omitempty" console:"-"` - RunsWithDelegatedTempTargets int `json:"runs_with_delegated_temp_targets,omitempty" console:"-"` - RunsWithMissingTemporaryIDMap int `json:"runs_with_missing_temporary_id_map,omitempty" console:"-"` - RunsWithInvalidTemporaryIDMap int `json:"runs_with_invalid_temporary_id_map,omitempty" console:"-"` - TotalTemporaryIDMappings int `json:"total_temporary_id_mappings,omitempty" console:"-"` - TotalChainedTargets int `json:"total_chained_targets,omitempty" console:"-"` - TotalChainedFollowupActions int `json:"total_chained_followup_actions,omitempty" console:"-"` - TotalClosedTempTargets int `json:"total_closed_temp_targets,omitempty" console:"-"` - TotalEpisodes int `json:"total_episodes" console:"header:Total Episodes"` - HighConfidenceEpisodes int `json:"high_confidence_episodes" console:"header:High Confidence Episodes"` - TotalGitHubAPICalls int `json:"total_github_api_calls,omitempty" console:"header:Total GitHub API Calls,format:number,omitempty"` + TotalRuns int `json:"total_runs" console:"header:Total Runs"` + TotalDuration string `json:"total_duration" console:"header:Total Duration"` + TotalAIC float64 `json:"total_aic,omitempty"` + TotalTokens int `json:"total_tokens,omitempty" console:"header:Total Tokens,format:number,omitempty"` + TotalActionMinutes float64 `json:"total_action_minutes" console:"header:Total Action Minutes"` + TotalTurns int `json:"total_turns" console:"header:Total Turns"` + TotalSteeringEvents int `json:"total_steering_events,omitempty" console:"header:Total Steering Events,format:number,omitempty"` + TotalErrors int `json:"total_errors" console:"header:Total Errors"` + TotalWarnings int `json:"total_warnings" console:"header:Total Warnings"` + TotalMissingTools int `json:"total_missing_tools" console:"header:Total Missing Tools"` + TotalMissingData int `json:"total_missing_data" console:"header:Total Missing Data"` + TotalSafeItems int `json:"total_safe_items" console:"header:Total Safe Items"` + // TotalDriverExitFailures counts failed runs with zero agent turns — the CLI wrapper + // or a pre/post-agent infrastructure step exited non-zero before the agent ran. + // These are infra-flakiness signals, not agent-logic regressions. + TotalDriverExitFailures int `json:"total_driver_exit_failures,omitempty" console:"header:Driver-Exit Failures,omitempty"` + // TotalAgentLogicFailures counts failed runs with one or more agent turns — the agent + // started and executed but the run still concluded as a failure. + TotalAgentLogicFailures int `json:"total_agent_logic_failures,omitempty" console:"header:Agent-Logic Failures,omitempty"` + RunsWithTemporaryIDChains int `json:"runs_with_temporary_id_chains,omitempty" console:"-"` + RunsWithDelegatedTempTargets int `json:"runs_with_delegated_temp_targets,omitempty" console:"-"` + RunsWithMissingTemporaryIDMap int `json:"runs_with_missing_temporary_id_map,omitempty" console:"-"` + RunsWithInvalidTemporaryIDMap int `json:"runs_with_invalid_temporary_id_map,omitempty" console:"-"` + TotalTemporaryIDMappings int `json:"total_temporary_id_mappings,omitempty" console:"-"` + TotalChainedTargets int `json:"total_chained_targets,omitempty" console:"-"` + TotalChainedFollowupActions int `json:"total_chained_followup_actions,omitempty" console:"-"` + TotalClosedTempTargets int `json:"total_closed_temp_targets,omitempty" console:"-"` + TotalEpisodes int `json:"total_episodes" console:"header:Total Episodes"` + HighConfidenceEpisodes int `json:"high_confidence_episodes" console:"header:High Confidence Episodes"` + TotalGitHubAPICalls int `json:"total_github_api_calls,omitempty" console:"header:Total GitHub API Calls,format:number,omitempty"` // EngineCounts maps engine_id (from aw_info.json) to the number of runs using that engine. // Use this field to accurately classify engine types — do NOT infer engines by scanning // lock files, which contain the word "copilot" in allowed-domains and workflow-source paths @@ -95,16 +102,21 @@ type LogsSummary struct { // RunData contains information about a single workflow run type RunData struct { - RunID int64 `json:"run_id" console:"header:Run ID"` - Number int `json:"number" console:"-"` - WorkflowName string `json:"workflow_name" console:"header:Workflow"` - WorkflowPath string `json:"workflow_path" console:"-"` - Agent string `json:"agent,omitempty" console:"header:Agent,omitempty"` - Engine string `json:"engine,omitempty" console:"-"` - EngineID string `json:"engine_id,omitempty" console:"-"` - Status string `json:"status" console:"header:Status"` - Conclusion string `json:"conclusion,omitempty" console:"-"` - Classification string `json:"classification" console:"-"` + RunID int64 `json:"run_id" console:"header:Run ID"` + Number int `json:"number" console:"-"` + WorkflowName string `json:"workflow_name" console:"header:Workflow"` + WorkflowPath string `json:"workflow_path" console:"-"` + Agent string `json:"agent,omitempty" console:"header:Agent,omitempty"` + Engine string `json:"engine,omitempty" console:"-"` + EngineID string `json:"engine_id,omitempty" console:"-"` + Status string `json:"status" console:"header:Status"` + Conclusion string `json:"conclusion,omitempty" console:"-"` + Classification string `json:"classification" console:"-"` + // FailureKind classifies the cause of a failed run. + // "driver_exit" – zero agent turns; the CLI wrapper or an infra step exited before the agent ran. + // "agent_logic" – one or more agent turns; the agent ran but the run still failed. + // "" – the run did not fail (success, cancelled, etc.). + FailureKind string `json:"failure_kind,omitempty" console:"-"` Duration string `json:"duration,omitempty" console:"header:Duration,omitempty"` ActionMinutes float64 `json:"action_minutes,omitempty" console:"header:Action Minutes,omitempty"` TokenUsage int `json:"token_usage,omitempty" console:"header:Tokens,format:number,omitempty"` @@ -167,6 +179,8 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation var totalMissingTools int var totalMissingData int var totalSafeItems int + var totalDriverExitFailures int + var totalAgentLogicFailures int var runsWithTemporaryIDChains int var runsWithDelegatedTempTargets int var runsWithMissingTemporaryIDMap int @@ -206,6 +220,16 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation totalMissingData += run.MissingDataCount totalSafeItems += run.SafeItemsCount + // Classify the failure kind for this run and accumulate rollup counts. + failureKind := "" + if isDriverExitFailure(run) { + failureKind = "driver_exit" + totalDriverExitFailures++ + } else if isFailureConclusion(run.Conclusion) { + failureKind = "agent_logic" + totalAgentLogicFailures++ + } + // Accumulate GitHub API call counts var gitHubAPICalls int if pr.GitHubRateLimitUsage != nil { @@ -272,6 +296,7 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation Status: run.Status, Conclusion: run.Conclusion, Classification: deriveRunClassification(comparison), + FailureKind: failureKind, TokenUsage: run.TokenUsage, AIC: 0, AmbientContext: ambientContext, @@ -357,6 +382,8 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation TotalMissingTools: totalMissingTools, TotalMissingData: totalMissingData, TotalSafeItems: totalSafeItems, + TotalDriverExitFailures: totalDriverExitFailures, + TotalAgentLogicFailures: totalAgentLogicFailures, RunsWithTemporaryIDChains: runsWithTemporaryIDChains, RunsWithDelegatedTempTargets: runsWithDelegatedTempTargets, RunsWithMissingTemporaryIDMap: runsWithMissingTemporaryIDMap, diff --git a/pkg/cli/logs_report_test.go b/pkg/cli/logs_report_test.go index 8e6b180c66a..2321593ffc2 100644 --- a/pkg/cli/logs_report_test.go +++ b/pkg/cli/logs_report_test.go @@ -1139,3 +1139,62 @@ func TestBuildLogsDataAggregatesTokensFromRunTokenUsage(t *testing.T) { t.Fatalf("Expected run[2].TokenUsage = 0, got %d", data.Runs[2].TokenUsage) } } + +// TestBuildLogsDataDriverExitFailureClassification verifies that buildLogsData correctly +// classifies failed runs as driver_exit (zero turns) vs agent_logic (non-zero turns) and +// accumulates the rollup counts in LogsSummary. +func TestBuildLogsDataDriverExitFailureClassification(t *testing.T) { + processedRuns := []ProcessedRun{ + {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf", Conclusion: "success", Turns: 4}}, + // driver-exit: failed, agent never ran + {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf", Conclusion: "failure", Turns: 0}}, + {Run: WorkflowRun{DatabaseID: 3, WorkflowName: "wf", Conclusion: "failure", Turns: 0}}, + // agent-logic: failed, agent ran + {Run: WorkflowRun{DatabaseID: 4, WorkflowName: "wf", Conclusion: "failure", Turns: 3}}, + } + + data := buildLogsData(processedRuns, "/tmp/logs", nil) + + if data.Summary.TotalDriverExitFailures != 2 { + t.Errorf("Expected TotalDriverExitFailures = 2, got %d", data.Summary.TotalDriverExitFailures) + } + if data.Summary.TotalAgentLogicFailures != 1 { + t.Errorf("Expected TotalAgentLogicFailures = 1, got %d", data.Summary.TotalAgentLogicFailures) + } + + // Verify per-run FailureKind + byID := make(map[int64]RunData) + for _, r := range data.Runs { + byID[r.RunID] = r + } + if byID[1].FailureKind != "" { + t.Errorf("run 1 (success): expected empty FailureKind, got %q", byID[1].FailureKind) + } + if byID[2].FailureKind != "driver_exit" { + t.Errorf("run 2 (failure, 0 turns): expected FailureKind=driver_exit, got %q", byID[2].FailureKind) + } + if byID[3].FailureKind != "driver_exit" { + t.Errorf("run 3 (failure, 0 turns): expected FailureKind=driver_exit, got %q", byID[3].FailureKind) + } + if byID[4].FailureKind != "agent_logic" { + t.Errorf("run 4 (failure, 3 turns): expected FailureKind=agent_logic, got %q", byID[4].FailureKind) + } +} + +// TestBuildLogsDataNoFailuresProducesZeroDriverExitCount verifies that zero-failure +// runs do not populate the driver-exit or agent-logic counters. +func TestBuildLogsDataNoFailuresProducesZeroDriverExitCount(t *testing.T) { + processedRuns := []ProcessedRun{ + {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf", Conclusion: "success", Turns: 5}}, + {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf", Conclusion: "success", Turns: 3}}, + } + + data := buildLogsData(processedRuns, "/tmp/logs", nil) + + if data.Summary.TotalDriverExitFailures != 0 { + t.Errorf("Expected TotalDriverExitFailures = 0, got %d", data.Summary.TotalDriverExitFailures) + } + if data.Summary.TotalAgentLogicFailures != 0 { + t.Errorf("Expected TotalAgentLogicFailures = 0, got %d", data.Summary.TotalAgentLogicFailures) + } +} From cbd6a8ae9c6d985a18eb7046c74ed3f372abb678 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:32:57 +0000 Subject: [PATCH 3/4] docs(adr): add draft ADR-47250 for driver-exit vs agent-logic failure classification Co-Authored-By: Claude Sonnet 4.6 --- ...ify-driver-exit-vs-agent-logic-failures.md | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/adr/47250-classify-driver-exit-vs-agent-logic-failures.md diff --git a/docs/adr/47250-classify-driver-exit-vs-agent-logic-failures.md b/docs/adr/47250-classify-driver-exit-vs-agent-logic-failures.md new file mode 100644 index 00000000000..5e9a1c1b419 --- /dev/null +++ b/docs/adr/47250-classify-driver-exit-vs-agent-logic-failures.md @@ -0,0 +1,50 @@ +# ADR-47250: Classify Driver-Exit (0-Turn) Failures Separately from Agent-Logic Failures + +**Date**: 2026-07-22 +**Status**: Draft +**Deciders**: Unknown (copilot-swe-agent, pelikhan) + +--- + +### Context + +Fleet health dashboards aggregate all failed workflow runs into a single failure count. Some failures occur before the agent ever runs — the CLI wrapper or a pre/post-agent infrastructure step exits non-zero with zero agent turns recorded. These infra-level failures are indistinguishable from genuine agent-logic failures (where the agent ran but the run still concluded as failure), causing health dashboards to overstate fleet ill-health and making it harder to detect actual agent regressions versus transient infrastructure flakiness. + +The existing `WorkflowRun` data model already tracks `Turns` (the number of agent conversation turns), which provides a reliable proxy: zero turns means the agent never ran, implying the failure originated in the driver layer. + +### Decision + +We will add a `driver_exit` vs `agent_logic` failure classifier throughout the health-metrics and logs-report surfaces. The classification rule is: a failed run with `Turns == 0` is a `driver_exit` failure; a failed run with `Turns > 0` is an `agent_logic` failure. This rule is implemented as `isDriverExitFailure(run WorkflowRun) bool` in `pkg/cli/logs_models.go` and propagated to `WorkflowHealth.DriverExitCount`, `WorkflowHealth.AgentLogicFailureCount`, `LogsSummary.TotalDriverExitFailures`, `LogsSummary.TotalAgentLogicFailures`, and the per-run `RunData.FailureKind` field. Operators can filter on `failure_kind` to separate infra flakiness from agent regressions without manually cross-referencing turn counts. + +### Alternatives Considered + +#### Alternative 1: Operator-Side Filtering (No Schema Change) + +Operators could filter `turns == 0` themselves when consuming the existing JSON output, without adding any new fields to the data model. This avoids expanding the public schema. + +**Why not chosen**: Every consumer would need to re-derive the classification independently, creating duplicated logic and risk of inconsistency. Embedding the classification in the output makes the distinction first-class and self-documenting for all consumers, including dashboards and automated alerts. + +#### Alternative 2: Richer Multi-Category Failure Taxonomy + +Instead of a binary `driver_exit`/`agent_logic` split keyed solely on turn count, introduce a broader `failure_category` enum (e.g., `infra_pre`, `infra_post`, `agent_logic`, `agent_timeout`) using additional signals such as log markers, job step names, or exit codes. + +**Why not chosen**: Additional signals are not consistently available across all run types and would require deeper changes to the data collection pipeline. The turn-count heuristic is immediately available, simple to verify in tests, and addresses the primary dashboard noise problem without waiting for richer instrumentation. + +### Consequences + +#### Positive +- Health dashboards no longer conflate infrastructure flakiness with agent regressions; fleet ill-health metrics become more actionable. +- The classification heuristic is simple (`turns == 0`) and fully testable without external dependencies, reducing maintenance burden. +- `failure_kind` is an additive, opt-in field (`omitempty`) so existing consumers that ignore it are unaffected. + +#### Negative +- The zero-turn heuristic is a proxy, not a guarantee: a run that fails at the agent's very first action (before completing a turn) would still report `Turns == 0` and be misclassified as `driver_exit` even if the agent did begin executing. +- `failure_kind`, `driver_exit_count`, and `agent_logic_failure_count` are new semi-stable JSON schema fields. Once dashboards and tooling build on them, changing the heuristic or renaming the fields will require a coordinated migration. + +#### Neutral +- `WorkflowHealth` gains two new exported fields that appear in JSON output; consumers doing strict schema validation may need minor updates. +- The implementation spans three files (`logs_models.go`, `health_metrics.go`, `logs_report.go`), keeping the classifier logic centralized in `logs_models.go` and consumed by the other two. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* From 8d4f46c13d7cb1c602ddf9e03c1a60c792729110 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:14:36 +0000 Subject: [PATCH 4/4] fix: add TurnsAvailable guard to driver-exit classifier, remove omitempty from rollup counters, fix cancelled comment - Add WorkflowRun.TurnsAvailable bool set only when extractLogMetrics succeeds, preventing ErrNoArtifacts runs and health-path runs (which never download artifact logs) from being mislabelled as driver_exit - Update isDriverExitFailure to require TurnsAvailable; update classification in health_metrics.go and logs_report.go to use TurnsAvailable || Turns > 0 for agent_logic so backfilled-turns runs are still classified correctly - Remove omitempty from TotalDriverExitFailures and TotalAgentLogicFailures so zero is always emitted (matching adjacent total_errors / total_missing_tools) - Fix FailureKind comment: cancelled is a failure conclusion, not a non-failure - Update tests: add TurnsAvailable:true to driver-exit fixtures, add test for the health-path (TurnsAvailable=false) and ErrNoArtifacts scenarios Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/health_metrics.go | 4 ++- pkg/cli/health_metrics_test.go | 40 ++++++++++++++++++++++------- pkg/cli/logs_models.go | 4 ++- pkg/cli/logs_report.go | 12 ++++++--- pkg/cli/logs_report_test.go | 46 ++++++++++++++++++++++++++++------ pkg/cli/logs_run_processor.go | 1 + 6 files changed, 85 insertions(+), 22 deletions(-) diff --git a/pkg/cli/health_metrics.go b/pkg/cli/health_metrics.go index ec8bfd545d4..f5216349a22 100644 --- a/pkg/cli/health_metrics.go +++ b/pkg/cli/health_metrics.go @@ -92,7 +92,9 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold failureCount++ if isDriverExitFailure(run) { driverExitCount++ - } else { + } else if run.TurnsAvailable || run.Turns > 0 { + // Only count as agent-logic when we have reliable turn data. + // Runs without artifact logs (TurnsAvailable=false, Turns=0) are left unclassified. agentLogicFailureCount++ } } diff --git a/pkg/cli/health_metrics_test.go b/pkg/cli/health_metrics_test.go index caabf9bd381..960767cccee 100644 --- a/pkg/cli/health_metrics_test.go +++ b/pkg/cli/health_metrics_test.go @@ -260,12 +260,12 @@ func TestFormatTokens(t *testing.T) { func TestCalculateWorkflowHealthDriverExitClassification(t *testing.T) { runs := []WorkflowRun{ - {Conclusion: "success", Duration: 2 * time.Minute, Turns: 3}, - // driver-exit: failed with zero turns - {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0}, - {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0}, + {Conclusion: "success", Duration: 2 * time.Minute, Turns: 3, TurnsAvailable: true}, + // driver-exit: failed with zero turns (TurnsAvailable confirms the count is real) + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0, TurnsAvailable: true}, + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0, TurnsAvailable: true}, // agent-logic: failed but agent ran (turns > 0) - {Conclusion: "failure", Duration: 2 * time.Minute, Turns: 5}, + {Conclusion: "failure", Duration: 2 * time.Minute, Turns: 5, TurnsAvailable: true}, } health := CalculateWorkflowHealth("test-workflow", runs, 80.0) @@ -289,6 +289,23 @@ func TestCalculateWorkflowHealthDriverExitCountZeroWhenNoFailures(t *testing.T) assert.Equal(t, 0, health.AgentLogicFailureCount, "agent-logic count should be zero when all runs succeed") } +func TestCalculateWorkflowHealthTurnsUnavailableSkipsClassification(t *testing.T) { + // Simulate the gh aw health path: GitHub API metadata only, no artifact logs, + // so TurnsAvailable is false for every run. Classification should be skipped and + // both DriverExitCount and AgentLogicFailureCount must stay at zero. + runs := []WorkflowRun{ + {Conclusion: "success", Duration: 2 * time.Minute}, + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0, TurnsAvailable: false}, + {Conclusion: "failure", Duration: 1 * time.Minute, Turns: 0, TurnsAvailable: false}, + } + + health := CalculateWorkflowHealth("test-workflow", runs, 80.0) + + assert.Equal(t, 2, health.FailureCount, "failure count should still be 2") + assert.Equal(t, 0, health.DriverExitCount, "driver-exit count should be zero when TurnsAvailable=false") + assert.Equal(t, 0, health.AgentLogicFailureCount, "agent-logic count should be zero when TurnsAvailable=false") +} + func TestIsDriverExitFailure(t *testing.T) { tests := []struct { name string @@ -297,22 +314,27 @@ func TestIsDriverExitFailure(t *testing.T) { }{ { name: "failure with zero turns is driver-exit", - run: WorkflowRun{Conclusion: "failure", Turns: 0}, + run: WorkflowRun{Conclusion: "failure", Turns: 0, TurnsAvailable: true}, expected: true, }, { name: "timed_out with zero turns is driver-exit", - run: WorkflowRun{Conclusion: "timed_out", Turns: 0}, + run: WorkflowRun{Conclusion: "timed_out", Turns: 0, TurnsAvailable: true}, expected: true, }, { name: "cancelled with zero turns is driver-exit", - run: WorkflowRun{Conclusion: "cancelled", Turns: 0}, + run: WorkflowRun{Conclusion: "cancelled", Turns: 0, TurnsAvailable: true}, expected: true, }, { name: "failure with non-zero turns is agent-logic (not driver-exit)", - run: WorkflowRun{Conclusion: "failure", Turns: 3}, + run: WorkflowRun{Conclusion: "failure", Turns: 3, TurnsAvailable: true}, + expected: false, + }, + { + name: "failure with zero turns but TurnsAvailable=false is unclassified", + run: WorkflowRun{Conclusion: "failure", Turns: 0, TurnsAvailable: false}, expected: false, }, { diff --git a/pkg/cli/logs_models.go b/pkg/cli/logs_models.go index d460c2c8638..44e078a5014 100644 --- a/pkg/cli/logs_models.go +++ b/pkg/cli/logs_models.go @@ -73,6 +73,7 @@ type WorkflowRun struct { ActionMinutes float64 // Billable Actions minutes estimated from wall-clock time TokenUsage int Turns int + TurnsAvailable bool // True when turn count was successfully read from artifact logs ErrorCount int WarningCount int MissingToolCount int @@ -354,6 +355,7 @@ func isFailureConclusion(conclusion string) bool { // indicates the CLI wrapper or a pre/post-agent infrastructure step exited non-zero // before the agent had a chance to run. Runs with Turns > 0 are classified as // agent-logic failures instead because the agent did execute. +// TurnsAvailable must be true to confirm the zero is real rather than missing data. func isDriverExitFailure(run WorkflowRun) bool { - return isFailureConclusion(run.Conclusion) && run.Turns == 0 + return isFailureConclusion(run.Conclusion) && run.TurnsAvailable && run.Turns == 0 } diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index 619d9cb8284..8469b52c5f5 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -69,10 +69,10 @@ type LogsSummary struct { // TotalDriverExitFailures counts failed runs with zero agent turns — the CLI wrapper // or a pre/post-agent infrastructure step exited non-zero before the agent ran. // These are infra-flakiness signals, not agent-logic regressions. - TotalDriverExitFailures int `json:"total_driver_exit_failures,omitempty" console:"header:Driver-Exit Failures,omitempty"` + TotalDriverExitFailures int `json:"total_driver_exit_failures" console:"header:Driver-Exit Failures"` // TotalAgentLogicFailures counts failed runs with one or more agent turns — the agent // started and executed but the run still concluded as a failure. - TotalAgentLogicFailures int `json:"total_agent_logic_failures,omitempty" console:"header:Agent-Logic Failures,omitempty"` + TotalAgentLogicFailures int `json:"total_agent_logic_failures" console:"header:Agent-Logic Failures"` RunsWithTemporaryIDChains int `json:"runs_with_temporary_id_chains,omitempty" console:"-"` RunsWithDelegatedTempTargets int `json:"runs_with_delegated_temp_targets,omitempty" console:"-"` RunsWithMissingTemporaryIDMap int `json:"runs_with_missing_temporary_id_map,omitempty" console:"-"` @@ -115,7 +115,7 @@ type RunData struct { // FailureKind classifies the cause of a failed run. // "driver_exit" – zero agent turns; the CLI wrapper or an infra step exited before the agent ran. // "agent_logic" – one or more agent turns; the agent ran but the run still failed. - // "" – the run did not fail (success, cancelled, etc.). + // "" – the run did not fail (success), or turn data was unavailable for classification. FailureKind string `json:"failure_kind,omitempty" console:"-"` Duration string `json:"duration,omitempty" console:"header:Duration,omitempty"` ActionMinutes float64 `json:"action_minutes,omitempty" console:"header:Action Minutes,omitempty"` @@ -221,11 +221,15 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation totalSafeItems += run.SafeItemsCount // Classify the failure kind for this run and accumulate rollup counts. + // isDriverExitFailure requires TurnsAvailable so that runs without artifact data + // (ErrNoArtifacts) are not wrongly labelled driver_exit. + // Agent-logic requires either reliable turn data (TurnsAvailable) or a confirmed + // non-zero turn count (e.g. backfilled from the usage-activity summary). failureKind := "" if isDriverExitFailure(run) { failureKind = "driver_exit" totalDriverExitFailures++ - } else if isFailureConclusion(run.Conclusion) { + } else if isFailureConclusion(run.Conclusion) && (run.TurnsAvailable || run.Turns > 0) { failureKind = "agent_logic" totalAgentLogicFailures++ } diff --git a/pkg/cli/logs_report_test.go b/pkg/cli/logs_report_test.go index 2321593ffc2..fa4466483b3 100644 --- a/pkg/cli/logs_report_test.go +++ b/pkg/cli/logs_report_test.go @@ -1141,16 +1141,16 @@ func TestBuildLogsDataAggregatesTokensFromRunTokenUsage(t *testing.T) { } // TestBuildLogsDataDriverExitFailureClassification verifies that buildLogsData correctly -// classifies failed runs as driver_exit (zero turns) vs agent_logic (non-zero turns) and -// accumulates the rollup counts in LogsSummary. +// classifies failed runs as driver_exit (zero turns, TurnsAvailable) vs agent_logic +// (non-zero turns) and accumulates the rollup counts in LogsSummary. func TestBuildLogsDataDriverExitFailureClassification(t *testing.T) { processedRuns := []ProcessedRun{ - {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf", Conclusion: "success", Turns: 4}}, - // driver-exit: failed, agent never ran - {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf", Conclusion: "failure", Turns: 0}}, - {Run: WorkflowRun{DatabaseID: 3, WorkflowName: "wf", Conclusion: "failure", Turns: 0}}, + {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf", Conclusion: "success", Turns: 4, TurnsAvailable: true}}, + // driver-exit: failed, agent never ran, artifact metrics confirmed 0 turns + {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf", Conclusion: "failure", Turns: 0, TurnsAvailable: true}}, + {Run: WorkflowRun{DatabaseID: 3, WorkflowName: "wf", Conclusion: "failure", Turns: 0, TurnsAvailable: true}}, // agent-logic: failed, agent ran - {Run: WorkflowRun{DatabaseID: 4, WorkflowName: "wf", Conclusion: "failure", Turns: 3}}, + {Run: WorkflowRun{DatabaseID: 4, WorkflowName: "wf", Conclusion: "failure", Turns: 3, TurnsAvailable: true}}, } data := buildLogsData(processedRuns, "/tmp/logs", nil) @@ -1181,6 +1181,38 @@ func TestBuildLogsDataDriverExitFailureClassification(t *testing.T) { } } +// TestBuildLogsDataNoArtifactsFailureUnclassified verifies that failed runs whose +// artifact download returned ErrNoArtifacts (TurnsAvailable=false, Turns=0) are left +// unclassified rather than mislabelled as driver_exit. +func TestBuildLogsDataNoArtifactsFailureUnclassified(t *testing.T) { + processedRuns := []ProcessedRun{ + // ErrNoArtifacts path: TurnsAvailable=false, Turns=0 — agent activity unknown + {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf", Conclusion: "failure", Turns: 0, TurnsAvailable: false}}, + // Normal driver-exit: TurnsAvailable=true confirms the zero + {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf", Conclusion: "failure", Turns: 0, TurnsAvailable: true}}, + } + + data := buildLogsData(processedRuns, "/tmp/logs", nil) + + if data.Summary.TotalDriverExitFailures != 1 { + t.Errorf("Expected TotalDriverExitFailures = 1, got %d", data.Summary.TotalDriverExitFailures) + } + if data.Summary.TotalAgentLogicFailures != 0 { + t.Errorf("Expected TotalAgentLogicFailures = 0, got %d", data.Summary.TotalAgentLogicFailures) + } + + byID := make(map[int64]RunData) + for _, r := range data.Runs { + byID[r.RunID] = r + } + if byID[1].FailureKind != "" { + t.Errorf("run 1 (no artifacts): expected empty FailureKind, got %q", byID[1].FailureKind) + } + if byID[2].FailureKind != "driver_exit" { + t.Errorf("run 2 (driver exit): expected FailureKind=driver_exit, got %q", byID[2].FailureKind) + } +} + // TestBuildLogsDataNoFailuresProducesZeroDriverExitCount verifies that zero-failure // runs do not populate the driver-exit or agent-logic counters. func TestBuildLogsDataNoFailuresProducesZeroDriverExitCount(t *testing.T) { diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 960e3e9e088..96930681589 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -368,6 +368,7 @@ func extractRunMetricsAndMetadata(result *DownloadResult, runOutputDir string, v // Update run with metrics so fingerprint computation uses the same data as the audit tool. result.Run.TokenUsage = metrics.TokenUsage result.Run.Turns = metrics.Turns + result.Run.TurnsAvailable = (metricsErr == nil) result.Run.AvgTimeBetweenTurns = metrics.AvgTimeBetweenTurns result.Run.LogsPath = runOutputDir