From 76a216387f28b0c7c7d4f62bec33466544731ed6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:35:04 +0000 Subject: [PATCH 1/3] feat: detect agent-job step outputs referenced in prompt body Add a new compile-time validation that errors when a workflow's markdown body (the agent prompt) references ${{ steps.STEP_ID.* }} outputs from steps defined in the frontmatter steps:/pre-steps:/pre-agent-steps:/ post-steps: sections. The prompt is rendered in the activation job; those steps execute in the agent job (after activation), so their outputs are unavailable when the prompt is assembled. The correct pattern is to write results to files under /tmp/gh-aw/agent/ and reference those file paths in the prompt. Built-in activation-job steps (e.g. steps.sanitized.outputs.text) are not flagged because they are not declared in the user-visible step lists. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_validators.go | 7 + .../steps_output_in_prompt_validation.go | 125 ++++++++ .../steps_output_in_prompt_validation_test.go | 266 ++++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 pkg/workflow/steps_output_in_prompt_validation.go create mode 100644 pkg/workflow/steps_output_in_prompt_validation_test.go diff --git a/pkg/workflow/compiler_validators.go b/pkg/workflow/compiler_validators.go index 53f0eb7abcc..4e66882f666 100644 --- a/pkg/workflow/compiler_validators.go +++ b/pkg/workflow/compiler_validators.go @@ -62,6 +62,13 @@ func (c *Compiler) validateExpressions(workflowData *WorkflowData, markdownPath // of the recommended /tmp/gh-aw/agent/ subtree. c.validatePromptTmpPaths(workflowData, markdownPath) + // Detect agent-job step outputs referenced in the prompt. The prompt is rendered + // in the activation job; agent-job steps run later and their outputs are not + // available at prompt-creation time. + if err := validateStepsOutputsNotInPrompt(workflowData); err != nil { + return formatCompilerError(markdownPath, "error", err.Error(), err) + } + return nil } diff --git a/pkg/workflow/steps_output_in_prompt_validation.go b/pkg/workflow/steps_output_in_prompt_validation.go new file mode 100644 index 00000000000..1ccf4cefbd2 --- /dev/null +++ b/pkg/workflow/steps_output_in_prompt_validation.go @@ -0,0 +1,125 @@ +// This file validates that agent-job step outputs are not referenced in the +// workflow prompt (markdown body). The prompt is built in the activation job; +// agent-job steps run after the activation job and their outputs are therefore +// unavailable when the prompt is assembled. Steps that need to pass data into +// the prompt should write results to files instead. + +package workflow + +import ( + "fmt" + "regexp" + "strings" + + "github.com/github/gh-aw/pkg/logger" + "github.com/goccy/go-yaml" +) + +var stepsOutputInPromptLog = logger.New("workflow:steps_output_in_prompt_validation") + +// stepsRefInPromptPattern matches ${{ steps.STEP_ID ... }} occurrences and +// captures the step ID so it can be compared against the agent-job step list. +var stepsRefInPromptPattern = regexp.MustCompile(`\$\{\{\s*steps\.([a-zA-Z0-9_-]+)`) + +// validateStepsOutputsNotInPrompt checks that no agent-job step outputs are +// referenced in the prompt body (MarkdownContent). The prompt is rendered in +// the activation job; steps defined in the frontmatter steps: / pre-steps: / +// pre-agent-steps: / post-steps: sections run inside the agent job (after +// activation) and their outputs are not available at prompt-creation time. +// +// Use files (e.g. /tmp/gh-aw/agent/result.txt) to pass data from agent-job +// steps into the prompt instead of step outputs. +func validateStepsOutputsNotInPrompt(workflowData *WorkflowData) error { + if workflowData == nil || workflowData.MarkdownContent == "" { + return nil + } + if !strings.Contains(workflowData.MarkdownContent, "${{ steps.") { + return nil + } + + agentStepIDs := extractAgentJobStepIDs(workflowData) + if len(agentStepIDs) == 0 { + stepsOutputInPromptLog.Print("No agent-job step IDs found; skipping prompt step-output check") + return nil + } + + matches := stepsRefInPromptPattern.FindAllStringSubmatch(workflowData.MarkdownContent, -1) + + seen := make(map[string]struct{}) + var offending []string + for _, match := range matches { + if len(match) < 2 { + continue + } + stepID := match[1] + if _, isAgentStep := agentStepIDs[stepID]; isAgentStep { + if _, already := seen[stepID]; !already { + offending = append(offending, stepID) + seen[stepID] = struct{}{} + } + } + } + + if len(offending) == 0 { + stepsOutputInPromptLog.Print("No agent-job step outputs found in prompt") + return nil + } + + stepsOutputInPromptLog.Printf("Found %d agent-job step output(s) in prompt: %v", len(offending), offending) + + return NewValidationError( + "steps-output-in-prompt", + "steps referenced in prompt: "+strings.Join(offending, ", "), + fmt.Sprintf( + "step(s) %s are defined in the agent job and run after the prompt is rendered in the activation job; "+ + "their outputs are not available at prompt-creation time", + strings.Join(offending, ", "), + ), + "Write step results to a file (e.g. /tmp/gh-aw/agent/result.txt) and reference that file path in the prompt instead of using ${{ steps.STEP_ID.outputs.* }}.", + ) +} + +// extractAgentJobStepIDs returns the set of step IDs defined across all +// agent-job step sections: steps:, pre-steps:, pre-agent-steps:, post-steps:. +func extractAgentJobStepIDs(workflowData *WorkflowData) map[string]struct{} { + ids := make(map[string]struct{}) + for _, stepsYAML := range []string{ + workflowData.CustomSteps, + workflowData.PreSteps, + workflowData.PreAgentSteps, + workflowData.PostSteps, + } { + if stepsYAML == "" { + continue + } + addStepIDsFromYAML(stepsYAML, ids) + } + return ids +} + +// addStepIDsFromYAML parses a YAML string that wraps a steps list under a +// top-level key (e.g. "steps:", "pre-steps:") and collects each step's id. +func addStepIDsFromYAML(stepsYAML string, ids map[string]struct{}) { + var wrapper map[string]any + if err := yaml.Unmarshal([]byte(stepsYAML), &wrapper); err != nil { + stepsOutputInPromptLog.Printf("Failed to parse steps YAML: %v", err) + return + } + // The root key varies ("steps", "pre-steps", "pre-agent-steps", "post-steps"), + // so iterate over all top-level values and look for step lists. + for _, value := range wrapper { + steps, ok := value.([]any) + if !ok { + continue + } + for _, step := range steps { + stepMap, ok := step.(map[string]any) + if !ok { + continue + } + if id, ok := stepMap["id"].(string); ok && id != "" { + ids[id] = struct{}{} + } + } + } +} diff --git a/pkg/workflow/steps_output_in_prompt_validation_test.go b/pkg/workflow/steps_output_in_prompt_validation_test.go new file mode 100644 index 00000000000..788b888a2a6 --- /dev/null +++ b/pkg/workflow/steps_output_in_prompt_validation_test.go @@ -0,0 +1,266 @@ +//go:build !integration + +package workflow + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// agentJobStepsYAML is a reusable YAML snippet that defines two agent-job steps +// with IDs "prepare" and "fetch_data". Used across multiple subtests. +const agentJobStepsYAML = `steps: + - name: Prepare workspace + id: prepare + run: | + mkdir -p /tmp/gh-aw/agent + - name: Fetch data + id: fetch_data + run: | + echo "result=42" >> $GITHUB_OUTPUT +` + +func TestValidateStepsOutputsNotInPrompt(t *testing.T) { + tests := []struct { + name string + markdownBody string + customSteps string + shouldError bool + errorContains string + }{ + { + name: "no steps, no expressions", + markdownBody: "# My Workflow\n\nDo something useful.", + customSteps: "", + shouldError: false, + }, + { + name: "steps with no outputs in prompt", + markdownBody: "# My Workflow\n\nRead results from /tmp/gh-aw/agent/result.txt.", + customSteps: agentJobStepsYAML, + shouldError: false, + }, + { + name: "steps with unrelated expressions in prompt", + markdownBody: "# My Workflow\n\n" + + "Repository: ${{ github.repository }}\n" + + "PR number: ${{ github.event.pull_request.number }}\n", + customSteps: agentJobStepsYAML, + shouldError: false, + }, + { + name: "steps.sanitized in prompt without agent-job steps", + markdownBody: "# My Workflow\n\n" + + "Content: ${{ steps.sanitized.outputs.text }}\n", + customSteps: "", + shouldError: false, + }, + { + name: "steps.sanitized in prompt with different agent-job steps", + markdownBody: "# My Workflow\n\n" + + "Content: ${{ steps.sanitized.outputs.text }}\n", + customSteps: agentJobStepsYAML, + shouldError: false, + }, + { + name: "agent-job step output referenced in prompt", + markdownBody: "# My Workflow\n\n" + + "The result is: ${{ steps.fetch_data.outputs.result }}\n", + customSteps: agentJobStepsYAML, + shouldError: true, + errorContains: "fetch_data", + }, + { + name: "multiple agent-job step outputs referenced in prompt", + markdownBody: "# My Workflow\n\n" + + "Prep: ${{ steps.prepare.outputs.status }}\n" + + "Result: ${{ steps.fetch_data.outputs.result }}\n", + customSteps: agentJobStepsYAML, + shouldError: true, + errorContains: "steps-output-in-prompt", + }, + { + name: "step without id not flagged even if referenced", + markdownBody: "# My Workflow\n\n" + + "Value: ${{ steps.unnamed_step.outputs.value }}\n", + customSteps: `steps: + - name: A step without an id + run: echo "no id here" +`, + shouldError: false, + }, + { + name: "needs.* expressions not affected", + markdownBody: "# My Workflow\n\n" + + "CI status: ${{ needs.check_ci.outputs.status }}\n", + customSteps: agentJobStepsYAML, + shouldError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workflowData := &WorkflowData{ + Name: "Test", + MarkdownContent: tt.markdownBody, + CustomSteps: tt.customSteps, + } + + err := validateStepsOutputsNotInPrompt(workflowData) + if tt.shouldError { + require.Error(t, err, "expected validateStepsOutputsNotInPrompt to return an error") + if tt.errorContains != "" { + assert.Contains(t, err.Error(), tt.errorContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExtractAgentJobStepIDs(t *testing.T) { + tests := []struct { + name string + customSteps string + preSteps string + preAgentSteps string + postSteps string + expectedIDs []string + notExpectedIDs []string + }{ + { + name: "empty steps", + customSteps: "", + expectedIDs: []string{}, + }, + { + name: "custom steps with IDs", + customSteps: agentJobStepsYAML, + expectedIDs: []string{"prepare", "fetch_data"}, + }, + { + name: "step without id is not collected", + customSteps: `steps: + - name: Setup + run: echo hi +`, + expectedIDs: []string{}, + notExpectedIDs: []string{"Setup"}, + }, + { + name: "IDs collected from multiple step sections", + customSteps: `steps: + - name: Main + id: main_step + run: echo main +`, + preSteps: `pre-steps: + - name: Pre + id: pre_step + run: echo pre +`, + postSteps: `post-steps: + - name: Post + id: post_step + run: echo post +`, + expectedIDs: []string{"main_step", "pre_step", "post_step"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workflowData := &WorkflowData{ + CustomSteps: tt.customSteps, + PreSteps: tt.preSteps, + PreAgentSteps: tt.preAgentSteps, + PostSteps: tt.postSteps, + } + + ids := extractAgentJobStepIDs(workflowData) + + for _, expectedID := range tt.expectedIDs { + assert.Contains(t, ids, expectedID, "expected step ID %q to be collected", expectedID) + } + for _, notExpectedID := range tt.notExpectedIDs { + assert.NotContains(t, ids, notExpectedID, "step ID %q should not be collected", notExpectedID) + } + }) + } +} + +// TestValidateStepsOutputsNotInPromptViaCompiler verifies that the validation +// is wired into the compilation pipeline and produces a compile error when a +// step output is referenced in the prompt body. +func TestValidateStepsOutputsNotInPromptViaCompiler(t *testing.T) { + tests := []struct { + name string + workflow string + shouldError bool + errorContains string + }{ + { + name: "agent-job step output referenced in prompt produces error", + workflow: `--- +on: push +permissions: + contents: read +engine: copilot +steps: + - name: Compute result + id: compute + run: echo "value=42" >> $GITHUB_OUTPUT +--- + +# My Workflow + +Process this value: ${{ steps.compute.outputs.value }} +`, + shouldError: true, + errorContains: "compute", + }, + { + name: "file-based approach compiles without error", + workflow: `--- +on: push +permissions: + contents: read +engine: copilot +steps: + - name: Compute result + id: compute + run: echo "42" > /tmp/gh-aw/agent/result.txt +--- + +# My Workflow + +Read the result from /tmp/gh-aw/agent/result.txt. +`, + shouldError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + workflowPath := filepath.Join(tmpDir, "test-workflow.md") + require.NoError(t, os.WriteFile(workflowPath, []byte(tt.workflow), 0644)) + + compiler := NewCompiler() + err := compiler.CompileWorkflow(workflowPath) + if tt.shouldError { + require.Error(t, err, "expected compilation to fail") + if tt.errorContains != "" { + assert.Contains(t, err.Error(), tt.errorContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} From 33cff22bfa11c30fcb9ab9c17e6f2b6d262de1a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:12:20 +0000 Subject: [PATCH 2/3] docs(adr): add draft ADR-48610 for compile-time step-output-in-prompt validation --- ...10-validate-agent-step-output-in-prompt.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/adr/48610-validate-agent-step-output-in-prompt.md diff --git a/docs/adr/48610-validate-agent-step-output-in-prompt.md b/docs/adr/48610-validate-agent-step-output-in-prompt.md new file mode 100644 index 00000000000..36f40be3865 --- /dev/null +++ b/docs/adr/48610-validate-agent-step-output-in-prompt.md @@ -0,0 +1,48 @@ +# ADR-48610: Compile-Time Validation of Agent-Job Step Outputs in Prompt Body + +**Date**: 2026-07-28 +**Status**: Draft +**Deciders**: pelikhan, copilot-swe-agent + +--- + +### Context + +gh-aw workflows have a two-job architecture: the **activation job** renders the prompt (markdown body) and starts the agent; the **agent job** runs the steps defined in `steps:`, `pre-steps:`, `pre-agent-steps:`, and `post-steps:` sections. Because the prompt is rendered before the agent job executes, any `${{ steps.STEP_ID.outputs.* }}` expression referencing an agent-job step silently resolves to an empty string at prompt-creation time. This is a confusing silent failure: the workflow compiles successfully, the agent receives an incomplete prompt, and the root cause is non-obvious. Authors familiar with standard GitHub Actions expect step outputs to be available in surrounding content, and this architectural constraint is not surfaced until runtime (if at all). + +### Decision + +We will add a compile-time validator (`validateStepsOutputsNotInPrompt`) that scans the prompt body for `${{ steps.STEP_ID.* }}` expressions and cross-references the step ID against the set of IDs declared in agent-job step sections. If a match is found, compilation is rejected with an actionable error that names the offending step and directs authors to the file-based alternative (`/tmp/gh-aw/agent/result.txt`). Built-in activation-job steps (e.g., `steps.sanitized`) are not flagged because they do not appear in user-visible step lists. The validator is wired into the existing `validateExpressions` gate in `compiler_validators.go`. + +### Alternatives Considered + +#### Alternative 1: Runtime Warning Without Compilation Failure + +Detect the pattern at compile time but emit only a log warning rather than a hard error, allowing the workflow to proceed. Authors would see a warning in CI logs but the workflow would still run with an empty prompt value. + +This was rejected because a warning is too easy to overlook, and the resulting runtime behavior (silently empty prompt variable) is indistinguishable from an intentional empty value. The goal is to prevent a class of hard-to-debug bugs, which requires failing loudly at the earliest opportunity. + +#### Alternative 2: Documentation Only, No Enforcement + +Document the two-job execution order in the workflow authoring guide and rely on authors to internalize the constraint without automated enforcement. + +This was rejected because it scales poorly: each new author must discover the limitation individually (typically after a confusing empty-prompt incident), and there is no mechanism to prevent the pattern from being introduced by automated tooling or copy-paste from standard GitHub Actions workflows. + +### Consequences + +#### Positive +- Prevents the silent empty-prompt class of bugs at the earliest point in the authoring cycle (compile time), before the workflow is ever run. +- Provides an actionable error message that names the offending step ID and links directly to the file-based alternative, reducing debugging time. +- Makes the two-job execution boundary explicit in the toolchain, reinforcing the architectural constraint for all authors. + +#### Negative +- Any workflow that currently relies on the silent-empty behavior (however unintentionally) will fail compilation after this change is deployed, requiring a migration to file-based data passing. +- The YAML parsing step in `addStepIDsFromYAML` adds a small compile-time cost for every workflow that contains step sections, even when the prompt contains no `${{ steps.* }}` expressions (mitigated by an early-exit fast path). + +#### Neutral +- The validator only flags agent-job step IDs that appear in the user-visible step YAML sections; built-in activation-job step references (e.g., `${{ steps.sanitized.outputs.text }}`) pass through unchanged, so existing valid workflows are unaffected. +- The file-based pattern (`/tmp/gh-aw/agent/result.txt`) that replaces the forbidden pattern was already the recommended approach; this change codifies that recommendation as an enforcement rule. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* From f22020b041592bc50bccf66704b619785c3fbb33 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:04:10 +0000 Subject: [PATCH 3/3] fix: scan all steps refs in multi-operand expressions Use a two-step approach: extract ${{ ... }} expression bodies first, then find all steps.STEP_ID occurrences within each body. Previously the regex anchored to ${{ so only the first operand in expressions like ${{ steps.a.outputs.x || steps.b.outputs.y }} was checked. Add test case covering the multi-operand expression scenario. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../steps_output_in_prompt_validation.go | 32 ++++++++++++------- .../steps_output_in_prompt_validation_test.go | 8 +++++ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/pkg/workflow/steps_output_in_prompt_validation.go b/pkg/workflow/steps_output_in_prompt_validation.go index 1ccf4cefbd2..b0caf75bc24 100644 --- a/pkg/workflow/steps_output_in_prompt_validation.go +++ b/pkg/workflow/steps_output_in_prompt_validation.go @@ -17,9 +17,13 @@ import ( var stepsOutputInPromptLog = logger.New("workflow:steps_output_in_prompt_validation") -// stepsRefInPromptPattern matches ${{ steps.STEP_ID ... }} occurrences and -// captures the step ID so it can be compared against the agent-job step list. -var stepsRefInPromptPattern = regexp.MustCompile(`\$\{\{\s*steps\.([a-zA-Z0-9_-]+)`) +// stepsExprBodyPattern extracts the body of each ${{ ... }} expression so that +// all operands can be inspected for steps references. +var stepsExprBodyPattern = regexp.MustCompile(`\$\{\{([\s\S]*?)\}\}`) + +// stepsRefInExprPattern matches every steps.STEP_ID occurrence inside an +// expression body (after stripping the surrounding ${{ }}). +var stepsRefInExprPattern = regexp.MustCompile(`\bsteps\.([a-zA-Z0-9_-]+)`) // validateStepsOutputsNotInPrompt checks that no agent-job step outputs are // referenced in the prompt body (MarkdownContent). The prompt is rendered in @@ -43,19 +47,23 @@ func validateStepsOutputsNotInPrompt(workflowData *WorkflowData) error { return nil } - matches := stepsRefInPromptPattern.FindAllStringSubmatch(workflowData.MarkdownContent, -1) - seen := make(map[string]struct{}) var offending []string - for _, match := range matches { - if len(match) < 2 { + for _, exprMatch := range stepsExprBodyPattern.FindAllStringSubmatch(workflowData.MarkdownContent, -1) { + if len(exprMatch) < 2 { continue } - stepID := match[1] - if _, isAgentStep := agentStepIDs[stepID]; isAgentStep { - if _, already := seen[stepID]; !already { - offending = append(offending, stepID) - seen[stepID] = struct{}{} + body := exprMatch[1] + for _, refMatch := range stepsRefInExprPattern.FindAllStringSubmatch(body, -1) { + if len(refMatch) < 2 { + continue + } + stepID := refMatch[1] + if _, isAgentStep := agentStepIDs[stepID]; isAgentStep { + if _, already := seen[stepID]; !already { + offending = append(offending, stepID) + seen[stepID] = struct{}{} + } } } } diff --git a/pkg/workflow/steps_output_in_prompt_validation_test.go b/pkg/workflow/steps_output_in_prompt_validation_test.go index 788b888a2a6..591bf60f994 100644 --- a/pkg/workflow/steps_output_in_prompt_validation_test.go +++ b/pkg/workflow/steps_output_in_prompt_validation_test.go @@ -100,6 +100,14 @@ func TestValidateStepsOutputsNotInPrompt(t *testing.T) { customSteps: agentJobStepsYAML, shouldError: false, }, + { + name: "agent-job step reference as non-first operand in expression", + markdownBody: "# My Workflow\n\n" + + "Value: ${{ steps.sanitized.outputs.text || steps.fetch_data.outputs.result }}\n", + customSteps: agentJobStepsYAML, + shouldError: true, + errorContains: "fetch_data", + }, } for _, tt := range tests {