Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions pkg/workflow/compiler_validators.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ func (c *Compiler) validateExpressions(workflowData *WorkflowData, markdownPath
// of the recommended /tmp/gh-aw/agent/ subtree.
c.validatePromptTmpPaths(workflowData, markdownPath)

// Warn when the prompt references list_code_scanning_alerts without the required
// bounding parameters (state: open and severity: critical,high).
c.validatePromptUnboundedCodeScanningAlerts(workflowData, markdownPath)

return nil
}

Expand Down Expand Up @@ -107,6 +111,43 @@ func (c *Compiler) validatePromptTmpPaths(workflowData *WorkflowData, markdownPa
}
}

// codeScanningAlertsNeedle is the MCP tool name to scan for in prompt content.
const codeScanningAlertsNeedle = "list_code_scanning_alerts"

// codeScanningAlertsStateParam is the required state parameter value.
const codeScanningAlertsStateParam = "state: open"

// codeScanningAlertsSeverityParam is the required severity parameter value.
const codeScanningAlertsSeverityParam = "severity: critical,high"

// warnPromptUnboundedCodeScanningAlerts returns a non-empty advisory message when
// content mentions list_code_scanning_alerts without the required bounding parameters
// (state: open and severity: critical,high). Unbounded queries produce oversized MCP
// responses that break downstream workflow runs.
// Returns an empty string when no problematic pattern is found.
func warnPromptUnboundedCodeScanningAlerts(content string) string {
if !strings.Contains(content, codeScanningAlertsNeedle) {
return ""
}
missingState := !strings.Contains(content, codeScanningAlertsStateParam)
missingSeverity := !strings.Contains(content, codeScanningAlertsSeverityParam)
if !missingState && !missingSeverity {
return ""
}
return "Prompt calls list_code_scanning_alerts without required bounding parameters. " +
"Always include state: open and severity: critical,high to avoid oversized MCP responses. " +
"Example: list_code_scanning_alerts with state: open and severity: critical,high."
}

// validatePromptUnboundedCodeScanningAlerts emits an advisory warning when the workflow
// markdown body calls list_code_scanning_alerts without state: open and severity: critical,high.
func (c *Compiler) validatePromptUnboundedCodeScanningAlerts(workflowData *WorkflowData, markdownPath string) {
if msg := warnPromptUnboundedCodeScanningAlerts(workflowData.MarkdownContent); msg != "" {
fmt.Fprintln(os.Stderr, formatCompilerMessage(markdownPath, "warning", msg))
c.IncrementWarningCount()
}
}

// validateFeatureConfig validates feature flags declared in the workflow frontmatter
// and applies any action-mode override specified via the "action-mode" feature flag.
func (c *Compiler) validateFeatureConfig(workflowData *WorkflowData, markdownPath string) error {
Expand Down
115 changes: 115 additions & 0 deletions pkg/workflow/compiler_validators_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -697,3 +697,118 @@ func TestValidatePromptTmpPaths(t *testing.T) {
})
}
}

// TestWarnPromptUnboundedCodeScanningAlerts tests the pure heuristic that detects
// list_code_scanning_alerts references without the required bounding parameters.
func TestWarnPromptUnboundedCodeScanningAlerts(t *testing.T) {
tests := []struct {
name string
content string
expectWarning bool
warnContains string
}{
{
name: "no mention of list_code_scanning_alerts — no warning",
content: "# Workflow\n\nList open pull requests.",
expectWarning: false,
},
{
name: "mentions tool with both required params — no warning",
content: "Call list_code_scanning_alerts with state: open and severity: critical,high.",
expectWarning: false,
},
{
name: "mentions tool without state: open — warning",
content: "Call list_code_scanning_alerts with severity: critical,high.",
expectWarning: true,
warnContains: "state: open",
},
{
name: "mentions tool without severity: critical,high — warning",
content: "Call list_code_scanning_alerts with state: open.",
expectWarning: true,
warnContains: "severity: critical,high",
},
{
name: "mentions tool without any bounding params — warning",
content: "Use list_code_scanning_alerts to fetch all alerts.",
expectWarning: true,
warnContains: "list_code_scanning_alerts",
},
{
name: "multi-line prompt with both params present — no warning",
content: "# Scan\n\nCall list_code_scanning_alerts.\nUse state: open and severity: critical,high.",
expectWarning: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msg := warnPromptUnboundedCodeScanningAlerts(tt.content)
if tt.expectWarning {
assert.NotEmpty(t, msg, "expected a warning message")
if tt.warnContains != "" {
assert.Contains(t, msg, tt.warnContains, "warning should mention the missing parameter")
}
} else {
assert.Empty(t, msg, "expected no warning message")
}
})
}
}

// TestValidatePromptUnboundedCodeScanningAlerts tests that the compiler method
// increments the warning counter when the markdown body calls list_code_scanning_alerts
// without the required bounding parameters.
func TestValidatePromptUnboundedCodeScanningAlerts(t *testing.T) {
tests := []struct {
name string
markdown string
expectWarn bool
}{
{
name: "no code scanning mention — no warning",
markdown: "# Hello\n\nList open issues.",
expectWarn: false,
},
{
name: "properly bounded query — no warning",
markdown: "Call list_code_scanning_alerts with state: open and severity: critical,high.",
expectWarn: false,
},
{
name: "unbounded query — warning",
markdown: "Call list_code_scanning_alerts to fetch all alerts.",
expectWarn: true,
},
{
name: "missing severity only — warning",
markdown: "Use list_code_scanning_alerts with state: open.",
expectWarn: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir := testutil.TempDir(t, "code-scanning-test")
markdownPath := filepath.Join(tmpDir, "workflow.md")

compiler := NewCompiler()
workflowData := &WorkflowData{
Name: "Test",
MarkdownContent: tt.markdown,
AI: "copilot",
}

before := compiler.GetWarningCount()
compiler.validatePromptUnboundedCodeScanningAlerts(workflowData, markdownPath)
after := compiler.GetWarningCount()

if tt.expectWarn {
assert.Greater(t, after, before, "warning count should have increased")
} else {
assert.Equal(t, before, after, "warning count should not have changed")
}
})
}
}