From cc95cfc3e2f80bfc78b270b7fd715a04e0a55e10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:49:43 +0000 Subject: [PATCH 1/2] Add evals AIC to generated footers Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/handle_noop_message.cjs | 6 +- actions/setup/js/handle_noop_message.test.cjs | 24 ++++++++ actions/setup/js/messages.test.cjs | 60 +++++++++++++++++++ actions/setup/js/messages_footer.cjs | 32 +++++++++- pkg/workflow/compiler_jobs_test.go | 1 + pkg/workflow/evals_job.go | 5 +- pkg/workflow/evals_job_test.go | 2 + .../notify_comment_conclusion_helpers.go | 6 ++ 8 files changed, 130 insertions(+), 6 deletions(-) diff --git a/actions/setup/js/handle_noop_message.cjs b/actions/setup/js/handle_noop_message.cjs index 14633486c27..c3d82a56b76 100644 --- a/actions/setup/js/handle_noop_message.cjs +++ b/actions/setup/js/handle_noop_message.cjs @@ -79,17 +79,19 @@ async function ensureAgentRunsIssue() { /** * Build the AIC suffix string for use in comment footers. - * Includes both agent and threat-detection AIC when available. + * Includes agent, threat-detection, and evals AIC when available. * Returns a string like " · 0.001 AIC" or "" when not available. * @returns {string} */ function buildAICSuffix() { const agentRaw = process.env.GH_AW_AIC; const detectionRaw = process.env.GH_AW_THREAT_DETECTION_AIC; + const evalsRaw = process.env.GH_AW_EVALS_AIC; const agentAIC = agentRaw ? Number.parseFloat(agentRaw) : NaN; const detectionAIC = detectionRaw ? Number.parseFloat(detectionRaw) : NaN; + const evalsAIC = evalsRaw ? Number.parseFloat(evalsRaw) : NaN; const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL); - const totalAIC = (Number.isFinite(agentAIC) && agentAIC > 0 ? agentAIC : 0) + (Number.isFinite(detectionAIC) && detectionAIC > 0 ? detectionAIC : 0); + const totalAIC = (Number.isFinite(agentAIC) && agentAIC > 0 ? agentAIC : 0) + (Number.isFinite(detectionAIC) && detectionAIC > 0 ? detectionAIC : 0) + (Number.isFinite(evalsAIC) && evalsAIC > 0 ? evalsAIC : 0); if (totalAIC <= 0) { return ""; } diff --git a/actions/setup/js/handle_noop_message.test.cjs b/actions/setup/js/handle_noop_message.test.cjs index 3bde070351e..4e57b160fc7 100644 --- a/actions/setup/js/handle_noop_message.test.cjs +++ b/actions/setup/js/handle_noop_message.test.cjs @@ -781,6 +781,30 @@ This issue helps you: expect(commentCall.body).toContain("sonnet46 0.125 AIC"); }); + it("should include evals AIC in the footer total when GH_AW_EVALS_AIC is set", async () => { + process.env.GH_AW_WORKFLOW_NAME = "Evals AIC Workflow"; + process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123"; + process.env.GH_AW_AGENT_CONCLUSION = "success"; + process.env.GH_AW_AIC = "0.100"; + process.env.GH_AW_EVALS_AIC = "0.025"; + process.env.GH_AW_ENGINE_MODEL = "claude-sonnet-4.6"; + + const outputFile = path.join(tempDir, "agent_output.json"); + fs.writeFileSync(outputFile, JSON.stringify({ items: [{ type: "noop", message: "No action needed" }] })); + process.env.GH_AW_AGENT_OUTPUT = outputFile; + + mockGithub.rest.search.issuesAndPullRequests.mockResolvedValue({ + data: { total_count: 1, items: [{ number: 1, node_id: "ID", html_url: "url" }] }, + }); + mockGithub.rest.issues.createComment.mockResolvedValue({ data: {} }); + + const { main } = await import("./handle_noop_message.cjs?t=" + Date.now()); + await main(); + + const commentCall = mockGithub.rest.issues.createComment.mock.calls[0][0]; + expect(commentCall.body).toContain("sonnet46 0.125 AIC"); + }); + it("should not include AIC suffix in comment footer when GH_AW_AIC is not set", async () => { process.env.GH_AW_WORKFLOW_NAME = "No Credits Workflow"; process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123"; diff --git a/actions/setup/js/messages.test.cjs b/actions/setup/js/messages.test.cjs index 5de160f29ab..8a71ee09439 100644 --- a/actions/setup/js/messages.test.cjs +++ b/actions/setup/js/messages.test.cjs @@ -40,6 +40,7 @@ describe("messages.cjs", () => { delete process.env.GH_AW_AIC; delete process.env.GH_AW_AMBIENT_CONTEXT; delete process.env.GH_AW_AGENT_AIC; + delete process.env.GH_AW_EVALS_AIC; delete process.env.GH_AW_THREAT_DETECTION_AIC; delete process.env.GH_AW_LABEL_COMMANDS; delete process.env.GH_AW_DETECTION_CONCLUSION; @@ -472,6 +473,20 @@ describe("messages.cjs", () => { expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · 0.125 AIC · ⌖ 0.025 AIC · ⊞ 900"); }); + it("should include evals AI Credits with a Unicode icon in the default footer", async () => { + process.env.GH_AW_AGENT_AIC = "0.125"; + process.env.GH_AW_EVALS_AIC = "0.01"; + + const { getFooterMessage } = await import("./messages.cjs"); + + const result = getFooterMessage({ + workflowName: "Test Workflow", + runUrl: "https://github.com/test/repo/actions/runs/123", + }); + + expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · 0.125 AIC · ◇ 0.01 AIC"); + }); + it("should not include effective tokens when GH_AW_EFFECTIVE_TOKENS is not set and not passed in context", async () => { delete process.env.GH_AW_EFFECTIVE_TOKENS; @@ -534,6 +549,23 @@ describe("messages.cjs", () => { expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ⌖ 0.25 AIC"); }); + it("should expose evals AI Credits entry in custom footer templates", async () => { + process.env.GH_AW_AGENT_AIC = "1.25"; + process.env.GH_AW_EVALS_AIC = "0.25"; + process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({ + footer: "> Custom: [{workflow_name}]({run_url}){ai_credits_suffix}", + }); + + const { getFooterMessage } = await import("./messages.cjs"); + + const result = getFooterMessage({ + workflowName: "Test Workflow", + runUrl: "https://github.com/test/repo/actions/runs/123", + }); + + expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC"); + }); + it("should include ambient context in ai_credits_suffix for custom footer templates", async () => { process.env.GH_AW_AGENT_AIC = "1.25"; process.env.GH_AW_THREAT_DETECTION_AIC = "0.25"; @@ -1156,6 +1188,20 @@ describe("messages.cjs", () => { expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · sonnet46 1.25 AIC · ⊞ 900"); }); + it("should include evals AI Credits in the default footer when available", async () => { + process.env.GH_AW_AIC = "1.25"; + process.env.GH_AW_EVALS_AIC = "0.25"; + + const { getFooterAgentFailureIssueMessage } = await import("./messages.cjs"); + + const result = getFooterAgentFailureIssueMessage({ + workflowName: "Test Workflow", + runUrl: "https://github.com/test/repo/actions/runs/123", + }); + + expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC"); + }); + it("should suppress env AIC fallback when explicit context AIC is zero", async () => { process.env.GH_AW_AIC = "1.25"; @@ -1256,6 +1302,20 @@ describe("messages.cjs", () => { expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · sonnet46 1.25 AIC · ⊞ 900"); }); + it("should include evals AI Credits in the default comment footer when available", async () => { + process.env.GH_AW_AIC = "1.25"; + process.env.GH_AW_EVALS_AIC = "0.25"; + + const { getFooterAgentFailureCommentMessage } = await import("./messages.cjs"); + + const result = getFooterAgentFailureCommentMessage({ + workflowName: "Test Workflow", + runUrl: "https://github.com/test/repo/actions/runs/123", + }); + + expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC"); + }); + it("should expose ai_credits_suffix in custom comment footer templates", async () => { process.env.GH_AW_AMBIENT_CONTEXT = "900"; process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({ diff --git a/actions/setup/js/messages_footer.cjs b/actions/setup/js/messages_footer.cjs index 5956bfd75eb..978adb4922d 100644 --- a/actions/setup/js/messages_footer.cjs +++ b/actions/setup/js/messages_footer.cjs @@ -101,6 +101,9 @@ function buildAICEntry(label, value, modelAlias) { * agentAiCredits: number|undefined, * agentAiCreditsFormatted: string|undefined, * agentAiCreditsSuffix: string, + * evalsAiCredits: number|undefined, + * evalsAiCreditsFormatted: string|undefined, + * evalsAiCreditsSuffix: string, * threatDetectionAiCredits: number|undefined, * threatDetectionAiCreditsFormatted: string|undefined, * threatDetectionAiCreditsSuffix: string @@ -110,14 +113,16 @@ function getAICFromEnv() { const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL); const totalAIC = parsePositiveAIC(process.env.GH_AW_AIC); const explicitAgentAIC = parsePositiveAIC(process.env.GH_AW_AGENT_AIC); + const evalsAIC = parsePositiveAIC(process.env.GH_AW_EVALS_AIC); const threatDetectionAIC = parsePositiveAIC(process.env.GH_AW_THREAT_DETECTION_AIC); const agentAIC = typeof explicitAgentAIC === "number" ? explicitAgentAIC : totalAIC; const agentEntry = buildAICEntry("", agentAIC, compressedModelName); + const evalsEntry = buildAICEntry("◇", evalsAIC); const threatDetectionEntry = buildAICEntry("⌖", threatDetectionAIC); - const useBreakdown = threatDetectionEntry.suffix.length > 0; - const aiCredits = useBreakdown ? (agentAIC || 0) + (threatDetectionAIC || 0) : typeof totalAIC === "number" ? totalAIC : agentAIC; + const useBreakdown = threatDetectionEntry.suffix.length > 0 || evalsEntry.suffix.length > 0; + const aiCredits = useBreakdown ? (agentAIC || 0) + (threatDetectionAIC || 0) + (evalsAIC || 0) : typeof totalAIC === "number" ? totalAIC : agentAIC; const aiCreditsFormatted = typeof aiCredits === "number" ? formatAIC(aiCredits) : undefined; - const aiCreditsSuffix = useBreakdown ? `${agentEntry.suffix}${threatDetectionEntry.suffix}` : buildAICEntry("", aiCredits, compressedModelName).suffix; + const aiCreditsSuffix = useBreakdown ? `${agentEntry.suffix}${threatDetectionEntry.suffix}${evalsEntry.suffix}` : buildAICEntry("", aiCredits, compressedModelName).suffix; return { aiCredits, @@ -127,6 +132,9 @@ function getAICFromEnv() { agentAiCredits: agentEntry.value, agentAiCreditsFormatted: agentEntry.formatted, agentAiCreditsSuffix: agentEntry.suffix, + evalsAiCredits: evalsEntry.value, + evalsAiCreditsFormatted: evalsEntry.formatted, + evalsAiCreditsSuffix: evalsEntry.suffix, threatDetectionAiCredits: threatDetectionEntry.value, threatDetectionAiCreditsFormatted: threatDetectionEntry.formatted, threatDetectionAiCreditsSuffix: threatDetectionEntry.suffix, @@ -167,6 +175,9 @@ function getFooterMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, @@ -205,6 +216,9 @@ function getFooterMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, @@ -387,6 +401,9 @@ function getFooterAgentFailureIssueMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, @@ -410,6 +427,9 @@ function getFooterAgentFailureIssueMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, @@ -463,6 +483,9 @@ function getFooterAgentFailureCommentMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, @@ -486,6 +509,9 @@ function getFooterAgentFailureCommentMessage(ctx) { agentAiCredits, agentAiCreditsFormatted, agentAiCreditsSuffix, + evalsAiCredits, + evalsAiCreditsFormatted, + evalsAiCreditsSuffix, threatDetectionAiCredits, threatDetectionAiCreditsFormatted, threatDetectionAiCreditsSuffix, diff --git a/pkg/workflow/compiler_jobs_test.go b/pkg/workflow/compiler_jobs_test.go index d1a7b89612a..a717d1c655f 100644 --- a/pkg/workflow/compiler_jobs_test.go +++ b/pkg/workflow/compiler_jobs_test.go @@ -4095,6 +4095,7 @@ Test content` conclusionSection := extractJobSection(yamlStr, "conclusion") require.NotEmpty(t, conclusionSection, "conclusion job should be present") assert.Contains(t, conclusionSection, "push_evals_state", "conclusion job should depend on push_evals_state") + assert.Contains(t, conclusionSection, "GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }}", "conclusion job should pass evals AIC to footer generation") } // TestBuildMainJobEngineEnvNeedsExpression verifies that when engine.env values contain diff --git a/pkg/workflow/evals_job.go b/pkg/workflow/evals_job.go index 72aca454f38..15f58e09a47 100644 --- a/pkg/workflow/evals_job.go +++ b/pkg/workflow/evals_job.go @@ -98,7 +98,10 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { RunsOn: c.indentYAMLLines(runsOn, " "), Environment: c.indentYAMLLines(data.Environment, " "), Permissions: permissions, - Steps: steps, + Outputs: map[string]string{ + "aic": fmt.Sprintf("${{ steps.%s.outputs.aic }}", constants.ParseMCPGatewayStepID), + }, + Steps: steps, } return job, nil diff --git a/pkg/workflow/evals_job_test.go b/pkg/workflow/evals_job_test.go index 25e59df92eb..702cbd33a46 100644 --- a/pkg/workflow/evals_job_test.go +++ b/pkg/workflow/evals_job_test.go @@ -34,6 +34,7 @@ func TestBuildEvalsJobNeedsWithoutDetection(t *testing.T) { assert.NotContains(t, job.Needs, string(constants.DetectionJobName)) assert.Contains(t, job.If, "needs.agent.result") assert.NotContains(t, job.If, "needs.safe_outputs.result") + assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"]) } func TestBuildEvalsJobNeedsWithDetection(t *testing.T) { @@ -63,4 +64,5 @@ func TestBuildEvalsJobNeedsWithDetection(t *testing.T) { assert.NotContains(t, job.Needs, string(constants.SafeOutputsJobName)) assert.Contains(t, job.If, "needs.agent.result") assert.NotContains(t, job.If, "needs.safe_outputs.result") + assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"]) } diff --git a/pkg/workflow/notify_comment_conclusion_helpers.go b/pkg/workflow/notify_comment_conclusion_helpers.go index bdc7bb8a1af..a7b738451db 100644 --- a/pkg/workflow/notify_comment_conclusion_helpers.go +++ b/pkg/workflow/notify_comment_conclusion_helpers.go @@ -79,6 +79,9 @@ func (c *Compiler) buildConclusionNoOpStep(data *WorkflowData, mainJobName strin if IsDetectionJobEnabled(data.SafeOutputs) { envVars = append(envVars, fmt.Sprintf(" GH_AW_THREAT_DETECTION_AIC: ${{ needs.%s.outputs.aic }}\n", constants.DetectionJobName)) } + if data.Evals != nil && data.Evals.HasEvals() { + envVars = append(envVars, fmt.Sprintf(" GH_AW_EVALS_AIC: ${{ needs.%s.outputs.aic }}\n", constants.EvalsJobName)) + } envVars = append(envVars, fmt.Sprintf(" GH_AW_AMBIENT_CONTEXT: ${{ needs.%s.outputs.ambient_context }}\n", mainJobName)) if data.WorkflowID != "" { envVars = append(envVars, fmt.Sprintf(" GH_AW_WORKFLOW_ID: %q\n", data.WorkflowID)) @@ -224,6 +227,9 @@ func (c *Compiler) buildAgentFailureCoreVars(data *WorkflowData, mainJobName str if IsDetectionJobEnabled(data.SafeOutputs) { envVars = append(envVars, fmt.Sprintf(" GH_AW_THREAT_DETECTION_AIC: ${{ needs.%s.outputs.aic }}\n", constants.DetectionJobName)) } + if data.Evals != nil && data.Evals.HasEvals() { + envVars = append(envVars, fmt.Sprintf(" GH_AW_EVALS_AIC: ${{ needs.%s.outputs.aic }}\n", constants.EvalsJobName)) + } if data.EngineConfig != nil && data.EngineConfig.MaxAICredits != 0 { envVars = append(envVars, fmt.Sprintf(" GH_AW_MAX_AI_CREDITS: %q\n", strconv.FormatInt(data.EngineConfig.MaxAICredits, 10))) } else { From 0e1b1383bbde11c49248742ceb9e32de9d802d02 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:17:30 +0000 Subject: [PATCH 2/2] Fix evals AIC output parsing and add combined footer test Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/messages.test.cjs | 18 ++++++++++++++++++ pkg/workflow/evals_job_test.go | 3 +++ pkg/workflow/evals_steps.go | 26 ++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/actions/setup/js/messages.test.cjs b/actions/setup/js/messages.test.cjs index 8a71ee09439..59aa458db20 100644 --- a/actions/setup/js/messages.test.cjs +++ b/actions/setup/js/messages.test.cjs @@ -566,6 +566,24 @@ describe("messages.cjs", () => { expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC"); }); + it("should expose detection and evals AI Credits entries in order in custom footer templates", async () => { + process.env.GH_AW_AGENT_AIC = "1.25"; + process.env.GH_AW_THREAT_DETECTION_AIC = "0.25"; + process.env.GH_AW_EVALS_AIC = "0.05"; + process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({ + footer: "> Custom: [{workflow_name}]({run_url}){ai_credits_suffix}", + }); + + const { getFooterMessage } = await import("./messages.cjs"); + + const result = getFooterMessage({ + workflowName: "Test Workflow", + runUrl: "https://github.com/test/repo/actions/runs/123", + }); + + expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ⌖ 0.25 AIC · ◇ 0.05 AIC"); + }); + it("should include ambient context in ai_credits_suffix for custom footer templates", async () => { process.env.GH_AW_AGENT_AIC = "1.25"; process.env.GH_AW_THREAT_DETECTION_AIC = "0.25"; diff --git a/pkg/workflow/evals_job_test.go b/pkg/workflow/evals_job_test.go index 702cbd33a46..c031c41191c 100644 --- a/pkg/workflow/evals_job_test.go +++ b/pkg/workflow/evals_job_test.go @@ -1,6 +1,7 @@ package workflow import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -35,6 +36,7 @@ func TestBuildEvalsJobNeedsWithoutDetection(t *testing.T) { assert.Contains(t, job.If, "needs.agent.result") assert.NotContains(t, job.If, "needs.safe_outputs.result") assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"]) + assert.Contains(t, strings.Join(job.Steps, ""), "id: parse-mcp-gateway\n") } func TestBuildEvalsJobNeedsWithDetection(t *testing.T) { @@ -65,4 +67,5 @@ func TestBuildEvalsJobNeedsWithDetection(t *testing.T) { assert.Contains(t, job.If, "needs.agent.result") assert.NotContains(t, job.If, "needs.safe_outputs.result") assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"]) + assert.Contains(t, strings.Join(job.Steps, ""), "id: parse-mcp-gateway\n") } diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 63a8f98c505..48ba8fadf68 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -60,22 +60,40 @@ func (c *Compiler) buildEvalsJobSteps(data *WorkflowData) []string { // Steps 6 & 7: Install engine and execute via AWF (network-restricted sandbox). steps = append(steps, c.buildEvalsEngineSteps(data)...) - // Step 8: Parse engine output and write evals.jsonl. + // Step 8: Parse MCP gateway logs to capture evals job AIC output. + steps = append(steps, c.buildParseMCPGatewayLogStep(data)...) + + // Step 9: Parse engine output and write evals.jsonl. steps = append(steps, c.buildParseEvalsResultsStep(data)...) - // Step 9: Redact secrets from evals results before upload. + // Step 10: Redact secrets from evals results before upload. steps = append(steps, c.buildRedactEvalsSecretsStep(data)...) - // Step 10: Render evals results as a progressive disclosure step summary section. + // Step 11: Render evals results as a progressive disclosure step summary section. // Runs after redaction so the published summary is always free of secrets. steps = append(steps, c.buildRenderEvalsSummaryStep(data)...) - // Step 11: Upload evals.jsonl as the evals artifact. + // Step 12: Upload evals.jsonl as the evals artifact. steps = append(steps, c.buildUploadEvalsArtifactStep(data)...) return steps } +func (c *Compiler) buildParseMCPGatewayLogStep(data *WorkflowData) []string { + return []string{ + " - name: Parse MCP Gateway logs for step summary\n", + " if: always()\n", + fmt.Sprintf(" id: %s\n", constants.ParseMCPGatewayStepID), + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " with:\n", + " script: |\n", + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n", + " setupGlobals(core, github, context, exec, io, getOctokit);\n", + " const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs');\n", + " await main();\n", + } +} + // buildPrepareEvalsFilesStep creates a step that copies agent output files into the // evals working directory so the JS harness can read them for context. func buildPrepareEvalsFilesStep() []string {