Skip to content

Show evals AIC in generated footer metrics #47260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 3 commits into from
Jul 22, 2026
Merged

Show evals AIC in generated footer metrics #47260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cc95cfc
Add evals AIC to generated footers
Copilot Jul 22, 2026
2c194fc
Merge branch 'main' into copilot/update-generated-footer-evals-aic-cost
pelikhan Jul 22, 2026
0e1b138
Fix evals AIC output parsing and add combined footer test
Copilot Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions actions/setup/js/handle_noop_message.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,17 +79,19 @@ async function ensureAgentRunsIssue() {

/**
* Build the AIC suffix string for use in comment footers.
* Includes both agent and threat-detection AIC when available.
* Includes agent, threat-detection, and evals AIC when available.
* Returns a string like " · 0.001 AIC" or "" when not available.
* @returns {string}
*/
function buildAICSuffix() {
const agentRaw = process.env.GH_AW_AIC;
const detectionRaw = process.env.GH_AW_THREAT_DETECTION_AIC;
const evalsRaw = process.env.GH_AW_EVALS_AIC;
const agentAIC = agentRaw ? Number.parseFloat(agentRaw) : NaN;
const detectionAIC = detectionRaw ? Number.parseFloat(detectionRaw) : NaN;
const evalsAIC = evalsRaw ? Number.parseFloat(evalsRaw) : NaN;
const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL);
const totalAIC = (Number.isFinite(agentAIC) && agentAIC > 0 ? agentAIC : 0) + (Number.isFinite(detectionAIC) && detectionAIC > 0 ? detectionAIC : 0);
const totalAIC = (Number.isFinite(agentAIC) && agentAIC > 0 ? agentAIC : 0) + (Number.isFinite(detectionAIC) && detectionAIC > 0 ? detectionAIC : 0) + (Number.isFinite(evalsAIC) && evalsAIC > 0 ? evalsAIC : 0);
if (totalAIC <= 0) {
return "";
}
Expand Down
24 changes: 24 additions & 0 deletions actions/setup/js/handle_noop_message.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -781,6 +781,30 @@ This issue helps you:
expect(commentCall.body).toContain("sonnet46 0.125 AIC");
});

it("should include evals AIC in the footer total when GH_AW_EVALS_AIC is set", async () => {
process.env.GH_AW_WORKFLOW_NAME = "Evals AIC Workflow";
process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123";
process.env.GH_AW_AGENT_CONCLUSION = "success";
process.env.GH_AW_AIC = "0.100";
process.env.GH_AW_EVALS_AIC = "0.025";
process.env.GH_AW_ENGINE_MODEL = "claude-sonnet-4.6";

const outputFile = path.join(tempDir, "agent_output.json");
fs.writeFileSync(outputFile, JSON.stringify({ items: [{ type: "noop", message: "No action needed" }] }));
process.env.GH_AW_AGENT_OUTPUT = outputFile;

mockGithub.rest.search.issuesAndPullRequests.mockResolvedValue({
data: { total_count: 1, items: [{ number: 1, node_id: "ID", html_url: "url" }] },
});
mockGithub.rest.issues.createComment.mockResolvedValue({ data: {} });

const { main } = await import("./handle_noop_message.cjs?t=" + Date.now());
await main();

const commentCall = mockGithub.rest.issues.createComment.mock.calls[0][0];
expect(commentCall.body).toContain("sonnet46 0.125 AIC");
});

it("should not include AIC suffix in comment footer when GH_AW_AIC is not set", async () => {
process.env.GH_AW_WORKFLOW_NAME = "No Credits Workflow";
process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123";
Expand Down
78 changes: 78 additions & 0 deletions actions/setup/js/messages.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ describe("messages.cjs", () => {
delete process.env.GH_AW_AIC;
delete process.env.GH_AW_AMBIENT_CONTEXT;
delete process.env.GH_AW_AGENT_AIC;
delete process.env.GH_AW_EVALS_AIC;
delete process.env.GH_AW_THREAT_DETECTION_AIC;
delete process.env.GH_AW_LABEL_COMMANDS;
delete process.env.GH_AW_DETECTION_CONCLUSION;
Expand Down Expand Up @@ -472,6 +473,20 @@ describe("messages.cjs", () => {
expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · 0.125 AIC · ⌖ 0.025 AIC · ⊞ 900");
});

it("should include evals AI Credits with a Unicode icon in the default footer", async () => {
process.env.GH_AW_AGENT_AIC = "0.125";
process.env.GH_AW_EVALS_AIC = "0.01";

const { getFooterMessage } = await import("./messages.cjs");

const result = getFooterMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · 0.125 AIC · ◇ 0.01 AIC");
});

it("should not include effective tokens when GH_AW_EFFECTIVE_TOKENS is not set and not passed in context", async () => {
delete process.env.GH_AW_EFFECTIVE_TOKENS;

Expand Down Expand Up @@ -534,6 +549,41 @@ describe("messages.cjs", () => {
expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ⌖ 0.25 AIC");
});

it("should expose evals AI Credits entry in custom footer templates", async () => {
process.env.GH_AW_AGENT_AIC = "1.25";
process.env.GH_AW_EVALS_AIC = "0.25";
process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({
footer: "> Custom: [{workflow_name}]({run_url}){ai_credits_suffix}",
});

const { getFooterMessage } = await import("./messages.cjs");

const result = getFooterMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC");
});

it("should expose detection and evals AI Credits entries in order in custom footer templates", async () => {
process.env.GH_AW_AGENT_AIC = "1.25";
process.env.GH_AW_THREAT_DETECTION_AIC = "0.25";
process.env.GH_AW_EVALS_AIC = "0.05";
process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({
footer: "> Custom: [{workflow_name}]({run_url}){ai_credits_suffix}",
});

const { getFooterMessage } = await import("./messages.cjs");

const result = getFooterMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ⌖ 0.25 AIC · ◇ 0.05 AIC");
});

it("should include ambient context in ai_credits_suffix for custom footer templates", async () => {
process.env.GH_AW_AGENT_AIC = "1.25";
process.env.GH_AW_THREAT_DETECTION_AIC = "0.25";

@github-actions github-actions Bot Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] No test covers the combined case where both GH_AW_THREAT_DETECTION_AIC and GH_AW_EVALS_AIC are set simultaneously — the ordering agent → detection → evals in the suffix and the combined total are untested together.

💡 Suggested test 
it("should include both detection and evals AIC entries in correct order", async () => {
  process.env.GH_AW_AGENT_AIC = "1.00";
  process.env.GH_AW_THREAT_DETECTION_AIC = "0.10";
  process.env.GH_AW_EVALS_AIC = "0.05";
  const { getFooterMessage } = await import("./messages.cjs");
  const result = getFooterMessage({ workflowName: "W", runUrl: "(u/redacted)" });
  // total = 1.15, detection before evals
  expect(result).toContain("⌖ 0.1 AIC · ◇ 0.05 AIC");
});

@copilot please address this.

Expand Down Expand Up @@ -1156,6 +1206,20 @@ describe("messages.cjs", () => {
expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · sonnet46 1.25 AIC · ⊞ 900");
});

it("should include evals AI Credits in the default footer when available", async () => {
process.env.GH_AW_AIC = "1.25";
process.env.GH_AW_EVALS_AIC = "0.25";

const { getFooterAgentFailureIssueMessage } = await import("./messages.cjs");

const result = getFooterAgentFailureIssueMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC");
});

it("should suppress env AIC fallback when explicit context AIC is zero", async () => {
process.env.GH_AW_AIC = "1.25";

Expand Down Expand Up @@ -1256,6 +1320,20 @@ describe("messages.cjs", () => {
expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · sonnet46 1.25 AIC · ⊞ 900");
});

it("should include evals AI Credits in the default comment footer when available", async () => {
process.env.GH_AW_AIC = "1.25";
process.env.GH_AW_EVALS_AIC = "0.25";

const { getFooterAgentFailureCommentMessage } = await import("./messages.cjs");

const result = getFooterAgentFailureCommentMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Generated from [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC · ◇ 0.25 AIC");
});

it("should expose ai_credits_suffix in custom comment footer templates", async () => {
process.env.GH_AW_AMBIENT_CONTEXT = "900";
process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({
Expand Down
32 changes: 29 additions & 3 deletions actions/setup/js/messages_footer.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ function buildAICEntry(label, value, modelAlias) {
* agentAiCredits: number|undefined,
* agentAiCreditsFormatted: string|undefined,
* agentAiCreditsSuffix: string,
* evalsAiCredits: number|undefined,
* evalsAiCreditsFormatted: string|undefined,
* evalsAiCreditsSuffix: string,
* threatDetectionAiCredits: number|undefined,
* threatDetectionAiCreditsFormatted: string|undefined,
* threatDetectionAiCreditsSuffix: string
Expand All @@ -110,14 +113,16 @@ function getAICFromEnv() {
const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL);
const totalAIC = parsePositiveAIC(process.env.GH_AW_AIC);
const explicitAgentAIC = parsePositiveAIC(process.env.GH_AW_AGENT_AIC);
const evalsAIC = parsePositiveAIC(process.env.GH_AW_EVALS_AIC);
const threatDetectionAIC = parsePositiveAIC(process.env.GH_AW_THREAT_DETECTION_AIC);
const agentAIC = typeof explicitAgentAIC === "number" ? explicitAgentAIC : totalAIC;
const agentEntry = buildAICEntry("", agentAIC, compressedModelName);
const evalsEntry = buildAICEntry("◇", evalsAIC);
const threatDetectionEntry = buildAICEntry("⌖", threatDetectionAIC);
const useBreakdown = threatDetectionEntry.suffix.length > 0;
const aiCredits = useBreakdown ? (agentAIC || 0) + (threatDetectionAIC || 0) : typeof totalAIC === "number" ? totalAIC : agentAIC;
const useBreakdown = threatDetectionEntry.suffix.length > 0 || evalsEntry.suffix.length > 0;
const aiCredits = useBreakdown ? (agentAIC || 0) + (threatDetectionAIC || 0) + (evalsAIC || 0) : typeof totalAIC === "number" ? totalAIC : agentAIC;
const aiCreditsFormatted = typeof aiCredits === "number" ? formatAIC(aiCredits) : undefined;
const aiCreditsSuffix = useBreakdown ? `${agentEntry.suffix}${threatDetectionEntry.suffix}` : buildAICEntry("", aiCredits, compressedModelName).suffix;
const aiCreditsSuffix = useBreakdown ? `${agentEntry.suffix}${threatDetectionEntry.suffix}${evalsEntry.suffix}` : buildAICEntry("", aiCredits, compressedModelName).suffix;

return {
aiCredits,
Expand All @@ -127,6 +132,9 @@ function getAICFromEnv() {
agentAiCredits: agentEntry.value,
agentAiCreditsFormatted: agentEntry.formatted,
agentAiCreditsSuffix: agentEntry.suffix,
evalsAiCredits: evalsEntry.value,
evalsAiCreditsFormatted: evalsEntry.formatted,
evalsAiCreditsSuffix: evalsEntry.suffix,
threatDetectionAiCredits: threatDetectionEntry.value,
threatDetectionAiCreditsFormatted: threatDetectionEntry.formatted,
threatDetectionAiCreditsSuffix: threatDetectionEntry.suffix,
Expand Down Expand Up @@ -167,6 +175,9 @@ function getFooterMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand Down Expand Up @@ -205,6 +216,9 @@ function getFooterMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand Down Expand Up @@ -387,6 +401,9 @@ function getFooterAgentFailureIssueMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand All @@ -410,6 +427,9 @@ function getFooterAgentFailureIssueMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand Down Expand Up @@ -463,6 +483,9 @@ function getFooterAgentFailureCommentMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand All @@ -486,6 +509,9 @@ function getFooterAgentFailureCommentMessage(ctx) {
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
evalsAiCredits,
evalsAiCreditsFormatted,
evalsAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
Expand Down
1 change: 1 addition & 0 deletions pkg/workflow/compiler_jobs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4095,6 +4095,7 @@ Test content`
conclusionSection := extractJobSection(yamlStr, "conclusion")
require.NotEmpty(t, conclusionSection, "conclusion job should be present")
assert.Contains(t, conclusionSection, "push_evals_state", "conclusion job should depend on push_evals_state")
assert.Contains(t, conclusionSection, "GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }}", "conclusion job should pass evals AIC to footer generation")
}

// TestBuildMainJobEngineEnvNeedsExpression verifies that when engine.env values contain
Expand Down
5 changes: 4 additions & 1 deletion pkg/workflow/evals_job.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) {
RunsOn: c.indentYAMLLines(runsOn, " "),
Environment: c.indentYAMLLines(data.Environment, " "),
Permissions: permissions,
Steps: steps,
Outputs: map[string]string{
"aic": fmt.Sprintf("${{ steps.%s.outputs.aic }}", constants.ParseMCPGatewayStepID),
},
Comment on lines +101 to +103
Steps: steps,
}

return job, nil
Expand Down
5 changes: 5 additions & 0 deletions pkg/workflow/evals_job_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package workflow

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -34,6 +35,8 @@ func TestBuildEvalsJobNeedsWithoutDetection(t *testing.T) {
assert.NotContains(t, job.Needs, string(constants.DetectionJobName))
assert.Contains(t, job.If, "needs.agent.result")
assert.NotContains(t, job.If, "needs.safe_outputs.result")
assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"])
assert.Contains(t, strings.Join(job.Steps, ""), "id: parse-mcp-gateway\n")
}

func TestBuildEvalsJobNeedsWithDetection(t *testing.T) {
Expand Down Expand Up @@ -63,4 +66,6 @@ func TestBuildEvalsJobNeedsWithDetection(t *testing.T) {
assert.NotContains(t, job.Needs, string(constants.SafeOutputsJobName))
assert.Contains(t, job.If, "needs.agent.result")
assert.NotContains(t, job.If, "needs.safe_outputs.result")
assert.Equal(t, "${{ steps.parse-mcp-gateway.outputs.aic }}", job.Outputs["aic"])
assert.Contains(t, strings.Join(job.Steps, ""), "id: parse-mcp-gateway\n")
}
26 changes: 22 additions & 4 deletions pkg/workflow/evals_steps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,22 +60,40 @@ func (c *Compiler) buildEvalsJobSteps(data *WorkflowData) []string {
// Steps 6 & 7: Install engine and execute via AWF (network-restricted sandbox).
steps = append(steps, c.buildEvalsEngineSteps(data)...)

// Step 8: Parse engine output and write evals.jsonl.
// Step 8: Parse MCP gateway logs to capture evals job AIC output.
steps = append(steps, c.buildParseMCPGatewayLogStep(data)...)

// Step 9: Parse engine output and write evals.jsonl.
steps = append(steps, c.buildParseEvalsResultsStep(data)...)

// Step 9: Redact secrets from evals results before upload.
// Step 10: Redact secrets from evals results before upload.
steps = append(steps, c.buildRedactEvalsSecretsStep(data)...)

// Step 10: Render evals results as a progressive disclosure step summary section.
// Step 11: Render evals results as a progressive disclosure step summary section.
// Runs after redaction so the published summary is always free of secrets.
steps = append(steps, c.buildRenderEvalsSummaryStep(data)...)

// Step 11: Upload evals.jsonl as the evals artifact.
// Step 12: Upload evals.jsonl as the evals artifact.
steps = append(steps, c.buildUploadEvalsArtifactStep(data)...)

return steps
}

func (c *Compiler) buildParseMCPGatewayLogStep(data *WorkflowData) []string {
return []string{
" - name: Parse MCP Gateway logs for step summary\n",
" if: always()\n",
fmt.Sprintf(" id: %s\n", constants.ParseMCPGatewayStepID),
fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)),
" with:\n",
" script: |\n",
" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n",
" setupGlobals(core, github, context, exec, io, getOctokit);\n",
" const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs');\n",
" await main();\n",
}
}

// buildPrepareEvalsFilesStep creates a step that copies agent output files into the
// evals working directory so the JS harness can read them for context.
func buildPrepareEvalsFilesStep() []string {
Expand Down
6 changes: 6 additions & 0 deletions pkg/workflow/notify_comment_conclusion_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ func (c *Compiler) buildConclusionNoOpStep(data *WorkflowData, mainJobName strin
if IsDetectionJobEnabled(data.SafeOutputs) {
envVars = append(envVars, fmt.Sprintf(" GH_AW_THREAT_DETECTION_AIC: ${{ needs.%s.outputs.aic }}\n", constants.DetectionJobName))
}
if data.Evals != nil && data.Evals.HasEvals() {
envVars = append(envVars, fmt.Sprintf(" GH_AW_EVALS_AIC: ${{ needs.%s.outputs.aic }}\n", constants.EvalsJobName))
}
envVars = append(envVars, fmt.Sprintf(" GH_AW_AMBIENT_CONTEXT: ${{ needs.%s.outputs.ambient_context }}\n", mainJobName))
if data.WorkflowID != "" {
envVars = append(envVars, fmt.Sprintf(" GH_AW_WORKFLOW_ID: %q\n", data.WorkflowID))
Expand Down Expand Up @@ -224,6 +227,9 @@ func (c *Compiler) buildAgentFailureCoreVars(data *WorkflowData, mainJobName str
if IsDetectionJobEnabled(data.SafeOutputs) {
envVars = append(envVars, fmt.Sprintf(" GH_AW_THREAT_DETECTION_AIC: ${{ needs.%s.outputs.aic }}\n", constants.DetectionJobName))
}
if data.Evals != nil && data.Evals.HasEvals() {
envVars = append(envVars, fmt.Sprintf(" GH_AW_EVALS_AIC: ${{ needs.%s.outputs.aic }}\n", constants.EvalsJobName))
}
if data.EngineConfig != nil && data.EngineConfig.MaxAICredits != 0 {
envVars = append(envVars, fmt.Sprintf(" GH_AW_MAX_AI_CREDITS: %q\n", strconv.FormatInt(data.EngineConfig.MaxAICredits, 10)))
} else {
Expand Down
Loading